From eb3f145e574faaaf05f387ccd3bd2fa16507dc84 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Wed, 24 Aug 2016 13:19:33 +0200 Subject: [PATCH 01/21] added UNet + example (road segmentation) --- .gitignore | 13 + examples/UNet/mass_roads_test.txt | 49 + examples/UNet/mass_roads_train.txt | 1108 ++++++++++++++++++++++ examples/UNet/mass_roads_validation.txt | 14 + examples/UNet/massachussets_road_segm.py | 333 +++++++ modelzoo/Unet.py | 62 ++ 6 files changed, 1579 insertions(+) create mode 100644 examples/UNet/mass_roads_test.txt create mode 100644 examples/UNet/mass_roads_train.txt create mode 100644 examples/UNet/mass_roads_validation.txt create mode 100644 examples/UNet/massachussets_road_segm.py create mode 100644 modelzoo/Unet.py diff --git a/.gitignore b/.gitignore index ba74660..a9bca33 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ __pycache__/ # C extensions *.so +# datasets +*.zip + # Distribution / packaging .Python env/ @@ -22,6 +25,7 @@ var/ *.egg-info/ .installed.cfg *.egg +.idea # PyInstaller # Usually these files are written by a python script from a template @@ -55,3 +59,12 @@ docs/_build/ # PyBuilder target/ + +examples/UNet/data* +examples/UNet/*.sh +examples/UNet/*.npy +examples/UNet/*.pkl +examples/UNet/tmp* +examples/UNet/best_parameters* +.gitignore~ + diff --git a/examples/UNet/mass_roads_test.txt b/examples/UNet/mass_roads_test.txt new file mode 100644 index 0000000..afd6d7a --- /dev/null +++ b/examples/UNet/mass_roads_test.txt @@ -0,0 +1,49 @@ +10378780_15.tiff +10828720_15.tiff +11128870_15.tiff +11278840_15.tiff +11728825_15.tiff +12328750_15.tiff +15928855_15.tiff +16078870_15.tiff +17878735_15.tiff +17878780_15.tiff +17878885_15.tiff +18028750_15.tiff +18178780_15.tiff +18328735_15.tiff +18328780_15.tiff +18328960_15.tiff +18478735_15.tiff +18478900_15.tiff +18478930_15.tiff +20278885_15.tiff +20728960_15.tiff +20878930_15.tiff +21479035_15.tiff +21779005_15.tiff +22078975_15.tiff +22228900_15.tiff +22229050_15.tiff +22528885_15.tiff +22529065_15.tiff +23278915_15.tiff +23278930_15.tiff +23428540_15.tiff +23428810_15.tiff +23429080_15.tiff +23729020_15.tiff +23878540_15.tiff +24029035_15.tiff +24478825_15.tiff +24478855_15.tiff +24479170_15.tiff +24479215_15.tiff +24628885_15.tiff +24629200_15.tiff +24779275_15.tiff +25079170_15.tiff +26278720_15.tiff +26428735_15.tiff +26578720_15.tiff +26878690_15.tiff diff --git a/examples/UNet/mass_roads_train.txt b/examples/UNet/mass_roads_train.txt new file mode 100644 index 0000000..144dd39 --- /dev/null +++ b/examples/UNet/mass_roads_train.txt @@ -0,0 +1,1108 @@ +10078660_15.tif +10078675_15.tif +10078690_15.tif +10078705_15.tif +10078720_15.tif +10078735_15.tif +10078750_15.tif +10228660_15.tif +10228675_15.tif +10228705_15.tif +10228720_15.tif +10228735_15.tif +10228750_15.tif +10228765_15.tif +10228780_15.tif +10228795_15.tif +10378660_15.tif +10378675_15.tif +10378690_15.tif +10378705_15.tif +10378720_15.tif +10378735_15.tif +10378750_15.tif +10378765_15.tif +10378795_15.tif +10528660_15.tif +10528675_15.tif +10528690_15.tif +10528705_15.tif +10528720_15.tif +10528735_15.tif +10528750_15.tif +10528765_15.tif +10528780_15.tif +10528795_15.tif +10678660_15.tif +10678675_15.tif +10678690_15.tif +10678705_15.tif +10678720_15.tif +10678735_15.tif +10678750_15.tif +10678765_15.tif +10678780_15.tif +10678795_15.tif +10678810_15.tif +10678825_15.tif +10678870_15.tif +10678885_15.tif +10828645_15.tif +10828660_15.tif +10828675_15.tif +10828690_15.tif +10828705_15.tif +10828735_15.tif +10828750_15.tif +10828765_15.tif +10828780_15.tif +10828795_15.tif +10828810_15.tif +10828825_15.tif +10828855_15.tif +10828870_15.tif +10828900_15.tif +10978645_15.tif +10978660_15.tif +10978675_15.tif +10978690_15.tif +10978705_15.tif +10978720_15.tif +10978750_15.tif +10978765_15.tif +10978780_15.tif +10978810_15.tif +10978825_15.tif +10978840_15.tif +10978855_15.tif +10978870_15.tif +10978885_15.tif +10978900_15.tif +10978915_15.tif +10978930_15.tif +10978945_15.tif +11128645_15.tif +11128660_15.tif +11128675_15.tif +11128690_15.tif +11128705_15.tif +11128720_15.tif +11128735_15.tif +11128750_15.tif +11128765_15.tif +11128780_15.tif +11128795_15.tif +11128810_15.tif +11128825_15.tif +11128840_15.tif +11128855_15.tif +11128885_15.tif +11128900_15.tif +11128915_15.tif +11128930_15.tif +11128945_15.tif +11278645_15.tif +11278660_15.tif +11278675_15.tif +11278690_15.tif +11278705_15.tif +11278720_15.tif +11278735_15.tif +11278750_15.tif +11278765_15.tif +11278780_15.tif +11278795_15.tif +11278810_15.tif +11278825_15.tif +11278855_15.tif +11278870_15.tif +11278885_15.tif +11278900_15.tif +11278915_15.tif +11278930_15.tif +11278945_15.tif +11428660_15.tif +11428675_15.tif +11428690_15.tif +11428705_15.tif +11428720_15.tif +11428735_15.tif +11428750_15.tif +11428765_15.tif +11428780_15.tif +11428795_15.tif +11428810_15.tif +11428825_15.tif +11428840_15.tif +11428855_15.tif +11428930_15.tif +11428945_15.tif +11578660_15.tif +11578675_15.tif +11578690_15.tif +11578705_15.tif +11578720_15.tif +11578735_15.tif +11578750_15.tif +11578765_15.tif +11578780_15.tif +11578795_15.tif +11578810_15.tif +11578825_15.tif +11578840_15.tif +11578855_15.tif +11728660_15.tif +11728675_15.tif +11728690_15.tif +11728705_15.tif +11728720_15.tif +11728735_15.tif +11728750_15.tif +11728765_15.tif +11728780_15.tif +11728795_15.tif +11728810_15.tif +11728840_15.tif +11728855_15.tif +11878660_15.tif +11878675_15.tif +11878690_15.tif +11878705_15.tif +11878720_15.tif +11878735_15.tif +11878750_15.tif +11878765_15.tif +11878780_15.tif +11878795_15.tif +12028660_15.tif +12028675_15.tif +12028690_15.tif +12028705_15.tif +12028720_15.tif +12028735_15.tif +12028750_15.tif +12028765_15.tif +12028780_15.tif +12178705_15.tif +12178720_15.tif +12178735_15.tif +12178750_15.tif +12178765_15.tif +12178780_15.tif +12328720_15.tif +12328735_15.tif +12328765_15.tif +12328780_15.tif +12478720_15.tif +12478735_15.tif +12478750_15.tif +12478765_15.tif +12478780_15.tif +12478795_15.tif +12478810_15.tif +12628735_15.tif +12628750_15.tif +12628765_15.tif +12628780_15.tif +12628795_15.tif +12628810_15.tif +15628825_15.tif +15628840_15.tif +15628855_15.tif +15628870_15.tif +15628885_15.tif +15628900_15.tif +15628915_15.tif +15628930_15.tif +15628945_15.tif +15628960_15.tif +15778825_15.tif +15778840_15.tif +15778855_15.tif +15778870_15.tif +15778885_15.tif +15778900_15.tif +15778915_15.tif +15778930_15.tif +15778945_15.tif +15778960_15.tif +15928825_15.tif +15928840_15.tif +15928870_15.tif +15928885_15.tif +15928900_15.tif +15928915_15.tif +15928930_15.tif +15928945_15.tif +15928960_15.tif +16078825_15.tif +16078840_15.tif +16078855_15.tif +16078885_15.tif +16078900_15.tif +16078915_15.tif +16078930_15.tif +16078945_15.tif +16078960_15.tif +16228825_15.tif +16228840_15.tif +16228855_15.tif +16228870_15.tif +16228885_15.tif +16228900_15.tif +16228915_15.tif +16228930_15.tif +16228945_15.tif +16228960_15.tif +16378825_15.tif +16378840_15.tif +16378855_15.tif +16828900_15.tif +16828915_15.tif +16828930_15.tif +16978870_15.tif +16978885_15.tif +16978915_15.tif +16978930_15.tif +16978945_15.tif +17128870_15.tif +17128885_15.tif +17128900_15.tif +17128915_15.tif +17128930_15.tif +17128945_15.tif +17128960_15.tif +17278750_15.tif +17278765_15.tif +17278780_15.tif +17278870_15.tif +17278885_15.tif +17278900_15.tif +17278915_15.tif +17278930_15.tif +17278945_15.tif +17278960_15.tif +17428735_15.tif +17428750_15.tif +17428765_15.tif +17428780_15.tif +17428855_15.tif +17428870_15.tif +17428885_15.tif +17428900_15.tif +17428915_15.tif +17428930_15.tif +17428945_15.tif +17428960_15.tif +17428975_15.tif +17578720_15.tif +17578735_15.tif +17578750_15.tif +17578765_15.tif +17578780_15.tif +17578795_15.tif +17578840_15.tif +17578855_15.tif +17578870_15.tif +17578885_15.tif +17578900_15.tif +17578915_15.tif +17578930_15.tif +17578945_15.tif +17578960_15.tif +17578975_15.tif +17578990_15.tif +17728705_15.tif +17728720_15.tif +17728735_15.tif +17728750_15.tif +17728765_15.tif +17728780_15.tif +17728795_15.tif +17728810_15.tif +17728855_15.tif +17728870_15.tif +17728885_15.tif +17728900_15.tif +17728915_15.tif +17728930_15.tif +17728945_15.tif +17728960_15.tif +17728975_15.tif +17728990_15.tif +17878705_15.tif +17878720_15.tif +17878750_15.tif +17878765_15.tif +17878795_15.tif +17878810_15.tif +17878855_15.tif +17878870_15.tif +17878900_15.tif +17878915_15.tif +17878930_15.tif +17878945_15.tif +17878960_15.tif +18028705_15.tif +18028720_15.tif +18028735_15.tif +18028765_15.tif +18028780_15.tif +18028795_15.tif +18028810_15.tif +18028825_15.tif +18028870_15.tif +18028885_15.tif +18028900_15.tif +18028915_15.tif +18028930_15.tif +18028960_15.tif +18178705_15.tif +18178720_15.tif +18178735_15.tif +18178750_15.tif +18178765_15.tif +18178795_15.tif +18178810_15.tif +18178825_15.tif +18178885_15.tif +18178900_15.tif +18178915_15.tif +18178930_15.tif +18178945_15.tif +18178960_15.tif +18178975_15.tif +18328720_15.tif +18328750_15.tif +18328765_15.tif +18328795_15.tif +18328810_15.tif +18328885_15.tif +18328900_15.tif +18328915_15.tif +18328930_15.tif +18328945_15.tif +18328975_15.tif +18478720_15.tif +18478750_15.tif +18478765_15.tif +18478780_15.tif +18478795_15.tif +18478885_15.tif +18478915_15.tif +18478945_15.tif +18478960_15.tif +18478975_15.tif +18628720_15.tif +18628735_15.tif +18628750_15.tif +18628765_15.tif +18628780_15.tif +18628795_15.tif +18628885_15.tif +18628900_15.tif +18628915_15.tif +18628930_15.tif +18778720_15.tif +18778735_15.tif +18778750_15.tif +18778765_15.tif +18778780_15.tif +18778795_15.tif +18928720_15.tif +18928735_15.tif +18928750_15.tif +18928765_15.tif +18928780_15.tif +18928795_15.tif +19078735_15.tif +19078750_15.tif +19078765_15.tif +19228735_15.tif +19228750_15.tif +19528630_15.tif +19528645_15.tif +19528660_15.tif +19528675_15.tif +19678630_15.tif +19678645_15.tif +19678660_15.tif +19678675_15.tif +19828630_15.tif +19828645_15.tif +19828660_15.tif +19828675_15.tif +19828885_15.tif +19828900_15.tif +19828915_15.tif +19978630_15.tif +19978645_15.tif +19978660_15.tif +19978675_15.tif +19978885_15.tif +19978900_15.tif +19978915_15.tif +19978930_15.tif +19978945_15.tif +20128870_15.tif +20128885_15.tif +20128900_15.tif +20128915_15.tif +20128930_15.tif +20128945_15.tif +20128960_15.tif +20128975_15.tif +20128990_15.tif +20129005_15.tif +20278870_15.tif +20278900_15.tif +20278915_15.tif +20278930_15.tif +20278945_15.tif +20278960_15.tif +20278975_15.tif +20278990_15.tif +20279005_15.tif +20428870_15.tif +20428885_15.tif +20428900_15.tif +20428915_15.tif +20428930_15.tif +20428945_15.tif +20428960_15.tif +20428975_15.tif +20428990_15.tif +20429005_15.tif +20578870_15.tif +20578885_15.tif +20578900_15.tif +20578915_15.tif +20578930_15.tif +20578945_15.tif +20578960_15.tif +20578975_15.tif +20578990_15.tif +20579005_15.tif +20728870_15.tif +20728885_15.tif +20728900_15.tif +20728915_15.tif +20728930_15.tif +20728945_15.tif +20728975_15.tif +20728990_15.tif +20729005_15.tif +20878900_15.tif +20878915_15.tif +20878945_15.tif +20878960_15.tif +20878975_15.tif +20878990_15.tif +20879005_15.tif +21028900_15.tif +21028915_15.tif +21028930_15.tif +21028945_15.tif +21028960_15.tif +21028975_15.tif +21178885_15.tif +21178900_15.tif +21178915_15.tif +21178930_15.tif +21178945_15.tif +21178960_15.tif +21178975_15.tif +21328870_15.tif +21328885_15.tif +21328900_15.tif +21328915_15.tif +21328930_15.tif +21328945_15.tif +21328960_15.tif +21328975_15.tif +21328990_15.tif +21329005_15.tif +21329020_15.tif +21329035_15.tif +21329050_15.tif +21478870_15.tif +21478885_15.tif +21478900_15.tif +21478915_15.tif +21478930_15.tif +21478945_15.tif +21478960_15.tif +21478975_15.tif +21478990_15.tif +21479005_15.tif +21479020_15.tif +21479050_15.tif +21479065_15.tif +21628870_15.tif +21628885_15.tif +21628900_15.tif +21628915_15.tif +21628930_15.tif +21628945_15.tif +21628960_15.tif +21628975_15.tif +21628990_15.tif +21629005_15.tif +21629020_15.tif +21629035_15.tif +21629050_15.tif +21629065_15.tif +21778900_15.tif +21778915_15.tif +21778930_15.tif +21778945_15.tif +21778960_15.tif +21778975_15.tif +21778990_15.tif +21779020_15.tif +21779035_15.tif +21779050_15.tif +21779065_15.tif +21779080_15.tif +21928900_15.tif +21928915_15.tif +21928930_15.tif +21928945_15.tif +21928960_15.tif +21928975_15.tif +21928990_15.tif +21929005_15.tif +21929035_15.tif +21929050_15.tif +21929065_15.tif +21929080_15.tif +22078900_15.tif +22078915_15.tif +22078930_15.tif +22078945_15.tif +22078960_15.tif +22078990_15.tif +22079005_15.tif +22079020_15.tif +22079035_15.tif +22079050_15.tif +22079065_15.tif +22079080_15.tif +22079365_15.tif +22079380_15.tif +22079395_15.tif +22079410_15.tif +22079425_15.tif +22228915_15.tif +22228930_15.tif +22228945_15.tif +22228960_15.tif +22228975_15.tif +22228990_15.tif +22229005_15.tif +22229020_15.tif +22229035_15.tif +22229065_15.tif +22229080_15.tif +22229365_15.tif +22229380_15.tif +22229395_15.tif +22229410_15.tif +22229425_15.tif +22229440_15.tif +22378885_15.tif +22378900_15.tif +22378915_15.tif +22378930_15.tif +22378945_15.tif +22378960_15.tif +22378975_15.tif +22378990_15.tif +22379005_15.tif +22379020_15.tif +22379035_15.tif +22379050_15.tif +22379065_15.tif +22379080_15.tif +22379380_15.tif +22379395_15.tif +22379410_15.tif +22379425_15.tif +22379440_15.tif +22528870_15.tif +22528915_15.tif +22528930_15.tif +22528945_15.tif +22528960_15.tif +22528975_15.tif +22528990_15.tif +22529005_15.tif +22529020_15.tif +22529035_15.tif +22529050_15.tif +22529080_15.tif +22529380_15.tif +22529395_15.tif +22529410_15.tif +22529425_15.tif +22529440_15.tif +22529455_15.tif +22529470_15.tif +22529485_15.tif +22678855_15.tif +22678870_15.tif +22678885_15.tif +22678900_15.tif +22678915_15.tif +22678930_15.tif +22678945_15.tif +22678960_15.tif +22678975_15.tif +22678990_15.tif +22679005_15.tif +22679020_15.tif +22679035_15.tif +22679050_15.tif +22679065_15.tif +22679080_15.tif +22679410_15.tif +22679425_15.tif +22679440_15.tif +22679455_15.tif +22679470_15.tif +22679485_15.tif +22828855_15.tif +22828870_15.tif +22828885_15.tif +22828900_15.tif +22828915_15.tif +22828930_15.tif +22828945_15.tif +22828960_15.tif +22828975_15.tif +22828990_15.tif +22829005_15.tif +22829020_15.tif +22829050_15.tif +22829065_15.tif +22829410_15.tif +22829425_15.tif +22829440_15.tif +22829455_15.tif +22829470_15.tif +22829485_15.tif +22978840_15.tif +22978855_15.tif +22978870_15.tif +22978885_15.tif +22978900_15.tif +22978915_15.tif +22978930_15.tif +22978945_15.tif +22978960_15.tif +22978975_15.tif +22979005_15.tif +22979020_15.tif +22979035_15.tif +22979050_15.tif +22979065_15.tif +22979410_15.tif +22979425_15.tif +22979440_15.tif +22979455_15.tif +22979470_15.tif +23128870_15.tif +23128885_15.tif +23128900_15.tif +23128915_15.tif +23128945_15.tif +23128960_15.tif +23128975_15.tif +23128990_15.tif +23129005_15.tif +23129020_15.tif +23129035_15.tif +23129050_15.tif +23129065_15.tif +23129110_15.tif +23129125_15.tif +23129140_15.tif +23129155_15.tif +23129170_15.tif +23129410_15.tif +23129425_15.tif +23129440_15.tif +23129455_15.tif +23278825_15.tif +23278840_15.tif +23278885_15.tif +23278900_15.tif +23278945_15.tif +23278960_15.tif +23278975_15.tif +23278990_15.tif +23279005_15.tif +23279020_15.tif +23279035_15.tif +23279050_15.tif +23279080_15.tif +23279095_15.tif +23279110_15.tif +23279125_15.tif +23279140_15.tif +23279155_15.tif +23279170_15.tif +23428510_15.tif +23428525_15.tif +23428555_15.tif +23428570_15.tif +23428585_15.tif +23428825_15.tif +23428840_15.tif +23428900_15.tif +23428915_15.tif +23428930_15.tif +23428945_15.tif +23428960_15.tif +23428975_15.tif +23428990_15.tif +23429005_15.tif +23429020_15.tif +23429035_15.tif +23429050_15.tif +23429065_15.tif +23429095_15.tif +23429110_15.tif +23429125_15.tif +23429140_15.tif +23429155_15.tif +23429170_15.tif +23578495_15.tif +23578510_15.tif +23578525_15.tif +23578540_15.tif +23578555_15.tif +23578570_15.tif +23578585_15.tif +23578600_15.tif +23578780_15.tif +23578795_15.tif +23578810_15.tif +23578825_15.tif +23578840_15.tif +23578915_15.tif +23578930_15.tif +23578945_15.tif +23578960_15.tif +23578975_15.tif +23578990_15.tif +23579005_15.tif +23579020_15.tif +23579035_15.tif +23579050_15.tif +23579065_15.tif +23579080_15.tif +23579095_15.tif +23579110_15.tif +23579125_15.tif +23579140_15.tif +23579155_15.tif +23728480_15.tif +23728495_15.tif +23728510_15.tif +23728525_15.tif +23728540_15.tif +23728555_15.tif +23728570_15.tif +23728600_15.tif +23728765_15.tif +23728780_15.tif +23728795_15.tif +23728810_15.tif +23728825_15.tif +23728840_15.tif +23728915_15.tif +23728930_15.tif +23728945_15.tif +23728960_15.tif +23728975_15.tif +23728990_15.tif +23729005_15.tif +23729035_15.tif +23729050_15.tif +23729065_15.tif +23729080_15.tif +23729095_15.tif +23729110_15.tif +23729230_15.tif +23729245_15.tif +23878480_15.tif +23878495_15.tif +23878510_15.tif +23878525_15.tif +23878555_15.tif +23878570_15.tif +23878585_15.tif +23878765_15.tif +23878780_15.tif +23878795_15.tif +23878810_15.tif +23878825_15.tif +23878915_15.tif +23878930_15.tif +23878945_15.tif +23878960_15.tif +23878975_15.tif +23878990_15.tif +23879005_15.tif +23879020_15.tif +23879035_15.tif +23879050_15.tif +23879065_15.tif +23879080_15.tif +23879095_15.tif +23879110_15.tif +23879200_15.tif +23879215_15.tif +23879230_15.tif +23879245_15.tif +24028480_15.tif +24028495_15.tif +24028510_15.tif +24028525_15.tif +24028540_15.tif +24028555_15.tif +24028795_15.tif +24028810_15.tif +24028825_15.tif +24028990_15.tif +24029005_15.tif +24029020_15.tif +24029050_15.tif +24029065_15.tif +24029080_15.tif +24029095_15.tif +24029110_15.tif +24029185_15.tif +24029200_15.tif +24029215_15.tif +24029230_15.tif +24029245_15.tif +24178480_15.tif +24178495_15.tif +24178510_15.tif +24178525_15.tif +24178540_15.tif +24178975_15.tif +24179005_15.tif +24179020_15.tif +24179035_15.tif +24179050_15.tif +24179065_15.tif +24179080_15.tif +24179095_15.tif +24179110_15.tif +24179170_15.tif +24179185_15.tif +24179200_15.tif +24179215_15.tif +24179230_15.tif +24179260_15.tif +24179275_15.tif +24328765_15.tif +24328780_15.tif +24328795_15.tif +24328825_15.tif +24328840_15.tif +24328855_15.tif +24328870_15.tif +24328885_15.tif +24329005_15.tif +24329020_15.tif +24329035_15.tif +24329050_15.tif +24329095_15.tif +24329110_15.tif +24329155_15.tif +24329170_15.tif +24329185_15.tif +24329200_15.tif +24329215_15.tif +24329230_15.tif +24329245_15.tif +24329260_15.tif +24329275_15.tif +24478765_15.tif +24478780_15.tif +24478795_15.tif +24478810_15.tif +24478840_15.tif +24478870_15.tif +24478885_15.tif +24478900_15.tif +24478990_15.tif +24479005_15.tif +24479095_15.tif +24479110_15.tif +24479155_15.tif +24479185_15.tif +24479200_15.tif +24479230_15.tif +24479245_15.tif +24479260_15.tif +24479275_15.tif +24479290_15.tif +24628780_15.tif +24628795_15.tif +24628810_15.tif +24628825_15.tif +24628840_15.tif +24628855_15.tif +24628870_15.tif +24628900_15.tif +24629155_15.tif +24629170_15.tif +24629185_15.tif +24629215_15.tif +24629230_15.tif +24629245_15.tif +24629260_15.tif +24629275_15.tif +24629290_15.tif +24629305_15.tif +24778780_15.tif +24778795_15.tif +24778810_15.tif +24778825_15.tif +24778840_15.tif +24778855_15.tif +24778870_15.tif +24778885_15.tif +24778900_15.tif +24778915_15.tif +24779155_15.tif +24779170_15.tif +24779185_15.tif +24779200_15.tif +24779215_15.tif +24779230_15.tif +24779245_15.tif +24779260_15.tif +24779290_15.tif +24779305_15.tif +24928900_15.tif +24929155_15.tif +24929185_15.tif +24929200_15.tif +24929215_15.tif +24929230_15.tif +24929260_15.tif +24929275_15.tif +24929290_15.tif +25079155_15.tif +25079185_15.tif +25079200_15.tif +25079215_15.tif +25079230_15.tif +25079245_15.tif +25079260_15.tif +25079275_15.tif +25079290_15.tif +25229155_15.tif +25229170_15.tif +25229185_15.tif +25229200_15.tif +25229215_15.tif +25229260_15.tif +25229275_15.tif +25229290_15.tif +25379185_15.tif +25379230_15.tif +25379245_15.tif +25379260_15.tif +25379275_15.tif +25379290_15.tif +25529170_15.tif +25529230_15.tif +25529245_15.tif +25529260_15.tif +25529275_15.tif +25529290_15.tif +25679230_15.tif +25679245_15.tif +25679260_15.tif +25679275_15.tif +25828765_15.tif +25828780_15.tif +25829245_15.tif +25829260_15.tif +25829275_15.tif +25829290_15.tif +25978735_15.tif +25978750_15.tif +25978765_15.tif +25978780_15.tif +25978795_15.tif +25979230_15.tif +25979245_15.tif +25979260_15.tif +25979275_15.tif +25979290_15.tif +26128705_15.tif +26128720_15.tif +26128735_15.tif +26128750_15.tif +26128765_15.tif +26128780_15.tif +26128795_15.tif +26128810_15.tif +26129245_15.tif +26129260_15.tif +26129275_15.tif +26129290_15.tif +26278705_15.tif +26278735_15.tif +26278750_15.tif +26278765_15.tif +26278780_15.tif +26278795_15.tif +26278810_15.tif +26279260_15.tif +26428690_15.tif +26428705_15.tif +26428720_15.tif +26428750_15.tif +26428765_15.tif +26428780_15.tif +26428795_15.tif +26429245_15.tif +26429260_15.tif +26429275_15.tif +26578675_15.tif +26578690_15.tif +26578705_15.tif +26578735_15.tif +26578750_15.tif +26578765_15.tif +26578780_15.tif +26578795_15.tif +26728675_15.tif +26728690_15.tif +26728705_15.tif +26728720_15.tif +26728735_15.tif +26728750_15.tif +26728765_15.tif +26728780_15.tif +26878675_15.tif +26878705_15.tif +26878720_15.tif +26878735_15.tif +26878750_15.tif +27028675_15.tif +27028690_15.tif +27028705_15.tif +27028720_15.tif +27178705_15.tif +99238660_15.tif +99238675_15.tif diff --git a/examples/UNet/mass_roads_validation.txt b/examples/UNet/mass_roads_validation.txt new file mode 100644 index 0000000..66464ac --- /dev/null +++ b/examples/UNet/mass_roads_validation.txt @@ -0,0 +1,14 @@ +10228690_15.tiff +10978735_15.tiff +10978795_15.tiff +18028945_15.tiff +21929020_15.tiff +22528900_15.tiff +22829035_15.tiff +22978990_15.tiff +23128930_15.tiff +24179245_15.tiff +24328810_15.tiff +24929245_15.tiff +25229230_15.tiff +25229245_15.tiff diff --git a/examples/UNet/massachussets_road_segm.py b/examples/UNet/massachussets_road_segm.py new file mode 100644 index 0000000..95288f4 --- /dev/null +++ b/examples/UNet/massachussets_road_segm.py @@ -0,0 +1,333 @@ +__author__ = 'Fabian Isensee' +import numpy as np +import lasagne +import os +import sys +import fnmatch +import matplotlib.pyplot as plt +sys.path.append("../../modelzoo/") +from Unet import * +import theano.tensor as T +import theano +import cPickle +from time import sleep + +def prep_folders(): + if not os.path.isdir("data"): + os.mkdir("data") + + if not os.path.isdir("data/validation"): + os.mkdir("data/validation") + if not os.path.isdir("data/training"): + os.mkdir("data/training") + if not os.path.isdir("data/test"): + os.mkdir("data/test") + + if not os.path.isdir("data/validation/sat_img"): + os.mkdir("data/validation/sat_img") + if not os.path.isdir("data/validation/map"): + os.mkdir("data/validation/map") + if not os.path.isdir("data/training/sat_img"): + os.mkdir("data/training/sat_img") + if not os.path.isdir("data/training/map"): + os.mkdir("data/training/map") + if not os.path.isdir("data/test/sat_img"): + os.mkdir("data/test/sat_img") + if not os.path.isdir("data/test/map"): + os.mkdir("data/test/map") + +def prep_urls(): + valid_data_url = valid_target_url = np.loadtxt("mass_roads_validation.txt", dtype=str) + valid_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/sat/" + valid_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/map/" + + train_data_url = train_target_url = np.loadtxt("mass_roads_train.txt", dtype=str) + train_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/sat/" + train_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/map/" + + test_data_url = test_target_url = np.loadtxt("mass_roads_test.txt", dtype=str) + test_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/sat/" + test_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/map/" + + f = open("mass_roads_train_data_download.sh", 'w') + g = open("mass_roads_train_target_download.sh", 'w') + for img_name in train_data_url: + f.write("wget -O data/training/sat_img/%sf "%img_name + train_data_str + img_name + "f" + "\n") + g.write("wget -O data/training/map/%s "%img_name + train_target_str + img_name + "\n") + f.close() + g.close() + + f = open("mass_roads_validation_data_download.sh", 'w') + g = open("mass_roads_validation_target_download.sh", 'w') + for img_name in valid_data_url: + f.write("wget -O data/validation/sat_img/%s "%img_name + valid_data_str + img_name + "\n") + g.write("wget -O data/validation/map/%s "%img_name[:-1] + valid_target_str + img_name[:-1] + "\n") + f.close() + g.close() + + f = open("mass_roads_test_data_download.sh", 'w') + g = open("mass_roads_test_target_download.sh", 'w') + for img_name in test_data_url: + f.write("wget -O data/test/sat_img/%s "%img_name + test_data_str + img_name + "\n") + g.write("wget -O data/test/map/%s "%img_name[:-1] + test_target_str + img_name[:-1] + "\n") + f.close() + g.close() + +def download_dataset(): + os.system("sh mass_roads_train_data_download.sh &") + os.system("sh mass_roads_train_target_download.sh &") + os.system("sh mass_roads_validation_data_download.sh &") + os.system("sh mass_roads_validation_target_download.sh &") + os.system("sh mass_roads_test_data_download.sh &") + os.system("sh mass_roads_test_target_download.sh &") + +def load_data(folder): + images_sat = [img for img in os.listdir(os.path.join(folder, "sat_img")) if fnmatch.fnmatch(img, "*.tif*")] + images_map = [img for img in os.listdir(os.path.join(folder, "map")) if fnmatch.fnmatch(img, "*.tif*")] + assert(len(images_sat) == len(images_map)) + images_sat.sort() + images_map.sort() + # images are 1500 by 1500 pixels each + data = np.zeros((len(images_sat), 3, 1500, 1500), dtype=np.uint8) + target = np.zeros((len(images_sat), 1, 1500, 1500), dtype=np.uint8) + ctr = 0 + for sat_im, map_im in zip(images_sat, images_map): + data[ctr] = plt.imread(os.path.join(folder, "sat_img", sat_im)).transpose((2, 0, 1)) + # target has values 0 and 255. make that 0 and 1 + target[ctr, 0] = plt.imread(os.path.join(folder, "map", map_im))/255 + ctr += 1 + return data, target + +def batch_generator(data, target, BATCH_SIZE): + ''' + just a simple batch iterator, no cropping, no rotation, no anything + ''' + np.random.seed() + idx = np.arange(data.shape[0]) + while True: + ids = np.random.choice(idx, BATCH_SIZE) + yield data[ids], target[ids] + +def random_crop_generator(generator, crop_size=(128, 128)): + ''' + yields a random crop of size crop_size + ''' + np.random.seed() + if type(crop_size) not in (tuple, list): + crop_size = [crop_size, crop_size] + elif len(crop_size) == 2: + crop_size = list(crop_size) + else: + raise ValueError("invalid crop_size") + for data, seg in generator: + lb_x = np.random.randint(0, data.shape[2]-crop_size[0]) + lb_y = np.random.randint(0, data.shape[3]-crop_size[1]) + data = data[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] + seg = seg[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] + yield data, seg + +def threaded_generator(generator, num_cached=10): + # this code is written by jan Schluter + # copied from https://github.com/benanne/Lasagne/issues/12 + import Queue + queue = Queue.Queue(maxsize=num_cached) + sentinel = object() # guaranteed unique reference + + # define producer (putting items into queue) + def producer(): + for item in generator: + queue.put(item) + queue.put(sentinel) + + # start producer (in a background thread) + import threading + thread = threading.Thread(target=producer) + thread.daemon = True + thread.start() + + # run as consumer (read items from queue, in current thread) + item = queue.get() + while item is not sentinel: + yield item + queue.task_done() + item = queue.get() + +def prepare_dataset(): + prep_folders() + prep_urls() + download_dataset() + # the dataset is now downloaded in the background. Once every few seconds we check if the download is done. We do + # this by checking whether tha last training image exists + while not os.path.isfile("data/training/map/99238675_15.tiff") and not os.path.isfile("data/training/sat_img/99238675_15.tiff"): + print "download seems to be running..." + sleep(5) + print "download done..." + try: + data_train, target_train = load_data("data/training") + data_valid, target_valid = load_data("data/validation") + data_test, target_test = load_data("data/test") + # loading np arrays is much faster than loading the images one by one every time + np.save("train_data.npy", data_train) + np.save("train_target.npy", target_train) + np.save("valid_data.npy", data_valid) + np.save("valid_target.npy", target_valid) + np.save("test_data.npy", data_test) + np.save("test_target.npy", target_test) + except: + print "something went wrong, maybe the download?" + + +def plot_some_results(pred_fn, test_generator, BATCH_SIZE): + fig_ctr = 0 + ctr = 0 + for data, seg in test_generator: + res = pred_fn(data).argmax(-1).reshape(BATCH_SIZE, 1, 128+64, 128+64) + for d, s, r in zip(data, seg, res): + plt.figure(figsize=(12, 6)) + plt.subplot(1, 3, 1) + plt.imshow(d.transpose(1,2,0)) + plt.subplot(1, 3, 2) + plt.imshow(s[0]) + plt.subplot(1, 3, 3) + plt.imshow(r[0]) + plt.savefig("road_segmentation_result_%03.0f.png"%fig_ctr) + plt.close() + fig_ctr += 1 + ctr += 1 + if ctr > 10: + break + + +def main(): + # only download dataset once. This takes a while. + if not os.path.isfile("test_target.npy"): + # heuristic that I included to make sure the dataset is only donwloaded and prepared once + prepare_dataset() + + # set some hyper parameters. You should not have to touch anything if you have 4GB or more VRAM + BATCH_SIZE = 24 + N_EPOCHS = 30 + N_BATCHES_PER_EPOCH = 100 + N_BATCHES_PER_EPOCH_valid = 15 + PATCH_SIZE = 128+64 + + # load the prepared data. They have been converted to np arrays because they are much faster to load than single image files. + # you will need some ram in order to have everything in memory. + # TODO is it worth the effort to refactor everything so that we use memmaps? + data_train = np.load("train_data.npy") + target_train = np.load("train_target.npy") + data_valid = np.load("valid_data.npy") + target_valid= np.load("valid_target.npy") + data_test = np.load("test_data.npy") + target_test = np.load("test_target.npy") + + # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). + # Did not test for other paddings TODO + net = build_UNet(n_input_channels=3, BATCH_SIZE=BATCH_SIZE, num_output_classes=2, pad='same', + nonlinearity=lasagne.nonlinearities.elu, input_dim=(PATCH_SIZE, PATCH_SIZE), + base_n_filters=16, do_dropout=False) + output_layer = net["output"] + + # if you wish to load pretrained weights you can uncomment this code and modify the file name + '''with open("UNet_params_noBN_ep%03.0f.pkl"%10, 'r') as f: + params = cPickle.load(f) + lasagne.layers.set_all_param_values(output_layer, params)''' + + x_sym = T.tensor4() + seg_sym = T.ivector() + w_sym = T.vector() + + # add some weight decay + l2_loss = lasagne.regularization.regularize_network_params(output_layer, lasagne.regularization.l2) * 1e-4 + + # the distinction between prediction_train and test is important only if we enable dropout + prediction_train = lasagne.layers.get_output(output_layer, x_sym, deterministic=False) + # we could use a binary loss but I stuck with categorical crossentropy so that less code has to be changed if your + # application has more than two classes + loss = lasagne.objectives.categorical_crossentropy(prediction_train, seg_sym) + loss *= w_sym + loss = loss.mean() + loss += l2_loss + acc_train = T.mean(T.eq(T.argmax(prediction_train, axis=1), seg_sym), dtype=theano.config.floatX) + + prediction_test = lasagne.layers.get_output(output_layer, x_sym, deterministic=True) + loss_val = lasagne.objectives.categorical_crossentropy(prediction_test, seg_sym) + + # we multiply our loss by a weight map. In this example the weight map only increases the loss for road pixels and + # decreases the loss for other pixels. We do this to ensure that the network puts more focus on getting the roads + # right + loss_val *= w_sym + loss_val = loss_val.mean() + loss_val += l2_loss + acc = T.mean(T.eq(T.argmax(prediction_test, axis=1), seg_sym), dtype=theano.config.floatX) + + # learning rate has to be a shared variablebecause we decrease it with every epoch + params = lasagne.layers.get_all_params(output_layer, trainable=True) + learning_rate = theano.shared(np.float32(0.001)) + updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate) + + train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates) + val_fn = theano.function([x_sym, seg_sym, w_sym], [loss_val, acc]) + # have a look at plot_some_results if you want to see how to use pred_fn + pred_fn = theano.function([x_sym], prediction_test) + + # this is np.sum(target_train == 0) and np.sum(target_train == 1). No need to compute this every time + class_frequencies = np.array([2374093357., 118906643.]) + # we are taking the log here because we want the net to focus more on the road pixels but not too much (otherwise + # it would not be penalized enough for missclassifying terrain pixels which results in too many false positives) + class_weights = np.log(class_frequencies[[1,0]]) + class_weights = class_weights / np.sum(class_weights) * 2. + class_weights = class_weights.astype(np.float32) + + # some data augmentation. If you want better results you should invest more effort here. I left rotations and + # deformations out for the sake of speed and simplicity + train_generator = random_crop_generator(batch_generator(data_train, target_train, BATCH_SIZE), PATCH_SIZE) + train_generator = threaded_generator(train_generator, num_cached=10) + + # there is no need for data augmentation on the validation. However we need patches of the same size which is why + # we are using the random crop generator here again + validation_generator = random_crop_generator(batch_generator(data_valid, target_valid, BATCH_SIZE), PATCH_SIZE) + validation_generator = threaded_generator(validation_generator, num_cached=10) + + # do the actual training + for epoch in range(N_EPOCHS): + print epoch + losses_train = [] + n_batches = 0 + accuracies_train = [] + for data, target in train_generator: + # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so + # that we can match it with the prediction via the crossentropy loss function + target_flat = target.flatten() + loss, acc = train_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) + losses_train.append(loss) + accuracies_train.append(acc) + n_batches += 1 + if n_batches > N_BATCHES_PER_EPOCH: + break + print "epoch: ", epoch, "\ntrain accuracy: ", np.mean(accuracies_train), " train loss: ", np.mean(losses_train) + + losses_val = [] + accuracies_val = [] + n_batches = 0 + for data, target in validation_generator: + target_flat = target.flatten() + loss, acc = val_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) + losses_val.append(loss) + accuracies_val.append(acc) + n_batches += 1 + if n_batches > N_BATCHES_PER_EPOCH_valid: + break + print "val accuracy: ", np.mean(accuracies_val), " val loss: ", np.mean(losses_val) + learning_rate *= 0.5 + # save trained weights after each epoch + with open("UNet_params_ep%03.0f.pkl"%epoch, 'w') as f: + cPickle.dump(lasagne.layers.get_all_param_values(output_layer), f) + + # create some png files showing (raw image, ground truth, prediction). Of course we use the test set here ;-) + test_gen = random_crop_generator(batch_generator(data_test, target_test, BATCH_SIZE), PATCH_SIZE) + plot_some_results(pred_fn, test_gen, BATCH_SIZE) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py new file mode 100644 index 0000000..9ffe50f --- /dev/null +++ b/modelzoo/Unet.py @@ -0,0 +1,62 @@ +__author__ = 'Fabian Isensee' +from collections import OrderedDict +from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, Deconv2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer +from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer +import lasagne + + +def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(128, 128), base_n_filters=64, do_dropout=False): + net = OrderedDict() + net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1])) + + net['contr_1_1'] = ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_1_2'] = ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) + net['pool1'] = Pool2DLayer(net['contr_1_2'], 2) + + net['contr_2_1'] = ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_2_2'] = ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) + net['pool2'] = Pool2DLayer(net['contr_2_2'], 2) + + net['contr_3_1'] = ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_3_2'] = ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) + net['pool3'] = Pool2DLayer(net['contr_3_2'], 2) + + net['contr_4_1'] = ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_4_2'] = ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) + l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) + # the paper does not really describe where and how dropout is added. Feel free to try more options + if do_dropout: + l = DropoutLayer(l, p=0.5) + + net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) + net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) + net['deconv1'] = Deconv2DLayer(net['encode_2'], base_n_filters*8, 2, 2) + + net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center")) + net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) + net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) + net['deconv2'] = Deconv2DLayer(net['expand_1_2'], base_n_filters*4, 2, 2) + + net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center")) + net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) + net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) + net['deconv3'] = Deconv2DLayer(net['expand_2_2'], base_n_filters*2, 2, 2) + + net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center")) + net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) + net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) + net['deconv4'] = Deconv2DLayer(net['expand_3_2'], base_n_filters, 2, 2) + + net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center")) + net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) + net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) + + net['segLayer'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) + net['dimshuffle'] = DimshuffleLayer(net['segLayer'], (1, 0, 2, 3)) + net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) + net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) + net['output'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax) + return net + + + From 4cd59a44ba483eff992c4d8427a7d9fb567cd53f Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Wed, 24 Aug 2016 14:40:14 +0200 Subject: [PATCH 02/21] Reduced number of plotted test images, Added a plot of the road probability map --- .gitignore | 4 ++-- examples/UNet/massachussets_road_segm.py | 27 +++++++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index a9bca33..2e4422a 100644 --- a/.gitignore +++ b/.gitignore @@ -64,7 +64,7 @@ examples/UNet/data* examples/UNet/*.sh examples/UNet/*.npy examples/UNet/*.pkl -examples/UNet/tmp* -examples/UNet/best_parameters* +examples/UNet/*.png +examples/UNet/readme.txt .gitignore~ diff --git a/examples/UNet/massachussets_road_segm.py b/examples/UNet/massachussets_road_segm.py index 95288f4..28a3b83 100644 --- a/examples/UNet/massachussets_road_segm.py +++ b/examples/UNet/massachussets_road_segm.py @@ -106,7 +106,7 @@ def batch_generator(data, target, BATCH_SIZE): idx = np.arange(data.shape[0]) while True: ids = np.random.choice(idx, BATCH_SIZE) - yield data[ids], target[ids] + yield np.array(data[ids]), np.array(target[ids]) def random_crop_generator(generator, crop_size=(128, 128)): ''' @@ -177,24 +177,28 @@ def prepare_dataset(): print "something went wrong, maybe the download?" -def plot_some_results(pred_fn, test_generator, BATCH_SIZE): +def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192): fig_ctr = 0 ctr = 0 for data, seg in test_generator: - res = pred_fn(data).argmax(-1).reshape(BATCH_SIZE, 1, 128+64, 128+64) - for d, s, r in zip(data, seg, res): + pred = pred_fn(data) + res = pred.argmax(-1).reshape(BATCH_SIZE, 1, PATCH_SIZE, PATCH_SIZE) + prob = pred.transpose(1, 0).reshape(2, BATCH_SIZE, PATCH_SIZE, PATCH_SIZE)[1] + for d, s, r, p in zip(data, seg, res, prob): plt.figure(figsize=(12, 6)) - plt.subplot(1, 3, 1) + plt.subplot(1, 4, 1) plt.imshow(d.transpose(1,2,0)) - plt.subplot(1, 3, 2) + plt.subplot(1, 4, 2) plt.imshow(s[0]) - plt.subplot(1, 3, 3) + plt.subplot(1, 4, 3) plt.imshow(r[0]) + plt.subplot(1, 4, 4) + plt.imshow(p) plt.savefig("road_segmentation_result_%03.0f.png"%fig_ctr) plt.close() fig_ctr += 1 ctr += 1 - if ctr > 10: + if ctr > 5: break @@ -229,7 +233,10 @@ def main(): output_layer = net["output"] # if you wish to load pretrained weights you can uncomment this code and modify the file name - '''with open("UNet_params_noBN_ep%03.0f.pkl"%10, 'r') as f: + # if you want, use my pretained weights (got around 96% accuracy and a loss of 0.11 using excessive data + # augmentation) + # https://www.dropbox.com/s/t6juf6o2ix7dntk/UNet_roadSegmentation_Params.zip?dl=0 + '''with open("UNet_params_ep0.pkl", 'r') as f: params = cPickle.load(f) lasagne.layers.set_all_param_values(output_layer, params)''' @@ -319,7 +326,7 @@ def main(): if n_batches > N_BATCHES_PER_EPOCH_valid: break print "val accuracy: ", np.mean(accuracies_val), " val loss: ", np.mean(losses_val) - learning_rate *= 0.5 + learning_rate *= 0.2 # save trained weights after each epoch with open("UNet_params_ep%03.0f.pkl"%epoch, 'w') as f: cPickle.dump(lasagne.layers.get_all_param_values(output_layer), f) From ea84aef1961e38eb72d4c0432ab3d3e5f98bfc62 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 13:25:10 +0200 Subject: [PATCH 03/21] renamed some layers of UNet to make their names more clear, added convenience function get_segmentation, changed upscaling in UNet from Deconv2DLayer to Upscale2DLayer --- examples/UNet/massachussets_road_segm.py | 44 +++++++++++------------- modelzoo/Unet.py | 19 +++++----- 2 files changed, 30 insertions(+), 33 deletions(-) diff --git a/examples/UNet/massachussets_road_segm.py b/examples/UNet/massachussets_road_segm.py index 28a3b83..8ff7916 100644 --- a/examples/UNet/massachussets_road_segm.py +++ b/examples/UNet/massachussets_road_segm.py @@ -177,29 +177,23 @@ def prepare_dataset(): print "something went wrong, maybe the download?" -def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192): +def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_images=10): fig_ctr = 0 - ctr = 0 for data, seg in test_generator: - pred = pred_fn(data) - res = pred.argmax(-1).reshape(BATCH_SIZE, 1, PATCH_SIZE, PATCH_SIZE) - prob = pred.transpose(1, 0).reshape(2, BATCH_SIZE, PATCH_SIZE, PATCH_SIZE)[1] - for d, s, r, p in zip(data, seg, res, prob): + res = pred_fn(data) + for d, s, r, p in zip(data, seg, res): plt.figure(figsize=(12, 6)) - plt.subplot(1, 4, 1) + plt.subplot(1, 3, 1) plt.imshow(d.transpose(1,2,0)) - plt.subplot(1, 4, 2) + plt.subplot(1, 3, 2) plt.imshow(s[0]) - plt.subplot(1, 4, 3) + plt.subplot(1, 3, 3) plt.imshow(r[0]) - plt.subplot(1, 4, 4) - plt.imshow(p) plt.savefig("road_segmentation_result_%03.0f.png"%fig_ctr) plt.close() fig_ctr += 1 - ctr += 1 - if ctr > 5: - break + if fig_ctr > n_images: + break def main(): @@ -230,7 +224,7 @@ def main(): net = build_UNet(n_input_channels=3, BATCH_SIZE=BATCH_SIZE, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(PATCH_SIZE, PATCH_SIZE), base_n_filters=16, do_dropout=False) - output_layer = net["output"] + output_layer_for_loss = net["output_flattened"] # if you wish to load pretrained weights you can uncomment this code and modify the file name # if you want, use my pretained weights (got around 96% accuracy and a loss of 0.11 using excessive data @@ -245,10 +239,10 @@ def main(): w_sym = T.vector() # add some weight decay - l2_loss = lasagne.regularization.regularize_network_params(output_layer, lasagne.regularization.l2) * 1e-4 + l2_loss = lasagne.regularization.regularize_network_params(output_layer_for_loss, lasagne.regularization.l2) * 1e-4 # the distinction between prediction_train and test is important only if we enable dropout - prediction_train = lasagne.layers.get_output(output_layer, x_sym, deterministic=False) + prediction_train = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=False) # we could use a binary loss but I stuck with categorical crossentropy so that less code has to be changed if your # application has more than two classes loss = lasagne.objectives.categorical_crossentropy(prediction_train, seg_sym) @@ -257,7 +251,7 @@ def main(): loss += l2_loss acc_train = T.mean(T.eq(T.argmax(prediction_train, axis=1), seg_sym), dtype=theano.config.floatX) - prediction_test = lasagne.layers.get_output(output_layer, x_sym, deterministic=True) + prediction_test = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=True) loss_val = lasagne.objectives.categorical_crossentropy(prediction_test, seg_sym) # we multiply our loss by a weight map. In this example the weight map only increases the loss for road pixels and @@ -269,14 +263,18 @@ def main(): acc = T.mean(T.eq(T.argmax(prediction_test, axis=1), seg_sym), dtype=theano.config.floatX) # learning rate has to be a shared variablebecause we decrease it with every epoch - params = lasagne.layers.get_all_params(output_layer, trainable=True) + params = lasagne.layers.get_all_params(output_layer_for_loss, trainable=True) learning_rate = theano.shared(np.float32(0.001)) updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate) + # create a convenience function to get the segmentation + seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym) + seg_output = seg_output.argmax(1) + train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates) val_fn = theano.function([x_sym, seg_sym, w_sym], [loss_val, acc]) - # have a look at plot_some_results if you want to see how to use pred_fn - pred_fn = theano.function([x_sym], prediction_test) + get_segmentation = theano.function([x_sym], seg_output) + # this is np.sum(target_train == 0) and np.sum(target_train == 1). No need to compute this every time class_frequencies = np.array([2374093357., 118906643.]) @@ -329,11 +327,11 @@ def main(): learning_rate *= 0.2 # save trained weights after each epoch with open("UNet_params_ep%03.0f.pkl"%epoch, 'w') as f: - cPickle.dump(lasagne.layers.get_all_param_values(output_layer), f) + cPickle.dump(lasagne.layers.get_all_param_values(output_layer_for_loss), f) # create some png files showing (raw image, ground truth, prediction). Of course we use the test set here ;-) test_gen = random_crop_generator(batch_generator(data_test, target_test, BATCH_SIZE), PATCH_SIZE) - plot_some_results(pred_fn, test_gen, BATCH_SIZE) + plot_some_results(get_segmentation, test_gen, BATCH_SIZE) if __name__ == "__main__": diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py index 9ffe50f..84b7dfa 100644 --- a/modelzoo/Unet.py +++ b/modelzoo/Unet.py @@ -1,6 +1,6 @@ __author__ = 'Fabian Isensee' from collections import OrderedDict -from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, Deconv2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer +from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer, Upscale2DLayer from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer import lasagne @@ -26,37 +26,36 @@ def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='s l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) # the paper does not really describe where and how dropout is added. Feel free to try more options if do_dropout: - l = DropoutLayer(l, p=0.5) + l = DropoutLayer(l, p=0.4) net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv1'] = Deconv2DLayer(net['encode_2'], base_n_filters*8, 2, 2) + net['deconv1'] = Upscale2DLayer(net['encode_2'], 2) net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center")) net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv2'] = Deconv2DLayer(net['expand_1_2'], base_n_filters*4, 2, 2) + net['deconv2'] = Upscale2DLayer(net['expand_1_2'], 2) net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center")) net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv3'] = Deconv2DLayer(net['expand_2_2'], base_n_filters*2, 2, 2) + net['deconv3'] = Upscale2DLayer(net['expand_2_2'], 2) net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center")) net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv4'] = Deconv2DLayer(net['expand_3_2'], base_n_filters, 2, 2) + net['deconv4'] = Upscale2DLayer(net['expand_3_2'], 2) net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center")) net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) - net['segLayer'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) + net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) net['dimshuffle'] = DimshuffleLayer(net['segLayer'], (1, 0, 2, 3)) net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) - net['output'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax) - return net - + net['output_flattened'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax) + return net From fcff2da57590c96ea3122507154948f9c8ceede0 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 13:35:41 +0200 Subject: [PATCH 04/21] code restructuring (now it's not cramped into a single .py file anymore). Also fixed a typo in massachusetts --- examples/UNet/generators.py | 56 ++++++ .../UNet/massachusetts_road_dataset_utils.py | 119 +++++++++++++ ...oad_segm.py => massachusetts_road_segm.py} | 168 +----------------- 3 files changed, 177 insertions(+), 166 deletions(-) create mode 100644 examples/UNet/generators.py create mode 100644 examples/UNet/massachusetts_road_dataset_utils.py rename examples/UNet/{massachussets_road_segm.py => massachusetts_road_segm.py} (54%) diff --git a/examples/UNet/generators.py b/examples/UNet/generators.py new file mode 100644 index 0000000..a17e7e8 --- /dev/null +++ b/examples/UNet/generators.py @@ -0,0 +1,56 @@ +__author__ = 'Fabian Isensee' +import numpy as np + +def batch_generator(data, target, BATCH_SIZE): + ''' + just a simple batch iterator, no cropping, no rotation, no anything + ''' + np.random.seed() + idx = np.arange(data.shape[0]) + while True: + ids = np.random.choice(idx, BATCH_SIZE) + yield np.array(data[ids]), np.array(target[ids]) + +def random_crop_generator(generator, crop_size=(128, 128)): + ''' + yields a random crop of size crop_size + ''' + np.random.seed() + if type(crop_size) not in (tuple, list): + crop_size = [crop_size, crop_size] + elif len(crop_size) == 2: + crop_size = list(crop_size) + else: + raise ValueError("invalid crop_size") + for data, seg in generator: + lb_x = np.random.randint(0, data.shape[2]-crop_size[0]) + lb_y = np.random.randint(0, data.shape[3]-crop_size[1]) + data = data[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] + seg = seg[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] + yield data, seg + +def threaded_generator(generator, num_cached=10): + # this code is written by jan Schluter + # copied from https://github.com/benanne/Lasagne/issues/12 + import Queue + queue = Queue.Queue(maxsize=num_cached) + sentinel = object() # guaranteed unique reference + + # define producer (putting items into queue) + def producer(): + for item in generator: + queue.put(item) + queue.put(sentinel) + + # start producer (in a background thread) + import threading + thread = threading.Thread(target=producer) + thread.daemon = True + thread.start() + + # run as consumer (read items from queue, in current thread) + item = queue.get() + while item is not sentinel: + yield item + queue.task_done() + item = queue.get() diff --git a/examples/UNet/massachusetts_road_dataset_utils.py b/examples/UNet/massachusetts_road_dataset_utils.py new file mode 100644 index 0000000..c987677 --- /dev/null +++ b/examples/UNet/massachusetts_road_dataset_utils.py @@ -0,0 +1,119 @@ +__author__ = 'Fabian Isensee' +import numpy as np +import os +import sys +import fnmatch +import matplotlib.pyplot as plt +sys.path.append("../../modelzoo/") +from time import sleep +from generators import * + +def prep_folders(): + if not os.path.isdir("data"): + os.mkdir("data") + + if not os.path.isdir("data/validation"): + os.mkdir("data/validation") + if not os.path.isdir("data/training"): + os.mkdir("data/training") + if not os.path.isdir("data/test"): + os.mkdir("data/test") + + if not os.path.isdir("data/validation/sat_img"): + os.mkdir("data/validation/sat_img") + if not os.path.isdir("data/validation/map"): + os.mkdir("data/validation/map") + if not os.path.isdir("data/training/sat_img"): + os.mkdir("data/training/sat_img") + if not os.path.isdir("data/training/map"): + os.mkdir("data/training/map") + if not os.path.isdir("data/test/sat_img"): + os.mkdir("data/test/sat_img") + if not os.path.isdir("data/test/map"): + os.mkdir("data/test/map") + +def prep_urls(): + valid_data_url = valid_target_url = np.loadtxt("mass_roads_validation.txt", dtype=str) + valid_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/sat/" + valid_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/map/" + + train_data_url = train_target_url = np.loadtxt("mass_roads_train.txt", dtype=str) + train_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/sat/" + train_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/map/" + + test_data_url = test_target_url = np.loadtxt("mass_roads_test.txt", dtype=str) + test_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/sat/" + test_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/map/" + + f = open("mass_roads_train_data_download.sh", 'w') + g = open("mass_roads_train_target_download.sh", 'w') + for img_name in train_data_url: + f.write("wget -O data/training/sat_img/%sf "%img_name + train_data_str + img_name + "f" + "\n") + g.write("wget -O data/training/map/%s "%img_name + train_target_str + img_name + "\n") + f.close() + g.close() + + f = open("mass_roads_validation_data_download.sh", 'w') + g = open("mass_roads_validation_target_download.sh", 'w') + for img_name in valid_data_url: + f.write("wget -O data/validation/sat_img/%s "%img_name + valid_data_str + img_name + "\n") + g.write("wget -O data/validation/map/%s "%img_name[:-1] + valid_target_str + img_name[:-1] + "\n") + f.close() + g.close() + + f = open("mass_roads_test_data_download.sh", 'w') + g = open("mass_roads_test_target_download.sh", 'w') + for img_name in test_data_url: + f.write("wget -O data/test/sat_img/%s "%img_name + test_data_str + img_name + "\n") + g.write("wget -O data/test/map/%s "%img_name[:-1] + test_target_str + img_name[:-1] + "\n") + f.close() + g.close() + +def download_dataset(): + os.system("sh mass_roads_train_data_download.sh &") + os.system("sh mass_roads_train_target_download.sh &") + os.system("sh mass_roads_validation_data_download.sh &") + os.system("sh mass_roads_validation_target_download.sh &") + os.system("sh mass_roads_test_data_download.sh &") + os.system("sh mass_roads_test_target_download.sh &") + +def load_data(folder): + images_sat = [img for img in os.listdir(os.path.join(folder, "sat_img")) if fnmatch.fnmatch(img, "*.tif*")] + images_map = [img for img in os.listdir(os.path.join(folder, "map")) if fnmatch.fnmatch(img, "*.tif*")] + assert(len(images_sat) == len(images_map)) + images_sat.sort() + images_map.sort() + # images are 1500 by 1500 pixels each + data = np.zeros((len(images_sat), 3, 1500, 1500), dtype=np.uint8) + target = np.zeros((len(images_sat), 1, 1500, 1500), dtype=np.uint8) + ctr = 0 + for sat_im, map_im in zip(images_sat, images_map): + data[ctr] = plt.imread(os.path.join(folder, "sat_img", sat_im)).transpose((2, 0, 1)) + # target has values 0 and 255. make that 0 and 1 + target[ctr, 0] = plt.imread(os.path.join(folder, "map", map_im))/255 + ctr += 1 + return data, target + +def prepare_dataset(): + prep_folders() + prep_urls() + download_dataset() + # the dataset is now downloaded in the background. Once every few seconds we check if the download is done. We do + # this by checking whether tha last training image exists + while not os.path.isfile("data/training/map/99238675_15.tiff") and not os.path.isfile("data/training/sat_img/99238675_15.tiff"): + print "download seems to be running..." + sleep(5) + print "download done..." + try: + data_train, target_train = load_data("data/training") + data_valid, target_valid = load_data("data/validation") + data_test, target_test = load_data("data/test") + # loading np arrays is much faster than loading the images one by one every time + np.save("train_data.npy", data_train) + np.save("train_target.npy", target_train) + np.save("valid_data.npy", data_valid) + np.save("valid_target.npy", target_valid) + np.save("test_data.npy", data_test) + np.save("test_target.npy", target_test) + except: + print "something went wrong, maybe the download?" \ No newline at end of file diff --git a/examples/UNet/massachussets_road_segm.py b/examples/UNet/massachusetts_road_segm.py similarity index 54% rename from examples/UNet/massachussets_road_segm.py rename to examples/UNet/massachusetts_road_segm.py index 8ff7916..e5049f1 100644 --- a/examples/UNet/massachussets_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -11,171 +11,8 @@ import theano import cPickle from time import sleep - -def prep_folders(): - if not os.path.isdir("data"): - os.mkdir("data") - - if not os.path.isdir("data/validation"): - os.mkdir("data/validation") - if not os.path.isdir("data/training"): - os.mkdir("data/training") - if not os.path.isdir("data/test"): - os.mkdir("data/test") - - if not os.path.isdir("data/validation/sat_img"): - os.mkdir("data/validation/sat_img") - if not os.path.isdir("data/validation/map"): - os.mkdir("data/validation/map") - if not os.path.isdir("data/training/sat_img"): - os.mkdir("data/training/sat_img") - if not os.path.isdir("data/training/map"): - os.mkdir("data/training/map") - if not os.path.isdir("data/test/sat_img"): - os.mkdir("data/test/sat_img") - if not os.path.isdir("data/test/map"): - os.mkdir("data/test/map") - -def prep_urls(): - valid_data_url = valid_target_url = np.loadtxt("mass_roads_validation.txt", dtype=str) - valid_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/sat/" - valid_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/map/" - - train_data_url = train_target_url = np.loadtxt("mass_roads_train.txt", dtype=str) - train_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/sat/" - train_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/map/" - - test_data_url = test_target_url = np.loadtxt("mass_roads_test.txt", dtype=str) - test_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/sat/" - test_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/map/" - - f = open("mass_roads_train_data_download.sh", 'w') - g = open("mass_roads_train_target_download.sh", 'w') - for img_name in train_data_url: - f.write("wget -O data/training/sat_img/%sf "%img_name + train_data_str + img_name + "f" + "\n") - g.write("wget -O data/training/map/%s "%img_name + train_target_str + img_name + "\n") - f.close() - g.close() - - f = open("mass_roads_validation_data_download.sh", 'w') - g = open("mass_roads_validation_target_download.sh", 'w') - for img_name in valid_data_url: - f.write("wget -O data/validation/sat_img/%s "%img_name + valid_data_str + img_name + "\n") - g.write("wget -O data/validation/map/%s "%img_name[:-1] + valid_target_str + img_name[:-1] + "\n") - f.close() - g.close() - - f = open("mass_roads_test_data_download.sh", 'w') - g = open("mass_roads_test_target_download.sh", 'w') - for img_name in test_data_url: - f.write("wget -O data/test/sat_img/%s "%img_name + test_data_str + img_name + "\n") - g.write("wget -O data/test/map/%s "%img_name[:-1] + test_target_str + img_name[:-1] + "\n") - f.close() - g.close() - -def download_dataset(): - os.system("sh mass_roads_train_data_download.sh &") - os.system("sh mass_roads_train_target_download.sh &") - os.system("sh mass_roads_validation_data_download.sh &") - os.system("sh mass_roads_validation_target_download.sh &") - os.system("sh mass_roads_test_data_download.sh &") - os.system("sh mass_roads_test_target_download.sh &") - -def load_data(folder): - images_sat = [img for img in os.listdir(os.path.join(folder, "sat_img")) if fnmatch.fnmatch(img, "*.tif*")] - images_map = [img for img in os.listdir(os.path.join(folder, "map")) if fnmatch.fnmatch(img, "*.tif*")] - assert(len(images_sat) == len(images_map)) - images_sat.sort() - images_map.sort() - # images are 1500 by 1500 pixels each - data = np.zeros((len(images_sat), 3, 1500, 1500), dtype=np.uint8) - target = np.zeros((len(images_sat), 1, 1500, 1500), dtype=np.uint8) - ctr = 0 - for sat_im, map_im in zip(images_sat, images_map): - data[ctr] = plt.imread(os.path.join(folder, "sat_img", sat_im)).transpose((2, 0, 1)) - # target has values 0 and 255. make that 0 and 1 - target[ctr, 0] = plt.imread(os.path.join(folder, "map", map_im))/255 - ctr += 1 - return data, target - -def batch_generator(data, target, BATCH_SIZE): - ''' - just a simple batch iterator, no cropping, no rotation, no anything - ''' - np.random.seed() - idx = np.arange(data.shape[0]) - while True: - ids = np.random.choice(idx, BATCH_SIZE) - yield np.array(data[ids]), np.array(target[ids]) - -def random_crop_generator(generator, crop_size=(128, 128)): - ''' - yields a random crop of size crop_size - ''' - np.random.seed() - if type(crop_size) not in (tuple, list): - crop_size = [crop_size, crop_size] - elif len(crop_size) == 2: - crop_size = list(crop_size) - else: - raise ValueError("invalid crop_size") - for data, seg in generator: - lb_x = np.random.randint(0, data.shape[2]-crop_size[0]) - lb_y = np.random.randint(0, data.shape[3]-crop_size[1]) - data = data[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] - seg = seg[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] - yield data, seg - -def threaded_generator(generator, num_cached=10): - # this code is written by jan Schluter - # copied from https://github.com/benanne/Lasagne/issues/12 - import Queue - queue = Queue.Queue(maxsize=num_cached) - sentinel = object() # guaranteed unique reference - - # define producer (putting items into queue) - def producer(): - for item in generator: - queue.put(item) - queue.put(sentinel) - - # start producer (in a background thread) - import threading - thread = threading.Thread(target=producer) - thread.daemon = True - thread.start() - - # run as consumer (read items from queue, in current thread) - item = queue.get() - while item is not sentinel: - yield item - queue.task_done() - item = queue.get() - -def prepare_dataset(): - prep_folders() - prep_urls() - download_dataset() - # the dataset is now downloaded in the background. Once every few seconds we check if the download is done. We do - # this by checking whether tha last training image exists - while not os.path.isfile("data/training/map/99238675_15.tiff") and not os.path.isfile("data/training/sat_img/99238675_15.tiff"): - print "download seems to be running..." - sleep(5) - print "download done..." - try: - data_train, target_train = load_data("data/training") - data_valid, target_valid = load_data("data/validation") - data_test, target_test = load_data("data/test") - # loading np arrays is much faster than loading the images one by one every time - np.save("train_data.npy", data_train) - np.save("train_target.npy", target_train) - np.save("valid_data.npy", data_valid) - np.save("valid_target.npy", target_valid) - np.save("test_data.npy", data_test) - np.save("test_target.npy", target_test) - except: - print "something went wrong, maybe the download?" - +from generators import batch_generator, threaded_generator, random_crop_generator +from massachusetts_road_dataset_utils import prepare_dataset def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_images=10): fig_ctr = 0 @@ -195,7 +32,6 @@ def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_i if fig_ctr > n_images: break - def main(): # only download dataset once. This takes a while. if not os.path.isfile("test_target.npy"): From 9cbb33ff8d3354a408f16590cfb7dc68dc543f91 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 14:05:19 +0200 Subject: [PATCH 05/21] now using urlretrieve for downloading the data. Also uses multiprocessing.Pool for miltithreaded download. Seems to work well! --- .../UNet/massachusetts_road_dataset_utils.py | 66 +++++++++---------- 1 file changed, 30 insertions(+), 36 deletions(-) diff --git a/examples/UNet/massachusetts_road_dataset_utils.py b/examples/UNet/massachusetts_road_dataset_utils.py index c987677..19253ad 100644 --- a/examples/UNet/massachusetts_road_dataset_utils.py +++ b/examples/UNet/massachusetts_road_dataset_utils.py @@ -1,12 +1,12 @@ __author__ = 'Fabian Isensee' -import numpy as np import os import sys import fnmatch import matplotlib.pyplot as plt sys.path.append("../../modelzoo/") -from time import sleep from generators import * +from multiprocessing.dummy import Pool +from urllib import urlretrieve def prep_folders(): if not os.path.isdir("data"): @@ -45,37 +45,32 @@ def prep_urls(): test_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/sat/" test_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/map/" - f = open("mass_roads_train_data_download.sh", 'w') - g = open("mass_roads_train_target_download.sh", 'w') + all_tasks = [] + + # save url along with the filename for each file for img_name in train_data_url: - f.write("wget -O data/training/sat_img/%sf "%img_name + train_data_str + img_name + "f" + "\n") - g.write("wget -O data/training/map/%s "%img_name + train_target_str + img_name + "\n") - f.close() - g.close() + all_tasks.append(tuple([train_data_str + img_name + "f", "data/training/sat_img/%sf"%img_name])) + all_tasks.append(tuple([train_target_str + img_name, "data/training/map/%s"%img_name])) - f = open("mass_roads_validation_data_download.sh", 'w') - g = open("mass_roads_validation_target_download.sh", 'w') for img_name in valid_data_url: - f.write("wget -O data/validation/sat_img/%s "%img_name + valid_data_str + img_name + "\n") - g.write("wget -O data/validation/map/%s "%img_name[:-1] + valid_target_str + img_name[:-1] + "\n") - f.close() - g.close() + all_tasks.append(tuple([valid_data_str + img_name, "data/validation/sat_img/%s"%img_name])) + all_tasks.append(tuple([valid_target_str + img_name[:-1], "data/validation/map/%s"%img_name[:-1]])) - f = open("mass_roads_test_data_download.sh", 'w') - g = open("mass_roads_test_target_download.sh", 'w') for img_name in test_data_url: - f.write("wget -O data/test/sat_img/%s "%img_name + test_data_str + img_name + "\n") - g.write("wget -O data/test/map/%s "%img_name[:-1] + test_target_str + img_name[:-1] + "\n") - f.close() - g.close() - -def download_dataset(): - os.system("sh mass_roads_train_data_download.sh &") - os.system("sh mass_roads_train_target_download.sh &") - os.system("sh mass_roads_validation_data_download.sh &") - os.system("sh mass_roads_validation_target_download.sh &") - os.system("sh mass_roads_test_data_download.sh &") - os.system("sh mass_roads_test_target_download.sh &") + all_tasks.append(tuple([test_data_str + img_name, "data/test/sat_img/%s"%img_name])) + all_tasks.append(tuple([test_target_str + img_name[:-1], "data/test/map/%s"%img_name[:-1]])) + + return all_tasks + +def download_dataset(all_tasks, num_workers=4): + def urlretrieve_star(args): + return urlretrieve(*args) + + pool = Pool(num_workers) + pool.map(urlretrieve_star, all_tasks) + pool.close() + pool.join() + def load_data(folder): images_sat = [img for img in os.listdir(os.path.join(folder, "sat_img")) if fnmatch.fnmatch(img, "*.tif*")] @@ -96,13 +91,9 @@ def load_data(folder): def prepare_dataset(): prep_folders() - prep_urls() - download_dataset() - # the dataset is now downloaded in the background. Once every few seconds we check if the download is done. We do - # this by checking whether tha last training image exists - while not os.path.isfile("data/training/map/99238675_15.tiff") and not os.path.isfile("data/training/sat_img/99238675_15.tiff"): - print "download seems to be running..." - sleep(5) + all_tasks = prep_urls() + download_dataset(all_tasks) + print "download done..." try: data_train, target_train = load_data("data/training") @@ -116,4 +107,7 @@ def prepare_dataset(): np.save("test_data.npy", data_test) np.save("test_target.npy", target_test) except: - print "something went wrong, maybe the download?" \ No newline at end of file + print "something went wrong, maybe the download?" + +if __name__ == "__main__": + prepare_dataset() \ No newline at end of file From f10200df2b3316a2746e3fbae4f50006cc151053 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 14:28:15 +0200 Subject: [PATCH 06/21] updated batch_iterator so that it now supports deterministic batch generation (needed for validation) --- examples/UNet/generators.py | 17 +++++++-- examples/UNet/massachusetts_road_segm.py | 48 ++++++++++++------------ 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/examples/UNet/generators.py b/examples/UNet/generators.py index a17e7e8..d7005b7 100644 --- a/examples/UNet/generators.py +++ b/examples/UNet/generators.py @@ -1,21 +1,30 @@ __author__ = 'Fabian Isensee' import numpy as np -def batch_generator(data, target, BATCH_SIZE): +def batch_generator(data, target, BATCH_SIZE, shuffle=False): ''' just a simple batch iterator, no cropping, no rotation, no anything ''' np.random.seed() idx = np.arange(data.shape[0]) + if shuffle: + np.random.shuffle() + idx_2 = np.array(idx) + # if BATCH_SIZE is larger than len(data) we need to artificially enlarge the idx array (loop around) + while BATCH_SIZE > len(idx): + idx_2 = np.concatenate((idx_2, idx)) + del(idx) while True: - ids = np.random.choice(idx, BATCH_SIZE) - yield np.array(data[ids]), np.array(target[ids]) + ctr = 0 + yield np.array(data[idx_2[ctr:ctr+BATCH_SIZE]]), np.array(target[idx_2[ctr:ctr+BATCH_SIZE]]) + ctr += BATCH_SIZE + if ctr >= data.shape[0]: + ctr -= data.shape[0] def random_crop_generator(generator, crop_size=(128, 128)): ''' yields a random crop of size crop_size ''' - np.random.seed() if type(crop_size) not in (tuple, list): crop_size = [crop_size, crop_size] elif len(crop_size) == 2: diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index e5049f1..ba9a079 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -47,24 +47,35 @@ def main(): # load the prepared data. They have been converted to np arrays because they are much faster to load than single image files. # you will need some ram in order to have everything in memory. - # TODO is it worth the effort to refactor everything so that we use memmaps? - data_train = np.load("train_data.npy") - target_train = np.load("train_target.npy") - data_valid = np.load("valid_data.npy") - target_valid= np.load("valid_target.npy") - data_test = np.load("test_data.npy") - target_test = np.load("test_target.npy") + # If you are having RAM issues, change mmap_mode to 'r'. This will not load the entire array into memory but rather + # read from disk the bits that we currently need + # (if you have, copy your repository including the data to an SSD, otherwise it will take a long time to + # generate batches) + mmap_mode = None + data_train = np.load("train_data.npy", mmap_mode=mmap_mode) + target_train = np.load("train_target.npy", mmap_mode=mmap_mode) + data_valid = np.load("valid_data.npy", mmap_mode=mmap_mode) + target_valid= np.load("valid_target.npy", mmap_mode=mmap_mode) + data_test = np.load("test_data.npy", mmap_mode=mmap_mode) + target_test = np.load("test_target.npy", mmap_mode=mmap_mode) # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). - # Did not test for other paddings TODO net = build_UNet(n_input_channels=3, BATCH_SIZE=BATCH_SIZE, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(PATCH_SIZE, PATCH_SIZE), base_n_filters=16, do_dropout=False) output_layer_for_loss = net["output_flattened"] + # this is np.sum(target_train == 0) and np.sum(target_train == 1). No need to compute this every time + class_frequencies = np.array([2374093357., 118906643.]) + # we are taking the log here because we want the net to focus more on the road pixels but not too much (otherwise + # it would not be penalized enough for missclassifying terrain pixels which results in too many false positives) + class_weights = np.log(class_frequencies[[1,0]]) + class_weights = class_weights / np.sum(class_weights) * 2. + class_weights = class_weights.astype(np.float32) + # if you wish to load pretrained weights you can uncomment this code and modify the file name # if you want, use my pretained weights (got around 96% accuracy and a loss of 0.11 using excessive data - # augmentation) + # augmentation: cropping, rotation, elastic deformation) # https://www.dropbox.com/s/t6juf6o2ix7dntk/UNet_roadSegmentation_Params.zip?dl=0 '''with open("UNet_params_ep0.pkl", 'r') as f: params = cPickle.load(f) @@ -72,7 +83,7 @@ def main(): x_sym = T.tensor4() seg_sym = T.ivector() - w_sym = T.vector() + w_sym = class_weights[seg_sym] # add some weight decay l2_loss = lasagne.regularization.regularize_network_params(output_layer_for_loss, lasagne.regularization.l2) * 1e-4 @@ -107,19 +118,10 @@ def main(): seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym) seg_output = seg_output.argmax(1) - train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates) - val_fn = theano.function([x_sym, seg_sym, w_sym], [loss_val, acc]) + train_fn = theano.function([x_sym, seg_sym], [loss, acc_train], updates=updates) + val_fn = theano.function([x_sym, seg_sym], [loss_val, acc]) get_segmentation = theano.function([x_sym], seg_output) - - # this is np.sum(target_train == 0) and np.sum(target_train == 1). No need to compute this every time - class_frequencies = np.array([2374093357., 118906643.]) - # we are taking the log here because we want the net to focus more on the road pixels but not too much (otherwise - # it would not be penalized enough for missclassifying terrain pixels which results in too many false positives) - class_weights = np.log(class_frequencies[[1,0]]) - class_weights = class_weights / np.sum(class_weights) * 2. - class_weights = class_weights.astype(np.float32) - # some data augmentation. If you want better results you should invest more effort here. I left rotations and # deformations out for the sake of speed and simplicity train_generator = random_crop_generator(batch_generator(data_train, target_train, BATCH_SIZE), PATCH_SIZE) @@ -140,7 +142,7 @@ def main(): # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so # that we can match it with the prediction via the crossentropy loss function target_flat = target.flatten() - loss, acc = train_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) + loss, acc = train_fn(data.astype(np.float32), target_flat) losses_train.append(loss) accuracies_train.append(acc) n_batches += 1 @@ -153,7 +155,7 @@ def main(): n_batches = 0 for data, target in validation_generator: target_flat = target.flatten() - loss, acc = val_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) + loss, acc = val_fn(data.astype(np.float32), target_flat) losses_val.append(loss) accuracies_val.append(acc) n_batches += 1 From db99d8227a72e860d3abeebf44d42f1edc50585b Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 14:31:04 +0200 Subject: [PATCH 07/21] added center crop generator --- examples/UNet/generators.py | 16 ++++++++++++++++ examples/UNet/massachusetts_road_segm.py | 6 +++--- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/examples/UNet/generators.py b/examples/UNet/generators.py index d7005b7..e1e1130 100644 --- a/examples/UNet/generators.py +++ b/examples/UNet/generators.py @@ -21,6 +21,22 @@ def batch_generator(data, target, BATCH_SIZE, shuffle=False): if ctr >= data.shape[0]: ctr -= data.shape[0] + +def center_crop_generator(generator, output_size): + ''' + yields center crop of size output_size (may be 1d or 2d) from data and seg + ''' + if type(output_size) not in (tuple, list): + center_crop = [output_size, output_size] + elif len(output_size) == 2: + center_crop = list(output_size) + else: + raise ValueError("invalid output_size") + for data, seg in generator: + center = np.array(data.shape[2:])/2 + yield data[:, :, int(center[0]-center_crop[0]/2.):int(center[0]+center_crop[0]/2.), int(center[1]-center_crop[1]/2.):int(center[1]+center_crop[1]/2.)], seg[:, :, int(center[0]-center_crop[0]/2.):int(center[0]+center_crop[0]/2.), int(center[1]-center_crop[1]/2.):int(center[1]+center_crop[1]/2.)] + + def random_crop_generator(generator, crop_size=(128, 128)): ''' yields a random crop of size crop_size diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index ba9a079..9c92f86 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -11,7 +11,7 @@ import theano import cPickle from time import sleep -from generators import batch_generator, threaded_generator, random_crop_generator +from generators import batch_generator, threaded_generator, random_crop_generator, center_crop_generator from massachusetts_road_dataset_utils import prepare_dataset def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_images=10): @@ -124,12 +124,12 @@ def main(): # some data augmentation. If you want better results you should invest more effort here. I left rotations and # deformations out for the sake of speed and simplicity - train_generator = random_crop_generator(batch_generator(data_train, target_train, BATCH_SIZE), PATCH_SIZE) + train_generator = random_crop_generator(batch_generator(data_train, target_train, BATCH_SIZE, shuffle=True), PATCH_SIZE) train_generator = threaded_generator(train_generator, num_cached=10) # there is no need for data augmentation on the validation. However we need patches of the same size which is why # we are using the random crop generator here again - validation_generator = random_crop_generator(batch_generator(data_valid, target_valid, BATCH_SIZE), PATCH_SIZE) + validation_generator = center_crop_generator(batch_generator(data_valid, target_valid, BATCH_SIZE, shuffle=False), PATCH_SIZE) validation_generator = threaded_generator(validation_generator, num_cached=10) # do the actual training From 90aa43da1247e80fa7d64e309296191344dab514 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 14:41:34 +0200 Subject: [PATCH 08/21] now uses compressed numpy arrays to store the dataset --- .gitignore | 2 +- .../UNet/massachusetts_road_dataset_utils.py | 10 ++++------ examples/UNet/massachusetts_road_segm.py | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.gitignore b/.gitignore index 2e4422a..d01177e 100644 --- a/.gitignore +++ b/.gitignore @@ -62,7 +62,7 @@ target/ examples/UNet/data* examples/UNet/*.sh -examples/UNet/*.npy +examples/UNet/*.npz examples/UNet/*.pkl examples/UNet/*.png examples/UNet/readme.txt diff --git a/examples/UNet/massachusetts_road_dataset_utils.py b/examples/UNet/massachusetts_road_dataset_utils.py index 19253ad..45ea777 100644 --- a/examples/UNet/massachusetts_road_dataset_utils.py +++ b/examples/UNet/massachusetts_road_dataset_utils.py @@ -100,12 +100,10 @@ def prepare_dataset(): data_valid, target_valid = load_data("data/validation") data_test, target_test = load_data("data/test") # loading np arrays is much faster than loading the images one by one every time - np.save("train_data.npy", data_train) - np.save("train_target.npy", target_train) - np.save("valid_data.npy", data_valid) - np.save("valid_target.npy", target_valid) - np.save("test_data.npy", data_test) - np.save("test_target.npy", target_test) + np.savez_compressed("road_segm_dataset.npz", + data_train="data_train", target_train="target_train", + data_valid="data_valid", target_valid="target_valid", + data_test="data_test", target_test="target_test") except: print "something went wrong, maybe the download?" diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index 9c92f86..d65767f 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -52,12 +52,14 @@ def main(): # (if you have, copy your repository including the data to an SSD, otherwise it will take a long time to # generate batches) mmap_mode = None - data_train = np.load("train_data.npy", mmap_mode=mmap_mode) - target_train = np.load("train_target.npy", mmap_mode=mmap_mode) - data_valid = np.load("valid_data.npy", mmap_mode=mmap_mode) - target_valid= np.load("valid_target.npy", mmap_mode=mmap_mode) - data_test = np.load("test_data.npy", mmap_mode=mmap_mode) - target_test = np.load("test_target.npy", mmap_mode=mmap_mode) + dataset = np.load("road_segm_dataset.npz", mmap_mode=mmap_mode) + data_train = dataset["data_train"] + target_train = dataset["target_train"] + data_valid = dataset["data_valid"] + target_valid = dataset["target_valid"] + data_test = dataset["data_test"] + target_test = dataset["target_test"] + # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). net = build_UNet(n_input_channels=3, BATCH_SIZE=BATCH_SIZE, num_output_classes=2, pad='same', @@ -141,7 +143,7 @@ def main(): for data, target in train_generator: # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so # that we can match it with the prediction via the crossentropy loss function - target_flat = target.flatten() + target_flat = target.ravel() loss, acc = train_fn(data.astype(np.float32), target_flat) losses_train.append(loss) accuracies_train.append(acc) @@ -154,7 +156,7 @@ def main(): accuracies_val = [] n_batches = 0 for data, target in validation_generator: - target_flat = target.flatten() + target_flat = target.ravel() loss, acc = val_fn(data.astype(np.float32), target_flat) losses_val.append(loss) accuracies_val.append(acc) From c657d51561c6cdeb38c9fb6071b5dbbf7c7a56f9 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 15:46:33 +0200 Subject: [PATCH 09/21] some minor fixes --- examples/UNet/massachusetts_road_dataset_utils.py | 6 +++--- examples/UNet/massachusetts_road_segm.py | 12 ++++++------ modelzoo/Unet.py | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/examples/UNet/massachusetts_road_dataset_utils.py b/examples/UNet/massachusetts_road_dataset_utils.py index 45ea777..e9ac19c 100644 --- a/examples/UNet/massachusetts_road_dataset_utils.py +++ b/examples/UNet/massachusetts_road_dataset_utils.py @@ -101,9 +101,9 @@ def prepare_dataset(): data_test, target_test = load_data("data/test") # loading np arrays is much faster than loading the images one by one every time np.savez_compressed("road_segm_dataset.npz", - data_train="data_train", target_train="target_train", - data_valid="data_valid", target_valid="target_valid", - data_test="data_test", target_test="target_test") + data_train=data_train, target_train=target_train, + data_valid=data_valid, target_valid=target_valid, + data_test=data_test, target_test=target_test) except: print "something went wrong, maybe the download?" diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index d65767f..c993b77 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -34,7 +34,7 @@ def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_i def main(): # only download dataset once. This takes a while. - if not os.path.isfile("test_target.npy"): + if not os.path.isfile("road_segm_dataset.npz"): # heuristic that I included to make sure the dataset is only donwloaded and prepared once prepare_dataset() @@ -85,7 +85,7 @@ def main(): x_sym = T.tensor4() seg_sym = T.ivector() - w_sym = class_weights[seg_sym] + w_sym = T.vector() # add some weight decay l2_loss = lasagne.regularization.regularize_network_params(output_layer_for_loss, lasagne.regularization.l2) * 1e-4 @@ -120,8 +120,8 @@ def main(): seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym) seg_output = seg_output.argmax(1) - train_fn = theano.function([x_sym, seg_sym], [loss, acc_train], updates=updates) - val_fn = theano.function([x_sym, seg_sym], [loss_val, acc]) + train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates) + val_fn = theano.function([x_sym, seg_sym, w_sym], [loss_val, acc]) get_segmentation = theano.function([x_sym], seg_output) # some data augmentation. If you want better results you should invest more effort here. I left rotations and @@ -144,7 +144,7 @@ def main(): # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so # that we can match it with the prediction via the crossentropy loss function target_flat = target.ravel() - loss, acc = train_fn(data.astype(np.float32), target_flat) + loss, acc = train_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) losses_train.append(loss) accuracies_train.append(acc) n_batches += 1 @@ -157,7 +157,7 @@ def main(): n_batches = 0 for data, target in validation_generator: target_flat = target.ravel() - loss, acc = val_fn(data.astype(np.float32), target_flat) + loss, acc = val_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) losses_val.append(loss) accuracies_val.append(acc) n_batches += 1 diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py index 84b7dfa..86dc752 100644 --- a/modelzoo/Unet.py +++ b/modelzoo/Unet.py @@ -52,7 +52,7 @@ def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='s net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) - net['dimshuffle'] = DimshuffleLayer(net['segLayer'], (1, 0, 2, 3)) + net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3)) net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) net['output_flattened'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax) From 49df413a2584b3366b405a86597860b9eca92d0b Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 26 Aug 2016 15:52:07 +0200 Subject: [PATCH 10/21] some more minor fixes. It now runs out of the box --- examples/UNet/generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/UNet/generators.py b/examples/UNet/generators.py index e1e1130..af1f421 100644 --- a/examples/UNet/generators.py +++ b/examples/UNet/generators.py @@ -8,7 +8,7 @@ def batch_generator(data, target, BATCH_SIZE, shuffle=False): np.random.seed() idx = np.arange(data.shape[0]) if shuffle: - np.random.shuffle() + np.random.shuffle(idx) idx_2 = np.array(idx) # if BATCH_SIZE is larger than len(data) we need to artificially enlarge the idx array (loop around) while BATCH_SIZE > len(idx): From f9c7fca8d2b1729a34415d31b466cf6adf1081d0 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 29 Aug 2016 11:06:20 +0200 Subject: [PATCH 11/21] switched back to uncompressed numpy arrays for saving the dataset (its faster to load and supports mmap) while being not much larger (10.5GB instead of 7.5GB with .npz). Also switched out batch_generator with f0k's suggestion. Added AUC score computation for validation set --- examples/UNet/generators.py | 17 +++++++-- .../UNet/massachusetts_road_dataset_utils.py | 10 +++-- examples/UNet/massachusetts_road_segm.py | 38 +++++++++---------- 3 files changed, 39 insertions(+), 26 deletions(-) diff --git a/examples/UNet/generators.py b/examples/UNet/generators.py index af1f421..b9bfc04 100644 --- a/examples/UNet/generators.py +++ b/examples/UNet/generators.py @@ -1,7 +1,18 @@ __author__ = 'Fabian Isensee' import numpy as np +import lasagne def batch_generator(data, target, BATCH_SIZE, shuffle=False): + if shuffle: + while True: + ids = np.random.choice(len(data), BATCH_SIZE) + yield data[ids], target[ids] + else: + for idx in range(0, len(data), BATCH_SIZE): + ids = slice(idx, idx + BATCH_SIZE) + yield data[ids], target[ids] + +def batch_generator_old(data, target, BATCH_SIZE, shuffle=False): ''' just a simple batch iterator, no cropping, no rotation, no anything ''' @@ -21,17 +32,17 @@ def batch_generator(data, target, BATCH_SIZE, shuffle=False): if ctr >= data.shape[0]: ctr -= data.shape[0] - def center_crop_generator(generator, output_size): ''' yields center crop of size output_size (may be 1d or 2d) from data and seg ''' - if type(output_size) not in (tuple, list): + '''if type(output_size) not in (tuple, list): center_crop = [output_size, output_size] elif len(output_size) == 2: center_crop = list(output_size) else: - raise ValueError("invalid output_size") + raise ValueError("invalid output_size")''' + center_crop = lasagne.utils.as_tuple(output_size, 2, int) for data, seg in generator: center = np.array(data.shape[2:])/2 yield data[:, :, int(center[0]-center_crop[0]/2.):int(center[0]+center_crop[0]/2.), int(center[1]-center_crop[1]/2.):int(center[1]+center_crop[1]/2.)], seg[:, :, int(center[0]-center_crop[0]/2.):int(center[0]+center_crop[0]/2.), int(center[1]-center_crop[1]/2.):int(center[1]+center_crop[1]/2.)] diff --git a/examples/UNet/massachusetts_road_dataset_utils.py b/examples/UNet/massachusetts_road_dataset_utils.py index e9ac19c..7988bc7 100644 --- a/examples/UNet/massachusetts_road_dataset_utils.py +++ b/examples/UNet/massachusetts_road_dataset_utils.py @@ -100,10 +100,12 @@ def prepare_dataset(): data_valid, target_valid = load_data("data/validation") data_test, target_test = load_data("data/test") # loading np arrays is much faster than loading the images one by one every time - np.savez_compressed("road_segm_dataset.npz", - data_train=data_train, target_train=target_train, - data_valid=data_valid, target_valid=target_valid, - data_test=data_test, target_test=target_test) + np.save("data_train.npy", data_train) + np.save("target_train.npy", target_train) + np.save("data_valid.npy", data_valid) + np.save("target_valid.npy", target_valid) + np.save("data_test.npy", data_test) + np.save("target_test.npy", target_test) except: print "something went wrong, maybe the download?" diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index c993b77..64dadce 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -13,6 +13,7 @@ from time import sleep from generators import batch_generator, threaded_generator, random_crop_generator, center_crop_generator from massachusetts_road_dataset_utils import prepare_dataset +from sklearn.metrics import roc_auc_score def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_images=10): fig_ctr = 0 @@ -42,7 +43,6 @@ def main(): BATCH_SIZE = 24 N_EPOCHS = 30 N_BATCHES_PER_EPOCH = 100 - N_BATCHES_PER_EPOCH_valid = 15 PATCH_SIZE = 128+64 # load the prepared data. They have been converted to np arrays because they are much faster to load than single image files. @@ -52,13 +52,12 @@ def main(): # (if you have, copy your repository including the data to an SSD, otherwise it will take a long time to # generate batches) mmap_mode = None - dataset = np.load("road_segm_dataset.npz", mmap_mode=mmap_mode) - data_train = dataset["data_train"] - target_train = dataset["target_train"] - data_valid = dataset["data_valid"] - target_valid = dataset["target_valid"] - data_test = dataset["data_test"] - target_test = dataset["target_test"] + data_train = np.load("data_train.npy", mmap_mode=mmap_mode) + target_train = np.load("target_train.npy", mmap_mode=mmap_mode) + data_valid = np.load("data_valid.npy", mmap_mode=mmap_mode) + target_valid = np.load("target_valid.npy", mmap_mode=mmap_mode) + data_test = np.load("data_test.npy", mmap_mode=mmap_mode) + target_test = np.load("target_test.npy", mmap_mode=mmap_mode) # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). @@ -117,23 +116,20 @@ def main(): updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate) # create a convenience function to get the segmentation - seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym) + seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym, deterministic=True) seg_output = seg_output.argmax(1) train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates) val_fn = theano.function([x_sym, seg_sym, w_sym], [loss_val, acc]) get_segmentation = theano.function([x_sym], seg_output) + # we need this for calculating the AUC score + get_class_probas = theano.function([x_sym], prediction_test) # some data augmentation. If you want better results you should invest more effort here. I left rotations and # deformations out for the sake of speed and simplicity train_generator = random_crop_generator(batch_generator(data_train, target_train, BATCH_SIZE, shuffle=True), PATCH_SIZE) train_generator = threaded_generator(train_generator, num_cached=10) - # there is no need for data augmentation on the validation. However we need patches of the same size which is why - # we are using the random crop generator here again - validation_generator = center_crop_generator(batch_generator(data_valid, target_valid, BATCH_SIZE, shuffle=False), PATCH_SIZE) - validation_generator = threaded_generator(validation_generator, num_cached=10) - # do the actual training for epoch in range(N_EPOCHS): print epoch @@ -141,6 +137,7 @@ def main(): n_batches = 0 accuracies_train = [] for data, target in train_generator: + print n_batches # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so # that we can match it with the prediction via the crossentropy loss function target_flat = target.ravel() @@ -154,16 +151,19 @@ def main(): losses_val = [] accuracies_val = [] - n_batches = 0 + auc_val = [] + # there is no need for data augmentation on the validation. However we need patches of the same size which is why + # we are using center crop generator + # since the validation generator does not loop around we need to reinstantiate it for every epoch + validation_generator = center_crop_generator(batch_generator(data_valid, target_valid, BATCH_SIZE, shuffle=False), PATCH_SIZE) + validation_generator = threaded_generator(validation_generator, num_cached=10) for data, target in validation_generator: target_flat = target.ravel() loss, acc = val_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) losses_val.append(loss) accuracies_val.append(acc) - n_batches += 1 - if n_batches > N_BATCHES_PER_EPOCH_valid: - break - print "val accuracy: ", np.mean(accuracies_val), " val loss: ", np.mean(losses_val) + auc_val.append(roc_auc_score(target_flat, get_class_probas(data)[:, 1])) + print "val accuracy: ", np.mean(accuracies_val), " val loss: ", np.mean(losses_val), " val AUC score: ", np.mean(auc_val) learning_rate *= 0.2 # save trained weights after each epoch with open("UNet_params_ep%03.0f.pkl"%epoch, 'w') as f: From b89f4d6b79d9a802207014eadd73c7f959a7aacb Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 29 Aug 2016 11:57:38 +0200 Subject: [PATCH 12/21] minor bugfixes, new pretrained weights --- .gitignore | 1 + examples/UNet/massachusetts_road_segm.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index d01177e..db78efc 100644 --- a/.gitignore +++ b/.gitignore @@ -63,6 +63,7 @@ target/ examples/UNet/data* examples/UNet/*.sh examples/UNet/*.npz +examples/UNet/*.npy examples/UNet/*.pkl examples/UNet/*.png examples/UNet/readme.txt diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index 64dadce..7eb66b1 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -15,18 +15,18 @@ from massachusetts_road_dataset_utils import prepare_dataset from sklearn.metrics import roc_auc_score -def plot_some_results(pred_fn, test_generator, BATCH_SIZE, PATCH_SIZE = 192, n_images=10): +def plot_some_results(pred_fn, test_generator, n_images=10): fig_ctr = 0 for data, seg in test_generator: res = pred_fn(data) - for d, s, r, p in zip(data, seg, res): + for d, s, r in zip(data, seg, res): plt.figure(figsize=(12, 6)) plt.subplot(1, 3, 1) plt.imshow(d.transpose(1,2,0)) plt.subplot(1, 3, 2) plt.imshow(s[0]) plt.subplot(1, 3, 3) - plt.imshow(r[0]) + plt.imshow(r) plt.savefig("road_segmentation_result_%03.0f.png"%fig_ctr) plt.close() fig_ctr += 1 @@ -75,10 +75,10 @@ def main(): class_weights = class_weights.astype(np.float32) # if you wish to load pretrained weights you can uncomment this code and modify the file name - # if you want, use my pretained weights (got around 96% accuracy and a loss of 0.11 using excessive data - # augmentation: cropping, rotation, elastic deformation) - # https://www.dropbox.com/s/t6juf6o2ix7dntk/UNet_roadSegmentation_Params.zip?dl=0 - '''with open("UNet_params_ep0.pkl", 'r') as f: + # if you want, use my pretained weights: + # val accuracy: 0.963513 val loss: 0.114994 val AUC score: 0.978996643458 + # https://www.dropbox.com/s/0vasqq491skf9iz/UNet_mass_road_segm_params.zip?dl=0 + '''with open("UNet_params_ep029.pkl", 'r') as f: params = cPickle.load(f) lasagne.layers.set_all_param_values(output_layer, params)''' @@ -171,7 +171,7 @@ def main(): # create some png files showing (raw image, ground truth, prediction). Of course we use the test set here ;-) test_gen = random_crop_generator(batch_generator(data_test, target_test, BATCH_SIZE), PATCH_SIZE) - plot_some_results(get_segmentation, test_gen, BATCH_SIZE) + plot_some_results(get_segmentation, test_gen, 30) if __name__ == "__main__": From 654c2d77c3b605706304e34dc0edf47be82bfd0c Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 29 Aug 2016 12:04:51 +0200 Subject: [PATCH 13/21] minor bug fixes and cleanup --- examples/UNet/massachusetts_road_segm.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index 7eb66b1..b3131a1 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -35,8 +35,8 @@ def plot_some_results(pred_fn, test_generator, n_images=10): def main(): # only download dataset once. This takes a while. - if not os.path.isfile("road_segm_dataset.npz"): - # heuristic that I included to make sure the dataset is only donwloaded and prepared once + # heuristic that I included to make sure the dataset is only donwloaded and prepared once + if not os.path.isfile("test_target.npy"): prepare_dataset() # set some hyper parameters. You should not have to touch anything if you have 4GB or more VRAM @@ -59,7 +59,6 @@ def main(): data_test = np.load("data_test.npy", mmap_mode=mmap_mode) target_test = np.load("target_test.npy", mmap_mode=mmap_mode) - # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). net = build_UNet(n_input_channels=3, BATCH_SIZE=BATCH_SIZE, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(PATCH_SIZE, PATCH_SIZE), @@ -137,7 +136,6 @@ def main(): n_batches = 0 accuracies_train = [] for data, target in train_generator: - print n_batches # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so # that we can match it with the prediction via the crossentropy loss function target_flat = target.ravel() From 4daee1ebe0253bc4864393ffa3646ba9971a06f3 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 29 Aug 2016 12:11:47 +0200 Subject: [PATCH 14/21] minor bug fix --- examples/UNet/massachusetts_road_segm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index b3131a1..3fd0fef 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -77,9 +77,9 @@ def main(): # if you want, use my pretained weights: # val accuracy: 0.963513 val loss: 0.114994 val AUC score: 0.978996643458 # https://www.dropbox.com/s/0vasqq491skf9iz/UNet_mass_road_segm_params.zip?dl=0 - '''with open("UNet_params_ep029.pkl", 'r') as f: + with open("UNet_params_ep029.pkl", 'r') as f: params = cPickle.load(f) - lasagne.layers.set_all_param_values(output_layer, params)''' + lasagne.layers.set_all_param_values(output_layer_for_loss, params) x_sym = T.tensor4() seg_sym = T.ivector() From 13938aad2b944463d6adf64290e8acf63c30344c Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 29 Aug 2016 12:13:06 +0200 Subject: [PATCH 15/21] forgot to comment loading of pretrained params --- examples/UNet/massachusetts_road_segm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index 3fd0fef..ed31d10 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -77,9 +77,9 @@ def main(): # if you want, use my pretained weights: # val accuracy: 0.963513 val loss: 0.114994 val AUC score: 0.978996643458 # https://www.dropbox.com/s/0vasqq491skf9iz/UNet_mass_road_segm_params.zip?dl=0 - with open("UNet_params_ep029.pkl", 'r') as f: + '''with open("UNet_params_ep029.pkl", 'r') as f: params = cPickle.load(f) - lasagne.layers.set_all_param_values(output_layer_for_loss, params) + lasagne.layers.set_all_param_values(output_layer_for_loss, params)''' x_sym = T.tensor4() seg_sym = T.ivector() From 1c8301549de41e7879e707a4ad2f4a5774b6d4e8 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 29 Aug 2016 15:37:22 +0200 Subject: [PATCH 16/21] renamed layers for clarity --- modelzoo/Unet.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py index 86dc752..e3c7df4 100644 --- a/modelzoo/Unet.py +++ b/modelzoo/Unet.py @@ -30,24 +30,24 @@ def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='s net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv1'] = Upscale2DLayer(net['encode_2'], 2) + net['upscale1'] = Upscale2DLayer(net['encode_2'], 2) - net['concat1'] = ConcatLayer([net['deconv1'], net['contr_4_2']], cropping=(None, None, "center", "center")) + net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']], cropping=(None, None, "center", "center")) net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv2'] = Upscale2DLayer(net['expand_1_2'], 2) + net['upscale2'] = Upscale2DLayer(net['expand_1_2'], 2) - net['concat2'] = ConcatLayer([net['deconv2'], net['contr_3_2']], cropping=(None, None, "center", "center")) + net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']], cropping=(None, None, "center", "center")) net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv3'] = Upscale2DLayer(net['expand_2_2'], 2) + net['upscale3'] = Upscale2DLayer(net['expand_2_2'], 2) - net['concat3'] = ConcatLayer([net['deconv3'], net['contr_2_2']], cropping=(None, None, "center", "center")) + net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']], cropping=(None, None, "center", "center")) net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) - net['deconv4'] = Upscale2DLayer(net['expand_3_2'], 2) + net['upscale4'] = Upscale2DLayer(net['expand_3_2'], 2) - net['concat4'] = ConcatLayer([net['deconv4'], net['contr_1_2']], cropping=(None, None, "center", "center")) + net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']], cropping=(None, None, "center", "center")) net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) From 5412a68d2c9035d40bfb8a64457923e7e1d55086 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Thu, 12 Jan 2017 14:05:39 +0100 Subject: [PATCH 17/21] python code for downloading and extracting pretrained weights, added more comments, less learning rate decay, fixed some typos, added try-except for import of dnn layers --- examples/UNet/massachusetts_road_segm.py | 38 ++++++++++++++---------- modelzoo/Unet.py | 5 +++- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index ed31d10..581c39e 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -36,14 +36,14 @@ def plot_some_results(pred_fn, test_generator, n_images=10): def main(): # only download dataset once. This takes a while. # heuristic that I included to make sure the dataset is only donwloaded and prepared once - if not os.path.isfile("test_target.npy"): + if not os.path.isfile("target_test.npy"): prepare_dataset() # set some hyper parameters. You should not have to touch anything if you have 4GB or more VRAM - BATCH_SIZE = 24 + BATCH_SIZE = 6 N_EPOCHS = 30 N_BATCHES_PER_EPOCH = 100 - PATCH_SIZE = 128+64 + PATCH_SIZE = 512 # load the prepared data. They have been converted to np arrays because they are much faster to load than single image files. # you will need some ram in order to have everything in memory. @@ -59,25 +59,33 @@ def main(): data_test = np.load("data_test.npy", mmap_mode=mmap_mode) target_test = np.load("target_test.npy", mmap_mode=mmap_mode) - # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). - net = build_UNet(n_input_channels=3, BATCH_SIZE=BATCH_SIZE, num_output_classes=2, pad='same', + # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). Keep in mind that this + # may not be ideal + net = build_UNet(n_input_channels=3, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(PATCH_SIZE, PATCH_SIZE), base_n_filters=16, do_dropout=False) output_layer_for_loss = net["output_flattened"] # this is np.sum(target_train == 0) and np.sum(target_train == 1). No need to compute this every time class_frequencies = np.array([2374093357., 118906643.]) - # we are taking the log here because we want the net to focus more on the road pixels but not too much (otherwise + # we will reweight the loss to put more focus on road pixels (because of class imbalance). This is a simple approach + # and could be improved if you also have a class imbalance in your experiments. + # we are taking **0.25 here because we want the net to focus more on the road pixels but not too much (otherwise # it would not be penalized enough for missclassifying terrain pixels which results in too many false positives) - class_weights = np.log(class_frequencies[[1,0]]) + class_weights = (class_frequencies[[1,0]])**0.25 class_weights = class_weights / np.sum(class_weights) * 2. class_weights = class_weights.astype(np.float32) - # if you wish to load pretrained weights you can uncomment this code and modify the file name - # if you want, use my pretained weights: + # if you wish to load pretrained weights you can uncomment this code # val accuracy: 0.963513 val loss: 0.114994 val AUC score: 0.978996643458 - # https://www.dropbox.com/s/0vasqq491skf9iz/UNet_mass_road_segm_params.zip?dl=0 - '''with open("UNet_params_ep029.pkl", 'r') as f: + '''if not os.path.isfile('UNet_params_ep029.pkl'): + import urllib + import zipfile + urllib.urlretrieve("https://s3.amazonaws.com/lasagne/recipes/pretrained/UNet_mass_road_segm_params.zip", 'pretrained_weights.zip') + zip_ref = zipfile.ZipFile('pretrained_weights.zip', 'r') + zip_ref.extractall("./") + zip_ref.close() + with open("UNet_params_ep029.pkl", 'r') as f: params = cPickle.load(f) lasagne.layers.set_all_param_values(output_layer_for_loss, params)''' @@ -101,7 +109,7 @@ def main(): prediction_test = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=True) loss_val = lasagne.objectives.categorical_crossentropy(prediction_test, seg_sym) - # we multiply our loss by a weight map. In this example the weight map only increases the loss for road pixels and + # we multiply our loss by a weight map. In this example the weight map simply increases the loss for road pixels and # decreases the loss for other pixels. We do this to ensure that the network puts more focus on getting the roads # right loss_val *= w_sym @@ -109,7 +117,7 @@ def main(): loss_val += l2_loss acc = T.mean(T.eq(T.argmax(prediction_test, axis=1), seg_sym), dtype=theano.config.floatX) - # learning rate has to be a shared variablebecause we decrease it with every epoch + # learning rate has to be a shared variable because we decrease it with every epoch params = lasagne.layers.get_all_params(output_layer_for_loss, trainable=True) learning_rate = theano.shared(np.float32(0.001)) updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate) @@ -162,14 +170,14 @@ def main(): accuracies_val.append(acc) auc_val.append(roc_auc_score(target_flat, get_class_probas(data)[:, 1])) print "val accuracy: ", np.mean(accuracies_val), " val loss: ", np.mean(losses_val), " val AUC score: ", np.mean(auc_val) - learning_rate *= 0.2 + learning_rate *= 0.8 # save trained weights after each epoch with open("UNet_params_ep%03.0f.pkl"%epoch, 'w') as f: cPickle.dump(lasagne.layers.get_all_param_values(output_layer_for_loss), f) # create some png files showing (raw image, ground truth, prediction). Of course we use the test set here ;-) test_gen = random_crop_generator(batch_generator(data_test, target_test, BATCH_SIZE), PATCH_SIZE) - plot_some_results(get_segmentation, test_gen, 30) + plot_some_results(get_segmentation, test_gen, 15) if __name__ == "__main__": diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py index e3c7df4..b4e83b7 100644 --- a/modelzoo/Unet.py +++ b/modelzoo/Unet.py @@ -1,7 +1,10 @@ __author__ = 'Fabian Isensee' from collections import OrderedDict from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer, Upscale2DLayer -from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer +try: + from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer +except ImportError: + from lasagne.layers import Conv2DLayer as ConvLayer import lasagne From d88900bd2c0634b98063ae7b9781c98154320c6b Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Thu, 12 Jan 2017 14:25:35 +0100 Subject: [PATCH 18/21] Unet now uses relu and He's initialization scheme as in the original UNet paper. Changed Upscale2DLayer to Deconv2DLaer as in paper. Changed Patch size to 512. Default setting for mmap_mode is now r as this is almost as fast as copying everything into memory --- examples/UNet/massachusetts_road_segm.py | 16 ++++---- modelzoo/Unet.py | 47 ++++++++++++------------ 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index 581c39e..ea5840a 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -23,10 +23,13 @@ def plot_some_results(pred_fn, test_generator, n_images=10): plt.figure(figsize=(12, 6)) plt.subplot(1, 3, 1) plt.imshow(d.transpose(1,2,0)) + plt.title("input patch") plt.subplot(1, 3, 2) plt.imshow(s[0]) + plt.title("ground truth") plt.subplot(1, 3, 3) plt.imshow(r) + plt.title("segmentation") plt.savefig("road_segmentation_result_%03.0f.png"%fig_ctr) plt.close() fig_ctr += 1 @@ -40,18 +43,15 @@ def main(): prepare_dataset() # set some hyper parameters. You should not have to touch anything if you have 4GB or more VRAM - BATCH_SIZE = 6 + BATCH_SIZE = 12 # this works if you have ~ 8GB VRAM. Use smaller BATCH_SIZE for other GPUs N_EPOCHS = 30 N_BATCHES_PER_EPOCH = 100 PATCH_SIZE = 512 # load the prepared data. They have been converted to np arrays because they are much faster to load than single image files. - # you will need some ram in order to have everything in memory. - # If you are having RAM issues, change mmap_mode to 'r'. This will not load the entire array into memory but rather - # read from disk the bits that we currently need - # (if you have, copy your repository including the data to an SSD, otherwise it will take a long time to - # generate batches) - mmap_mode = None + # This code will not load the entire array into memory but rather read from disk the bits that we currently need + # set mmap_mode to None if you want to load the data into RAM + mmap_mode = 'r' data_train = np.load("data_train.npy", mmap_mode=mmap_mode) target_train = np.load("target_train.npy", mmap_mode=mmap_mode) data_valid = np.load("data_valid.npy", mmap_mode=mmap_mode) @@ -62,7 +62,7 @@ def main(): # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). Keep in mind that this # may not be ideal net = build_UNet(n_input_channels=3, BATCH_SIZE=None, num_output_classes=2, pad='same', - nonlinearity=lasagne.nonlinearities.elu, input_dim=(PATCH_SIZE, PATCH_SIZE), + nonlinearity=lasagne.nonlinearities.rectify, input_dim=(PATCH_SIZE, PATCH_SIZE), base_n_filters=16, do_dropout=False) output_layer_for_loss = net["output_flattened"] diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py index b4e83b7..f046e89 100644 --- a/modelzoo/Unet.py +++ b/modelzoo/Unet.py @@ -1,58 +1,59 @@ __author__ = 'Fabian Isensee' from collections import OrderedDict -from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer, Upscale2DLayer +from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer, Deconv2DLayer try: from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer except ImportError: from lasagne.layers import Conv2DLayer as ConvLayer import lasagne +from lasagne.init import HeNormal def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(128, 128), base_n_filters=64, do_dropout=False): net = OrderedDict() net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1])) - net['contr_1_1'] = ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) - net['contr_1_2'] = ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_1_1'] = ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_1_2'] = ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) net['pool1'] = Pool2DLayer(net['contr_1_2'], 2) - net['contr_2_1'] = ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) - net['contr_2_2'] = ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_2_1'] = ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_2_2'] = ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) net['pool2'] = Pool2DLayer(net['contr_2_2'], 2) - net['contr_3_1'] = ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) - net['contr_3_2'] = ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_3_1'] = ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_3_2'] = ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) net['pool3'] = Pool2DLayer(net['contr_3_2'], 2) - net['contr_4_1'] = ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) - net['contr_4_2'] = ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) + net['contr_4_1'] = ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_4_2'] = ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) # the paper does not really describe where and how dropout is added. Feel free to try more options if do_dropout: l = DropoutLayer(l, p=0.4) - net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) - net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad) - net['upscale1'] = Upscale2DLayer(net['encode_2'], 2) + net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['upscale1'] = Deconv2DLayer(net['encode_2'], base_n_filters*16, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']], cropping=(None, None, "center", "center")) - net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) - net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad) - net['upscale2'] = Upscale2DLayer(net['expand_1_2'], 2) + net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['upscale2'] = Deconv2DLayer(net['expand_1_2'], base_n_filters*8, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']], cropping=(None, None, "center", "center")) - net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) - net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad) - net['upscale3'] = Upscale2DLayer(net['expand_2_2'], 2) + net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['upscale3'] = Deconv2DLayer(net['expand_2_2'], base_n_filters*4, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']], cropping=(None, None, "center", "center")) - net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) - net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad) - net['upscale4'] = Upscale2DLayer(net['expand_3_2'], 2) + net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['upscale4'] = Deconv2DLayer(net['expand_3_2'], base_n_filters*2, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']], cropping=(None, None, "center", "center")) - net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) - net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad) + net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3)) From 570b18e25c3e9efab720d5b576541468074f2d63 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Mon, 16 Jan 2017 18:03:25 +0100 Subject: [PATCH 19/21] new pretrained weights --- examples/UNet/massachusetts_road_segm.py | 17 +++++---- modelzoo/Unet.py | 47 ++++++++++++------------ 2 files changed, 33 insertions(+), 31 deletions(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index ea5840a..f26a2b4 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -43,8 +43,8 @@ def main(): prepare_dataset() # set some hyper parameters. You should not have to touch anything if you have 4GB or more VRAM - BATCH_SIZE = 12 # this works if you have ~ 8GB VRAM. Use smaller BATCH_SIZE for other GPUs - N_EPOCHS = 30 + BATCH_SIZE = 8 # this works if you have ~ 8GB VRAM. Use smaller BATCH_SIZE for other GPUs + N_EPOCHS = 50 N_BATCHES_PER_EPOCH = 100 PATCH_SIZE = 512 @@ -77,15 +77,16 @@ def main(): class_weights = class_weights.astype(np.float32) # if you wish to load pretrained weights you can uncomment this code - # val accuracy: 0.963513 val loss: 0.114994 val AUC score: 0.978996643458 - '''if not os.path.isfile('UNet_params_ep029.pkl'): + # val accuracy: 0.966384 val loss: 0.0947428 val AUC score: 0.980004909707 + # you can also change the lower part of this code to load your own pretrained params + '''if not os.path.isfile('UNet_params_pretrained.pkl'): import urllib import zipfile urllib.urlretrieve("https://s3.amazonaws.com/lasagne/recipes/pretrained/UNet_mass_road_segm_params.zip", 'pretrained_weights.zip') zip_ref = zipfile.ZipFile('pretrained_weights.zip', 'r') zip_ref.extractall("./") zip_ref.close() - with open("UNet_params_ep029.pkl", 'r') as f: + with open("UNet_params_pretrained.pkl", 'r') as f: params = cPickle.load(f) lasagne.layers.set_all_param_values(output_layer_for_loss, params)''' @@ -97,7 +98,7 @@ def main(): l2_loss = lasagne.regularization.regularize_network_params(output_layer_for_loss, lasagne.regularization.l2) * 1e-4 # the distinction between prediction_train and test is important only if we enable dropout - prediction_train = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=False) + prediction_train = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=False, batch_norm_update_averages=False, batch_norm_use_averages=False) # we could use a binary loss but I stuck with categorical crossentropy so that less code has to be changed if your # application has more than two classes loss = lasagne.objectives.categorical_crossentropy(prediction_train, seg_sym) @@ -106,7 +107,7 @@ def main(): loss += l2_loss acc_train = T.mean(T.eq(T.argmax(prediction_train, axis=1), seg_sym), dtype=theano.config.floatX) - prediction_test = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=True) + prediction_test = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=True, batch_norm_update_averages=False, batch_norm_use_averages=False) loss_val = lasagne.objectives.categorical_crossentropy(prediction_test, seg_sym) # we multiply our loss by a weight map. In this example the weight map simply increases the loss for road pixels and @@ -138,7 +139,7 @@ def main(): train_generator = threaded_generator(train_generator, num_cached=10) # do the actual training - for epoch in range(N_EPOCHS): + for epoch in np.arange(0, N_EPOCHS): print epoch losses_train = [] n_batches = 0 diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py index f046e89..c530bf8 100644 --- a/modelzoo/Unet.py +++ b/modelzoo/Unet.py @@ -1,6 +1,7 @@ __author__ = 'Fabian Isensee' from collections import OrderedDict -from lasagne.layers import InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, DropoutLayer, Deconv2DLayer +from lasagne.layers import (InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, + DropoutLayer, Deconv2DLayer, batch_norm) try: from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer except ImportError: @@ -13,47 +14,47 @@ def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='s net = OrderedDict() net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1])) - net['contr_1_1'] = ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['contr_1_2'] = ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_1_1'] = batch_norm(ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_1_2'] = batch_norm(ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) net['pool1'] = Pool2DLayer(net['contr_1_2'], 2) - net['contr_2_1'] = ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['contr_2_2'] = ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_2_1'] = batch_norm(ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_2_2'] = batch_norm(ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) net['pool2'] = Pool2DLayer(net['contr_2_2'], 2) - net['contr_3_1'] = ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['contr_3_2'] = ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_3_1'] = batch_norm(ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_3_2'] = batch_norm(ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) net['pool3'] = Pool2DLayer(net['contr_3_2'], 2) - net['contr_4_1'] = ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['contr_4_2'] = ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['contr_4_1'] = batch_norm(ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_4_2'] = batch_norm(ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) # the paper does not really describe where and how dropout is added. Feel free to try more options if do_dropout: l = DropoutLayer(l, p=0.4) - net['encode_1'] = ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['encode_2'] = ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['upscale1'] = Deconv2DLayer(net['encode_2'], base_n_filters*16, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) + net['encode_1'] = batch_norm(ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['encode_2'] = batch_norm(ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale1'] = batch_norm(Deconv2DLayer(net['encode_2'], base_n_filters*16, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']], cropping=(None, None, "center", "center")) - net['expand_1_1'] = ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['expand_1_2'] = ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['upscale2'] = Deconv2DLayer(net['expand_1_2'], base_n_filters*8, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) + net['expand_1_1'] = batch_norm(ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_1_2'] = batch_norm(ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale2'] = batch_norm(Deconv2DLayer(net['expand_1_2'], base_n_filters*8, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']], cropping=(None, None, "center", "center")) - net['expand_2_1'] = ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['expand_2_2'] = ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['upscale3'] = Deconv2DLayer(net['expand_2_2'], base_n_filters*4, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) + net['expand_2_1'] = batch_norm(ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_2_2'] = batch_norm(ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale3'] = batch_norm(Deconv2DLayer(net['expand_2_2'], base_n_filters*4, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']], cropping=(None, None, "center", "center")) - net['expand_3_1'] = ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['expand_3_2'] = ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['upscale4'] = Deconv2DLayer(net['expand_3_2'], base_n_filters*2, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu")) + net['expand_3_1'] = batch_norm(ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_3_2'] = batch_norm(ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale4'] = batch_norm(Deconv2DLayer(net['expand_3_2'], base_n_filters*2, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']], cropping=(None, None, "center", "center")) - net['expand_4_1'] = ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) - net['expand_4_2'] = ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu")) + net['expand_4_1'] = batch_norm(ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_4_2'] = batch_norm(ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3)) From d25461731732487ae8a6dc9ea6c338ef5986018b Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Wed, 24 Aug 2016 13:19:33 +0200 Subject: [PATCH 20/21] This commit adds the UNet segmentation network to lasagne's recipes. UNet is implemented almost identically to the paper. Solely the padding has been set to 'same' (instead of 'valid') and we make use of batch normalization. The functionality of UNet is demonstrated at the example of road segmentation on a pablicly available dataset. added UNet + example (road segmentation) Reduced number of plotted test images, Added a plot of the road probability map renamed some layers of UNet to make their names more clear, added convenience function get_segmentation, changed upscaling in UNet from Deconv2DLayer to Upscale2DLayer code restructuring (now it's not cramped into a single .py file anymore). Also fixed a typo in massachusetts now using urlretrieve for downloading the data. Also uses multiprocessing.Pool for miltithreaded download. Seems to work well! updated batch_iterator so that it now supports deterministic batch generation (needed for validation) added center crop generator now uses compressed numpy arrays to store the dataset some minor fixes some more minor fixes. It now runs out of the box switched back to uncompressed numpy arrays for saving the dataset (its faster to load and supports mmap) while being not much larger (10.5GB instead of 7.5GB with .npz). Also switched out batch_generator with f0k's suggestion. Added AUC score computation for validation set minor bugfixes, new pretrained weights minor bug fixes and cleanup minor bug fix forgot to comment loading of pretrained params renamed layers for clarity python code for downloading and extracting pretrained weights, added more comments, less learning rate decay, fixed some typos, added try-except for import of dnn layers Unet now uses relu and He's initialization scheme as in the original UNet paper. Changed Upscale2DLayer to Deconv2DLaer as in paper. Changed Patch size to 512. Default setting for mmap_mode is now r as this is almost as fast as copying everything into memory new pretrained weights --- .gitignore | 14 + examples/UNet/generators.py | 92 ++ examples/UNet/mass_roads_test.txt | 49 + examples/UNet/mass_roads_train.txt | 1108 +++++++++++++++++ examples/UNet/mass_roads_validation.txt | 14 + .../UNet/massachusetts_road_dataset_utils.py | 113 ++ examples/UNet/massachusetts_road_segm.py | 185 +++ modelzoo/Unet.py | 66 + 8 files changed, 1641 insertions(+) create mode 100644 examples/UNet/generators.py create mode 100644 examples/UNet/mass_roads_test.txt create mode 100644 examples/UNet/mass_roads_train.txt create mode 100644 examples/UNet/mass_roads_validation.txt create mode 100644 examples/UNet/massachusetts_road_dataset_utils.py create mode 100644 examples/UNet/massachusetts_road_segm.py create mode 100644 modelzoo/Unet.py diff --git a/.gitignore b/.gitignore index ba74660..db78efc 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ __pycache__/ # C extensions *.so +# datasets +*.zip + # Distribution / packaging .Python env/ @@ -22,6 +25,7 @@ var/ *.egg-info/ .installed.cfg *.egg +.idea # PyInstaller # Usually these files are written by a python script from a template @@ -55,3 +59,13 @@ docs/_build/ # PyBuilder target/ + +examples/UNet/data* +examples/UNet/*.sh +examples/UNet/*.npz +examples/UNet/*.npy +examples/UNet/*.pkl +examples/UNet/*.png +examples/UNet/readme.txt +.gitignore~ + diff --git a/examples/UNet/generators.py b/examples/UNet/generators.py new file mode 100644 index 0000000..b9bfc04 --- /dev/null +++ b/examples/UNet/generators.py @@ -0,0 +1,92 @@ +__author__ = 'Fabian Isensee' +import numpy as np +import lasagne + +def batch_generator(data, target, BATCH_SIZE, shuffle=False): + if shuffle: + while True: + ids = np.random.choice(len(data), BATCH_SIZE) + yield data[ids], target[ids] + else: + for idx in range(0, len(data), BATCH_SIZE): + ids = slice(idx, idx + BATCH_SIZE) + yield data[ids], target[ids] + +def batch_generator_old(data, target, BATCH_SIZE, shuffle=False): + ''' + just a simple batch iterator, no cropping, no rotation, no anything + ''' + np.random.seed() + idx = np.arange(data.shape[0]) + if shuffle: + np.random.shuffle(idx) + idx_2 = np.array(idx) + # if BATCH_SIZE is larger than len(data) we need to artificially enlarge the idx array (loop around) + while BATCH_SIZE > len(idx): + idx_2 = np.concatenate((idx_2, idx)) + del(idx) + while True: + ctr = 0 + yield np.array(data[idx_2[ctr:ctr+BATCH_SIZE]]), np.array(target[idx_2[ctr:ctr+BATCH_SIZE]]) + ctr += BATCH_SIZE + if ctr >= data.shape[0]: + ctr -= data.shape[0] + +def center_crop_generator(generator, output_size): + ''' + yields center crop of size output_size (may be 1d or 2d) from data and seg + ''' + '''if type(output_size) not in (tuple, list): + center_crop = [output_size, output_size] + elif len(output_size) == 2: + center_crop = list(output_size) + else: + raise ValueError("invalid output_size")''' + center_crop = lasagne.utils.as_tuple(output_size, 2, int) + for data, seg in generator: + center = np.array(data.shape[2:])/2 + yield data[:, :, int(center[0]-center_crop[0]/2.):int(center[0]+center_crop[0]/2.), int(center[1]-center_crop[1]/2.):int(center[1]+center_crop[1]/2.)], seg[:, :, int(center[0]-center_crop[0]/2.):int(center[0]+center_crop[0]/2.), int(center[1]-center_crop[1]/2.):int(center[1]+center_crop[1]/2.)] + + +def random_crop_generator(generator, crop_size=(128, 128)): + ''' + yields a random crop of size crop_size + ''' + if type(crop_size) not in (tuple, list): + crop_size = [crop_size, crop_size] + elif len(crop_size) == 2: + crop_size = list(crop_size) + else: + raise ValueError("invalid crop_size") + for data, seg in generator: + lb_x = np.random.randint(0, data.shape[2]-crop_size[0]) + lb_y = np.random.randint(0, data.shape[3]-crop_size[1]) + data = data[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] + seg = seg[:, :, lb_x:lb_x+crop_size[0], lb_y:lb_y+crop_size[1]] + yield data, seg + +def threaded_generator(generator, num_cached=10): + # this code is written by jan Schluter + # copied from https://github.com/benanne/Lasagne/issues/12 + import Queue + queue = Queue.Queue(maxsize=num_cached) + sentinel = object() # guaranteed unique reference + + # define producer (putting items into queue) + def producer(): + for item in generator: + queue.put(item) + queue.put(sentinel) + + # start producer (in a background thread) + import threading + thread = threading.Thread(target=producer) + thread.daemon = True + thread.start() + + # run as consumer (read items from queue, in current thread) + item = queue.get() + while item is not sentinel: + yield item + queue.task_done() + item = queue.get() diff --git a/examples/UNet/mass_roads_test.txt b/examples/UNet/mass_roads_test.txt new file mode 100644 index 0000000..afd6d7a --- /dev/null +++ b/examples/UNet/mass_roads_test.txt @@ -0,0 +1,49 @@ +10378780_15.tiff +10828720_15.tiff +11128870_15.tiff +11278840_15.tiff +11728825_15.tiff +12328750_15.tiff +15928855_15.tiff +16078870_15.tiff +17878735_15.tiff +17878780_15.tiff +17878885_15.tiff +18028750_15.tiff +18178780_15.tiff +18328735_15.tiff +18328780_15.tiff +18328960_15.tiff +18478735_15.tiff +18478900_15.tiff +18478930_15.tiff +20278885_15.tiff +20728960_15.tiff +20878930_15.tiff +21479035_15.tiff +21779005_15.tiff +22078975_15.tiff +22228900_15.tiff +22229050_15.tiff +22528885_15.tiff +22529065_15.tiff +23278915_15.tiff +23278930_15.tiff +23428540_15.tiff +23428810_15.tiff +23429080_15.tiff +23729020_15.tiff +23878540_15.tiff +24029035_15.tiff +24478825_15.tiff +24478855_15.tiff +24479170_15.tiff +24479215_15.tiff +24628885_15.tiff +24629200_15.tiff +24779275_15.tiff +25079170_15.tiff +26278720_15.tiff +26428735_15.tiff +26578720_15.tiff +26878690_15.tiff diff --git a/examples/UNet/mass_roads_train.txt b/examples/UNet/mass_roads_train.txt new file mode 100644 index 0000000..144dd39 --- /dev/null +++ b/examples/UNet/mass_roads_train.txt @@ -0,0 +1,1108 @@ +10078660_15.tif +10078675_15.tif +10078690_15.tif +10078705_15.tif +10078720_15.tif +10078735_15.tif +10078750_15.tif +10228660_15.tif +10228675_15.tif +10228705_15.tif +10228720_15.tif +10228735_15.tif +10228750_15.tif +10228765_15.tif +10228780_15.tif +10228795_15.tif +10378660_15.tif +10378675_15.tif +10378690_15.tif +10378705_15.tif +10378720_15.tif +10378735_15.tif +10378750_15.tif +10378765_15.tif +10378795_15.tif +10528660_15.tif +10528675_15.tif +10528690_15.tif +10528705_15.tif +10528720_15.tif +10528735_15.tif +10528750_15.tif +10528765_15.tif +10528780_15.tif +10528795_15.tif +10678660_15.tif +10678675_15.tif +10678690_15.tif +10678705_15.tif +10678720_15.tif +10678735_15.tif +10678750_15.tif +10678765_15.tif +10678780_15.tif +10678795_15.tif +10678810_15.tif +10678825_15.tif +10678870_15.tif +10678885_15.tif +10828645_15.tif +10828660_15.tif +10828675_15.tif +10828690_15.tif +10828705_15.tif +10828735_15.tif +10828750_15.tif +10828765_15.tif +10828780_15.tif +10828795_15.tif +10828810_15.tif +10828825_15.tif +10828855_15.tif +10828870_15.tif +10828900_15.tif +10978645_15.tif +10978660_15.tif +10978675_15.tif +10978690_15.tif +10978705_15.tif +10978720_15.tif +10978750_15.tif +10978765_15.tif +10978780_15.tif +10978810_15.tif +10978825_15.tif +10978840_15.tif +10978855_15.tif +10978870_15.tif +10978885_15.tif +10978900_15.tif +10978915_15.tif +10978930_15.tif +10978945_15.tif +11128645_15.tif +11128660_15.tif +11128675_15.tif +11128690_15.tif +11128705_15.tif +11128720_15.tif +11128735_15.tif +11128750_15.tif +11128765_15.tif +11128780_15.tif +11128795_15.tif +11128810_15.tif +11128825_15.tif +11128840_15.tif +11128855_15.tif +11128885_15.tif +11128900_15.tif +11128915_15.tif +11128930_15.tif +11128945_15.tif +11278645_15.tif +11278660_15.tif +11278675_15.tif +11278690_15.tif +11278705_15.tif +11278720_15.tif +11278735_15.tif +11278750_15.tif +11278765_15.tif +11278780_15.tif +11278795_15.tif +11278810_15.tif +11278825_15.tif +11278855_15.tif +11278870_15.tif +11278885_15.tif +11278900_15.tif +11278915_15.tif +11278930_15.tif +11278945_15.tif +11428660_15.tif +11428675_15.tif +11428690_15.tif +11428705_15.tif +11428720_15.tif +11428735_15.tif +11428750_15.tif +11428765_15.tif +11428780_15.tif +11428795_15.tif +11428810_15.tif +11428825_15.tif +11428840_15.tif +11428855_15.tif +11428930_15.tif +11428945_15.tif +11578660_15.tif +11578675_15.tif +11578690_15.tif +11578705_15.tif +11578720_15.tif +11578735_15.tif +11578750_15.tif +11578765_15.tif +11578780_15.tif +11578795_15.tif +11578810_15.tif +11578825_15.tif +11578840_15.tif +11578855_15.tif +11728660_15.tif +11728675_15.tif +11728690_15.tif +11728705_15.tif +11728720_15.tif +11728735_15.tif +11728750_15.tif +11728765_15.tif +11728780_15.tif +11728795_15.tif +11728810_15.tif +11728840_15.tif +11728855_15.tif +11878660_15.tif +11878675_15.tif +11878690_15.tif +11878705_15.tif +11878720_15.tif +11878735_15.tif +11878750_15.tif +11878765_15.tif +11878780_15.tif +11878795_15.tif +12028660_15.tif +12028675_15.tif +12028690_15.tif +12028705_15.tif +12028720_15.tif +12028735_15.tif +12028750_15.tif +12028765_15.tif +12028780_15.tif +12178705_15.tif +12178720_15.tif +12178735_15.tif +12178750_15.tif +12178765_15.tif +12178780_15.tif +12328720_15.tif +12328735_15.tif +12328765_15.tif +12328780_15.tif +12478720_15.tif +12478735_15.tif +12478750_15.tif +12478765_15.tif +12478780_15.tif +12478795_15.tif +12478810_15.tif +12628735_15.tif +12628750_15.tif +12628765_15.tif +12628780_15.tif +12628795_15.tif +12628810_15.tif +15628825_15.tif +15628840_15.tif +15628855_15.tif +15628870_15.tif +15628885_15.tif +15628900_15.tif +15628915_15.tif +15628930_15.tif +15628945_15.tif +15628960_15.tif +15778825_15.tif +15778840_15.tif +15778855_15.tif +15778870_15.tif +15778885_15.tif +15778900_15.tif +15778915_15.tif +15778930_15.tif +15778945_15.tif +15778960_15.tif +15928825_15.tif +15928840_15.tif +15928870_15.tif +15928885_15.tif +15928900_15.tif +15928915_15.tif +15928930_15.tif +15928945_15.tif +15928960_15.tif +16078825_15.tif +16078840_15.tif +16078855_15.tif +16078885_15.tif +16078900_15.tif +16078915_15.tif +16078930_15.tif +16078945_15.tif +16078960_15.tif +16228825_15.tif +16228840_15.tif +16228855_15.tif +16228870_15.tif +16228885_15.tif +16228900_15.tif +16228915_15.tif +16228930_15.tif +16228945_15.tif +16228960_15.tif +16378825_15.tif +16378840_15.tif +16378855_15.tif +16828900_15.tif +16828915_15.tif +16828930_15.tif +16978870_15.tif +16978885_15.tif +16978915_15.tif +16978930_15.tif +16978945_15.tif +17128870_15.tif +17128885_15.tif +17128900_15.tif +17128915_15.tif +17128930_15.tif +17128945_15.tif +17128960_15.tif +17278750_15.tif +17278765_15.tif +17278780_15.tif +17278870_15.tif +17278885_15.tif +17278900_15.tif +17278915_15.tif +17278930_15.tif +17278945_15.tif +17278960_15.tif +17428735_15.tif +17428750_15.tif +17428765_15.tif +17428780_15.tif +17428855_15.tif +17428870_15.tif +17428885_15.tif +17428900_15.tif +17428915_15.tif +17428930_15.tif +17428945_15.tif +17428960_15.tif +17428975_15.tif +17578720_15.tif +17578735_15.tif +17578750_15.tif +17578765_15.tif +17578780_15.tif +17578795_15.tif +17578840_15.tif +17578855_15.tif +17578870_15.tif +17578885_15.tif +17578900_15.tif +17578915_15.tif +17578930_15.tif +17578945_15.tif +17578960_15.tif +17578975_15.tif +17578990_15.tif +17728705_15.tif +17728720_15.tif +17728735_15.tif +17728750_15.tif +17728765_15.tif +17728780_15.tif +17728795_15.tif +17728810_15.tif +17728855_15.tif +17728870_15.tif +17728885_15.tif +17728900_15.tif +17728915_15.tif +17728930_15.tif +17728945_15.tif +17728960_15.tif +17728975_15.tif +17728990_15.tif +17878705_15.tif +17878720_15.tif +17878750_15.tif +17878765_15.tif +17878795_15.tif +17878810_15.tif +17878855_15.tif +17878870_15.tif +17878900_15.tif +17878915_15.tif +17878930_15.tif +17878945_15.tif +17878960_15.tif +18028705_15.tif +18028720_15.tif +18028735_15.tif +18028765_15.tif +18028780_15.tif +18028795_15.tif +18028810_15.tif +18028825_15.tif +18028870_15.tif +18028885_15.tif +18028900_15.tif +18028915_15.tif +18028930_15.tif +18028960_15.tif +18178705_15.tif +18178720_15.tif +18178735_15.tif +18178750_15.tif +18178765_15.tif +18178795_15.tif +18178810_15.tif +18178825_15.tif +18178885_15.tif +18178900_15.tif +18178915_15.tif +18178930_15.tif +18178945_15.tif +18178960_15.tif +18178975_15.tif +18328720_15.tif +18328750_15.tif +18328765_15.tif +18328795_15.tif +18328810_15.tif +18328885_15.tif +18328900_15.tif +18328915_15.tif +18328930_15.tif +18328945_15.tif +18328975_15.tif +18478720_15.tif +18478750_15.tif +18478765_15.tif +18478780_15.tif +18478795_15.tif +18478885_15.tif +18478915_15.tif +18478945_15.tif +18478960_15.tif +18478975_15.tif +18628720_15.tif +18628735_15.tif +18628750_15.tif +18628765_15.tif +18628780_15.tif +18628795_15.tif +18628885_15.tif +18628900_15.tif +18628915_15.tif +18628930_15.tif +18778720_15.tif +18778735_15.tif +18778750_15.tif +18778765_15.tif +18778780_15.tif +18778795_15.tif +18928720_15.tif +18928735_15.tif +18928750_15.tif +18928765_15.tif +18928780_15.tif +18928795_15.tif +19078735_15.tif +19078750_15.tif +19078765_15.tif +19228735_15.tif +19228750_15.tif +19528630_15.tif +19528645_15.tif +19528660_15.tif +19528675_15.tif +19678630_15.tif +19678645_15.tif +19678660_15.tif +19678675_15.tif +19828630_15.tif +19828645_15.tif +19828660_15.tif +19828675_15.tif +19828885_15.tif +19828900_15.tif +19828915_15.tif +19978630_15.tif +19978645_15.tif +19978660_15.tif +19978675_15.tif +19978885_15.tif +19978900_15.tif +19978915_15.tif +19978930_15.tif +19978945_15.tif +20128870_15.tif +20128885_15.tif +20128900_15.tif +20128915_15.tif +20128930_15.tif +20128945_15.tif +20128960_15.tif +20128975_15.tif +20128990_15.tif +20129005_15.tif +20278870_15.tif +20278900_15.tif +20278915_15.tif +20278930_15.tif +20278945_15.tif +20278960_15.tif +20278975_15.tif +20278990_15.tif +20279005_15.tif +20428870_15.tif +20428885_15.tif +20428900_15.tif +20428915_15.tif +20428930_15.tif +20428945_15.tif +20428960_15.tif +20428975_15.tif +20428990_15.tif +20429005_15.tif +20578870_15.tif +20578885_15.tif +20578900_15.tif +20578915_15.tif +20578930_15.tif +20578945_15.tif +20578960_15.tif +20578975_15.tif +20578990_15.tif +20579005_15.tif +20728870_15.tif +20728885_15.tif +20728900_15.tif +20728915_15.tif +20728930_15.tif +20728945_15.tif +20728975_15.tif +20728990_15.tif +20729005_15.tif +20878900_15.tif +20878915_15.tif +20878945_15.tif +20878960_15.tif +20878975_15.tif +20878990_15.tif +20879005_15.tif +21028900_15.tif +21028915_15.tif +21028930_15.tif +21028945_15.tif +21028960_15.tif +21028975_15.tif +21178885_15.tif +21178900_15.tif +21178915_15.tif +21178930_15.tif +21178945_15.tif +21178960_15.tif +21178975_15.tif +21328870_15.tif +21328885_15.tif +21328900_15.tif +21328915_15.tif +21328930_15.tif +21328945_15.tif +21328960_15.tif +21328975_15.tif +21328990_15.tif +21329005_15.tif +21329020_15.tif +21329035_15.tif +21329050_15.tif +21478870_15.tif +21478885_15.tif +21478900_15.tif +21478915_15.tif +21478930_15.tif +21478945_15.tif +21478960_15.tif +21478975_15.tif +21478990_15.tif +21479005_15.tif +21479020_15.tif +21479050_15.tif +21479065_15.tif +21628870_15.tif +21628885_15.tif +21628900_15.tif +21628915_15.tif +21628930_15.tif +21628945_15.tif +21628960_15.tif +21628975_15.tif +21628990_15.tif +21629005_15.tif +21629020_15.tif +21629035_15.tif +21629050_15.tif +21629065_15.tif +21778900_15.tif +21778915_15.tif +21778930_15.tif +21778945_15.tif +21778960_15.tif +21778975_15.tif +21778990_15.tif +21779020_15.tif +21779035_15.tif +21779050_15.tif +21779065_15.tif +21779080_15.tif +21928900_15.tif +21928915_15.tif +21928930_15.tif +21928945_15.tif +21928960_15.tif +21928975_15.tif +21928990_15.tif +21929005_15.tif +21929035_15.tif +21929050_15.tif +21929065_15.tif +21929080_15.tif +22078900_15.tif +22078915_15.tif +22078930_15.tif +22078945_15.tif +22078960_15.tif +22078990_15.tif +22079005_15.tif +22079020_15.tif +22079035_15.tif +22079050_15.tif +22079065_15.tif +22079080_15.tif +22079365_15.tif +22079380_15.tif +22079395_15.tif +22079410_15.tif +22079425_15.tif +22228915_15.tif +22228930_15.tif +22228945_15.tif +22228960_15.tif +22228975_15.tif +22228990_15.tif +22229005_15.tif +22229020_15.tif +22229035_15.tif +22229065_15.tif +22229080_15.tif +22229365_15.tif +22229380_15.tif +22229395_15.tif +22229410_15.tif +22229425_15.tif +22229440_15.tif +22378885_15.tif +22378900_15.tif +22378915_15.tif +22378930_15.tif +22378945_15.tif +22378960_15.tif +22378975_15.tif +22378990_15.tif +22379005_15.tif +22379020_15.tif +22379035_15.tif +22379050_15.tif +22379065_15.tif +22379080_15.tif +22379380_15.tif +22379395_15.tif +22379410_15.tif +22379425_15.tif +22379440_15.tif +22528870_15.tif +22528915_15.tif +22528930_15.tif +22528945_15.tif +22528960_15.tif +22528975_15.tif +22528990_15.tif +22529005_15.tif +22529020_15.tif +22529035_15.tif +22529050_15.tif +22529080_15.tif +22529380_15.tif +22529395_15.tif +22529410_15.tif +22529425_15.tif +22529440_15.tif +22529455_15.tif +22529470_15.tif +22529485_15.tif +22678855_15.tif +22678870_15.tif +22678885_15.tif +22678900_15.tif +22678915_15.tif +22678930_15.tif +22678945_15.tif +22678960_15.tif +22678975_15.tif +22678990_15.tif +22679005_15.tif +22679020_15.tif +22679035_15.tif +22679050_15.tif +22679065_15.tif +22679080_15.tif +22679410_15.tif +22679425_15.tif +22679440_15.tif +22679455_15.tif +22679470_15.tif +22679485_15.tif +22828855_15.tif +22828870_15.tif +22828885_15.tif +22828900_15.tif +22828915_15.tif +22828930_15.tif +22828945_15.tif +22828960_15.tif +22828975_15.tif +22828990_15.tif +22829005_15.tif +22829020_15.tif +22829050_15.tif +22829065_15.tif +22829410_15.tif +22829425_15.tif +22829440_15.tif +22829455_15.tif +22829470_15.tif +22829485_15.tif +22978840_15.tif +22978855_15.tif +22978870_15.tif +22978885_15.tif +22978900_15.tif +22978915_15.tif +22978930_15.tif +22978945_15.tif +22978960_15.tif +22978975_15.tif +22979005_15.tif +22979020_15.tif +22979035_15.tif +22979050_15.tif +22979065_15.tif +22979410_15.tif +22979425_15.tif +22979440_15.tif +22979455_15.tif +22979470_15.tif +23128870_15.tif +23128885_15.tif +23128900_15.tif +23128915_15.tif +23128945_15.tif +23128960_15.tif +23128975_15.tif +23128990_15.tif +23129005_15.tif +23129020_15.tif +23129035_15.tif +23129050_15.tif +23129065_15.tif +23129110_15.tif +23129125_15.tif +23129140_15.tif +23129155_15.tif +23129170_15.tif +23129410_15.tif +23129425_15.tif +23129440_15.tif +23129455_15.tif +23278825_15.tif +23278840_15.tif +23278885_15.tif +23278900_15.tif +23278945_15.tif +23278960_15.tif +23278975_15.tif +23278990_15.tif +23279005_15.tif +23279020_15.tif +23279035_15.tif +23279050_15.tif +23279080_15.tif +23279095_15.tif +23279110_15.tif +23279125_15.tif +23279140_15.tif +23279155_15.tif +23279170_15.tif +23428510_15.tif +23428525_15.tif +23428555_15.tif +23428570_15.tif +23428585_15.tif +23428825_15.tif +23428840_15.tif +23428900_15.tif +23428915_15.tif +23428930_15.tif +23428945_15.tif +23428960_15.tif +23428975_15.tif +23428990_15.tif +23429005_15.tif +23429020_15.tif +23429035_15.tif +23429050_15.tif +23429065_15.tif +23429095_15.tif +23429110_15.tif +23429125_15.tif +23429140_15.tif +23429155_15.tif +23429170_15.tif +23578495_15.tif +23578510_15.tif +23578525_15.tif +23578540_15.tif +23578555_15.tif +23578570_15.tif +23578585_15.tif +23578600_15.tif +23578780_15.tif +23578795_15.tif +23578810_15.tif +23578825_15.tif +23578840_15.tif +23578915_15.tif +23578930_15.tif +23578945_15.tif +23578960_15.tif +23578975_15.tif +23578990_15.tif +23579005_15.tif +23579020_15.tif +23579035_15.tif +23579050_15.tif +23579065_15.tif +23579080_15.tif +23579095_15.tif +23579110_15.tif +23579125_15.tif +23579140_15.tif +23579155_15.tif +23728480_15.tif +23728495_15.tif +23728510_15.tif +23728525_15.tif +23728540_15.tif +23728555_15.tif +23728570_15.tif +23728600_15.tif +23728765_15.tif +23728780_15.tif +23728795_15.tif +23728810_15.tif +23728825_15.tif +23728840_15.tif +23728915_15.tif +23728930_15.tif +23728945_15.tif +23728960_15.tif +23728975_15.tif +23728990_15.tif +23729005_15.tif +23729035_15.tif +23729050_15.tif +23729065_15.tif +23729080_15.tif +23729095_15.tif +23729110_15.tif +23729230_15.tif +23729245_15.tif +23878480_15.tif +23878495_15.tif +23878510_15.tif +23878525_15.tif +23878555_15.tif +23878570_15.tif +23878585_15.tif +23878765_15.tif +23878780_15.tif +23878795_15.tif +23878810_15.tif +23878825_15.tif +23878915_15.tif +23878930_15.tif +23878945_15.tif +23878960_15.tif +23878975_15.tif +23878990_15.tif +23879005_15.tif +23879020_15.tif +23879035_15.tif +23879050_15.tif +23879065_15.tif +23879080_15.tif +23879095_15.tif +23879110_15.tif +23879200_15.tif +23879215_15.tif +23879230_15.tif +23879245_15.tif +24028480_15.tif +24028495_15.tif +24028510_15.tif +24028525_15.tif +24028540_15.tif +24028555_15.tif +24028795_15.tif +24028810_15.tif +24028825_15.tif +24028990_15.tif +24029005_15.tif +24029020_15.tif +24029050_15.tif +24029065_15.tif +24029080_15.tif +24029095_15.tif +24029110_15.tif +24029185_15.tif +24029200_15.tif +24029215_15.tif +24029230_15.tif +24029245_15.tif +24178480_15.tif +24178495_15.tif +24178510_15.tif +24178525_15.tif +24178540_15.tif +24178975_15.tif +24179005_15.tif +24179020_15.tif +24179035_15.tif +24179050_15.tif +24179065_15.tif +24179080_15.tif +24179095_15.tif +24179110_15.tif +24179170_15.tif +24179185_15.tif +24179200_15.tif +24179215_15.tif +24179230_15.tif +24179260_15.tif +24179275_15.tif +24328765_15.tif +24328780_15.tif +24328795_15.tif +24328825_15.tif +24328840_15.tif +24328855_15.tif +24328870_15.tif +24328885_15.tif +24329005_15.tif +24329020_15.tif +24329035_15.tif +24329050_15.tif +24329095_15.tif +24329110_15.tif +24329155_15.tif +24329170_15.tif +24329185_15.tif +24329200_15.tif +24329215_15.tif +24329230_15.tif +24329245_15.tif +24329260_15.tif +24329275_15.tif +24478765_15.tif +24478780_15.tif +24478795_15.tif +24478810_15.tif +24478840_15.tif +24478870_15.tif +24478885_15.tif +24478900_15.tif +24478990_15.tif +24479005_15.tif +24479095_15.tif +24479110_15.tif +24479155_15.tif +24479185_15.tif +24479200_15.tif +24479230_15.tif +24479245_15.tif +24479260_15.tif +24479275_15.tif +24479290_15.tif +24628780_15.tif +24628795_15.tif +24628810_15.tif +24628825_15.tif +24628840_15.tif +24628855_15.tif +24628870_15.tif +24628900_15.tif +24629155_15.tif +24629170_15.tif +24629185_15.tif +24629215_15.tif +24629230_15.tif +24629245_15.tif +24629260_15.tif +24629275_15.tif +24629290_15.tif +24629305_15.tif +24778780_15.tif +24778795_15.tif +24778810_15.tif +24778825_15.tif +24778840_15.tif +24778855_15.tif +24778870_15.tif +24778885_15.tif +24778900_15.tif +24778915_15.tif +24779155_15.tif +24779170_15.tif +24779185_15.tif +24779200_15.tif +24779215_15.tif +24779230_15.tif +24779245_15.tif +24779260_15.tif +24779290_15.tif +24779305_15.tif +24928900_15.tif +24929155_15.tif +24929185_15.tif +24929200_15.tif +24929215_15.tif +24929230_15.tif +24929260_15.tif +24929275_15.tif +24929290_15.tif +25079155_15.tif +25079185_15.tif +25079200_15.tif +25079215_15.tif +25079230_15.tif +25079245_15.tif +25079260_15.tif +25079275_15.tif +25079290_15.tif +25229155_15.tif +25229170_15.tif +25229185_15.tif +25229200_15.tif +25229215_15.tif +25229260_15.tif +25229275_15.tif +25229290_15.tif +25379185_15.tif +25379230_15.tif +25379245_15.tif +25379260_15.tif +25379275_15.tif +25379290_15.tif +25529170_15.tif +25529230_15.tif +25529245_15.tif +25529260_15.tif +25529275_15.tif +25529290_15.tif +25679230_15.tif +25679245_15.tif +25679260_15.tif +25679275_15.tif +25828765_15.tif +25828780_15.tif +25829245_15.tif +25829260_15.tif +25829275_15.tif +25829290_15.tif +25978735_15.tif +25978750_15.tif +25978765_15.tif +25978780_15.tif +25978795_15.tif +25979230_15.tif +25979245_15.tif +25979260_15.tif +25979275_15.tif +25979290_15.tif +26128705_15.tif +26128720_15.tif +26128735_15.tif +26128750_15.tif +26128765_15.tif +26128780_15.tif +26128795_15.tif +26128810_15.tif +26129245_15.tif +26129260_15.tif +26129275_15.tif +26129290_15.tif +26278705_15.tif +26278735_15.tif +26278750_15.tif +26278765_15.tif +26278780_15.tif +26278795_15.tif +26278810_15.tif +26279260_15.tif +26428690_15.tif +26428705_15.tif +26428720_15.tif +26428750_15.tif +26428765_15.tif +26428780_15.tif +26428795_15.tif +26429245_15.tif +26429260_15.tif +26429275_15.tif +26578675_15.tif +26578690_15.tif +26578705_15.tif +26578735_15.tif +26578750_15.tif +26578765_15.tif +26578780_15.tif +26578795_15.tif +26728675_15.tif +26728690_15.tif +26728705_15.tif +26728720_15.tif +26728735_15.tif +26728750_15.tif +26728765_15.tif +26728780_15.tif +26878675_15.tif +26878705_15.tif +26878720_15.tif +26878735_15.tif +26878750_15.tif +27028675_15.tif +27028690_15.tif +27028705_15.tif +27028720_15.tif +27178705_15.tif +99238660_15.tif +99238675_15.tif diff --git a/examples/UNet/mass_roads_validation.txt b/examples/UNet/mass_roads_validation.txt new file mode 100644 index 0000000..66464ac --- /dev/null +++ b/examples/UNet/mass_roads_validation.txt @@ -0,0 +1,14 @@ +10228690_15.tiff +10978735_15.tiff +10978795_15.tiff +18028945_15.tiff +21929020_15.tiff +22528900_15.tiff +22829035_15.tiff +22978990_15.tiff +23128930_15.tiff +24179245_15.tiff +24328810_15.tiff +24929245_15.tiff +25229230_15.tiff +25229245_15.tiff diff --git a/examples/UNet/massachusetts_road_dataset_utils.py b/examples/UNet/massachusetts_road_dataset_utils.py new file mode 100644 index 0000000..7988bc7 --- /dev/null +++ b/examples/UNet/massachusetts_road_dataset_utils.py @@ -0,0 +1,113 @@ +__author__ = 'Fabian Isensee' +import os +import sys +import fnmatch +import matplotlib.pyplot as plt +sys.path.append("../../modelzoo/") +from generators import * +from multiprocessing.dummy import Pool +from urllib import urlretrieve + +def prep_folders(): + if not os.path.isdir("data"): + os.mkdir("data") + + if not os.path.isdir("data/validation"): + os.mkdir("data/validation") + if not os.path.isdir("data/training"): + os.mkdir("data/training") + if not os.path.isdir("data/test"): + os.mkdir("data/test") + + if not os.path.isdir("data/validation/sat_img"): + os.mkdir("data/validation/sat_img") + if not os.path.isdir("data/validation/map"): + os.mkdir("data/validation/map") + if not os.path.isdir("data/training/sat_img"): + os.mkdir("data/training/sat_img") + if not os.path.isdir("data/training/map"): + os.mkdir("data/training/map") + if not os.path.isdir("data/test/sat_img"): + os.mkdir("data/test/sat_img") + if not os.path.isdir("data/test/map"): + os.mkdir("data/test/map") + +def prep_urls(): + valid_data_url = valid_target_url = np.loadtxt("mass_roads_validation.txt", dtype=str) + valid_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/sat/" + valid_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/valid/map/" + + train_data_url = train_target_url = np.loadtxt("mass_roads_train.txt", dtype=str) + train_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/sat/" + train_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/train/map/" + + test_data_url = test_target_url = np.loadtxt("mass_roads_test.txt", dtype=str) + test_data_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/sat/" + test_target_str = "https://www.cs.toronto.edu/~vmnih/data/mass_roads/test/map/" + + all_tasks = [] + + # save url along with the filename for each file + for img_name in train_data_url: + all_tasks.append(tuple([train_data_str + img_name + "f", "data/training/sat_img/%sf"%img_name])) + all_tasks.append(tuple([train_target_str + img_name, "data/training/map/%s"%img_name])) + + for img_name in valid_data_url: + all_tasks.append(tuple([valid_data_str + img_name, "data/validation/sat_img/%s"%img_name])) + all_tasks.append(tuple([valid_target_str + img_name[:-1], "data/validation/map/%s"%img_name[:-1]])) + + for img_name in test_data_url: + all_tasks.append(tuple([test_data_str + img_name, "data/test/sat_img/%s"%img_name])) + all_tasks.append(tuple([test_target_str + img_name[:-1], "data/test/map/%s"%img_name[:-1]])) + + return all_tasks + +def download_dataset(all_tasks, num_workers=4): + def urlretrieve_star(args): + return urlretrieve(*args) + + pool = Pool(num_workers) + pool.map(urlretrieve_star, all_tasks) + pool.close() + pool.join() + + +def load_data(folder): + images_sat = [img for img in os.listdir(os.path.join(folder, "sat_img")) if fnmatch.fnmatch(img, "*.tif*")] + images_map = [img for img in os.listdir(os.path.join(folder, "map")) if fnmatch.fnmatch(img, "*.tif*")] + assert(len(images_sat) == len(images_map)) + images_sat.sort() + images_map.sort() + # images are 1500 by 1500 pixels each + data = np.zeros((len(images_sat), 3, 1500, 1500), dtype=np.uint8) + target = np.zeros((len(images_sat), 1, 1500, 1500), dtype=np.uint8) + ctr = 0 + for sat_im, map_im in zip(images_sat, images_map): + data[ctr] = plt.imread(os.path.join(folder, "sat_img", sat_im)).transpose((2, 0, 1)) + # target has values 0 and 255. make that 0 and 1 + target[ctr, 0] = plt.imread(os.path.join(folder, "map", map_im))/255 + ctr += 1 + return data, target + +def prepare_dataset(): + prep_folders() + all_tasks = prep_urls() + download_dataset(all_tasks) + + print "download done..." + try: + data_train, target_train = load_data("data/training") + data_valid, target_valid = load_data("data/validation") + data_test, target_test = load_data("data/test") + # loading np arrays is much faster than loading the images one by one every time + np.save("data_train.npy", data_train) + np.save("target_train.npy", target_train) + np.save("data_valid.npy", data_valid) + np.save("target_valid.npy", target_valid) + np.save("data_test.npy", data_test) + np.save("target_test.npy", target_test) + except: + print "something went wrong, maybe the download?" + +if __name__ == "__main__": + prepare_dataset() \ No newline at end of file diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py new file mode 100644 index 0000000..f26a2b4 --- /dev/null +++ b/examples/UNet/massachusetts_road_segm.py @@ -0,0 +1,185 @@ +__author__ = 'Fabian Isensee' +import numpy as np +import lasagne +import os +import sys +import fnmatch +import matplotlib.pyplot as plt +sys.path.append("../../modelzoo/") +from Unet import * +import theano.tensor as T +import theano +import cPickle +from time import sleep +from generators import batch_generator, threaded_generator, random_crop_generator, center_crop_generator +from massachusetts_road_dataset_utils import prepare_dataset +from sklearn.metrics import roc_auc_score + +def plot_some_results(pred_fn, test_generator, n_images=10): + fig_ctr = 0 + for data, seg in test_generator: + res = pred_fn(data) + for d, s, r in zip(data, seg, res): + plt.figure(figsize=(12, 6)) + plt.subplot(1, 3, 1) + plt.imshow(d.transpose(1,2,0)) + plt.title("input patch") + plt.subplot(1, 3, 2) + plt.imshow(s[0]) + plt.title("ground truth") + plt.subplot(1, 3, 3) + plt.imshow(r) + plt.title("segmentation") + plt.savefig("road_segmentation_result_%03.0f.png"%fig_ctr) + plt.close() + fig_ctr += 1 + if fig_ctr > n_images: + break + +def main(): + # only download dataset once. This takes a while. + # heuristic that I included to make sure the dataset is only donwloaded and prepared once + if not os.path.isfile("target_test.npy"): + prepare_dataset() + + # set some hyper parameters. You should not have to touch anything if you have 4GB or more VRAM + BATCH_SIZE = 8 # this works if you have ~ 8GB VRAM. Use smaller BATCH_SIZE for other GPUs + N_EPOCHS = 50 + N_BATCHES_PER_EPOCH = 100 + PATCH_SIZE = 512 + + # load the prepared data. They have been converted to np arrays because they are much faster to load than single image files. + # This code will not load the entire array into memory but rather read from disk the bits that we currently need + # set mmap_mode to None if you want to load the data into RAM + mmap_mode = 'r' + data_train = np.load("data_train.npy", mmap_mode=mmap_mode) + target_train = np.load("target_train.npy", mmap_mode=mmap_mode) + data_valid = np.load("data_valid.npy", mmap_mode=mmap_mode) + target_valid = np.load("target_valid.npy", mmap_mode=mmap_mode) + data_test = np.load("data_test.npy", mmap_mode=mmap_mode) + target_test = np.load("target_test.npy", mmap_mode=mmap_mode) + + # we are using pad='same' for simplicity (otherwise we would have to crop our ground truth). Keep in mind that this + # may not be ideal + net = build_UNet(n_input_channels=3, BATCH_SIZE=None, num_output_classes=2, pad='same', + nonlinearity=lasagne.nonlinearities.rectify, input_dim=(PATCH_SIZE, PATCH_SIZE), + base_n_filters=16, do_dropout=False) + output_layer_for_loss = net["output_flattened"] + + # this is np.sum(target_train == 0) and np.sum(target_train == 1). No need to compute this every time + class_frequencies = np.array([2374093357., 118906643.]) + # we will reweight the loss to put more focus on road pixels (because of class imbalance). This is a simple approach + # and could be improved if you also have a class imbalance in your experiments. + # we are taking **0.25 here because we want the net to focus more on the road pixels but not too much (otherwise + # it would not be penalized enough for missclassifying terrain pixels which results in too many false positives) + class_weights = (class_frequencies[[1,0]])**0.25 + class_weights = class_weights / np.sum(class_weights) * 2. + class_weights = class_weights.astype(np.float32) + + # if you wish to load pretrained weights you can uncomment this code + # val accuracy: 0.966384 val loss: 0.0947428 val AUC score: 0.980004909707 + # you can also change the lower part of this code to load your own pretrained params + '''if not os.path.isfile('UNet_params_pretrained.pkl'): + import urllib + import zipfile + urllib.urlretrieve("https://s3.amazonaws.com/lasagne/recipes/pretrained/UNet_mass_road_segm_params.zip", 'pretrained_weights.zip') + zip_ref = zipfile.ZipFile('pretrained_weights.zip', 'r') + zip_ref.extractall("./") + zip_ref.close() + with open("UNet_params_pretrained.pkl", 'r') as f: + params = cPickle.load(f) + lasagne.layers.set_all_param_values(output_layer_for_loss, params)''' + + x_sym = T.tensor4() + seg_sym = T.ivector() + w_sym = T.vector() + + # add some weight decay + l2_loss = lasagne.regularization.regularize_network_params(output_layer_for_loss, lasagne.regularization.l2) * 1e-4 + + # the distinction between prediction_train and test is important only if we enable dropout + prediction_train = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=False, batch_norm_update_averages=False, batch_norm_use_averages=False) + # we could use a binary loss but I stuck with categorical crossentropy so that less code has to be changed if your + # application has more than two classes + loss = lasagne.objectives.categorical_crossentropy(prediction_train, seg_sym) + loss *= w_sym + loss = loss.mean() + loss += l2_loss + acc_train = T.mean(T.eq(T.argmax(prediction_train, axis=1), seg_sym), dtype=theano.config.floatX) + + prediction_test = lasagne.layers.get_output(output_layer_for_loss, x_sym, deterministic=True, batch_norm_update_averages=False, batch_norm_use_averages=False) + loss_val = lasagne.objectives.categorical_crossentropy(prediction_test, seg_sym) + + # we multiply our loss by a weight map. In this example the weight map simply increases the loss for road pixels and + # decreases the loss for other pixels. We do this to ensure that the network puts more focus on getting the roads + # right + loss_val *= w_sym + loss_val = loss_val.mean() + loss_val += l2_loss + acc = T.mean(T.eq(T.argmax(prediction_test, axis=1), seg_sym), dtype=theano.config.floatX) + + # learning rate has to be a shared variable because we decrease it with every epoch + params = lasagne.layers.get_all_params(output_layer_for_loss, trainable=True) + learning_rate = theano.shared(np.float32(0.001)) + updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate) + + # create a convenience function to get the segmentation + seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym, deterministic=True) + seg_output = seg_output.argmax(1) + + train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates) + val_fn = theano.function([x_sym, seg_sym, w_sym], [loss_val, acc]) + get_segmentation = theano.function([x_sym], seg_output) + # we need this for calculating the AUC score + get_class_probas = theano.function([x_sym], prediction_test) + + # some data augmentation. If you want better results you should invest more effort here. I left rotations and + # deformations out for the sake of speed and simplicity + train_generator = random_crop_generator(batch_generator(data_train, target_train, BATCH_SIZE, shuffle=True), PATCH_SIZE) + train_generator = threaded_generator(train_generator, num_cached=10) + + # do the actual training + for epoch in np.arange(0, N_EPOCHS): + print epoch + losses_train = [] + n_batches = 0 + accuracies_train = [] + for data, target in train_generator: + # the output of the net has shape (BATCH_SIZE, N_CLASSES). We therefore need to flatten the segmentation so + # that we can match it with the prediction via the crossentropy loss function + target_flat = target.ravel() + loss, acc = train_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) + losses_train.append(loss) + accuracies_train.append(acc) + n_batches += 1 + if n_batches > N_BATCHES_PER_EPOCH: + break + print "epoch: ", epoch, "\ntrain accuracy: ", np.mean(accuracies_train), " train loss: ", np.mean(losses_train) + + losses_val = [] + accuracies_val = [] + auc_val = [] + # there is no need for data augmentation on the validation. However we need patches of the same size which is why + # we are using center crop generator + # since the validation generator does not loop around we need to reinstantiate it for every epoch + validation_generator = center_crop_generator(batch_generator(data_valid, target_valid, BATCH_SIZE, shuffle=False), PATCH_SIZE) + validation_generator = threaded_generator(validation_generator, num_cached=10) + for data, target in validation_generator: + target_flat = target.ravel() + loss, acc = val_fn(data.astype(np.float32), target_flat, class_weights[target_flat]) + losses_val.append(loss) + accuracies_val.append(acc) + auc_val.append(roc_auc_score(target_flat, get_class_probas(data)[:, 1])) + print "val accuracy: ", np.mean(accuracies_val), " val loss: ", np.mean(losses_val), " val AUC score: ", np.mean(auc_val) + learning_rate *= 0.8 + # save trained weights after each epoch + with open("UNet_params_ep%03.0f.pkl"%epoch, 'w') as f: + cPickle.dump(lasagne.layers.get_all_param_values(output_layer_for_loss), f) + + # create some png files showing (raw image, ground truth, prediction). Of course we use the test set here ;-) + test_gen = random_crop_generator(batch_generator(data_test, target_test, BATCH_SIZE), PATCH_SIZE) + plot_some_results(get_segmentation, test_gen, 15) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/modelzoo/Unet.py b/modelzoo/Unet.py new file mode 100644 index 0000000..c530bf8 --- /dev/null +++ b/modelzoo/Unet.py @@ -0,0 +1,66 @@ +__author__ = 'Fabian Isensee' +from collections import OrderedDict +from lasagne.layers import (InputLayer, ConcatLayer, Pool2DLayer, ReshapeLayer, DimshuffleLayer, NonlinearityLayer, + DropoutLayer, Deconv2DLayer, batch_norm) +try: + from lasagne.layers.dnn import Conv2DDNNLayer as ConvLayer +except ImportError: + from lasagne.layers import Conv2DLayer as ConvLayer +import lasagne +from lasagne.init import HeNormal + + +def build_UNet(n_input_channels=1, BATCH_SIZE=None, num_output_classes=2, pad='same', nonlinearity=lasagne.nonlinearities.elu, input_dim=(128, 128), base_n_filters=64, do_dropout=False): + net = OrderedDict() + net['input'] = InputLayer((BATCH_SIZE, n_input_channels, input_dim[0], input_dim[1])) + + net['contr_1_1'] = batch_norm(ConvLayer(net['input'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_1_2'] = batch_norm(ConvLayer(net['contr_1_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['pool1'] = Pool2DLayer(net['contr_1_2'], 2) + + net['contr_2_1'] = batch_norm(ConvLayer(net['pool1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_2_2'] = batch_norm(ConvLayer(net['contr_2_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['pool2'] = Pool2DLayer(net['contr_2_2'], 2) + + net['contr_3_1'] = batch_norm(ConvLayer(net['pool2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_3_2'] = batch_norm(ConvLayer(net['contr_3_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['pool3'] = Pool2DLayer(net['contr_3_2'], 2) + + net['contr_4_1'] = batch_norm(ConvLayer(net['pool3'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['contr_4_2'] = batch_norm(ConvLayer(net['contr_4_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + l = net['pool4'] = Pool2DLayer(net['contr_4_2'], 2) + # the paper does not really describe where and how dropout is added. Feel free to try more options + if do_dropout: + l = DropoutLayer(l, p=0.4) + + net['encode_1'] = batch_norm(ConvLayer(l, base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['encode_2'] = batch_norm(ConvLayer(net['encode_1'], base_n_filters*16, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale1'] = batch_norm(Deconv2DLayer(net['encode_2'], base_n_filters*16, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) + + net['concat1'] = ConcatLayer([net['upscale1'], net['contr_4_2']], cropping=(None, None, "center", "center")) + net['expand_1_1'] = batch_norm(ConvLayer(net['concat1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_1_2'] = batch_norm(ConvLayer(net['expand_1_1'], base_n_filters*8, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale2'] = batch_norm(Deconv2DLayer(net['expand_1_2'], base_n_filters*8, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) + + net['concat2'] = ConcatLayer([net['upscale2'], net['contr_3_2']], cropping=(None, None, "center", "center")) + net['expand_2_1'] = batch_norm(ConvLayer(net['concat2'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_2_2'] = batch_norm(ConvLayer(net['expand_2_1'], base_n_filters*4, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale3'] = batch_norm(Deconv2DLayer(net['expand_2_2'], base_n_filters*4, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) + + net['concat3'] = ConcatLayer([net['upscale3'], net['contr_2_2']], cropping=(None, None, "center", "center")) + net['expand_3_1'] = batch_norm(ConvLayer(net['concat3'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_3_2'] = batch_norm(ConvLayer(net['expand_3_1'], base_n_filters*2, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['upscale4'] = batch_norm(Deconv2DLayer(net['expand_3_2'], base_n_filters*2, 2, 2, crop="valid", nonlinearity=nonlinearity, W=HeNormal(gain="relu"))) + + net['concat4'] = ConcatLayer([net['upscale4'], net['contr_1_2']], cropping=(None, None, "center", "center")) + net['expand_4_1'] = batch_norm(ConvLayer(net['concat4'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + net['expand_4_2'] = batch_norm(ConvLayer(net['expand_4_1'], base_n_filters, 3, nonlinearity=nonlinearity, pad=pad, W=HeNormal(gain="relu"))) + + net['output_segmentation'] = ConvLayer(net['expand_4_2'], num_output_classes, 1, nonlinearity=None) + net['dimshuffle'] = DimshuffleLayer(net['output_segmentation'], (1, 0, 2, 3)) + net['reshapeSeg'] = ReshapeLayer(net['dimshuffle'], (num_output_classes, -1)) + net['dimshuffle2'] = DimshuffleLayer(net['reshapeSeg'], (1, 0)) + net['output_flattened'] = NonlinearityLayer(net['dimshuffle2'], nonlinearity=lasagne.nonlinearities.softmax) + + return net + From 4b6cec2ffb2686b19186a8ad4462794b42a5c1c5 Mon Sep 17 00:00:00 2001 From: Fabian Isensee Date: Fri, 17 Mar 2017 18:28:40 +0100 Subject: [PATCH 21/21] fixed a bug with the seg_output. There wehe some kwargs missing leading to mostly empty segmentation maps --- examples/UNet/massachusetts_road_segm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/UNet/massachusetts_road_segm.py b/examples/UNet/massachusetts_road_segm.py index f26a2b4..c2293d6 100644 --- a/examples/UNet/massachusetts_road_segm.py +++ b/examples/UNet/massachusetts_road_segm.py @@ -124,7 +124,7 @@ def main(): updates = lasagne.updates.adam(loss, params, learning_rate=learning_rate) # create a convenience function to get the segmentation - seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym, deterministic=True) + seg_output = lasagne.layers.get_output(net["output_segmentation"], x_sym, deterministic=True, batch_norm_update_averages=False, batch_norm_use_averages=False) seg_output = seg_output.argmax(1) train_fn = theano.function([x_sym, seg_sym, w_sym], [loss, acc_train], updates=updates)