From 583d105cc47a83aaa042f6ccb0e6f869199abb7c Mon Sep 17 00:00:00 2001 From: Carter Parks Date: Wed, 3 Mar 2021 12:12:28 -0700 Subject: [PATCH 1/2] remove north america get-datafiles and change datasource to nasa archive --- get-datafiles-northamerica.sh | 22 ------- get-datafiles-world.sh | 92 ++++++++++++++++++++++++++++-- get-datafiles.sh | 104 ---------------------------------- 3 files changed, 86 insertions(+), 132 deletions(-) delete mode 100755 get-datafiles-northamerica.sh delete mode 100755 get-datafiles.sh diff --git a/get-datafiles-northamerica.sh b/get-datafiles-northamerica.sh deleted file mode 100755 index a425cb9..0000000 --- a/get-datafiles-northamerica.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/sh - -# get the SRTM data files and convert them for splat use -# results in ~3.6 GB disk space consumed - -# License: Public domain / CC-0 - -# for state/county borders -- US only. -BORDERDIR=splat-datafiles/borders/ - -BORDERURL="https://web.archive.org/web/20130331144934/http://www.census.gov/geo/cob/bdy/co/co00ascii/co99_d00_ascii.zip" - -mkdir -p $BORDERDIR -cd $BORDERDIR -echo "retrieving state/county borders.." -wget -N -nv $BORDERURL -echo "unpacking state/county borders.." -unzip -o *.zip -cd - - -./get-datafiles.sh "North_America" - diff --git a/get-datafiles-world.sh b/get-datafiles-world.sh index f80c60e..3fee576 100755 --- a/get-datafiles-world.sh +++ b/get-datafiles-world.sh @@ -1,12 +1,92 @@ #!/bin/sh +# get the SRTM data files and convert them for splat use + # License: Public domain / CC-0 -./get-datafiles-northamerica.sh -./get-datafiles.sh "South_America" -./get-datafiles.sh "Africa" -./get-datafiles.sh "Eurasia" -./get-datafiles.sh "Australia" -./get-datafiles.sh "Islands" +# path to topgraphy datafiles +TOPOFILEDIR=splat-datafiles/sdf/ +# local hgt file archive +HGTFILEDIR=splat-datafiles/hgtzip/ + +echo "Please sign up for a NASA Earthdata account at: \nhttps://urs.earthdata.nasa.gov/users/new" +read -p 'NASA Earthdata Username: ' NASA_USERNAME +read -sp 'NASA Earthdata Password: ' NASA_PASSWORD +NASA_USERNAME="carterparks" +NASA_PASSWORD="D5Jc794HV0ip" + +for PAGE in 1 2 3 4 5 6 +do + INDEXURL="https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL3.003/2000.02.11/SRTMGL3_page_$PAGE.html" + + INDEXFILE=`mktemp` + + if [ ! -x `which srtm2sdf` ]; then + echo "error: not found in path: srtm2sdf splat conversion utility" + exit 1 + fi + + if [ ! -x `which readlink` ]; then + echo "error: not found in path: readlink" + exit 1 + fi + + if [ ! -x `which wget` ]; then + echo "error: not found in path: wget" + exit 1 + fi + + if [ ! -x `which unzip` ]; then + echo "error: not found in path: unzip" + exit 1 + fi + + if [ ! -x `which bzip2` ]; then + echo "error: not found in path: bzip2" + exit 1 + fi + + echo "getting index.." + wget -q -O - $INDEXURL | \ + sed -r -e '/hgt.zip\"/!d; s/.* ([NSWE0-9]+\.?hgt\.zip).*$/\1/; s/.*//' \ + > $INDEXFILE + echo $INDEXFILE + mkdir -p $HGTFILEDIR + mkdir -p $TOPOFILEDIR + + echo "retrieving files.." + cd $HGTFILEDIR + wget --http-user=$NASA_USERNAME --http-password=$NASA_PASSWORD -nv -N -B $INDEXURL -i $INDEXFILE + cd - + + rm $INDEXFILE +done + +# to minimize disk space required, run srtm2sdf on each file as it is unzipped. + +HGTREALPATH=`readlink -f $HGTFILEDIR` +TOPOREALPATH=`readlink -f $TOPOFILEDIR` +PWD=`pwd` + +echo "unpacking hgt files.." +cd $HGTFILEDIR +for e in *.zip ; do + echo $e + nice unzip -o $e + HGTFILE=`echo $e | sed -r -e 's/\.?.SRTMGL3.hgt.zip/.hgt/'` + if [ -r $HGTFILE ]; then + cd $TOPOREALPATH + nice srtm2sdf -d /dev/null $HGTREALPATH/$HGTFILE + echo "compressing.." + nice bzip2 -f -- *.sdf + echo "deleting hgt file.." + cd $HGTREALPATH + rm $HGTFILE + fi +done + +cd $PWD + +echo "Complete. The files in $HGTFILEDIR may be removed." diff --git a/get-datafiles.sh b/get-datafiles.sh deleted file mode 100755 index 45062f6..0000000 --- a/get-datafiles.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/sh - -# get the SRTM data files and convert them for splat use - -# License: Public domain / CC-0 - -# Takes one parameter: the continent to retrieve. Valid values: -# -# Africa -# Australia -# Eurasia -# Islands -# North_America -# South_America - -# path to topgraphy datafiles -TOPOFILEDIR=splat-datafiles/sdf/ -# local hgt file archive -HGTFILEDIR=splat-datafiles/hgtzip/ - -CONTINENT=$1 -case $CONTINENT in - North_America|South_America|Africa|Eurasia|Australia|Islands) - echo $CONTINENT - ;; - *) - echo "Invalid continent: $CONTINENT" - exit 1 - ;; -esac - - -INDEXURL="http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/${CONTINENT}/" - -INDEXFILE=`mktemp` - -if [ ! -x `which srtm2sdf` ]; then - echo "error: not found in path: srtm2sdf splat conversion utility" - exit 1 -fi - -if [ ! -x `which readlink` ]; then - echo "error: not found in path: readlink" - exit 1 -fi - -if [ ! -x `which wget` ]; then - echo "error: not found in path: wget" - exit 1 -fi - -if [ ! -x `which unzip` ]; then - echo "error: not found in path: unzip" - exit 1 -fi - -if [ ! -x `which bzip2` ]; then - echo "error: not found in path: bzip2" - exit 1 -fi - -echo "getting index.." -wget -q -O - $INDEXURL | \ - sed -r -e '/hgt.zip/!d; s/.* ([NSWE0-9]+\.?hgt\.zip).*$/\1/;' \ - > $INDEXFILE - -mkdir -p $HGTFILEDIR -mkdir -p $TOPOFILEDIR - -echo "retrieving files.." -cd $HGTFILEDIR -wget -nv -N -B $INDEXURL -i $INDEXFILE -cd - - -rm $INDEXFILE - -# to minimize disk space required, run srtm2sdf on each file as it is unzipped. - -HGTREALPATH=`readlink -f $HGTFILEDIR` -TOPOREALPATH=`readlink -f $TOPOFILEDIR` -PWD=`pwd` - -echo "unpacking hgt files.." -cd $HGTFILEDIR -for e in *.zip ; do - echo $e - nice unzip -o $e - HGTFILE=`echo $e | sed -r -e 's/\.?hgt.zip/.hgt/'` - if [ -r $HGTFILE ]; then - cd $TOPOREALPATH - nice srtm2sdf -d /dev/null $HGTREALPATH/$HGTFILE - echo "compressing.." - nice bzip2 -f -- *.sdf - echo "deleting hgt file.." - cd $HGTREALPATH - rm $HGTFILE - fi -done - -cd $PWD - -echo "Complete. The files in $HGTFILEDIR may be removed." - - From f1d14bd18fd7b3950a216f8d80cd6d1549144beb Mon Sep 17 00:00:00 2001 From: Carter Parks Date: Wed, 3 Mar 2021 13:11:40 -0700 Subject: [PATCH 2/2] doh --- get-datafiles-world.sh | 54 +++++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/get-datafiles-world.sh b/get-datafiles-world.sh index 3fee576..e760935 100755 --- a/get-datafiles-world.sh +++ b/get-datafiles-world.sh @@ -9,43 +9,40 @@ TOPOFILEDIR=splat-datafiles/sdf/ # local hgt file archive HGTFILEDIR=splat-datafiles/hgtzip/ +if [ ! -x `which srtm2sdf` ]; then + echo "error: not found in path: srtm2sdf splat conversion utility" + exit 1 +fi + +if [ ! -x `which readlink` ]; then + echo "error: not found in path: readlink" + exit 1 +fi + +if [ ! -x `which wget` ]; then + echo "error: not found in path: wget" + exit 1 +fi + +if [ ! -x `which unzip` ]; then + echo "error: not found in path: unzip" + exit 1 +fi + +if [ ! -x `which bzip2` ]; then + echo "error: not found in path: bzip2" + exit 1 +fi + echo "Please sign up for a NASA Earthdata account at: \nhttps://urs.earthdata.nasa.gov/users/new" read -p 'NASA Earthdata Username: ' NASA_USERNAME read -sp 'NASA Earthdata Password: ' NASA_PASSWORD -NASA_USERNAME="carterparks" -NASA_PASSWORD="D5Jc794HV0ip" for PAGE in 1 2 3 4 5 6 do INDEXURL="https://e4ftl01.cr.usgs.gov/MEASURES/SRTMGL3.003/2000.02.11/SRTMGL3_page_$PAGE.html" - INDEXFILE=`mktemp` - if [ ! -x `which srtm2sdf` ]; then - echo "error: not found in path: srtm2sdf splat conversion utility" - exit 1 - fi - - if [ ! -x `which readlink` ]; then - echo "error: not found in path: readlink" - exit 1 - fi - - if [ ! -x `which wget` ]; then - echo "error: not found in path: wget" - exit 1 - fi - - if [ ! -x `which unzip` ]; then - echo "error: not found in path: unzip" - exit 1 - fi - - if [ ! -x `which bzip2` ]; then - echo "error: not found in path: bzip2" - exit 1 - fi - echo "getting index.." wget -q -O - $INDEXURL | \ sed -r -e '/hgt.zip\"/!d; s/.* ([NSWE0-9]+\.?hgt\.zip).*$/\1/; s/.*//' \ @@ -63,7 +60,6 @@ do done # to minimize disk space required, run srtm2sdf on each file as it is unzipped. - HGTREALPATH=`readlink -f $HGTFILEDIR` TOPOREALPATH=`readlink -f $TOPOFILEDIR` PWD=`pwd`