From ee785fafd51c6892cf3d65926b238920f281e3ec Mon Sep 17 00:00:00 2001 From: Oliver Wisler Date: Sat, 20 Feb 2016 02:58:52 +0100 Subject: [PATCH 1/3] worked on get-datafiles to download srtm1 data, not completed Worked on the get-datafiles to download srtm1gl data, not completed --- get-datafiles.sh | 181 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 166 insertions(+), 15 deletions(-) diff --git a/get-datafiles.sh b/get-datafiles.sh index 45062f6..4fdb687 100755 --- a/get-datafiles.sh +++ b/get-datafiles.sh @@ -18,7 +18,85 @@ TOPOFILEDIR=splat-datafiles/sdf/ # local hgt file archive HGTFILEDIR=splat-datafiles/hgtzip/ -CONTINENT=$1 +# TODO remove ./ for prod +SRTM2SDF_HD=./srtm2sdf-hd +SRTM2SDF=./srtm2sdf + +SRTM3URL="http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" +SRTM1URL="http://e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" + + +SRTM2SDF_CMD=$SRTM2SDF + +#wether or not hgt files shall be directly extracted (saves diskspace) +USE_HIGHRES=false +DIRECT_CONVERSION=false +CONTINENT=unknown + +function helptext { +cat <&2 + helptext + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + helptext + exit 1 + ;; + esac +done + + case $CONTINENT in North_America|South_America|Africa|Eurasia|Australia|Islands) echo $CONTINENT @@ -29,12 +107,18 @@ case $CONTINENT in ;; esac - -INDEXURL="http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/${CONTINENT}/" +#set url to download tiles from: +if [ "$USE_HIGHRES" = true ] +then + INDEXURL=$SRTM1URL +else + INDEXURL=${SRTM3URL}${CONTINENT}/ +fi INDEXFILE=`mktemp` +FILELIST=`mktemp` -if [ ! -x `which srtm2sdf` ]; then +if [ ! -x `which $SRTM2SDF` ]; then echo "error: not found in path: srtm2sdf splat conversion utility" exit 1 fi @@ -59,27 +143,94 @@ if [ ! -x `which bzip2` ]; then exit 1 fi -echo "getting index.." -wget -q -O - $INDEXURL | \ +echo "getting index.. from $INDEXURL" +wget -q -O - $INDEXURL > $INDEXFILE + +if [ "$USE_HIGHRES" = true ] +then + #random magic stolen from the internet + grep -F '.hgt.zip<' $INDEXFILE | sed -e 's@.*href="@@g' -e 's/">.*//g' > $FILELIST +else + wget -q -O - $INDEXURL | \ sed -r -e '/hgt.zip/!d; s/.* ([NSWE0-9]+\.?hgt\.zip).*$/\1/;' \ - > $INDEXFILE + > $FILELIST +fi mkdir -p $HGTFILEDIR mkdir -p $TOPOFILEDIR echo "retrieving files.." -cd $HGTFILEDIR -wget -nv -N -B $INDEXURL -i $INDEXFILE -cd - - -rm $INDEXFILE - -# to minimize disk space required, run srtm2sdf on each file as it is unzipped. +#cd $HGTFILEDIR +#head $INDEXFILE +#convert to absolute path because srtm2sdf need cd HGTREALPATH=`readlink -f $HGTFILEDIR` TOPOREALPATH=`readlink -f $TOPOFILEDIR` + PWD=`pwd` +# TODO delete for prod: +SRTM2SDF_CMD=$PWD/$SRTM2SDF_CMD + +for FILE in $(cat $FILELIST); +do + echo $FILE + if [ "$USE_HIGHRES" = true ] + then + HGTFILE=${FILE%SRTMGL1.hgt.zip}hgt + else + HGTFILE=${FILE%.zip} + fi + + + wget -P $HGTFILEDIR -nv -N $INDEXURL$FILE + + #in direct conversion mode, directly make an sdf and delete all downloaded files + if [ "$DIRECT_CONVERSION" = true ] ; + then + echo "Unzip $FILE and then delete zip" + nice unzip -o $HGTFILEDIR/$FILE -d $TOPOFILEDIR + rm $HGTFILEDIR/$FILE + + + + #only execute if file exists: + if [ -r $TOPOFILEDIR/$HGTFILE ]; + then + echo "Convert $HGTFILE to SDF" + cd $TOPOFILEDIR + nice $SRTM2SDF_CMD -d /dev/null $HGTFILE + cd - + echo "compressing.." + #sadly i am too lazy to figure out srtm2sdf naming schemes + for SDF in $TOPOFILEDIR/*.sdf + do + if test -f "$SDF" + then + echo "Compress $SDF" + nice bzip2 -f -- $SDF + fi + done + echo "deleting hgt file $TOPOFILEDIR/$HGTFILE" + rm $TOPOFILEDIR/$HGTFILE + fi + fi +done + +# TODO +rm $INDEXFILE +rm $FILELIST + + +#nothing to do in direct conversion mode +if [ "$DIRECT_CONVERSION" = true ] ; +then + exit 0; +fi + + +# to minimize disk space required, run srtm2sdf on each file as it is unzipped. + echo "unpacking hgt files.." cd $HGTFILEDIR for e in *.zip ; do @@ -88,7 +239,7 @@ for e in *.zip ; do HGTFILE=`echo $e | sed -r -e 's/\.?hgt.zip/.hgt/'` if [ -r $HGTFILE ]; then cd $TOPOREALPATH - nice srtm2sdf -d /dev/null $HGTREALPATH/$HGTFILE + nice $SRTM2SDF_CMD -d /dev/null $HGTREALPATH/$HGTFILE echo "compressing.." nice bzip2 -f -- *.sdf echo "deleting hgt file.." From 912ea446962baaed398c4e3c7e8da9f610294c92 Mon Sep 17 00:00:00 2001 From: Oliver Wisler Date: Sat, 20 Feb 2016 21:50:13 +0100 Subject: [PATCH 2/3] Completed get-datafiles.sh script --- get-datafiles-northamerica.sh | 2 +- get-datafiles-world.sh | 10 +- get-datafiles.sh | 167 ++++++++++++++++++---------------- 3 files changed, 95 insertions(+), 84 deletions(-) diff --git a/get-datafiles-northamerica.sh b/get-datafiles-northamerica.sh index ae8904a..2db94ed 100755 --- a/get-datafiles-northamerica.sh +++ b/get-datafiles-northamerica.sh @@ -18,5 +18,5 @@ echo "unpacking state/county borders.." unzip -o *.zip cd - -./get-datafiles.sh "North_America" +./get-datafiles.sh -c "North_America" diff --git a/get-datafiles-world.sh b/get-datafiles-world.sh index f80c60e..fcc5ed3 100755 --- a/get-datafiles-world.sh +++ b/get-datafiles-world.sh @@ -3,10 +3,10 @@ # License: Public domain / CC-0 ./get-datafiles-northamerica.sh -./get-datafiles.sh "South_America" -./get-datafiles.sh "Africa" -./get-datafiles.sh "Eurasia" -./get-datafiles.sh "Australia" -./get-datafiles.sh "Islands" +./get-datafiles.sh -c "South_America" +./get-datafiles.sh -c "Africa" +./get-datafiles.sh -c "Eurasia" +./get-datafiles.sh -c "Australia" +./get-datafiles.sh -c "Islands" diff --git a/get-datafiles.sh b/get-datafiles.sh index 4fdb687..bdca80e 100755 --- a/get-datafiles.sh +++ b/get-datafiles.sh @@ -18,28 +18,65 @@ TOPOFILEDIR=splat-datafiles/sdf/ # local hgt file archive HGTFILEDIR=splat-datafiles/hgtzip/ -# TODO remove ./ for prod -SRTM2SDF_HD=./srtm2sdf-hd -SRTM2SDF=./srtm2sdf +SRTM2SDF_HD=srtm2sdf-hd +SRTM2SDF=srtm2sdf +INDEXFILE=`mktemp` +FILELIST=`mktemp` + +#URLs from where to fetch the tiles SRTM3URL="http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" SRTM1URL="http://e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" +#Default options: +USE_HIGHRES=false SRTM2SDF_CMD=$SRTM2SDF -#wether or not hgt files shall be directly extracted (saves diskspace) -USE_HIGHRES=false DIRECT_CONVERSION=false CONTINENT=unknown +USE_LONRANGE=false +USE_LATRANGE=false + + +# Check if all prerequisites are installed + +if [ ! -x `which $SRTM2SDF` ]; then + echo "error: not found in path: srtm2sdf splat conversion utility" + exit 1 +fi + +if [ ! -x `which readlink` ]; then + echo "error: not found in path: readlink" + exit 1 +fi + +if [ ! -x `which wget` ]; then + echo "error: not found in path: wget" + exit 1 +fi + +if [ ! -x `which unzip` ]; then + echo "error: not found in path: unzip" + exit 1 +fi + +if [ ! -x `which bzip2` ]; then + echo "error: not found in path: bzip2" + exit 1 +fi + + function helptext { cat <&2 helptext @@ -96,53 +130,31 @@ while getopts ":dc:rh" opt; do esac done +#set url to download tiles from: +if [ "$USE_HIGHRES" = true ] +then + SRTM2SDF_CMD=$SRTM2SDF_HD -case $CONTINENT in + #unfortunately there is no listing per continent + INDEXURL=$SRTM1URL + +else + case $CONTINENT in North_America|South_America|Africa|Eurasia|Australia|Islands) echo $CONTINENT ;; *) echo "Invalid continent: $CONTINENT" + helptext exit 1 ;; -esac - -#set url to download tiles from: -if [ "$USE_HIGHRES" = true ] -then - INDEXURL=$SRTM1URL -else + esac INDEXURL=${SRTM3URL}${CONTINENT}/ fi -INDEXFILE=`mktemp` -FILELIST=`mktemp` - -if [ ! -x `which $SRTM2SDF` ]; then - echo "error: not found in path: srtm2sdf splat conversion utility" - exit 1 -fi -if [ ! -x `which readlink` ]; then - echo "error: not found in path: readlink" - exit 1 -fi - -if [ ! -x `which wget` ]; then - echo "error: not found in path: wget" - exit 1 -fi - -if [ ! -x `which unzip` ]; then - echo "error: not found in path: unzip" - exit 1 -fi - -if [ ! -x `which bzip2` ]; then - echo "error: not found in path: bzip2" - exit 1 -fi +# Start to download tiles: echo "getting index.. from $INDEXURL" wget -q -O - $INDEXURL > $INDEXFILE @@ -156,57 +168,48 @@ else > $FILELIST fi +#cp $FILELIST ./filelist + mkdir -p $HGTFILEDIR mkdir -p $TOPOFILEDIR -echo "retrieving files.." -#cd $HGTFILEDIR -#head $INDEXFILE +echo "retrieving files..." +FILECOUNT=`wc -l $FILELIST` +echo "Starting processing of ${FILECOUNT} Files" -#convert to absolute path because srtm2sdf need cd +#convert to absolute path because srtm2sdf does not accept output path arguments HGTREALPATH=`readlink -f $HGTFILEDIR` TOPOREALPATH=`readlink -f $TOPOFILEDIR` - PWD=`pwd` -# TODO delete for prod: -SRTM2SDF_CMD=$PWD/$SRTM2SDF_CMD - -for FILE in $(cat $FILELIST); -do - echo $FILE - if [ "$USE_HIGHRES" = true ] - then +for FILE in $(cat $FILELIST); do + echo "Downloading: ${FILE}" + if [ "$USE_HIGHRES" = true ]; then HGTFILE=${FILE%SRTMGL1.hgt.zip}hgt else HGTFILE=${FILE%.zip} fi - + #download the tile wget -P $HGTFILEDIR -nv -N $INDEXURL$FILE - #in direct conversion mode, directly make an sdf and delete all downloaded files - if [ "$DIRECT_CONVERSION" = true ] ; - then + #in direct conversion mode, directly make an sdf.bz2 and delete all downloaded files + if [ "$DIRECT_CONVERSION" = true ] ; then echo "Unzip $FILE and then delete zip" nice unzip -o $HGTFILEDIR/$FILE -d $TOPOFILEDIR rm $HGTFILEDIR/$FILE - #only execute if file exists: - if [ -r $TOPOFILEDIR/$HGTFILE ]; - then + if [ -r $TOPOFILEDIR/$HGTFILE ]; then echo "Convert $HGTFILE to SDF" cd $TOPOFILEDIR nice $SRTM2SDF_CMD -d /dev/null $HGTFILE cd - echo "compressing.." #sadly i am too lazy to figure out srtm2sdf naming schemes - for SDF in $TOPOFILEDIR/*.sdf - do - if test -f "$SDF" - then + for SDF in $TOPOFILEDIR/*.sdf ; do + if test -f "$SDF" ; then echo "Compress $SDF" nice bzip2 -f -- $SDF fi @@ -215,28 +218,36 @@ do rm $TOPOFILEDIR/$HGTFILE fi fi + + # TODO comment for prod: + #break; done -# TODO +#delete tempfiles rm $INDEXFILE rm $FILELIST -#nothing to do in direct conversion mode -if [ "$DIRECT_CONVERSION" = true ] ; -then + +# Exit in direct conversion mode, because everything is done +if [ "$DIRECT_CONVERSION" = true ]; then + echo "Downloading finished, have fun!" exit 0; fi +#Conventional processing after all tiles are downloaded # to minimize disk space required, run srtm2sdf on each file as it is unzipped. - echo "unpacking hgt files.." cd $HGTFILEDIR for e in *.zip ; do echo $e nice unzip -o $e - HGTFILE=`echo $e | sed -r -e 's/\.?hgt.zip/.hgt/'` + if [ "$USE_HIGHRES" = true ]; then + HGTFILE=${FILE%SRTMGL1.hgt.zip}hgt + else + HGTFILE=${FILE%.zip} + fi if [ -r $HGTFILE ]; then cd $TOPOREALPATH nice $SRTM2SDF_CMD -d /dev/null $HGTREALPATH/$HGTFILE From 299a2fc5c9b83d00b5085cac4ea627fa0f773cca Mon Sep 17 00:00:00 2001 From: Oliver Wisler Date: Sat, 20 Feb 2016 22:04:30 +0100 Subject: [PATCH 3/3] Changed /bin/sh to /bin/bash to make it run in debian --- get-datafiles.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get-datafiles.sh b/get-datafiles.sh index bdca80e..19c6c39 100755 --- a/get-datafiles.sh +++ b/get-datafiles.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # get the SRTM data files and convert them for splat use