diff --git a/get-datafiles-northamerica.sh b/get-datafiles-northamerica.sh index ae8904a..2db94ed 100755 --- a/get-datafiles-northamerica.sh +++ b/get-datafiles-northamerica.sh @@ -18,5 +18,5 @@ echo "unpacking state/county borders.." unzip -o *.zip cd - -./get-datafiles.sh "North_America" +./get-datafiles.sh -c "North_America" diff --git a/get-datafiles-world.sh b/get-datafiles-world.sh index f80c60e..fcc5ed3 100755 --- a/get-datafiles-world.sh +++ b/get-datafiles-world.sh @@ -3,10 +3,10 @@ # License: Public domain / CC-0 ./get-datafiles-northamerica.sh -./get-datafiles.sh "South_America" -./get-datafiles.sh "Africa" -./get-datafiles.sh "Eurasia" -./get-datafiles.sh "Australia" -./get-datafiles.sh "Islands" +./get-datafiles.sh -c "South_America" +./get-datafiles.sh -c "Africa" +./get-datafiles.sh -c "Eurasia" +./get-datafiles.sh -c "Australia" +./get-datafiles.sh -c "Islands" diff --git a/get-datafiles.sh b/get-datafiles.sh index 45062f6..19c6c39 100755 --- a/get-datafiles.sh +++ b/get-datafiles.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # get the SRTM data files and convert them for splat use @@ -18,23 +18,30 @@ TOPOFILEDIR=splat-datafiles/sdf/ # local hgt file archive HGTFILEDIR=splat-datafiles/hgtzip/ -CONTINENT=$1 -case $CONTINENT in - North_America|South_America|Africa|Eurasia|Australia|Islands) - echo $CONTINENT - ;; - *) - echo "Invalid continent: $CONTINENT" - exit 1 - ;; -esac +SRTM2SDF_HD=srtm2sdf-hd +SRTM2SDF=srtm2sdf +INDEXFILE=`mktemp` +FILELIST=`mktemp` -INDEXURL="http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/${CONTINENT}/" +#URLs from where to fetch the tiles +SRTM3URL="http://dds.cr.usgs.gov/srtm/version2_1/SRTM3/" +SRTM1URL="http://e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" -INDEXFILE=`mktemp` -if [ ! -x `which srtm2sdf` ]; then +#Default options: +USE_HIGHRES=false +SRTM2SDF_CMD=$SRTM2SDF + +DIRECT_CONVERSION=false +CONTINENT=unknown +USE_LONRANGE=false +USE_LATRANGE=false + + +# Check if all prerequisites are installed + +if [ ! -x `which $SRTM2SDF` ]; then echo "error: not found in path: srtm2sdf splat conversion utility" exit 1 fi @@ -59,36 +66,191 @@ if [ ! -x `which bzip2` ]; then exit 1 fi -echo "getting index.." -wget -q -O - $INDEXURL | \ + + +function helptext { +cat <&2 + helptext + exit 1 + ;; + :) + echo "Option -$OPTARG requires an argument." >&2 + helptext + exit 1 + ;; + esac +done + +#set url to download tiles from: +if [ "$USE_HIGHRES" = true ] +then + SRTM2SDF_CMD=$SRTM2SDF_HD + + #unfortunately there is no listing per continent + INDEXURL=$SRTM1URL + +else + case $CONTINENT in + North_America|South_America|Africa|Eurasia|Australia|Islands) + echo $CONTINENT + ;; + *) + echo "Invalid continent: $CONTINENT" + helptext + exit 1 + ;; + esac + INDEXURL=${SRTM3URL}${CONTINENT}/ +fi + + + +# Start to download tiles: +echo "getting index.. from $INDEXURL" +wget -q -O - $INDEXURL > $INDEXFILE + +if [ "$USE_HIGHRES" = true ] +then + #random magic stolen from the internet + grep -F '.hgt.zip<' $INDEXFILE | sed -e 's@.*href="@@g' -e 's/">.*//g' > $FILELIST +else + wget -q -O - $INDEXURL | \ sed -r -e '/hgt.zip/!d; s/.* ([NSWE0-9]+\.?hgt\.zip).*$/\1/;' \ - > $INDEXFILE + > $FILELIST +fi + +#cp $FILELIST ./filelist mkdir -p $HGTFILEDIR mkdir -p $TOPOFILEDIR -echo "retrieving files.." -cd $HGTFILEDIR -wget -nv -N -B $INDEXURL -i $INDEXFILE -cd - - -rm $INDEXFILE - -# to minimize disk space required, run srtm2sdf on each file as it is unzipped. +echo "retrieving files..." +FILECOUNT=`wc -l $FILELIST` +echo "Starting processing of ${FILECOUNT} Files" +#convert to absolute path because srtm2sdf does not accept output path arguments HGTREALPATH=`readlink -f $HGTFILEDIR` TOPOREALPATH=`readlink -f $TOPOFILEDIR` PWD=`pwd` +for FILE in $(cat $FILELIST); do + echo "Downloading: ${FILE}" + if [ "$USE_HIGHRES" = true ]; then + HGTFILE=${FILE%SRTMGL1.hgt.zip}hgt + else + HGTFILE=${FILE%.zip} + fi + + #download the tile + wget -P $HGTFILEDIR -nv -N $INDEXURL$FILE + + #in direct conversion mode, directly make an sdf.bz2 and delete all downloaded files + if [ "$DIRECT_CONVERSION" = true ] ; then + echo "Unzip $FILE and then delete zip" + nice unzip -o $HGTFILEDIR/$FILE -d $TOPOFILEDIR + rm $HGTFILEDIR/$FILE + + + #only execute if file exists: + if [ -r $TOPOFILEDIR/$HGTFILE ]; then + echo "Convert $HGTFILE to SDF" + cd $TOPOFILEDIR + nice $SRTM2SDF_CMD -d /dev/null $HGTFILE + cd - + echo "compressing.." + #sadly i am too lazy to figure out srtm2sdf naming schemes + for SDF in $TOPOFILEDIR/*.sdf ; do + if test -f "$SDF" ; then + echo "Compress $SDF" + nice bzip2 -f -- $SDF + fi + done + echo "deleting hgt file $TOPOFILEDIR/$HGTFILE" + rm $TOPOFILEDIR/$HGTFILE + fi + fi + + # TODO comment for prod: + #break; +done + +#delete tempfiles +rm $INDEXFILE +rm $FILELIST + + + +# Exit in direct conversion mode, because everything is done +if [ "$DIRECT_CONVERSION" = true ]; then + echo "Downloading finished, have fun!" + exit 0; +fi + + +#Conventional processing after all tiles are downloaded +# to minimize disk space required, run srtm2sdf on each file as it is unzipped. echo "unpacking hgt files.." cd $HGTFILEDIR for e in *.zip ; do echo $e nice unzip -o $e - HGTFILE=`echo $e | sed -r -e 's/\.?hgt.zip/.hgt/'` + if [ "$USE_HIGHRES" = true ]; then + HGTFILE=${FILE%SRTMGL1.hgt.zip}hgt + else + HGTFILE=${FILE%.zip} + fi if [ -r $HGTFILE ]; then cd $TOPOREALPATH - nice srtm2sdf -d /dev/null $HGTREALPATH/$HGTFILE + nice $SRTM2SDF_CMD -d /dev/null $HGTREALPATH/$HGTFILE echo "compressing.." nice bzip2 -f -- *.sdf echo "deleting hgt file.."