Poi downloader

Point of interest downloaderis poiplaza.com, visokiems GPS'sininkams...

#!/bin/bash
username='pirdunas'
password='landunas'
working_dir='/home/devnull/poi/poi'
cookies="$working_dir/cookies.txt"
countries="$working_dir/countries"
downloads="$working_dir/downloads"
url='http://poiplaza.com'
UA='Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.58 Safari/537.36'
 
poi_page="$working_dir/poi_page.html"
tmp_list="$working_dir/tmp_list.html"
 
cd $working_dir
 
# Login function.
function login() {
    wget --save-cookies $cookies --load-cookies $cookies --keep-session-cookies -U "$UA" --post-data "trytologin=1&username=$username&userpassword=$password" -O /dev/null $url
}
 
# Get list of POIs for specific country.
# Function is recursive.
function get_poi_list() {
    wget --save-cookies $cookies --load-cookies $cookies --keep-session-cookies -U "$UA" -O $poi_page $1
    country_name=$(cat $poi_page | grep -oE '<h1 style=.f[^<>]*>[^<>]+' | cut -d'>' -f2)
    cat $poi_page | grep -oE "div.s.*href=.\?p=sdb[^<>]*>[^<>]+" >> $tmp_list
    next_page=$(cat $poi_page | grep -oE -m1 "8249\;.*\?p=".* | cut -d'"' -f4)
    while [[ ! -z $next_page ]]; do
        echo "NEXT PAGE: "$next_page
        get_poi_list $url/$next_page
    done
}
 
# Downloads POI zip files for various devices.
function get_poi_files() {
    while read line; do
        # Rewrite links for download.
        download_page=$(echo "$line" | cut -d'"' -f4 | sed s/sdb/download/ | sed s/lstpg=ds/lstpg2=sdb\&/)
        # Catalogize downloads by POI.
        poi_name=$(echo "$line" | cut -d'>' -f3)
        download_location="$downloads/$region_name/$country_name/$poi_name"
        if [[ ! -d $download_location ]]; then
            mkdir -p "$download_location"
        fi
        # Tons of flags, RTFM.
        # 503 download limit 1 / 1 min.
        wget -r -np -nd -nc -c -l1 -w 60 --save-cookies $cookies --load-cookies $cookies --keep-session-cookies -erobots=off -U "$UA" -A "zip" -P "$download_location" $url/$download_page
    done < $tmp_list
    # Clean things up before goin to another country.
    if [[ -e "$tmp_list" ]]; then
        rm "$tmp_list"
    fi
}
 
# Clean things up before working if there's a need.
function cleanup() {
    if [[ -e "$poi_page" ]]; then
        rm "$poi_page"
    fi
    if [[ -e "$tmp_list" ]]; then
        rm "$tmp_list"
    fi
    if [[ -e "$cookies" ]]; then
        rm "$cookies"
    fi
}
 
# Tasks beeing ran before working.
function init() {
    cleanup
    login
}
 
# Teh core.
function runtime() {
    init
    for f in $countries/*; do
        region_name=$(basename $f)
        while read g ; do
            get_poi_list $url/$g
            get_poi_files
        done < $f
    done
}
 

#runtime
get_poi_list
get_poi_files