import os
import sys
import time
import json
import requests
import argparse
import urllib.request
import threading
import cv2
import math
from scipy.spatial import distance
import logging

key = None
img_size = '200x225'
max_threads = 120
resquest_per_second = 40.0  # limit is 50/sec, up to 2500/day
nb_of_no_crosswalk_foreach_crosswalk = 2.2
get_path_every_n_crosswalks = 1
max_nb_of_waypoints = 20
max_attempts = 5


def log_and_print(message):
    print(message)
    logging.info(message)


def get_crosswalks(region):
    # REQUEST URL
    base_api_osm = 'https://overpass-api.de/api/'
    node_query = '"highway"~"crossing"'
    params = 'interpreter?data=[out:json];node[{}]({});out body;'.format(node_query, ','.join(map(str, region)))
    request_url = base_api_osm + params
    response = requests.get(request_url).json()
    return [{'id': i, 'lat': e['lat'], 'lng': e['lon']} for i, e in enumerate(response['elements'])]


def thread_download_static_map(location, type, region_name):
    base_api_static_maps = 'https://maps.googleapis.com/maps/api/staticmap'
    request_url = '{}?center={}&zoom=20&size={}&maptype=satellite&key={}'.format(
        base_api_static_maps, ','.join(map(str, [location['lat'], location['lng']])), img_size, key
    )
    urllib.request.urlretrieve(request_url, os.path.join(region_name, type, '{}.png'.format(location['id'])))


def get_directions_polylines(waypoints):
    orig = waypoints[0]
    dest = waypoints[-1]
    orig = '{},{}'.format(orig['lat'], orig['lng'])
    dest = '{},{}'.format(dest['lat'], dest['lng'])
    waypoints_str = '|'.join(['{},{}'.format(w['lat'], w['lng']) for w in waypoints[1:-1]])

    base_api = 'https://maps.googleapis.com/maps/api/directions/json'
    api_params = 'origin={}&destination={}&waypoints={}&key={}'.format(orig, dest, waypoints_str, key)
    request_url = '{}?{}'.format(base_api, api_params)

    polyline_data = {"request_url": request_url, "overview_polyline": ""}
    for nb_attempts in range(max_attempts):
        try:
            response = requests.get(request_url).json()
            polyline_data["overview_polyline"] = response['routes'][0]['overview_polyline']['points']
            break
        except:
            log_and_print('Request #{} to Direction API failed. Trying again...'.format(nb_attempts))
            time.sleep(1.0/max_threads)
    return polyline_data


def download_images(crosswalks, img_type, region_name):
    # get the images of img_type = {crosswalks, no-crosswalks}
    mythreads = []
    total = 0
    for i, crosswalk in enumerate(crosswalks):
        t = threading.Thread(target=thread_download_static_map, args=(crosswalk, img_type, region_name))
        mythreads.append(t)
        t.start()
        if (len(mythreads) >= max_threads) or (i == len(crosswalks) - 1):
            for t in mythreads:
                t.join()
            total += len(mythreads)
            mythreads[:] = []  # clear the thread's list
            log_and_print('Downloaded {} images already...'.format(total))
    log_and_print('Done: downloaded {} images...'.format(total))


def download_polylines_points(crosswalks, region):
    # get the path between these crosswalks
    min_points = int(nb_of_no_crosswalk_foreach_crosswalk * len(crosswalks))

    all_points = []
    no_crosswalk_points = []
    paths_data = []
    nb_duplicates = 0

    nb_of_paths = int(math.ceil((len(crosswalks) - 1) / float((max_nb_of_waypoints + 1))))
    log_and_print('Downloading {} paths from {} crosswalks'.format(nb_of_paths, len(crosswalks)))
    for i in range(nb_of_paths):

        path_start = i*(max_nb_of_waypoints+1)
        path_end = min((i+1)*(max_nb_of_waypoints+1), len(crosswalks)-1)
        waypoints = [crosswalks[j] for j in range(path_start, path_end+1)]

        path_data = get_directions_polylines(waypoints)
        paths_data.append(path_data)

        polylines_points, nb_polylines_duplicates = decode_polylines([path_data['overview_polyline']])
        no_crosswalk_polylines_points = remove_close_to_crosswalk(polylines_points, crosswalks, region)

        all_points.extend(polylines_points)
        no_crosswalk_points.extend(no_crosswalk_polylines_points)
        nb_duplicates += nb_polylines_duplicates

        log_and_print('Got {} points so far. We need at least {}'.format(len(no_crosswalk_points), min_points))
        if len(no_crosswalk_points) > min_points:
            break

    # sets an ID for the no-crosswalks
    for i in range(len(no_crosswalk_points)):
        no_crosswalk_points[i]['id'] = i

    log_and_print('Download of polylines finished! Nb of points: {}'.format(len(no_crosswalk_points)))

    return paths_data, all_points, nb_duplicates, no_crosswalk_points


def save_to_json(content, fname):
    with open(fname, 'w') as hf:
        json.dump(content, hf, indent=4, sort_keys=True)


def decode_polyline(polyline_str):
    """Pass a Google Maps encoded polyline string; returns list of lat/lon pairs"""
    index, lat, lng = 0, 0, 0
    coordinates = []
    changes = {'latitude': 0, 'longitude': 0}

    # Coordinates have variable length when encoded, so just keep
    # track of whether we've hit the end of the string. In each
    # while loop iteration, a single coordinate is decoded.
    while index < len(polyline_str):
        # Gather lat/lon changes, store them in a dictionary to apply them later
        for unit in ['latitude', 'longitude']:
            shift, result = 0, 0

            while index < len(polyline_str):
                byte = ord(polyline_str[index]) - 63
                index += 1
                result |= (byte & 0x1f) << shift
                shift += 5
                if not byte >= 0x20:
                    break

            if result & 1:
                changes[unit] = ~(result >> 1)
            else:
                changes[unit] = (result >> 1)

        lat += changes['latitude']
        lng += changes['longitude']

        coordinates.append((lat / 100000.0, lng / 100000.0))

    return coordinates


def decode_polylines(polylines):
    all_points = []
    for i, p in enumerate(polylines):
        points_polyline = decode_polyline(p)
        points_polyline = generate_samples_interpoler(points_polyline)
        all_points.extend(points_polyline)
    no_duplicate_points = list(set(all_points))
    return no_duplicate_points, len(all_points) - len(no_duplicate_points)


def remove_close_to_crosswalk(points, crosswalks, region):
    min_dist = 0.0003  # approx 33 meters
    max_dist = 0.0006  # approx 66 meters
    no_crosswalk_points = []
    for i, point in enumerate(points):
        dist_to_crosswalk = get_min_dist_to_crosswalk(point, crosswalks)
        if is_inside_region(point, region, min_dist) and min_dist < dist_to_crosswalk < max_dist:
            no_crosswalk_points.append({'lat': point[0], 'lng': point[1]})
    return no_crosswalk_points


def get_min_dist_to_crosswalk(location, crosswalks):
    min_dist = 360
    for crosswalk in crosswalks:
        dist = distance.euclidean(location, (crosswalk['lat'], crosswalk['lng']))
        if dist < min_dist:
            min_dist = dist
    return min_dist


def is_inside_region(point, region, safe_margin):
    lower_bound = point[0] > (region[0] + safe_margin)
    upper_bound = point[0] < (region[2] - safe_margin)
    left_bound = point[1] > (region[1] + safe_margin)
    right_bound = point[1] < (region[3] - safe_margin)
    return lower_bound and upper_bound and left_bound and right_bound


def generate_samples_interpoler(points):
    max_sample_step = 0.00015  # approx. 16 meters
    new_points = []
    for i in range(len(points) - 1):
        dist = distance.euclidean(points[i], points[i+1])
        num_steps = int(round(dist / max_sample_step))
        num_steps = 1 if num_steps <= 1 else num_steps

        step_size_lat = (points[i+1][0] - points[i][0]) / num_steps
        step_size_lng = (points[i+1][1] - points[i][1]) / num_steps

        for j in range(num_steps):
            new_points.append((points[i][0] + j*step_size_lat, points[i][1] + j*step_size_lng))
        new_points.append(points[i+1])
    return new_points


def crop_images(region_name, download_crosswalks, download_no_crosswalks):
    deleted_files = 0
    if download_crosswalks:
        for f in os.listdir("{}/crosswalk".format(region_name)):
            if f.endswith(".png"):
                full_path_img = os.path.abspath(os.path.join("{}/crosswalk".format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1

    if download_no_crosswalks:
        for f in os.listdir("{}/no-crosswalk".format(region_name)):
            if f.endswith(".png"):
                full_path_img = os.path.abspath(os.path.join("{}/no-crosswalk".format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1

    log_and_print('Deleted images: {}'.format(deleted_files))


def check_setup(region_name, download_crosswalk, download_no_crosswalk):
    directories = ['{}']

    if download_crosswalk:
        directories.append('{}/crosswalk')
    if download_no_crosswalk:
        directories.append('{}/no-crosswalk')

    for d in directories:
        _dir = d.format(region_name)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
    # init logger
    log_fname = os.path.join(region_name, 'run.log')
    logging.basicConfig(filename=log_fname, filemode='w', format='[%(asctime)s] %(name)s - %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
    logging.info('----------------------------------------')

    # disable log messages from the requests modules
    logging.getLogger("requests").setLevel(logging.CRITICAL)
    logging.getLogger("urllib3").setLevel(logging.CRITICAL)


def load_regions(regions_name):
    with open('regions.json') as hf:
        return json.load(hf)[regions_name]


def main(region_name, download_crosswalk, download_no_crosswalk, api_key):

    global key
    key = api_key

    regions = load_regions(region_name)

    # build download str
    download_str = "crosswalks" if download_crosswalk else "no-crosswalks"
    if download_crosswalk and download_no_crosswalk:
        download_str = "crosswalks and no-crosswalks"

    for region_data in regions:
        # init
        # =========================================================================================================
        start = time.time()
        region = region_data['region']
        region_name = region_data['name']
        check_setup(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('Downloading {} from {}: {}'.format(download_str, region_name, ','.join(map(str, region))))
        log_and_print('Using key: {}'.format(key))

        # 1. download crosswalk images
        # =========================================================================================================
        if download_crosswalk:
            # download the crosswalk locations using OpenStreetMap
            crosswalks = get_crosswalks(region_data['region'])
            save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))

            log_and_print('Downloading {} crosswalk images'.format(len(crosswalks)))
            download_images(crosswalks, 'crosswalk', region_name)
        else:
            log_and_print('Crosswalk images are not gonna be downloaded this time')

        # 2. download no-crosswalk images
        # =========================================================================================================
        if download_no_crosswalk:
            # we still need the crosswalk points
            if not download_crosswalk:
                if not os.path.isfile('{}/crosswalks.json'.format(region_name)):
                    crosswalks = get_crosswalks(region_data['region'])
                    save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))
                else:
                    with open('{}/crosswalks.json'.format(region_name)) as hf:
                        crosswalks = json.load(hf)

            if not os.path.isfile('{}/no_crosswalks.json'.format(region_name)):
                # download the polylines
                polylines, all_points, nb_duplicates, no_crosswalk_points = download_polylines_points(crosswalks, region)
                save_to_json(polylines, '{}/paths.json'.format(region_name))
                save_to_json(no_crosswalk_points, '{}/no_crosswalks.json'.format(region_name))
            else:
                with open('{}/no_crosswalks.json'.format(region_name)) as hf:
                    no_crosswalk_points = json.load(hf)
                with open('{}/paths.json'.format(region_name)) as hf:
                    polylines = json.load(hf)
                nb_duplicates = 'files were loaded'
                all_points = 'files were loaded'

            log_and_print('Downloading {} no-crosswalk images'.format(len(no_crosswalk_points)))
            download_images(no_crosswalk_points, 'no-crosswalk', region_name)

        # if any of the images were downloaded, crop the images
        # =========================================================================================================
        if download_crosswalk or download_no_crosswalk:
            log_and_print('Removing Google from the images')
            crop_images(region_name, download_crosswalk, download_no_crosswalk)

        # summary
        # =========================================================================================================
        log_and_print('-' * 50)
        log_and_print('SUMMARY - REGION {}'.format(region_name))
        if download_no_crosswalk:
            log_and_print('Number of paths: {}'.format(len(polylines)))
            log_and_print('Removed {} duplicate points from the paths'.format(nb_duplicates))
            log_and_print('Number of all points: {}'.format(len(all_points)))
            log_and_print('Number of no-crosswalk points: {}'.format(len(no_crosswalk_points)))
        if download_crosswalk:
            log_and_print('Number of crosswalk points: {}'.format(len(crosswalks)))
        log_and_print('-'*50)
        log_and_print('{} finished: {:.2f} seconds'.format(region_name, (time.time() - start)))


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Download crosswalk satellite imagery from a given region.')
    parser.add_argument('--region', type=str, required=True, help='Region name: regions.json')
    parser.add_argument('--positive', dest='positive', action='store_true', help='Download positive samples')
    parser.add_argument('--negative', dest='negative', action='store_true', help='Download negative samples')
    parser.add_argument('--key', type=str, required=True, help='API_KEY: console.developers.google.com')
    parser.set_defaults(positive=False)
    parser.set_defaults(negative=False)
    argv = parser.parse_args()

    if not (argv.positive or argv.negative):
        print('Nothing to do.')
        sys.exit()

    main(argv.region, argv.positive, argv.negative, argv.key)

d4ed9d5d9558e8b8b55b0067622315391969be0e
<mask token>


def get_crosswalks(region):
    base_api_osm = 'https://overpass-api.de/api/'
    node_query = '"highway"~"crossing"'
    params = 'interpreter?data=[out:json];node[{}]({});out body;'.format(
        node_query, ','.join(map(str, region)))
    request_url = base_api_osm + params
    response = requests.get(request_url).json()
    return [{'id': i, 'lat': e['lat'], 'lng': e['lon']} for i, e in
        enumerate(response['elements'])]


def thread_download_static_map(location, type, region_name):
    base_api_static_maps = 'https://maps.googleapis.com/maps/api/staticmap'
    request_url = ('{}?center={}&zoom=20&size={}&maptype=satellite&key={}'.
        format(base_api_static_maps, ','.join(map(str, [location['lat'],
        location['lng']])), img_size, key))
    urllib.request.urlretrieve(request_url, os.path.join(region_name, type,
        '{}.png'.format(location['id'])))


def get_directions_polylines(waypoints):
    orig = waypoints[0]
    dest = waypoints[-1]
    orig = '{},{}'.format(orig['lat'], orig['lng'])
    dest = '{},{}'.format(dest['lat'], dest['lng'])
    waypoints_str = '|'.join(['{},{}'.format(w['lat'], w['lng']) for w in
        waypoints[1:-1]])
    base_api = 'https://maps.googleapis.com/maps/api/directions/json'
    api_params = 'origin={}&destination={}&waypoints={}&key={}'.format(orig,
        dest, waypoints_str, key)
    request_url = '{}?{}'.format(base_api, api_params)
    polyline_data = {'request_url': request_url, 'overview_polyline': ''}
    for nb_attempts in range(max_attempts):
        try:
            response = requests.get(request_url).json()
            polyline_data['overview_polyline'] = response['routes'][0][
                'overview_polyline']['points']
            break
        except:
            log_and_print(
                'Request #{} to Direction API failed. Trying again...'.
                format(nb_attempts))
            time.sleep(1.0 / max_threads)
    return polyline_data


<mask token>


def decode_polyline(polyline_str):
    """Pass a Google Maps encoded polyline string; returns list of lat/lon pairs"""
    index, lat, lng = 0, 0, 0
    coordinates = []
    changes = {'latitude': 0, 'longitude': 0}
    while index < len(polyline_str):
        for unit in ['latitude', 'longitude']:
            shift, result = 0, 0
            while index < len(polyline_str):
                byte = ord(polyline_str[index]) - 63
                index += 1
                result |= (byte & 31) << shift
                shift += 5
                if not byte >= 32:
                    break
            if result & 1:
                changes[unit] = ~(result >> 1)
            else:
                changes[unit] = result >> 1
        lat += changes['latitude']
        lng += changes['longitude']
        coordinates.append((lat / 100000.0, lng / 100000.0))
    return coordinates


<mask token>


def remove_close_to_crosswalk(points, crosswalks, region):
    min_dist = 0.0003
    max_dist = 0.0006
    no_crosswalk_points = []
    for i, point in enumerate(points):
        dist_to_crosswalk = get_min_dist_to_crosswalk(point, crosswalks)
        if is_inside_region(point, region, min_dist
            ) and min_dist < dist_to_crosswalk < max_dist:
            no_crosswalk_points.append({'lat': point[0], 'lng': point[1]})
    return no_crosswalk_points


def get_min_dist_to_crosswalk(location, crosswalks):
    min_dist = 360
    for crosswalk in crosswalks:
        dist = distance.euclidean(location, (crosswalk['lat'], crosswalk[
            'lng']))
        if dist < min_dist:
            min_dist = dist
    return min_dist


def is_inside_region(point, region, safe_margin):
    lower_bound = point[0] > region[0] + safe_margin
    upper_bound = point[0] < region[2] - safe_margin
    left_bound = point[1] > region[1] + safe_margin
    right_bound = point[1] < region[3] - safe_margin
    return lower_bound and upper_bound and left_bound and right_bound


<mask token>


def crop_images(region_name, download_crosswalks, download_no_crosswalks):
    deleted_files = 0
    if download_crosswalks:
        for f in os.listdir('{}/crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join('{}/crosswalk'
                    .format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    if download_no_crosswalks:
        for f in os.listdir('{}/no-crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join(
                    '{}/no-crosswalk'.format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    log_and_print('Deleted images: {}'.format(deleted_files))


<mask token>

<mask token>


def get_crosswalks(region):
    base_api_osm = 'https://overpass-api.de/api/'
    node_query = '"highway"~"crossing"'
    params = 'interpreter?data=[out:json];node[{}]({});out body;'.format(
        node_query, ','.join(map(str, region)))
    request_url = base_api_osm + params
    response = requests.get(request_url).json()
    return [{'id': i, 'lat': e['lat'], 'lng': e['lon']} for i, e in
        enumerate(response['elements'])]


def thread_download_static_map(location, type, region_name):
    base_api_static_maps = 'https://maps.googleapis.com/maps/api/staticmap'
    request_url = ('{}?center={}&zoom=20&size={}&maptype=satellite&key={}'.
        format(base_api_static_maps, ','.join(map(str, [location['lat'],
        location['lng']])), img_size, key))
    urllib.request.urlretrieve(request_url, os.path.join(region_name, type,
        '{}.png'.format(location['id'])))


def get_directions_polylines(waypoints):
    orig = waypoints[0]
    dest = waypoints[-1]
    orig = '{},{}'.format(orig['lat'], orig['lng'])
    dest = '{},{}'.format(dest['lat'], dest['lng'])
    waypoints_str = '|'.join(['{},{}'.format(w['lat'], w['lng']) for w in
        waypoints[1:-1]])
    base_api = 'https://maps.googleapis.com/maps/api/directions/json'
    api_params = 'origin={}&destination={}&waypoints={}&key={}'.format(orig,
        dest, waypoints_str, key)
    request_url = '{}?{}'.format(base_api, api_params)
    polyline_data = {'request_url': request_url, 'overview_polyline': ''}
    for nb_attempts in range(max_attempts):
        try:
            response = requests.get(request_url).json()
            polyline_data['overview_polyline'] = response['routes'][0][
                'overview_polyline']['points']
            break
        except:
            log_and_print(
                'Request #{} to Direction API failed. Trying again...'.
                format(nb_attempts))
            time.sleep(1.0 / max_threads)
    return polyline_data


def download_images(crosswalks, img_type, region_name):
    mythreads = []
    total = 0
    for i, crosswalk in enumerate(crosswalks):
        t = threading.Thread(target=thread_download_static_map, args=(
            crosswalk, img_type, region_name))
        mythreads.append(t)
        t.start()
        if len(mythreads) >= max_threads or i == len(crosswalks) - 1:
            for t in mythreads:
                t.join()
            total += len(mythreads)
            mythreads[:] = []
            log_and_print('Downloaded {} images already...'.format(total))
    log_and_print('Done: downloaded {} images...'.format(total))


def download_polylines_points(crosswalks, region):
    min_points = int(nb_of_no_crosswalk_foreach_crosswalk * len(crosswalks))
    all_points = []
    no_crosswalk_points = []
    paths_data = []
    nb_duplicates = 0
    nb_of_paths = int(math.ceil((len(crosswalks) - 1) / float(
        max_nb_of_waypoints + 1)))
    log_and_print('Downloading {} paths from {} crosswalks'.format(
        nb_of_paths, len(crosswalks)))
    for i in range(nb_of_paths):
        path_start = i * (max_nb_of_waypoints + 1)
        path_end = min((i + 1) * (max_nb_of_waypoints + 1), len(crosswalks) - 1
            )
        waypoints = [crosswalks[j] for j in range(path_start, path_end + 1)]
        path_data = get_directions_polylines(waypoints)
        paths_data.append(path_data)
        polylines_points, nb_polylines_duplicates = decode_polylines([
            path_data['overview_polyline']])
        no_crosswalk_polylines_points = remove_close_to_crosswalk(
            polylines_points, crosswalks, region)
        all_points.extend(polylines_points)
        no_crosswalk_points.extend(no_crosswalk_polylines_points)
        nb_duplicates += nb_polylines_duplicates
        log_and_print('Got {} points so far. We need at least {}'.format(
            len(no_crosswalk_points), min_points))
        if len(no_crosswalk_points) > min_points:
            break
    for i in range(len(no_crosswalk_points)):
        no_crosswalk_points[i]['id'] = i
    log_and_print('Download of polylines finished! Nb of points: {}'.format
        (len(no_crosswalk_points)))
    return paths_data, all_points, nb_duplicates, no_crosswalk_points


<mask token>


def decode_polyline(polyline_str):
    """Pass a Google Maps encoded polyline string; returns list of lat/lon pairs"""
    index, lat, lng = 0, 0, 0
    coordinates = []
    changes = {'latitude': 0, 'longitude': 0}
    while index < len(polyline_str):
        for unit in ['latitude', 'longitude']:
            shift, result = 0, 0
            while index < len(polyline_str):
                byte = ord(polyline_str[index]) - 63
                index += 1
                result |= (byte & 31) << shift
                shift += 5
                if not byte >= 32:
                    break
            if result & 1:
                changes[unit] = ~(result >> 1)
            else:
                changes[unit] = result >> 1
        lat += changes['latitude']
        lng += changes['longitude']
        coordinates.append((lat / 100000.0, lng / 100000.0))
    return coordinates


def decode_polylines(polylines):
    all_points = []
    for i, p in enumerate(polylines):
        points_polyline = decode_polyline(p)
        points_polyline = generate_samples_interpoler(points_polyline)
        all_points.extend(points_polyline)
    no_duplicate_points = list(set(all_points))
    return no_duplicate_points, len(all_points) - len(no_duplicate_points)


def remove_close_to_crosswalk(points, crosswalks, region):
    min_dist = 0.0003
    max_dist = 0.0006
    no_crosswalk_points = []
    for i, point in enumerate(points):
        dist_to_crosswalk = get_min_dist_to_crosswalk(point, crosswalks)
        if is_inside_region(point, region, min_dist
            ) and min_dist < dist_to_crosswalk < max_dist:
            no_crosswalk_points.append({'lat': point[0], 'lng': point[1]})
    return no_crosswalk_points


def get_min_dist_to_crosswalk(location, crosswalks):
    min_dist = 360
    for crosswalk in crosswalks:
        dist = distance.euclidean(location, (crosswalk['lat'], crosswalk[
            'lng']))
        if dist < min_dist:
            min_dist = dist
    return min_dist


def is_inside_region(point, region, safe_margin):
    lower_bound = point[0] > region[0] + safe_margin
    upper_bound = point[0] < region[2] - safe_margin
    left_bound = point[1] > region[1] + safe_margin
    right_bound = point[1] < region[3] - safe_margin
    return lower_bound and upper_bound and left_bound and right_bound


def generate_samples_interpoler(points):
    max_sample_step = 0.00015
    new_points = []
    for i in range(len(points) - 1):
        dist = distance.euclidean(points[i], points[i + 1])
        num_steps = int(round(dist / max_sample_step))
        num_steps = 1 if num_steps <= 1 else num_steps
        step_size_lat = (points[i + 1][0] - points[i][0]) / num_steps
        step_size_lng = (points[i + 1][1] - points[i][1]) / num_steps
        for j in range(num_steps):
            new_points.append((points[i][0] + j * step_size_lat, points[i][
                1] + j * step_size_lng))
        new_points.append(points[i + 1])
    return new_points


def crop_images(region_name, download_crosswalks, download_no_crosswalks):
    deleted_files = 0
    if download_crosswalks:
        for f in os.listdir('{}/crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join('{}/crosswalk'
                    .format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    if download_no_crosswalks:
        for f in os.listdir('{}/no-crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join(
                    '{}/no-crosswalk'.format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    log_and_print('Deleted images: {}'.format(deleted_files))


<mask token>


def load_regions(regions_name):
    with open('regions.json') as hf:
        return json.load(hf)[regions_name]


def main(region_name, download_crosswalk, download_no_crosswalk, api_key):
    global key
    key = api_key
    regions = load_regions(region_name)
    download_str = 'crosswalks' if download_crosswalk else 'no-crosswalks'
    if download_crosswalk and download_no_crosswalk:
        download_str = 'crosswalks and no-crosswalks'
    for region_data in regions:
        start = time.time()
        region = region_data['region']
        region_name = region_data['name']
        check_setup(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('Downloading {} from {}: {}'.format(download_str,
            region_name, ','.join(map(str, region))))
        log_and_print('Using key: {}'.format(key))
        if download_crosswalk:
            crosswalks = get_crosswalks(region_data['region'])
            save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))
            log_and_print('Downloading {} crosswalk images'.format(len(
                crosswalks)))
            download_images(crosswalks, 'crosswalk', region_name)
        else:
            log_and_print(
                'Crosswalk images are not gonna be downloaded this time')
        if download_no_crosswalk:
            if not download_crosswalk:
                if not os.path.isfile('{}/crosswalks.json'.format(region_name)
                    ):
                    crosswalks = get_crosswalks(region_data['region'])
                    save_to_json(crosswalks, '{}/crosswalks.json'.format(
                        region_name))
                else:
                    with open('{}/crosswalks.json'.format(region_name)) as hf:
                        crosswalks = json.load(hf)
            if not os.path.isfile('{}/no_crosswalks.json'.format(region_name)):
                (polylines, all_points, nb_duplicates, no_crosswalk_points
                    ) = download_polylines_points(crosswalks, region)
                save_to_json(polylines, '{}/paths.json'.format(region_name))
                save_to_json(no_crosswalk_points, '{}/no_crosswalks.json'.
                    format(region_name))
            else:
                with open('{}/no_crosswalks.json'.format(region_name)) as hf:
                    no_crosswalk_points = json.load(hf)
                with open('{}/paths.json'.format(region_name)) as hf:
                    polylines = json.load(hf)
                nb_duplicates = 'files were loaded'
                all_points = 'files were loaded'
            log_and_print('Downloading {} no-crosswalk images'.format(len(
                no_crosswalk_points)))
            download_images(no_crosswalk_points, 'no-crosswalk', region_name)
        if download_crosswalk or download_no_crosswalk:
            log_and_print('Removing Google from the images')
            crop_images(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('-' * 50)
        log_and_print('SUMMARY - REGION {}'.format(region_name))
        if download_no_crosswalk:
            log_and_print('Number of paths: {}'.format(len(polylines)))
            log_and_print('Removed {} duplicate points from the paths'.
                format(nb_duplicates))
            log_and_print('Number of all points: {}'.format(len(all_points)))
            log_and_print('Number of no-crosswalk points: {}'.format(len(
                no_crosswalk_points)))
        if download_crosswalk:
            log_and_print('Number of crosswalk points: {}'.format(len(
                crosswalks)))
        log_and_print('-' * 50)
        log_and_print('{} finished: {:.2f} seconds'.format(region_name, 
            time.time() - start))


<mask token>

<mask token>


def log_and_print(message):
    print(message)
    logging.info(message)


def get_crosswalks(region):
    base_api_osm = 'https://overpass-api.de/api/'
    node_query = '"highway"~"crossing"'
    params = 'interpreter?data=[out:json];node[{}]({});out body;'.format(
        node_query, ','.join(map(str, region)))
    request_url = base_api_osm + params
    response = requests.get(request_url).json()
    return [{'id': i, 'lat': e['lat'], 'lng': e['lon']} for i, e in
        enumerate(response['elements'])]


def thread_download_static_map(location, type, region_name):
    base_api_static_maps = 'https://maps.googleapis.com/maps/api/staticmap'
    request_url = ('{}?center={}&zoom=20&size={}&maptype=satellite&key={}'.
        format(base_api_static_maps, ','.join(map(str, [location['lat'],
        location['lng']])), img_size, key))
    urllib.request.urlretrieve(request_url, os.path.join(region_name, type,
        '{}.png'.format(location['id'])))


def get_directions_polylines(waypoints):
    orig = waypoints[0]
    dest = waypoints[-1]
    orig = '{},{}'.format(orig['lat'], orig['lng'])
    dest = '{},{}'.format(dest['lat'], dest['lng'])
    waypoints_str = '|'.join(['{},{}'.format(w['lat'], w['lng']) for w in
        waypoints[1:-1]])
    base_api = 'https://maps.googleapis.com/maps/api/directions/json'
    api_params = 'origin={}&destination={}&waypoints={}&key={}'.format(orig,
        dest, waypoints_str, key)
    request_url = '{}?{}'.format(base_api, api_params)
    polyline_data = {'request_url': request_url, 'overview_polyline': ''}
    for nb_attempts in range(max_attempts):
        try:
            response = requests.get(request_url).json()
            polyline_data['overview_polyline'] = response['routes'][0][
                'overview_polyline']['points']
            break
        except:
            log_and_print(
                'Request #{} to Direction API failed. Trying again...'.
                format(nb_attempts))
            time.sleep(1.0 / max_threads)
    return polyline_data


def download_images(crosswalks, img_type, region_name):
    mythreads = []
    total = 0
    for i, crosswalk in enumerate(crosswalks):
        t = threading.Thread(target=thread_download_static_map, args=(
            crosswalk, img_type, region_name))
        mythreads.append(t)
        t.start()
        if len(mythreads) >= max_threads or i == len(crosswalks) - 1:
            for t in mythreads:
                t.join()
            total += len(mythreads)
            mythreads[:] = []
            log_and_print('Downloaded {} images already...'.format(total))
    log_and_print('Done: downloaded {} images...'.format(total))


def download_polylines_points(crosswalks, region):
    min_points = int(nb_of_no_crosswalk_foreach_crosswalk * len(crosswalks))
    all_points = []
    no_crosswalk_points = []
    paths_data = []
    nb_duplicates = 0
    nb_of_paths = int(math.ceil((len(crosswalks) - 1) / float(
        max_nb_of_waypoints + 1)))
    log_and_print('Downloading {} paths from {} crosswalks'.format(
        nb_of_paths, len(crosswalks)))
    for i in range(nb_of_paths):
        path_start = i * (max_nb_of_waypoints + 1)
        path_end = min((i + 1) * (max_nb_of_waypoints + 1), len(crosswalks) - 1
            )
        waypoints = [crosswalks[j] for j in range(path_start, path_end + 1)]
        path_data = get_directions_polylines(waypoints)
        paths_data.append(path_data)
        polylines_points, nb_polylines_duplicates = decode_polylines([
            path_data['overview_polyline']])
        no_crosswalk_polylines_points = remove_close_to_crosswalk(
            polylines_points, crosswalks, region)
        all_points.extend(polylines_points)
        no_crosswalk_points.extend(no_crosswalk_polylines_points)
        nb_duplicates += nb_polylines_duplicates
        log_and_print('Got {} points so far. We need at least {}'.format(
            len(no_crosswalk_points), min_points))
        if len(no_crosswalk_points) > min_points:
            break
    for i in range(len(no_crosswalk_points)):
        no_crosswalk_points[i]['id'] = i
    log_and_print('Download of polylines finished! Nb of points: {}'.format
        (len(no_crosswalk_points)))
    return paths_data, all_points, nb_duplicates, no_crosswalk_points


<mask token>


def decode_polyline(polyline_str):
    """Pass a Google Maps encoded polyline string; returns list of lat/lon pairs"""
    index, lat, lng = 0, 0, 0
    coordinates = []
    changes = {'latitude': 0, 'longitude': 0}
    while index < len(polyline_str):
        for unit in ['latitude', 'longitude']:
            shift, result = 0, 0
            while index < len(polyline_str):
                byte = ord(polyline_str[index]) - 63
                index += 1
                result |= (byte & 31) << shift
                shift += 5
                if not byte >= 32:
                    break
            if result & 1:
                changes[unit] = ~(result >> 1)
            else:
                changes[unit] = result >> 1
        lat += changes['latitude']
        lng += changes['longitude']
        coordinates.append((lat / 100000.0, lng / 100000.0))
    return coordinates


def decode_polylines(polylines):
    all_points = []
    for i, p in enumerate(polylines):
        points_polyline = decode_polyline(p)
        points_polyline = generate_samples_interpoler(points_polyline)
        all_points.extend(points_polyline)
    no_duplicate_points = list(set(all_points))
    return no_duplicate_points, len(all_points) - len(no_duplicate_points)


def remove_close_to_crosswalk(points, crosswalks, region):
    min_dist = 0.0003
    max_dist = 0.0006
    no_crosswalk_points = []
    for i, point in enumerate(points):
        dist_to_crosswalk = get_min_dist_to_crosswalk(point, crosswalks)
        if is_inside_region(point, region, min_dist
            ) and min_dist < dist_to_crosswalk < max_dist:
            no_crosswalk_points.append({'lat': point[0], 'lng': point[1]})
    return no_crosswalk_points


def get_min_dist_to_crosswalk(location, crosswalks):
    min_dist = 360
    for crosswalk in crosswalks:
        dist = distance.euclidean(location, (crosswalk['lat'], crosswalk[
            'lng']))
        if dist < min_dist:
            min_dist = dist
    return min_dist


def is_inside_region(point, region, safe_margin):
    lower_bound = point[0] > region[0] + safe_margin
    upper_bound = point[0] < region[2] - safe_margin
    left_bound = point[1] > region[1] + safe_margin
    right_bound = point[1] < region[3] - safe_margin
    return lower_bound and upper_bound and left_bound and right_bound


def generate_samples_interpoler(points):
    max_sample_step = 0.00015
    new_points = []
    for i in range(len(points) - 1):
        dist = distance.euclidean(points[i], points[i + 1])
        num_steps = int(round(dist / max_sample_step))
        num_steps = 1 if num_steps <= 1 else num_steps
        step_size_lat = (points[i + 1][0] - points[i][0]) / num_steps
        step_size_lng = (points[i + 1][1] - points[i][1]) / num_steps
        for j in range(num_steps):
            new_points.append((points[i][0] + j * step_size_lat, points[i][
                1] + j * step_size_lng))
        new_points.append(points[i + 1])
    return new_points


def crop_images(region_name, download_crosswalks, download_no_crosswalks):
    deleted_files = 0
    if download_crosswalks:
        for f in os.listdir('{}/crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join('{}/crosswalk'
                    .format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    if download_no_crosswalks:
        for f in os.listdir('{}/no-crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join(
                    '{}/no-crosswalk'.format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    log_and_print('Deleted images: {}'.format(deleted_files))


def check_setup(region_name, download_crosswalk, download_no_crosswalk):
    directories = ['{}']
    if download_crosswalk:
        directories.append('{}/crosswalk')
    if download_no_crosswalk:
        directories.append('{}/no-crosswalk')
    for d in directories:
        _dir = d.format(region_name)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
    log_fname = os.path.join(region_name, 'run.log')
    logging.basicConfig(filename=log_fname, filemode='w', format=
        '[%(asctime)s] %(name)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S',
        level=logging.DEBUG)
    logging.info('----------------------------------------')
    logging.getLogger('requests').setLevel(logging.CRITICAL)
    logging.getLogger('urllib3').setLevel(logging.CRITICAL)


def load_regions(regions_name):
    with open('regions.json') as hf:
        return json.load(hf)[regions_name]


def main(region_name, download_crosswalk, download_no_crosswalk, api_key):
    global key
    key = api_key
    regions = load_regions(region_name)
    download_str = 'crosswalks' if download_crosswalk else 'no-crosswalks'
    if download_crosswalk and download_no_crosswalk:
        download_str = 'crosswalks and no-crosswalks'
    for region_data in regions:
        start = time.time()
        region = region_data['region']
        region_name = region_data['name']
        check_setup(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('Downloading {} from {}: {}'.format(download_str,
            region_name, ','.join(map(str, region))))
        log_and_print('Using key: {}'.format(key))
        if download_crosswalk:
            crosswalks = get_crosswalks(region_data['region'])
            save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))
            log_and_print('Downloading {} crosswalk images'.format(len(
                crosswalks)))
            download_images(crosswalks, 'crosswalk', region_name)
        else:
            log_and_print(
                'Crosswalk images are not gonna be downloaded this time')
        if download_no_crosswalk:
            if not download_crosswalk:
                if not os.path.isfile('{}/crosswalks.json'.format(region_name)
                    ):
                    crosswalks = get_crosswalks(region_data['region'])
                    save_to_json(crosswalks, '{}/crosswalks.json'.format(
                        region_name))
                else:
                    with open('{}/crosswalks.json'.format(region_name)) as hf:
                        crosswalks = json.load(hf)
            if not os.path.isfile('{}/no_crosswalks.json'.format(region_name)):
                (polylines, all_points, nb_duplicates, no_crosswalk_points
                    ) = download_polylines_points(crosswalks, region)
                save_to_json(polylines, '{}/paths.json'.format(region_name))
                save_to_json(no_crosswalk_points, '{}/no_crosswalks.json'.
                    format(region_name))
            else:
                with open('{}/no_crosswalks.json'.format(region_name)) as hf:
                    no_crosswalk_points = json.load(hf)
                with open('{}/paths.json'.format(region_name)) as hf:
                    polylines = json.load(hf)
                nb_duplicates = 'files were loaded'
                all_points = 'files were loaded'
            log_and_print('Downloading {} no-crosswalk images'.format(len(
                no_crosswalk_points)))
            download_images(no_crosswalk_points, 'no-crosswalk', region_name)
        if download_crosswalk or download_no_crosswalk:
            log_and_print('Removing Google from the images')
            crop_images(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('-' * 50)
        log_and_print('SUMMARY - REGION {}'.format(region_name))
        if download_no_crosswalk:
            log_and_print('Number of paths: {}'.format(len(polylines)))
            log_and_print('Removed {} duplicate points from the paths'.
                format(nb_duplicates))
            log_and_print('Number of all points: {}'.format(len(all_points)))
            log_and_print('Number of no-crosswalk points: {}'.format(len(
                no_crosswalk_points)))
        if download_crosswalk:
            log_and_print('Number of crosswalk points: {}'.format(len(
                crosswalks)))
        log_and_print('-' * 50)
        log_and_print('{} finished: {:.2f} seconds'.format(region_name, 
            time.time() - start))


<mask token>

<mask token>


def log_and_print(message):
    print(message)
    logging.info(message)


def get_crosswalks(region):
    base_api_osm = 'https://overpass-api.de/api/'
    node_query = '"highway"~"crossing"'
    params = 'interpreter?data=[out:json];node[{}]({});out body;'.format(
        node_query, ','.join(map(str, region)))
    request_url = base_api_osm + params
    response = requests.get(request_url).json()
    return [{'id': i, 'lat': e['lat'], 'lng': e['lon']} for i, e in
        enumerate(response['elements'])]


def thread_download_static_map(location, type, region_name):
    base_api_static_maps = 'https://maps.googleapis.com/maps/api/staticmap'
    request_url = ('{}?center={}&zoom=20&size={}&maptype=satellite&key={}'.
        format(base_api_static_maps, ','.join(map(str, [location['lat'],
        location['lng']])), img_size, key))
    urllib.request.urlretrieve(request_url, os.path.join(region_name, type,
        '{}.png'.format(location['id'])))


def get_directions_polylines(waypoints):
    orig = waypoints[0]
    dest = waypoints[-1]
    orig = '{},{}'.format(orig['lat'], orig['lng'])
    dest = '{},{}'.format(dest['lat'], dest['lng'])
    waypoints_str = '|'.join(['{},{}'.format(w['lat'], w['lng']) for w in
        waypoints[1:-1]])
    base_api = 'https://maps.googleapis.com/maps/api/directions/json'
    api_params = 'origin={}&destination={}&waypoints={}&key={}'.format(orig,
        dest, waypoints_str, key)
    request_url = '{}?{}'.format(base_api, api_params)
    polyline_data = {'request_url': request_url, 'overview_polyline': ''}
    for nb_attempts in range(max_attempts):
        try:
            response = requests.get(request_url).json()
            polyline_data['overview_polyline'] = response['routes'][0][
                'overview_polyline']['points']
            break
        except:
            log_and_print(
                'Request #{} to Direction API failed. Trying again...'.
                format(nb_attempts))
            time.sleep(1.0 / max_threads)
    return polyline_data


def download_images(crosswalks, img_type, region_name):
    mythreads = []
    total = 0
    for i, crosswalk in enumerate(crosswalks):
        t = threading.Thread(target=thread_download_static_map, args=(
            crosswalk, img_type, region_name))
        mythreads.append(t)
        t.start()
        if len(mythreads) >= max_threads or i == len(crosswalks) - 1:
            for t in mythreads:
                t.join()
            total += len(mythreads)
            mythreads[:] = []
            log_and_print('Downloaded {} images already...'.format(total))
    log_and_print('Done: downloaded {} images...'.format(total))


def download_polylines_points(crosswalks, region):
    min_points = int(nb_of_no_crosswalk_foreach_crosswalk * len(crosswalks))
    all_points = []
    no_crosswalk_points = []
    paths_data = []
    nb_duplicates = 0
    nb_of_paths = int(math.ceil((len(crosswalks) - 1) / float(
        max_nb_of_waypoints + 1)))
    log_and_print('Downloading {} paths from {} crosswalks'.format(
        nb_of_paths, len(crosswalks)))
    for i in range(nb_of_paths):
        path_start = i * (max_nb_of_waypoints + 1)
        path_end = min((i + 1) * (max_nb_of_waypoints + 1), len(crosswalks) - 1
            )
        waypoints = [crosswalks[j] for j in range(path_start, path_end + 1)]
        path_data = get_directions_polylines(waypoints)
        paths_data.append(path_data)
        polylines_points, nb_polylines_duplicates = decode_polylines([
            path_data['overview_polyline']])
        no_crosswalk_polylines_points = remove_close_to_crosswalk(
            polylines_points, crosswalks, region)
        all_points.extend(polylines_points)
        no_crosswalk_points.extend(no_crosswalk_polylines_points)
        nb_duplicates += nb_polylines_duplicates
        log_and_print('Got {} points so far. We need at least {}'.format(
            len(no_crosswalk_points), min_points))
        if len(no_crosswalk_points) > min_points:
            break
    for i in range(len(no_crosswalk_points)):
        no_crosswalk_points[i]['id'] = i
    log_and_print('Download of polylines finished! Nb of points: {}'.format
        (len(no_crosswalk_points)))
    return paths_data, all_points, nb_duplicates, no_crosswalk_points


def save_to_json(content, fname):
    with open(fname, 'w') as hf:
        json.dump(content, hf, indent=4, sort_keys=True)


def decode_polyline(polyline_str):
    """Pass a Google Maps encoded polyline string; returns list of lat/lon pairs"""
    index, lat, lng = 0, 0, 0
    coordinates = []
    changes = {'latitude': 0, 'longitude': 0}
    while index < len(polyline_str):
        for unit in ['latitude', 'longitude']:
            shift, result = 0, 0
            while index < len(polyline_str):
                byte = ord(polyline_str[index]) - 63
                index += 1
                result |= (byte & 31) << shift
                shift += 5
                if not byte >= 32:
                    break
            if result & 1:
                changes[unit] = ~(result >> 1)
            else:
                changes[unit] = result >> 1
        lat += changes['latitude']
        lng += changes['longitude']
        coordinates.append((lat / 100000.0, lng / 100000.0))
    return coordinates


def decode_polylines(polylines):
    all_points = []
    for i, p in enumerate(polylines):
        points_polyline = decode_polyline(p)
        points_polyline = generate_samples_interpoler(points_polyline)
        all_points.extend(points_polyline)
    no_duplicate_points = list(set(all_points))
    return no_duplicate_points, len(all_points) - len(no_duplicate_points)


def remove_close_to_crosswalk(points, crosswalks, region):
    min_dist = 0.0003
    max_dist = 0.0006
    no_crosswalk_points = []
    for i, point in enumerate(points):
        dist_to_crosswalk = get_min_dist_to_crosswalk(point, crosswalks)
        if is_inside_region(point, region, min_dist
            ) and min_dist < dist_to_crosswalk < max_dist:
            no_crosswalk_points.append({'lat': point[0], 'lng': point[1]})
    return no_crosswalk_points


def get_min_dist_to_crosswalk(location, crosswalks):
    min_dist = 360
    for crosswalk in crosswalks:
        dist = distance.euclidean(location, (crosswalk['lat'], crosswalk[
            'lng']))
        if dist < min_dist:
            min_dist = dist
    return min_dist


def is_inside_region(point, region, safe_margin):
    lower_bound = point[0] > region[0] + safe_margin
    upper_bound = point[0] < region[2] - safe_margin
    left_bound = point[1] > region[1] + safe_margin
    right_bound = point[1] < region[3] - safe_margin
    return lower_bound and upper_bound and left_bound and right_bound


def generate_samples_interpoler(points):
    max_sample_step = 0.00015
    new_points = []
    for i in range(len(points) - 1):
        dist = distance.euclidean(points[i], points[i + 1])
        num_steps = int(round(dist / max_sample_step))
        num_steps = 1 if num_steps <= 1 else num_steps
        step_size_lat = (points[i + 1][0] - points[i][0]) / num_steps
        step_size_lng = (points[i + 1][1] - points[i][1]) / num_steps
        for j in range(num_steps):
            new_points.append((points[i][0] + j * step_size_lat, points[i][
                1] + j * step_size_lng))
        new_points.append(points[i + 1])
    return new_points


def crop_images(region_name, download_crosswalks, download_no_crosswalks):
    deleted_files = 0
    if download_crosswalks:
        for f in os.listdir('{}/crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join('{}/crosswalk'
                    .format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    if download_no_crosswalks:
        for f in os.listdir('{}/no-crosswalk'.format(region_name)):
            if f.endswith('.png'):
                full_path_img = os.path.abspath(os.path.join(
                    '{}/no-crosswalk'.format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1
    log_and_print('Deleted images: {}'.format(deleted_files))


def check_setup(region_name, download_crosswalk, download_no_crosswalk):
    directories = ['{}']
    if download_crosswalk:
        directories.append('{}/crosswalk')
    if download_no_crosswalk:
        directories.append('{}/no-crosswalk')
    for d in directories:
        _dir = d.format(region_name)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
    log_fname = os.path.join(region_name, 'run.log')
    logging.basicConfig(filename=log_fname, filemode='w', format=
        '[%(asctime)s] %(name)s - %(message)s', datefmt='%Y-%m-%d %H:%M:%S',
        level=logging.DEBUG)
    logging.info('----------------------------------------')
    logging.getLogger('requests').setLevel(logging.CRITICAL)
    logging.getLogger('urllib3').setLevel(logging.CRITICAL)


def load_regions(regions_name):
    with open('regions.json') as hf:
        return json.load(hf)[regions_name]


def main(region_name, download_crosswalk, download_no_crosswalk, api_key):
    global key
    key = api_key
    regions = load_regions(region_name)
    download_str = 'crosswalks' if download_crosswalk else 'no-crosswalks'
    if download_crosswalk and download_no_crosswalk:
        download_str = 'crosswalks and no-crosswalks'
    for region_data in regions:
        start = time.time()
        region = region_data['region']
        region_name = region_data['name']
        check_setup(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('Downloading {} from {}: {}'.format(download_str,
            region_name, ','.join(map(str, region))))
        log_and_print('Using key: {}'.format(key))
        if download_crosswalk:
            crosswalks = get_crosswalks(region_data['region'])
            save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))
            log_and_print('Downloading {} crosswalk images'.format(len(
                crosswalks)))
            download_images(crosswalks, 'crosswalk', region_name)
        else:
            log_and_print(
                'Crosswalk images are not gonna be downloaded this time')
        if download_no_crosswalk:
            if not download_crosswalk:
                if not os.path.isfile('{}/crosswalks.json'.format(region_name)
                    ):
                    crosswalks = get_crosswalks(region_data['region'])
                    save_to_json(crosswalks, '{}/crosswalks.json'.format(
                        region_name))
                else:
                    with open('{}/crosswalks.json'.format(region_name)) as hf:
                        crosswalks = json.load(hf)
            if not os.path.isfile('{}/no_crosswalks.json'.format(region_name)):
                (polylines, all_points, nb_duplicates, no_crosswalk_points
                    ) = download_polylines_points(crosswalks, region)
                save_to_json(polylines, '{}/paths.json'.format(region_name))
                save_to_json(no_crosswalk_points, '{}/no_crosswalks.json'.
                    format(region_name))
            else:
                with open('{}/no_crosswalks.json'.format(region_name)) as hf:
                    no_crosswalk_points = json.load(hf)
                with open('{}/paths.json'.format(region_name)) as hf:
                    polylines = json.load(hf)
                nb_duplicates = 'files were loaded'
                all_points = 'files were loaded'
            log_and_print('Downloading {} no-crosswalk images'.format(len(
                no_crosswalk_points)))
            download_images(no_crosswalk_points, 'no-crosswalk', region_name)
        if download_crosswalk or download_no_crosswalk:
            log_and_print('Removing Google from the images')
            crop_images(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('-' * 50)
        log_and_print('SUMMARY - REGION {}'.format(region_name))
        if download_no_crosswalk:
            log_and_print('Number of paths: {}'.format(len(polylines)))
            log_and_print('Removed {} duplicate points from the paths'.
                format(nb_duplicates))
            log_and_print('Number of all points: {}'.format(len(all_points)))
            log_and_print('Number of no-crosswalk points: {}'.format(len(
                no_crosswalk_points)))
        if download_crosswalk:
            log_and_print('Number of crosswalk points: {}'.format(len(
                crosswalks)))
        log_and_print('-' * 50)
        log_and_print('{} finished: {:.2f} seconds'.format(region_name, 
            time.time() - start))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=
        'Download crosswalk satellite imagery from a given region.')
    parser.add_argument('--region', type=str, required=True, help=
        'Region name: regions.json')
    parser.add_argument('--positive', dest='positive', action='store_true',
        help='Download positive samples')
    parser.add_argument('--negative', dest='negative', action='store_true',
        help='Download negative samples')
    parser.add_argument('--key', type=str, required=True, help=
        'API_KEY: console.developers.google.com')
    parser.set_defaults(positive=False)
    parser.set_defaults(negative=False)
    argv = parser.parse_args()
    if not (argv.positive or argv.negative):
        print('Nothing to do.')
        sys.exit()
    main(argv.region, argv.positive, argv.negative, argv.key)

import os
import sys
import time
import json
import requests
import argparse
import urllib.request
import threading
import cv2
import math
from scipy.spatial import distance
import logging

key = None
img_size = '200x225'
max_threads = 120
resquest_per_second = 40.0  # limit is 50/sec, up to 2500/day
nb_of_no_crosswalk_foreach_crosswalk = 2.2
get_path_every_n_crosswalks = 1
max_nb_of_waypoints = 20
max_attempts = 5


def log_and_print(message):
    print(message)
    logging.info(message)


def get_crosswalks(region):
    # REQUEST URL
    base_api_osm = 'https://overpass-api.de/api/'
    node_query = '"highway"~"crossing"'
    params = 'interpreter?data=[out:json];node[{}]({});out body;'.format(node_query, ','.join(map(str, region)))
    request_url = base_api_osm + params
    response = requests.get(request_url).json()
    return [{'id': i, 'lat': e['lat'], 'lng': e['lon']} for i, e in enumerate(response['elements'])]


def thread_download_static_map(location, type, region_name):
    base_api_static_maps = 'https://maps.googleapis.com/maps/api/staticmap'
    request_url = '{}?center={}&zoom=20&size={}&maptype=satellite&key={}'.format(
        base_api_static_maps, ','.join(map(str, [location['lat'], location['lng']])), img_size, key
    )
    urllib.request.urlretrieve(request_url, os.path.join(region_name, type, '{}.png'.format(location['id'])))


def get_directions_polylines(waypoints):
    orig = waypoints[0]
    dest = waypoints[-1]
    orig = '{},{}'.format(orig['lat'], orig['lng'])
    dest = '{},{}'.format(dest['lat'], dest['lng'])
    waypoints_str = '|'.join(['{},{}'.format(w['lat'], w['lng']) for w in waypoints[1:-1]])

    base_api = 'https://maps.googleapis.com/maps/api/directions/json'
    api_params = 'origin={}&destination={}&waypoints={}&key={}'.format(orig, dest, waypoints_str, key)
    request_url = '{}?{}'.format(base_api, api_params)

    polyline_data = {"request_url": request_url, "overview_polyline": ""}
    for nb_attempts in range(max_attempts):
        try:
            response = requests.get(request_url).json()
            polyline_data["overview_polyline"] = response['routes'][0]['overview_polyline']['points']
            break
        except:
            log_and_print('Request #{} to Direction API failed. Trying again...'.format(nb_attempts))
            time.sleep(1.0/max_threads)
    return polyline_data


def download_images(crosswalks, img_type, region_name):
    # get the images of img_type = {crosswalks, no-crosswalks}
    mythreads = []
    total = 0
    for i, crosswalk in enumerate(crosswalks):
        t = threading.Thread(target=thread_download_static_map, args=(crosswalk, img_type, region_name))
        mythreads.append(t)
        t.start()
        if (len(mythreads) >= max_threads) or (i == len(crosswalks) - 1):
            for t in mythreads:
                t.join()
            total += len(mythreads)
            mythreads[:] = []  # clear the thread's list
            log_and_print('Downloaded {} images already...'.format(total))
    log_and_print('Done: downloaded {} images...'.format(total))


def download_polylines_points(crosswalks, region):
    # get the path between these crosswalks
    min_points = int(nb_of_no_crosswalk_foreach_crosswalk * len(crosswalks))

    all_points = []
    no_crosswalk_points = []
    paths_data = []
    nb_duplicates = 0

    nb_of_paths = int(math.ceil((len(crosswalks) - 1) / float((max_nb_of_waypoints + 1))))
    log_and_print('Downloading {} paths from {} crosswalks'.format(nb_of_paths, len(crosswalks)))
    for i in range(nb_of_paths):

        path_start = i*(max_nb_of_waypoints+1)
        path_end = min((i+1)*(max_nb_of_waypoints+1), len(crosswalks)-1)
        waypoints = [crosswalks[j] for j in range(path_start, path_end+1)]

        path_data = get_directions_polylines(waypoints)
        paths_data.append(path_data)

        polylines_points, nb_polylines_duplicates = decode_polylines([path_data['overview_polyline']])
        no_crosswalk_polylines_points = remove_close_to_crosswalk(polylines_points, crosswalks, region)

        all_points.extend(polylines_points)
        no_crosswalk_points.extend(no_crosswalk_polylines_points)
        nb_duplicates += nb_polylines_duplicates

        log_and_print('Got {} points so far. We need at least {}'.format(len(no_crosswalk_points), min_points))
        if len(no_crosswalk_points) > min_points:
            break

    # sets an ID for the no-crosswalks
    for i in range(len(no_crosswalk_points)):
        no_crosswalk_points[i]['id'] = i

    log_and_print('Download of polylines finished! Nb of points: {}'.format(len(no_crosswalk_points)))

    return paths_data, all_points, nb_duplicates, no_crosswalk_points


def save_to_json(content, fname):
    with open(fname, 'w') as hf:
        json.dump(content, hf, indent=4, sort_keys=True)


def decode_polyline(polyline_str):
    """Pass a Google Maps encoded polyline string; returns list of lat/lon pairs"""
    index, lat, lng = 0, 0, 0
    coordinates = []
    changes = {'latitude': 0, 'longitude': 0}

    # Coordinates have variable length when encoded, so just keep
    # track of whether we've hit the end of the string. In each
    # while loop iteration, a single coordinate is decoded.
    while index < len(polyline_str):
        # Gather lat/lon changes, store them in a dictionary to apply them later
        for unit in ['latitude', 'longitude']:
            shift, result = 0, 0

            while index < len(polyline_str):
                byte = ord(polyline_str[index]) - 63
                index += 1
                result |= (byte & 0x1f) << shift
                shift += 5
                if not byte >= 0x20:
                    break

            if result & 1:
                changes[unit] = ~(result >> 1)
            else:
                changes[unit] = (result >> 1)

        lat += changes['latitude']
        lng += changes['longitude']

        coordinates.append((lat / 100000.0, lng / 100000.0))

    return coordinates


def decode_polylines(polylines):
    all_points = []
    for i, p in enumerate(polylines):
        points_polyline = decode_polyline(p)
        points_polyline = generate_samples_interpoler(points_polyline)
        all_points.extend(points_polyline)
    no_duplicate_points = list(set(all_points))
    return no_duplicate_points, len(all_points) - len(no_duplicate_points)


def remove_close_to_crosswalk(points, crosswalks, region):
    min_dist = 0.0003  # approx 33 meters
    max_dist = 0.0006  # approx 66 meters
    no_crosswalk_points = []
    for i, point in enumerate(points):
        dist_to_crosswalk = get_min_dist_to_crosswalk(point, crosswalks)
        if is_inside_region(point, region, min_dist) and min_dist < dist_to_crosswalk < max_dist:
            no_crosswalk_points.append({'lat': point[0], 'lng': point[1]})
    return no_crosswalk_points


def get_min_dist_to_crosswalk(location, crosswalks):
    min_dist = 360
    for crosswalk in crosswalks:
        dist = distance.euclidean(location, (crosswalk['lat'], crosswalk['lng']))
        if dist < min_dist:
            min_dist = dist
    return min_dist


def is_inside_region(point, region, safe_margin):
    lower_bound = point[0] > (region[0] + safe_margin)
    upper_bound = point[0] < (region[2] - safe_margin)
    left_bound = point[1] > (region[1] + safe_margin)
    right_bound = point[1] < (region[3] - safe_margin)
    return lower_bound and upper_bound and left_bound and right_bound


def generate_samples_interpoler(points):
    max_sample_step = 0.00015  # approx. 16 meters
    new_points = []
    for i in range(len(points) - 1):
        dist = distance.euclidean(points[i], points[i+1])
        num_steps = int(round(dist / max_sample_step))
        num_steps = 1 if num_steps <= 1 else num_steps

        step_size_lat = (points[i+1][0] - points[i][0]) / num_steps
        step_size_lng = (points[i+1][1] - points[i][1]) / num_steps

        for j in range(num_steps):
            new_points.append((points[i][0] + j*step_size_lat, points[i][1] + j*step_size_lng))
        new_points.append(points[i+1])
    return new_points


def crop_images(region_name, download_crosswalks, download_no_crosswalks):
    deleted_files = 0
    if download_crosswalks:
        for f in os.listdir("{}/crosswalk".format(region_name)):
            if f.endswith(".png"):
                full_path_img = os.path.abspath(os.path.join("{}/crosswalk".format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1

    if download_no_crosswalks:
        for f in os.listdir("{}/no-crosswalk".format(region_name)):
            if f.endswith(".png"):
                full_path_img = os.path.abspath(os.path.join("{}/no-crosswalk".format(region_name), f))
                img = cv2.imread(full_path_img)
                try:
                    if img.shape[0] != img.shape[1]:
                        diff = abs(img.shape[1] - img.shape[0])
                        new_img = img[:-diff, :]
                        cv2.imwrite(full_path_img, new_img)
                except:
                    os.remove(full_path_img)
                    deleted_files += 1

    log_and_print('Deleted images: {}'.format(deleted_files))


def check_setup(region_name, download_crosswalk, download_no_crosswalk):
    directories = ['{}']

    if download_crosswalk:
        directories.append('{}/crosswalk')
    if download_no_crosswalk:
        directories.append('{}/no-crosswalk')

    for d in directories:
        _dir = d.format(region_name)
        if not os.path.exists(_dir):
            os.makedirs(_dir)
    # init logger
    log_fname = os.path.join(region_name, 'run.log')
    logging.basicConfig(filename=log_fname, filemode='w', format='[%(asctime)s] %(name)s - %(message)s',
                        datefmt='%Y-%m-%d %H:%M:%S', level=logging.DEBUG)
    logging.info('----------------------------------------')

    # disable log messages from the requests modules
    logging.getLogger("requests").setLevel(logging.CRITICAL)
    logging.getLogger("urllib3").setLevel(logging.CRITICAL)


def load_regions(regions_name):
    with open('regions.json') as hf:
        return json.load(hf)[regions_name]


def main(region_name, download_crosswalk, download_no_crosswalk, api_key):

    global key
    key = api_key

    regions = load_regions(region_name)

    # build download str
    download_str = "crosswalks" if download_crosswalk else "no-crosswalks"
    if download_crosswalk and download_no_crosswalk:
        download_str = "crosswalks and no-crosswalks"

    for region_data in regions:
        # init
        # =========================================================================================================
        start = time.time()
        region = region_data['region']
        region_name = region_data['name']
        check_setup(region_name, download_crosswalk, download_no_crosswalk)
        log_and_print('Downloading {} from {}: {}'.format(download_str, region_name, ','.join(map(str, region))))
        log_and_print('Using key: {}'.format(key))

        # 1. download crosswalk images
        # =========================================================================================================
        if download_crosswalk:
            # download the crosswalk locations using OpenStreetMap
            crosswalks = get_crosswalks(region_data['region'])
            save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))

            log_and_print('Downloading {} crosswalk images'.format(len(crosswalks)))
            download_images(crosswalks, 'crosswalk', region_name)
        else:
            log_and_print('Crosswalk images are not gonna be downloaded this time')

        # 2. download no-crosswalk images
        # =========================================================================================================
        if download_no_crosswalk:
            # we still need the crosswalk points
            if not download_crosswalk:
                if not os.path.isfile('{}/crosswalks.json'.format(region_name)):
                    crosswalks = get_crosswalks(region_data['region'])
                    save_to_json(crosswalks, '{}/crosswalks.json'.format(region_name))
                else:
                    with open('{}/crosswalks.json'.format(region_name)) as hf:
                        crosswalks = json.load(hf)

            if not os.path.isfile('{}/no_crosswalks.json'.format(region_name)):
                # download the polylines
                polylines, all_points, nb_duplicates, no_crosswalk_points = download_polylines_points(crosswalks, region)
                save_to_json(polylines, '{}/paths.json'.format(region_name))
                save_to_json(no_crosswalk_points, '{}/no_crosswalks.json'.format(region_name))
            else:
                with open('{}/no_crosswalks.json'.format(region_name)) as hf:
                    no_crosswalk_points = json.load(hf)
                with open('{}/paths.json'.format(region_name)) as hf:
                    polylines = json.load(hf)
                nb_duplicates = 'files were loaded'
                all_points = 'files were loaded'

            log_and_print('Downloading {} no-crosswalk images'.format(len(no_crosswalk_points)))
            download_images(no_crosswalk_points, 'no-crosswalk', region_name)

        # if any of the images were downloaded, crop the images
        # =========================================================================================================
        if download_crosswalk or download_no_crosswalk:
            log_and_print('Removing Google from the images')
            crop_images(region_name, download_crosswalk, download_no_crosswalk)

        # summary
        # =========================================================================================================
        log_and_print('-' * 50)
        log_and_print('SUMMARY - REGION {}'.format(region_name))
        if download_no_crosswalk:
            log_and_print('Number of paths: {}'.format(len(polylines)))
            log_and_print('Removed {} duplicate points from the paths'.format(nb_duplicates))
            log_and_print('Number of all points: {}'.format(len(all_points)))
            log_and_print('Number of no-crosswalk points: {}'.format(len(no_crosswalk_points)))
        if download_crosswalk:
            log_and_print('Number of crosswalk points: {}'.format(len(crosswalks)))
        log_and_print('-'*50)
        log_and_print('{} finished: {:.2f} seconds'.format(region_name, (time.time() - start)))


if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Download crosswalk satellite imagery from a given region.')
    parser.add_argument('--region', type=str, required=True, help='Region name: regions.json')
    parser.add_argument('--positive', dest='positive', action='store_true', help='Download positive samples')
    parser.add_argument('--negative', dest='negative', action='store_true', help='Download negative samples')
    parser.add_argument('--key', type=str, required=True, help='API_KEY: console.developers.google.com')
    parser.set_defaults(positive=False)
    parser.set_defaults(negative=False)
    argv = parser.parse_args()

    if not (argv.positive or argv.negative):
        print('Nothing to do.')
        sys.exit()

    main(argv.region, argv.positive, argv.negative, argv.key)
