Merge pull request #153 from developmentseed/enh/152-auth

HTTP authentication

Author: wronk

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: circleci/python:3.5
+      - image: circleci/python:3.6-stretch
 
     working_directory: ~/label-maker
     steps:

CHANGES.txt

@@ -1,13 +1,15 @@
-0.6.0 (2018-11-06)
+0.6.1 (2019-11-11)
+------------------
+- Added ability to use HTTP Authentication for TMS endpoints (#152)
+
+0.6.0 (2019-11-06)
 ------------------
 - Use sys.exectuable in place of python string (#124)
 - Correct script reference to fix bug in skynet train example (#129)
 - Add s3 requirement to rasterio (#137)
 - users can split data into more groups than train and test, for example train/test/validate, and specify the ratio for
 each split (#149)
 
-
-
 0.5.1 (2018-11-12)
 ------------------
 - Skip invalid or empty geometries which prevent segmentation rendering (#118)

docs/parameters.rst

@@ -34,6 +34,9 @@ Here is the full list of configuration parameters you can specify in a ``config.
 
  		Remote files like a `WMS endpoint <http://www.opengeospatial.org/standards/wms>`_ ``GetMap`` request. Fill out all necessary parameters except ``bbox`` which should be set as ``{bbox}``. Ex: ``'https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?SERVICE=WMS&REQUEST=GetMap&VERSION=1.1.1&LAYERS=0&STYLES=&FORMAT=image%2Fjpeg&TRANSPARENT=false&HEIGHT=256&WIDTH=256&SRS=EPSG%3A3857&BBOX={bbox}'``
 
+**http_auth**: list
+	Optional parameter to specify a username and password for restricted WMS services. For example, ``['my_username', 'my_password']``.
+
 **background_ratio**: float
 	Specify how many background (or "negative") training examples to create when there is only one class specified with the ``classes`` parameter. Label Maker will generate ``background_ratio`` times the number of images matching the one class.
 

label_maker/images.py

@@ -72,4 +72,4 @@ def class_test(value):
     image_function = get_image_function(imagery)
 
     for tile in tiles:
-        image_function(tile, imagery, tiles_dir, imagery_offset)
+        image_function(tile, imagery, tiles_dir, imagery_offset, kwargs)

label_maker/main.py

@@ -93,6 +93,10 @@ def cli():
         config['country'] = op.splitext(op.basename(config.get('geojson')))[0]
         config['bounding_box'] = get_bounds(json.load(open(config.get('geojson'), 'r')))
 
+    # Convert HTTP auth from list to tuple if it exists
+    if 'http_auth' in config.keys():
+        config['http_auth'] = tuple(config['http_auth'])
+
     if cmd == 'download':
         download_mbtiles(dest_folder=dest_folder, **config)
     elif cmd == 'labels':

label_maker/package.py

@@ -9,43 +9,46 @@
 from label_maker.utils import is_tif
 
 
-def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_names=['train', 'test'],
-                      split_vals=[0.8, .2], **kwargs):
+def package_directory(dest_folder, classes, imagery, ml_type, seed=False,
+                      split_names=('train', 'test'), split_vals=(0.8, .2),
+                      **kwargs):
     """Generate an .npz file containing arrays for training machine learning algorithms
 
     Parameters
     ------------
     dest_folder: str
         Folder to save labels, tiles, and final numpy arrays into
     classes: list
-        A list of classes for machine learning training. Each class is defined as a dict
-        with two required properties:
+        A list of classes for machine learning training. Each class is defined
+        as a dict with two required properties:
           - name: class name
           - filter: A Mapbox GL Filter.
         See the README for more details
     imagery: str
         Imagery template to download satellite images from.
         Ex: http://a.tiles.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token=ACCESS_TOKEN
     ml_type: str
-        Defines the type of machine learning. One of "classification", "object-detection", or "segmentation"
+        Defines the type of machine learning. One of "classification",
+        "object-detection", or "segmentation"
     seed: int
         Random generator seed. Optional, use to make results reproducible.
-    split_vals: list
-        Default: [0.8, 0.2]
-        Percentage of data to put in each catagory listed in split_names.
-        Must be floats and must sum to one.
-    split_names: list
-        Default: ['train', 'test']
+    split_vals: tuple
+        Percentage of data to put in each catagory listed in split_names. Must
+        be floats and must sum to one. Default: (0.8, 0.2)
+    split_names: tupel
+        Default: ('train', 'test')
         List of names for each subset of the data.
     **kwargs: dict
-        Other properties from CLI config passed as keywords to other utility functions
+        Other properties from CLI config passed as keywords to other utility
+        functions.
     """
     # if a seed is given, use it
     if seed:
         np.random.seed(seed)
 
     if len(split_names) != len(split_vals):
-        raise ValueError('`split_names` and `split_vals` must be the same length. Please update your config.')
+        raise ValueError('`split_names` and `split_vals` must be the same '
+                         'length. Please update your config.')
     if not np.isclose(sum(split_vals), 1):
         raise ValueError('`split_vals` must sum to one. Please update your config.')
 
@@ -105,7 +108,8 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_
     split_n_samps = [len(x_vals) * val for val in split_vals]
 
     if np.any(split_n_samps == 0):
-        raise ValueError('split must not generate zero samples per partition, change ratio of values in config file.')
+        raise ValueError('Split must not generate zero samples per partition. '
+                         'Change ratio of values in config file.')
 
     # Convert into a cumulative sum to get indices
     split_inds = np.cumsum(split_n_samps).astype(np.integer)
@@ -117,8 +121,8 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_
     save_dict = {}
 
     for si, split_name in enumerate(split_names):
-        save_dict[f'x_{split_name}'] = split_arrs_x[si]
-        save_dict[f'y_{split_name}'] = split_arrs_y[si]
+        save_dict['x_{}'.format(split_name)] = split_arrs_x[si]
+        save_dict['y_{}'.format(split_name)] = split_arrs_y[si]
 
     np.savez(op.join(dest_folder, 'data.npz'), **save_dict)
-    print('Saving packaged file to {}'.format(op.join(dest_folder, 'data.npz')))
+    print('Saving packaged file to {}'.format(op.join(dest_folder, 'data.npz')))

label_maker/preview.py

@@ -64,7 +64,8 @@ def preview(dest_folder, number, classes, imagery, ml_type, imagery_offset=False
             if n >= number:
                 break
 
-            tile_img = image_function(tile, imagery, class_dir, imagery_offset)
+            tile_img = image_function(tile, imagery, class_dir, imagery_offset,
+                                      kwargs)
 
             if ml_type == 'object-detection':
                 img = Image.open(tile_img)

label_maker/utils.py

@@ -24,17 +24,18 @@ def class_match(ml_type, label, i):
         return np.count_nonzero(label == i)
     return None
 
-def download_tile_tms(tile, imagery, folder, *args):
+def download_tile_tms(tile, imagery, folder, kwargs):
     """Download a satellite image tile from a tms endpoint"""
     o = urlparse(imagery)
     _, image_format = op.splitext(o.path)
-    r = requests.get(url(tile.split('-'), imagery))
+    r = requests.get(url(tile.split('-'), imagery),
+                     auth=kwargs.get('http_auth'))
     tile_img = op.join(folder, '{}{}'.format(tile, image_format))
     with open(tile_img, 'wb')as w:
         w.write(r.content)
     return tile_img
 
-def get_tile_tif(tile, imagery, folder, imagery_offset):
+def get_tile_tif(tile, imagery, folder, imagery_offset, kwargs):
     """
     Read a GeoTIFF with a window corresponding to a TMS tile
 
@@ -87,7 +88,7 @@ def get_tile_tif(tile, imagery, folder, imagery_offset):
 
     return tile_img
 
-def get_tile_wms(tile, imagery, folder, imagery_offset):
+def get_tile_wms(tile, imagery, folder, imagery_offset, kwargs):
     """
     Read a WMS endpoint with query parameters corresponding to a TMS tile
 
@@ -118,7 +119,7 @@ def get_tile_wms(tile, imagery, folder, imagery_offset):
 
     # request the image with the transformed bounding box and save
     wms_url = imagery.replace('{bbox}', ','.join([str(b) for b in bbox]))
-    r = requests.get(wms_url)
+    r = requests.get(wms_url, auth=kwargs.get('http_auth'))
     tile_img = op.join(folder, '{}.{}'.format(tile, image_format))
     with open(tile_img, 'wb') as w:
         w.write(r.content)

label_maker/validate.py

@@ -27,6 +27,7 @@
     'zoom': {'type': 'integer', 'required': True},
     'classes': {'type': 'list', 'schema': class_schema, 'required': True},
     'imagery': {'type': 'string', 'required': True},
+    'http_auth': {'type': 'list', 'schema': {'type': 'string'}},
     'background_ratio': {'type': 'float'},
     'ml_type': {'allowed': ['classification', 'object-detection', 'segmentation'], 'required': True},
     'seed': {'type': 'integer'},

label_maker/version.py

@@ -1,2 +1,2 @@
 """Library verison"""
-__version__ = '0.6.0'
+__version__ = '0.6.1'

test/fixtures/146-195-9.jpeg

@@ -81,7 +81,7 @@ def test_get_tile_tif(self):
         if not op.isdir(tiles_dir):
             makedirs(tiles_dir)
 
-        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, None)
+        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, None, {})
         test_tile = Image.open('test/tiles/{}.jpg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
@@ -95,7 +95,7 @@ def test_get_tile_tif_offset(self):
         if not op.isdir(tiles_dir):
             makedirs(tiles_dir)
 
-        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, [128, 64])
+        get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, [128, 64], {})
         test_tile = Image.open('test/tiles/{}.jpg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}_offset.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
@@ -109,7 +109,7 @@ def test_get_tile_vrt(self):
         if not op.isdir(tiles_dir):
             makedirs(tiles_dir)
 
-        get_tile_tif(tile, 'test/fixtures/drone.vrt', tiles_dir, None)
+        get_tile_tif(tile, 'test/fixtures/drone.vrt', tiles_dir, None, {})
         test_tile = Image.open('test/tiles/{}.jpg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)
@@ -125,7 +125,7 @@ def test_get_tile_wms(self):
 
         nasa_url = 'https://gibs.earthdata.nasa.gov/wms/epsg4326/best/wms.cgi?SERVICE=WMS&REQUEST=GetMap&layers=MODIS_Aqua_CorrectedReflectance_TrueColor&version=1.3.0&crs=EPSG:4326&transparent=false&width=256&height=256&bbox={bbox}&format=image/jpeg&time=2019-03-05'
 
-        get_tile_wms(tile, nasa_url, tiles_dir, None)
+        get_tile_wms(tile, nasa_url, tiles_dir, None, {})
         test_tile = Image.open('test/tiles/{}.jpeg'.format(tile))
         fixture_tile = Image.open('test/fixtures/{}.jpeg'.format(tile))
         self.assertEqual(test_tile, fixture_tile)