diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a468cf..622f370 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,7 +2,7 @@ version: 2 jobs: build: docker: - - image: circleci/python:3.5 + - image: circleci/python:3.6-stretch working_directory: ~/label-maker steps: diff --git a/CHANGES.txt b/CHANGES.txt index 98ebe13..38ab3e9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,8 @@ -0.6.0 (2018-11-06) +0.6.1 (2019-11-11) +------------------ +- Added ability to use HTTP Authentication for TMS endpoints (#152) + +0.6.0 (2019-11-06) ------------------ - Use sys.exectuable in place of python string (#124) - Correct script reference to fix bug in skynet train example (#129) @@ -6,8 +10,6 @@ - users can split data into more groups than train and test, for example train/test/validate, and specify the ratio for each split (#149) - - 0.5.1 (2018-11-12) ------------------ - Skip invalid or empty geometries which prevent segmentation rendering (#118) diff --git a/docs/parameters.rst b/docs/parameters.rst index ea987cf..904dd29 100644 --- a/docs/parameters.rst +++ b/docs/parameters.rst @@ -34,6 +34,9 @@ Here is the full list of configuration parameters you can specify in a ``config. Remote files like a `WMS endpoint `_ ``GetMap`` request. Fill out all necessary parameters except ``bbox`` which should be set as ``{bbox}``. Ex: ``'https://basemap.nationalmap.gov/arcgis/services/USGSImageryOnly/MapServer/WMSServer?SERVICE=WMS&REQUEST=GetMap&VERSION=1.1.1&LAYERS=0&STYLES=&FORMAT=image%2Fjpeg&TRANSPARENT=false&HEIGHT=256&WIDTH=256&SRS=EPSG%3A3857&BBOX={bbox}'`` +**http_auth**: list + Optional parameter to specify a username and password for restricted WMS services. For example, ``['my_username', 'my_password']``. + **background_ratio**: float Specify how many background (or "negative") training examples to create when there is only one class specified with the ``classes`` parameter. Label Maker will generate ``background_ratio`` times the number of images matching the one class. diff --git a/label_maker/images.py b/label_maker/images.py index 354bcc9..ebf0975 100644 --- a/label_maker/images.py +++ b/label_maker/images.py @@ -72,4 +72,4 @@ def class_test(value): image_function = get_image_function(imagery) for tile in tiles: - image_function(tile, imagery, tiles_dir, imagery_offset) + image_function(tile, imagery, tiles_dir, imagery_offset, kwargs) diff --git a/label_maker/main.py b/label_maker/main.py index 79fd11b..45854b3 100644 --- a/label_maker/main.py +++ b/label_maker/main.py @@ -93,6 +93,10 @@ def cli(): config['country'] = op.splitext(op.basename(config.get('geojson')))[0] config['bounding_box'] = get_bounds(json.load(open(config.get('geojson'), 'r'))) + # Convert HTTP auth from list to tuple if it exists + if 'http_auth' in config.keys(): + config['http_auth'] = tuple(config['http_auth']) + if cmd == 'download': download_mbtiles(dest_folder=dest_folder, **config) elif cmd == 'labels': diff --git a/label_maker/package.py b/label_maker/package.py index 480b4a8..578ed7b 100644 --- a/label_maker/package.py +++ b/label_maker/package.py @@ -9,8 +9,9 @@ from label_maker.utils import is_tif -def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_names=['train', 'test'], - split_vals=[0.8, .2], **kwargs): +def package_directory(dest_folder, classes, imagery, ml_type, seed=False, + split_names=('train', 'test'), split_vals=(0.8, .2), + **kwargs): """Generate an .npz file containing arrays for training machine learning algorithms Parameters @@ -18,8 +19,8 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_ dest_folder: str Folder to save labels, tiles, and final numpy arrays into classes: list - A list of classes for machine learning training. Each class is defined as a dict - with two required properties: + A list of classes for machine learning training. Each class is defined + as a dict with two required properties: - name: class name - filter: A Mapbox GL Filter. See the README for more details @@ -27,25 +28,27 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_ Imagery template to download satellite images from. Ex: http://a.tiles.mapbox.com/v4/mapbox.satellite/{z}/{x}/{y}.jpg?access_token=ACCESS_TOKEN ml_type: str - Defines the type of machine learning. One of "classification", "object-detection", or "segmentation" + Defines the type of machine learning. One of "classification", + "object-detection", or "segmentation" seed: int Random generator seed. Optional, use to make results reproducible. - split_vals: list - Default: [0.8, 0.2] - Percentage of data to put in each catagory listed in split_names. - Must be floats and must sum to one. - split_names: list - Default: ['train', 'test'] + split_vals: tuple + Percentage of data to put in each catagory listed in split_names. Must + be floats and must sum to one. Default: (0.8, 0.2) + split_names: tupel + Default: ('train', 'test') List of names for each subset of the data. **kwargs: dict - Other properties from CLI config passed as keywords to other utility functions + Other properties from CLI config passed as keywords to other utility + functions. """ # if a seed is given, use it if seed: np.random.seed(seed) if len(split_names) != len(split_vals): - raise ValueError('`split_names` and `split_vals` must be the same length. Please update your config.') + raise ValueError('`split_names` and `split_vals` must be the same ' + 'length. Please update your config.') if not np.isclose(sum(split_vals), 1): raise ValueError('`split_vals` must sum to one. Please update your config.') @@ -105,7 +108,8 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_ split_n_samps = [len(x_vals) * val for val in split_vals] if np.any(split_n_samps == 0): - raise ValueError('split must not generate zero samples per partition, change ratio of values in config file.') + raise ValueError('Split must not generate zero samples per partition. ' + 'Change ratio of values in config file.') # Convert into a cumulative sum to get indices split_inds = np.cumsum(split_n_samps).astype(np.integer) @@ -117,8 +121,8 @@ def package_directory(dest_folder, classes, imagery, ml_type, seed=False, split_ save_dict = {} for si, split_name in enumerate(split_names): - save_dict[f'x_{split_name}'] = split_arrs_x[si] - save_dict[f'y_{split_name}'] = split_arrs_y[si] + save_dict['x_{}'.format(split_name)] = split_arrs_x[si] + save_dict['y_{}'.format(split_name)] = split_arrs_y[si] np.savez(op.join(dest_folder, 'data.npz'), **save_dict) - print('Saving packaged file to {}'.format(op.join(dest_folder, 'data.npz'))) \ No newline at end of file + print('Saving packaged file to {}'.format(op.join(dest_folder, 'data.npz'))) diff --git a/label_maker/preview.py b/label_maker/preview.py index 92de100..131b0a2 100644 --- a/label_maker/preview.py +++ b/label_maker/preview.py @@ -64,7 +64,8 @@ def preview(dest_folder, number, classes, imagery, ml_type, imagery_offset=False if n >= number: break - tile_img = image_function(tile, imagery, class_dir, imagery_offset) + tile_img = image_function(tile, imagery, class_dir, imagery_offset, + kwargs) if ml_type == 'object-detection': img = Image.open(tile_img) diff --git a/label_maker/utils.py b/label_maker/utils.py index bdc82ef..cce8ca0 100644 --- a/label_maker/utils.py +++ b/label_maker/utils.py @@ -24,17 +24,18 @@ def class_match(ml_type, label, i): return np.count_nonzero(label == i) return None -def download_tile_tms(tile, imagery, folder, *args): +def download_tile_tms(tile, imagery, folder, kwargs): """Download a satellite image tile from a tms endpoint""" o = urlparse(imagery) _, image_format = op.splitext(o.path) - r = requests.get(url(tile.split('-'), imagery)) + r = requests.get(url(tile.split('-'), imagery), + auth=kwargs.get('http_auth')) tile_img = op.join(folder, '{}{}'.format(tile, image_format)) with open(tile_img, 'wb')as w: w.write(r.content) return tile_img -def get_tile_tif(tile, imagery, folder, imagery_offset): +def get_tile_tif(tile, imagery, folder, imagery_offset, kwargs): """ Read a GeoTIFF with a window corresponding to a TMS tile @@ -87,7 +88,7 @@ def get_tile_tif(tile, imagery, folder, imagery_offset): return tile_img -def get_tile_wms(tile, imagery, folder, imagery_offset): +def get_tile_wms(tile, imagery, folder, imagery_offset, kwargs): """ Read a WMS endpoint with query parameters corresponding to a TMS tile @@ -118,7 +119,7 @@ def get_tile_wms(tile, imagery, folder, imagery_offset): # request the image with the transformed bounding box and save wms_url = imagery.replace('{bbox}', ','.join([str(b) for b in bbox])) - r = requests.get(wms_url) + r = requests.get(wms_url, auth=kwargs.get('http_auth')) tile_img = op.join(folder, '{}.{}'.format(tile, image_format)) with open(tile_img, 'wb') as w: w.write(r.content) diff --git a/label_maker/validate.py b/label_maker/validate.py index 13c34fb..fd9c574 100644 --- a/label_maker/validate.py +++ b/label_maker/validate.py @@ -27,6 +27,7 @@ 'zoom': {'type': 'integer', 'required': True}, 'classes': {'type': 'list', 'schema': class_schema, 'required': True}, 'imagery': {'type': 'string', 'required': True}, + 'http_auth': {'type': 'list', 'schema': {'type': 'string'}}, 'background_ratio': {'type': 'float'}, 'ml_type': {'allowed': ['classification', 'object-detection', 'segmentation'], 'required': True}, 'seed': {'type': 'integer'}, diff --git a/label_maker/version.py b/label_maker/version.py index 2fa3502..fa5fd07 100644 --- a/label_maker/version.py +++ b/label_maker/version.py @@ -1,2 +1,2 @@ """Library verison""" -__version__ = '0.6.0' +__version__ = '0.6.1' diff --git a/test/fixtures/146-195-9.jpeg b/test/fixtures/146-195-9.jpeg index dae1188..3130abb 100644 Binary files a/test/fixtures/146-195-9.jpeg and b/test/fixtures/146-195-9.jpeg differ diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index dd8bd6f..62eb089 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -81,7 +81,7 @@ def test_get_tile_tif(self): if not op.isdir(tiles_dir): makedirs(tiles_dir) - get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, None) + get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, None, {}) test_tile = Image.open('test/tiles/{}.jpg'.format(tile)) fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile)) self.assertEqual(test_tile, fixture_tile) @@ -95,7 +95,7 @@ def test_get_tile_tif_offset(self): if not op.isdir(tiles_dir): makedirs(tiles_dir) - get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, [128, 64]) + get_tile_tif(tile, 'test/fixtures/drone.tif', tiles_dir, [128, 64], {}) test_tile = Image.open('test/tiles/{}.jpg'.format(tile)) fixture_tile = Image.open('test/fixtures/{}_offset.jpg'.format(tile)) self.assertEqual(test_tile, fixture_tile) @@ -109,7 +109,7 @@ def test_get_tile_vrt(self): if not op.isdir(tiles_dir): makedirs(tiles_dir) - get_tile_tif(tile, 'test/fixtures/drone.vrt', tiles_dir, None) + get_tile_tif(tile, 'test/fixtures/drone.vrt', tiles_dir, None, {}) test_tile = Image.open('test/tiles/{}.jpg'.format(tile)) fixture_tile = Image.open('test/fixtures/{}.jpg'.format(tile)) self.assertEqual(test_tile, fixture_tile) @@ -125,7 +125,7 @@ def test_get_tile_wms(self): nasa_url = 'https://gibs.earthdata.nasa.gov/wms/epsg4326/best/wms.cgi?SERVICE=WMS&REQUEST=GetMap&layers=MODIS_Aqua_CorrectedReflectance_TrueColor&version=1.3.0&crs=EPSG:4326&transparent=false&width=256&height=256&bbox={bbox}&format=image/jpeg&time=2019-03-05' - get_tile_wms(tile, nasa_url, tiles_dir, None) + get_tile_wms(tile, nasa_url, tiles_dir, None, {}) test_tile = Image.open('test/tiles/{}.jpeg'.format(tile)) fixture_tile = Image.open('test/fixtures/{}.jpeg'.format(tile)) self.assertEqual(test_tile, fixture_tile)