Merge pull request #153 from developmentseed/develop

version 0.12.0

Author: Scisco

.travis.yml

@@ -32,7 +32,6 @@ deploy:
     repo: developmentseed/landsat-util
     branch:
       - master
-      - develop
 
 after_deploy:
   if [ "$TRAVIS_BRANCH" == "master" ]; then

CHANGES.txt

@@ -1,6 +1,15 @@
 Changes
 =======
 
+0.12.0 (2016-02-18)
+------------------
+- Add USGS download fallback closes #89
+
+0.11.0 (2016-01-12)
+------------------
+- a hotfix for search command not showing outputs #137
+- add support for geojson outputs #68
+
 0.10.0 (2016-01-05)
 ------------------
 - add support for bare json output

Dockerfile

@@ -1,6 +1,6 @@
 FROM    ubuntu:14.04
 RUN     apt-get -y update
-RUN     apt-get install --yes python-pip python-skimage python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev libglib2.0-dev zlib1g-dev python-pycurl
+RUN     apt-get install --yes git-core python-pip python-skimage python-numpy python-scipy libgdal-dev libatlas-base-dev gfortran libfreetype6-dev libglib2.0-dev zlib1g-dev python-pycurl
 ADD     landsat /usr/local/lib/python2.7/dist-packages/landsat
 ADD     bin/landsat /usr/local/bin/
 ADD     . /landsat

docs/commands.rst

@@ -39,6 +39,8 @@ Commands
 
                     --json              Returns a bare JSON response
 
+                    --geojson             Returns a geojson response
+
                     -h, --help          Show this help message and exit
 
             Download:

docs/overview.rst

@@ -36,9 +36,13 @@ Search by latitude and longitude::
 
     $: landsat search --lat 38.9004204 --lon -77.0237117
 
-Search by latitude and longitude with pure json output::
+Search by latitude and longitude with pure json output (you should install geojsonio-cli first)::
 
-    $: landsat search --lat 38.9004204 --lon -77.0237117 --json
+    $: landsat search --lat 38.9004204 --lon -77.0237117 --geojson | geojosnio
+
+Show search output on geojsonio::
+
+    $: landsat search
 
 Download
 ++++++++

landsat/__init__.py

@@ -1 +1 @@
-__version__ = '0.10.0'
+__version__ = '0.12.0'

landsat/downloader.py

@@ -1,9 +1,11 @@
 # Landsat Util
 # License: CC0 1.0 Universal
+from xml.etree import ElementTree
 from os.path import join, exists, getsize
 
-from homura import download as fetch
 import requests
+from usgs import api, USGSError
+from homura import download as fetch
 
 from utils import check_create_folder, url_builder
 from mixins import VerbosityMixin
@@ -20,13 +22,20 @@ class IncorrectSceneId(Exception):
     pass
 
 
+class USGSInventoryAccessMissing(Exception):
+    """ Exception for when User does not have Inventory Service access """
+    pass
+
+
 class Downloader(VerbosityMixin):
     """ The downloader class """
 
-    def __init__(self, verbose=False, download_dir=None):
+    def __init__(self, verbose=False, download_dir=None, usgs_user=None, usgs_pass=None):
         self.download_dir = download_dir if download_dir else settings.DOWNLOAD_DIR
         self.google = settings.GOOGLE_STORAGE
         self.s3 = settings.S3_LANDSAT
+        self.usgs_user = usgs_user
+        self.usgs_pass = usgs_pass
 
         # Make sure download directory exist
         check_create_folder(self.download_dir)
@@ -110,7 +119,22 @@ def google_storage(self, scene, path):
             return self.fetch(url, path, filename)
 
         else:
-            raise RemoteFileDoesntExist('%s is not available on Google Storage' % filename)
+            # download from usgs if login information is provided
+            if self.usgs_user and self.usgs_pass:
+                try:
+                    api_key = api.login(self.usgs_user, self.usgs_pass)
+                except USGSError as e:
+                    error_tree = ElementTree.fromstring(str(e.message))
+                    error_text = error_tree.find("SOAP-ENV:Body/SOAP-ENV:Fault/faultstring", api.NAMESPACES).text
+                    raise USGSInventoryAccessMissing(error_text)
+
+                download_url = api.download('LANDSAT_8', 'EE', [scene], api_key=api_key)
+                if download_url:
+                    return self.fetch(download_url[0], path, filename)
+
+                raise RemoteFileDoesntExist('%s is not available on AWS S3, Google or USGS Earth Explorer' % filename)
+
+            raise RemoteFileDoesntExist('%s is not available on AWS S3 or Google Storage' % filename)
 
     def amazon_s3(self, scene, band, path):
         """

landsat/landsat.py

@@ -3,7 +3,6 @@
 # Landsat Util
 # License: CC0 1.0 Universal
 
-import sys
 import argparse
 import textwrap
 import json
@@ -16,7 +15,7 @@
 import pycurl
 from boto.exception import NoAuthHandlerFound
 
-from downloader import Downloader, IncorrectSceneId
+from downloader import Downloader, IncorrectSceneId, RemoteFileDoesntExist, USGSInventoryAccessMissing
 from search import Search
 from uploader import Uploader
 from utils import reformat_date, convert_to_integer_list, timer, exit, get_file, convert_to_float_list
@@ -62,6 +61,8 @@
 
                 --json              Returns a bare JSON response
 
+                --geojson           Returns a geojson response
+
                 -h, --help          Show this help message and exit
 
         Download:
@@ -108,6 +109,12 @@
 
                 --force-unzip       Force unzip tar file
 
+                --username          USGS Eros account Username (only works if the account has special
+                                    inventory access). Username and password as a fallback if the image
+                                    is not found on AWS S3 or Google Storage
+
+                --password          USGS Eros account Password
+
         Process:
             landsat.py process path [-h] [-b --bands] [-p --pansharpen]
 
@@ -193,6 +200,7 @@ def args_options():
     parser_search.add_argument('--lon', type=float, help='The longitude')
     parser_search.add_argument('--address', type=str, help='The address')
     parser_search.add_argument('--json', action='store_true', help='Returns a bare JSON response')
+    parser_search.add_argument('--geojson', action='store_true', help='Returns a geojson response')
 
     parser_download = subparsers.add_parser('download',
                                             help='Download images from Google Storage')
@@ -218,6 +226,10 @@ def args_options():
                                  '50.2682767372753')
     parser_download.add_argument('-u', '--upload', action='store_true',
                                  help='Upload to S3 after the image processing completed')
+    parser_download.add_argument('--username', help='USGS Eros account Username (only works if the account has' +
+                                 ' special inventory access). Username and password as a fallback if the image' +
+                                 'is not found on AWS S3 or Google Storage')
+    parser_download.add_argument('--password', help='USGS Eros username, used as a fallback')
     parser_download.add_argument('--key', help='Amazon S3 Access Key (You can also be set AWS_ACCESS_KEY_ID as '
                                  'Environment Variables)')
     parser_download.add_argument('--secret', help='Amazon S3 Secret Key (You can also be set AWS_SECRET_ACCESS_KEY '
@@ -328,38 +340,45 @@ def main(args):
                               limit=args.limit,
                               start_date=args.start,
                               end_date=args.end,
-                              cloud_max=args.cloud)
+                              cloud_max=args.cloud,
+                              geojson=args.geojson)
 
-            if result['status'] == 'SUCCESS':
-                if args.json:
-                    return json.dumps(result)
+            if 'status' in result:
 
-                if args.latest > 0:
-                    datelist = []
-                    for i in range(0, result['total_returned']):
-                        datelist.append((result['results'][i]['date'], result['results'][i]))
+                if result['status'] == 'SUCCESS':
+                    if args.json:
+                        return json.dumps(result)
 
-                    datelist.sort(key=lambda tup: tup[0], reverse=True)
-                    datelist = datelist[:args.latest]
+                    if args.latest > 0:
+                        datelist = []
+                        for i in range(0, result['total_returned']):
+                            datelist.append((result['results'][i]['date'], result['results'][i]))
 
-                    result['results'] = []
-                    for i in range(0, len(datelist)):
-                        result['results'].append(datelist[i][1])
-                        result['total_returned'] = len(datelist)
+                        datelist.sort(key=lambda tup: tup[0], reverse=True)
+                        datelist = datelist[:args.latest]
 
-                else:
-                    v.output('%s items were found' % result['total'], normal=True, arrow=True)
+                        result['results'] = []
+                        for i in range(0, len(datelist)):
+                            result['results'].append(datelist[i][1])
+                            result['total_returned'] = len(datelist)
 
-                if result['total'] > 100:
-                    return ['Over 100 results. Please narrow your search', 1]
-                else:
-                    v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
-                return ['Search completed!']
+                    else:
+                        v.output('%s items were found' % result['total'], normal=True, arrow=True)
+
+                    if result['total'] > 100:
+                        return ['Over 100 results. Please narrow your search', 1]
+                    else:
+                        v.output(json.dumps(result, sort_keys=True, indent=4), normal=True, color='green')
+                    return ['Search completed!']
+
+                elif result['status'] == 'error':
+                    return [result['message'], 1]
+
+            if args.geojson:
+                return json.dumps(result)
 
-            elif result['status'] == 'error':
-                return [result['message'], 1]
         elif args.subs == 'download':
-            d = Downloader(download_dir=args.dest)
+            d = Downloader(download_dir=args.dest, usgs_user=args.username, usgs_pass=args.password)
             try:
                 bands = convert_to_integer_list(args.bands)
 
@@ -406,6 +425,8 @@ def main(args):
                     return ['Download Completed', 0]
             except IncorrectSceneId:
                 return ['The SceneID provided was incorrect', 1]
+            except (RemoteFileDoesntExist, USGSInventoryAccessMissing) as e:
+                return [e.message, 1]
 
 
 def process_image(path, bands=None, verbose=False, pansharpen=False, ndvi=False, force_unzip=None,
@@ -459,10 +480,8 @@ def __main__():
     global parser
     parser = args_options()
     args = parser.parse_args()
-    if args.subs == 'search':
-        if args.json:
-            print main(args)
-            sys.exit(0)
+    if args.subs == 'search' and (hasattr(args, 'json') or hasattr(args, 'geojson')):
+            print(main(args))
     else:
         with timer():
             exit(*main(args))

landsat/search.py

@@ -16,7 +16,7 @@ def __init__(self):
         self.api_url = settings.API_URL
 
     def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=None, end_date=None, cloud_min=None,
-               cloud_max=None, limit=1):
+               cloud_max=None, limit=1, geojson=False):
         """
         The main method of Search class. It searches Development Seed's Landsat API.
 
@@ -56,6 +56,10 @@ def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=N
             integer specigying the maximum results return.
         :type limit:
             integer
+        :param geojson:
+            boolean specifying whether to return a geojson object
+        :type geojson:
+            boolean
 
         :returns:
             dict
@@ -98,18 +102,51 @@ def search(self, paths_rows=None, lat=None, lon=None, address=None, start_date=N
             result['message'] = r_dict['error']['message']
 
         elif 'meta' in r_dict:
-            result['status'] = u'SUCCESS'
-            result['total'] = r_dict['meta']['results']['total']
-            result['limit'] = r_dict['meta']['results']['limit']
-            result['total_returned'] = len(r_dict['results'])
-            result['results'] = [{'sceneID': i['sceneID'],
-                                  'sat_type': u'L8',
-                                  'path': three_digit(i['path']),
-                                  'row': three_digit(i['row']),
-                                  'thumbnail': i['browseURL'],
-                                  'date': i['acquisitionDate'],
-                                  'cloud': i['cloudCoverFull']}
-                                 for i in r_dict['results']]
+            if geojson:
+                result = {
+                    'type': 'FeatureCollection',
+                    'features': []
+                }
+                for r in r_dict['results']:
+                    feature = {
+                        'type': 'Feature',
+                        'properties': {
+                            'sceneID': r['sceneID'],
+                            'row': three_digit(r['row']),
+                            'path': three_digit(r['path']),
+                            'thumbnail': r['browseURL'],
+                            'date': r['acquisitionDate'],
+                            'cloud': r['cloudCoverFull']
+                        },
+                        'geometry': {
+                            'type': 'Polygon',
+                            'coordinates': [
+                                [
+                                    [r['upperLeftCornerLongitude'], r['upperLeftCornerLatitude']],
+                                    [r['lowerLeftCornerLongitude'], r['lowerLeftCornerLatitude']],
+                                    [r['lowerRightCornerLongitude'], r['lowerRightCornerLatitude']],
+                                    [r['upperRightCornerLongitude'], r['upperRightCornerLatitude']],
+                                    [r['upperLeftCornerLongitude'], r['upperLeftCornerLatitude']]
+                                ]
+                            ]
+                        }
+                    }
+
+                    result['features'].append(feature)
+
+            else:
+                result['status'] = u'SUCCESS'
+                result['total'] = r_dict['meta']['results']['total']
+                result['limit'] = r_dict['meta']['results']['limit']
+                result['total_returned'] = len(r_dict['results'])
+                result['results'] = [{'sceneID': i['sceneID'],
+                                      'sat_type': u'L8',
+                                      'path': three_digit(i['path']),
+                                      'row': three_digit(i['row']),
+                                      'thumbnail': i['browseURL'],
+                                      'date': i['acquisitionDate'],
+                                      'cloud': i['cloudCoverFull']}
+                                     for i in r_dict['results']]
 
         return result
 

requirements/docker.txt

@@ -2,8 +2,10 @@ requests==2.7.0
 python-dateutil>=2.4.2
 termcolor>=1.1.0
 rasterio>=0.27.0
-six==1.9.0
+six==1.8.0
 homura>=0.1.2
 boto>=2.38.0
 polyline==1.1
 geocoder>=1.5.1
+jsonschema==2.5.1
+git+git://github.com/developmentseed/usgs@develop

setup.py

@@ -35,19 +35,23 @@ def readme():
     license='CCO',
     platforms='Posix; MacOS X; Windows',
     install_requires=[
+        'usgs==0.1.9',
         'requests==2.7.0',
         'python-dateutil>=2.4.2',
         'numpy>=1.9.3',
         'termcolor>=1.1.0',
         'rasterio>=0.26.0',
-        'six==1.9.0',
+        'six==1.8.0',
         'scipy>=0.16.0',
         'scikit-image>=0.11.3',
         'homura>=0.1.2',
         'boto>=2.38.0',
         'polyline==1.1',
         'geocoder>=1.5.1'
     ],
+    dependency_links=[
+        "git+https://github.com/developmentseed/usgs@develop#egg=usgs-0.1.9"
+    ],
     test_suite='nose.collector',
     tests_require=test_requirements
 )