diff --git a/notebooks/.ipynb_checkpoints/4a-download-large-area-checkpoint.ipynb b/notebooks/.ipynb_checkpoints/4a-download-large-area-checkpoint.ipynb index fa2789a..8435ecf 100644 --- a/notebooks/.ipynb_checkpoints/4a-download-large-area-checkpoint.ipynb +++ b/notebooks/.ipynb_checkpoints/4a-download-large-area-checkpoint.ipynb @@ -42,10 +42,10 @@ "source": [ "# If using example raw data\n", "import os\n", - "if not os.path.exists(\"../data/raw/\"):\n", - " os.makedirs(\"../data/raw/\")\n", + "if not os.path.exists(\"../data/example/raw\"):\n", + " os.makedirs(\"../data/example/raw/\")\n", " \n", - "landscape = 'example-raw'\n", + "landscape = 'example'\n", "OUTPUT_FOLDER = '../data/{}/'.format(landscape)\n", "coords = (13.727334, -90.015579)\n", "coords = (coords[1], coords[0])" @@ -58,9 +58,9 @@ "outputs": [], "source": [ "# Download example raw data - only if you don't have an API key!\n", - "!curl https://restoration-monitoring-external.s3.amazonaws.com/restoration-mapper/example/example-raw.zip \\\n", - " -o ../data/data.zip\n", - "!unzip ../data/data.zip -d ../data/" + "!curl https://restoration-monitoring-external.s3.amazonaws.com/restoration-mapper/example/example.zip \\\n", + " -o ../data/example/raw/data.zip\n", + "!unzip ../data/example/raw/data.zip -d ../data/example/raw/" ] }, { @@ -117,11 +117,14 @@ "metadata": {}, "outputs": [], "source": [ - "with open(\"../config.yaml\", 'r') as stream:\n", - " key = (yaml.safe_load(stream))\n", - " API_KEY = key['key']\n", - " AWSKEY = key['awskey']\n", - " AWSSECRET = key['awssecret']" + "if os.path.exists(\"../config.yaml\"):\n", + " with open(\"../config.yaml\", 'r') as stream:\n", + " key = (yaml.safe_load(stream))\n", + " API_KEY = key['key']\n", + " AWSKEY = key['awskey']\n", + " AWSSECRET = key['awssecret']\n", + "else:\n", + " API_KEY = \"none\"" ] }, { @@ -272,7 +275,7 @@ "}\n", "\n", "landscape = 'brazil-buffer'\n", - "OUTPUT_FOLDER = '../tile_data/{}/'.format(landscape)\n", + "OUTPUT_FOLDER = '../data/{}/'.format(landscape)\n", "coords = landscapes[landscape]\n", "coords = (coords[1], coords[0])\n", "print(OUTPUT_FOLDER, coords)" diff --git a/notebooks/4a-download-large-area.ipynb b/notebooks/4a-download-large-area.ipynb index fa2789a..8435ecf 100644 --- a/notebooks/4a-download-large-area.ipynb +++ b/notebooks/4a-download-large-area.ipynb @@ -42,10 +42,10 @@ "source": [ "# If using example raw data\n", "import os\n", - "if not os.path.exists(\"../data/raw/\"):\n", - " os.makedirs(\"../data/raw/\")\n", + "if not os.path.exists(\"../data/example/raw\"):\n", + " os.makedirs(\"../data/example/raw/\")\n", " \n", - "landscape = 'example-raw'\n", + "landscape = 'example'\n", "OUTPUT_FOLDER = '../data/{}/'.format(landscape)\n", "coords = (13.727334, -90.015579)\n", "coords = (coords[1], coords[0])" @@ -58,9 +58,9 @@ "outputs": [], "source": [ "# Download example raw data - only if you don't have an API key!\n", - "!curl https://restoration-monitoring-external.s3.amazonaws.com/restoration-mapper/example/example-raw.zip \\\n", - " -o ../data/data.zip\n", - "!unzip ../data/data.zip -d ../data/" + "!curl https://restoration-monitoring-external.s3.amazonaws.com/restoration-mapper/example/example.zip \\\n", + " -o ../data/example/raw/data.zip\n", + "!unzip ../data/example/raw/data.zip -d ../data/example/raw/" ] }, { @@ -117,11 +117,14 @@ "metadata": {}, "outputs": [], "source": [ - "with open(\"../config.yaml\", 'r') as stream:\n", - " key = (yaml.safe_load(stream))\n", - " API_KEY = key['key']\n", - " AWSKEY = key['awskey']\n", - " AWSSECRET = key['awssecret']" + "if os.path.exists(\"../config.yaml\"):\n", + " with open(\"../config.yaml\", 'r') as stream:\n", + " key = (yaml.safe_load(stream))\n", + " API_KEY = key['key']\n", + " AWSKEY = key['awskey']\n", + " AWSSECRET = key['awssecret']\n", + "else:\n", + " API_KEY = \"none\"" ] }, { @@ -272,7 +275,7 @@ "}\n", "\n", "landscape = 'brazil-buffer'\n", - "OUTPUT_FOLDER = '../tile_data/{}/'.format(landscape)\n", + "OUTPUT_FOLDER = '../data/{}/'.format(landscape)\n", "coords = landscapes[landscape]\n", "coords = (coords[1], coords[0])\n", "print(OUTPUT_FOLDER, coords)" diff --git a/notebooks/4b-predict-large-area.ipynb b/notebooks/4b-predict-large-area.ipynb index 59bb530..6ddb8ca 100644 --- a/notebooks/4b-predict-large-area.ipynb +++ b/notebooks/4b-predict-large-area.ipynb @@ -17,6 +17,402 @@ " * **Writing COG**" ] }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "absl-py==0.7.1\r\n", + "affine==2.3.0\r\n", + "ago==0.0.93\r\n", + "anaconda-client==1.7.2\r\n", + "anaconda-navigator==1.9.7\r\n", + "appnope==0.1.0\r\n", + "arrow==0.14.2\r\n", + "asn1crypto==0.24.0\r\n", + "astor==0.8.0\r\n", + "astropy==4.0.1.post1\r\n", + "attrs==19.3.0\r\n", + "Automat==0.7.0\r\n", + "awscli==1.16.227\r\n", + "backcall==0.1.0\r\n", + "beautifulsoup4==4.7.1\r\n", + "binaryornot==0.4.4\r\n", + "bleach==3.1.0\r\n", + "blis==0.2.4\r\n", + "bokeh==2.1.1\r\n", + "boto3==1.9.166\r\n", + "botocore==1.12.217\r\n", + "bs4==0.0.1\r\n", + "cachetools==4.1.0\r\n", + "cchardet==2.1.6\r\n", + "certifi==2019.6.16\r\n", + "cffi==1.12.3\r\n", + "chardet==3.0.4\r\n", + "Click==7.0\r\n", + "click-plugins==1.1.1\r\n", + "cligj==0.5.0\r\n", + "clyent==1.2.2\r\n", + "colorama==0.3.9\r\n", + "conda==4.7.5\r\n", + "conda-build==3.17.8\r\n", + "conda-package-handling==1.3.10\r\n", + "conda-verify==3.1.1\r\n", + "confuse==1.1.0\r\n", + "constantly==15.1.0\r\n", + "cookiecutter==1.6.0\r\n", + "cryptography==2.7\r\n", + "cssselect==1.1.0\r\n", + "cycler==0.10.0\r\n", + "cymem==2.0.2\r\n", + "decorator==4.4.0\r\n", + "defusedxml==0.6.0\r\n", + "descartes==1.1.0\r\n", + "dill==0.3.0\r\n", + "distro==1.5.0\r\n", + "docutils==0.15.2\r\n", + "dotmap==1.3.8\r\n", + "editdistance==0.5.3\r\n", + "elasticsearch==7.0.4\r\n", + "en-core-web-sm==2.1.0\r\n", + "entrypoints==0.3\r\n", + "fastprogress==0.2.3\r\n", + "feedfinder2==0.0.4\r\n", + "feedparser==5.2.1\r\n", + "filelock==3.0.12\r\n", + "Fiona==1.8.13.post1\r\n", + "Flask==1.1.1\r\n", + "future==0.17.1\r\n", + "gast==0.2.2\r\n", + "GDAL==2.3.3\r\n", + "gdelt==0.1.10.6\r\n", + "geographiclib==1.49\r\n", + "geopandas==0.7.0\r\n", + "geopy==1.20.0\r\n", + "getch==1.0\r\n", + "giveme5w1h==1.0.16\r\n", + "glob2==0.7\r\n", + "google-auth==1.18.0\r\n", + "google-auth-oauthlib==0.4.1\r\n", + "google-pasta==0.1.7\r\n", + "googletrans==2.4.0\r\n", + "grpcio==1.23.0\r\n", + "h5py==2.9.0\r\n", + "hjson==3.0.1\r\n", + "htmlmin==0.1.12\r\n", + "hurry.filesize==0.9\r\n", + "hyperlink==19.0.0\r\n", + "idna==2.8\r\n", + "ImageHash==4.1.0\r\n", + "importlib-metadata==1.6.0\r\n", + "incremental==17.5.0\r\n", + "ipykernel==5.1.1\r\n", + "ipython==7.6.0\r\n", + "ipython-genutils==0.2.0\r\n", + "ipywidgets==7.5.1\r\n", + "itsdangerous==1.1.0\r\n", + "jedi==0.13.3\r\n", + "jieba==0.42.1\r\n", + "jieba3k==0.35.1\r\n", + "Jinja2==2.11.2\r\n", + "jinja2-time==0.2.0\r\n", + "jmespath==0.9.4\r\n", + "joblib==0.13.2\r\n", + "jsonschema==3.0.1\r\n", + "jupyter==1.0.0\r\n", + "jupyter-client==6.1.3\r\n", + "jupyter-console==6.0.0\r\n", + "jupyter-core==4.6.3\r\n", + "Keras==2.2.5\r\n", + "Keras-Applications==1.0.8\r\n", + "keras-bert==0.84.0\r\n", + "keras-embed-sim==0.7.0\r\n", + "keras-layer-normalization==0.14.0\r\n", + "keras-multi-head==0.27.0\r\n", + "keras-pos-embd==0.11.0\r\n", + "keras-position-wise-feed-forward==0.6.0\r\n", + "Keras-Preprocessing==1.1.0\r\n", + "keras-self-attention==0.46.0\r\n", + "keras-transformer==0.37.0\r\n", + "keyboard==0.13.3\r\n", + "kiwisolver==1.1.0\r\n", + "ktrain==0.7.2\r\n", + "langdetect==1.0.7\r\n", + "libarchive-c==2.8\r\n", + "lief==0.9.0\r\n", + "llvmlite==0.29.0\r\n", + "lxml==4.3.4\r\n", + "Markdown==3.1.1\r\n", + "MarkupSafe==1.1.1\r\n", + "matplotlib==3.2.1\r\n", + "mercantile==1.1.2\r\n", + "missingno==0.4.2\r\n", + "mistune==0.8.4\r\n", + "mkl-fft==1.0.12\r\n", + "mkl-random==1.0.2\r\n", + "more-itertools==8.3.0\r\n", + "munch==2.5.0\r\n", + "murmurhash==1.0.2\r\n", + "navigator-updater==0.2.1\r\n", + "nbconvert==5.5.0\r\n", + "nbformat==4.4.0\r\n", + "networkx==2.3\r\n", + "news-please==1.4.23\r\n", + "newspaper3k==0.2.8\r\n", + "nltk==3.4.4\r\n", + "notebook==5.7.8\r\n", + "numba==0.45.0\r\n", + "numbskull==0.1.1\r\n", + "numpy==1.16.4\r\n", + "oauthlib==3.1.0\r\n", + "olefile==0.46\r\n", + "opt-einsum==3.2.1\r\n", + "packaging==20.4\r\n", + "pandas==1.0.4\r\n", + "pandas-profiling==2.8.0\r\n", + "pandas-summary==0.0.7\r\n", + "pandocfilters==1.4.2\r\n", + "parsedatetime==2.4\r\n", + "parsel==1.5.2\r\n", + "parso==0.5.0\r\n", + "pexpect==4.7.0\r\n", + "phik==0.10.0\r\n", + "pickleshare==0.7.5\r\n", + "Pillow==6.0.0\r\n", + "pkginfo==1.5.0.1\r\n", + "plac==0.9.6\r\n", + "plotly==4.1.0\r\n", + "pluggy==0.13.1\r\n", + "poyo==0.4.2\r\n", + "preshed==2.0.1\r\n", + "prometheus-client==0.7.1\r\n", + "prompt-toolkit==2.0.9\r\n", + "protobuf==3.9.0\r\n", + "psutil==5.6.3\r\n", + "ptyprocess==0.6.0\r\n", + "py==1.8.1\r\n", + "pyasn1==0.4.6\r\n", + "pyasn1-modules==0.2.6\r\n", + "pycorenlp==0.3.0\r\n", + "pycosat==0.6.3\r\n", + "pycparser==2.19\r\n", + "PyDispatcher==2.0.5\r\n", + "Pygments==2.4.2\r\n", + "PyHamcrest==1.9.0\r\n", + "PyMySQL==0.9.3\r\n", + "pynput==1.4.2\r\n", + "pyobjc==5.2\r\n", + "pyobjc-core==5.2\r\n", + "pyobjc-framework-Accounts==5.2\r\n", + "pyobjc-framework-AddressBook==5.2\r\n", + "pyobjc-framework-AdSupport==5.2\r\n", + "pyobjc-framework-AppleScriptKit==5.2\r\n", + "pyobjc-framework-AppleScriptObjC==5.2\r\n", + "pyobjc-framework-ApplicationServices==5.2\r\n", + "pyobjc-framework-Automator==5.2\r\n", + "pyobjc-framework-AVFoundation==5.2\r\n", + "pyobjc-framework-AVKit==5.2\r\n", + "pyobjc-framework-BusinessChat==5.2\r\n", + "pyobjc-framework-CalendarStore==5.2\r\n", + "pyobjc-framework-CFNetwork==5.2\r\n", + "pyobjc-framework-CloudKit==5.2\r\n", + "pyobjc-framework-Cocoa==5.2\r\n", + "pyobjc-framework-Collaboration==5.2\r\n", + "pyobjc-framework-ColorSync==5.2\r\n", + "pyobjc-framework-Contacts==5.2\r\n", + "pyobjc-framework-ContactsUI==5.2\r\n", + "pyobjc-framework-CoreAudio==5.2\r\n", + "pyobjc-framework-CoreAudioKit==5.2\r\n", + "pyobjc-framework-CoreBluetooth==5.2\r\n", + "pyobjc-framework-CoreData==5.2\r\n", + "pyobjc-framework-CoreLocation==5.2\r\n", + "pyobjc-framework-CoreMedia==5.2\r\n", + "pyobjc-framework-CoreMediaIO==5.2\r\n", + "pyobjc-framework-CoreML==5.2\r\n", + "pyobjc-framework-CoreServices==5.2\r\n", + "pyobjc-framework-CoreSpotlight==5.2\r\n", + "pyobjc-framework-CoreText==5.2\r\n", + "pyobjc-framework-CoreWLAN==5.2\r\n", + "pyobjc-framework-CryptoTokenKit==5.2\r\n", + "pyobjc-framework-DictionaryServices==5.2\r\n", + "pyobjc-framework-DiscRecording==5.2\r\n", + "pyobjc-framework-DiscRecordingUI==5.2\r\n", + "pyobjc-framework-DiskArbitration==5.2\r\n", + "pyobjc-framework-DVDPlayback==5.2\r\n", + "pyobjc-framework-EventKit==5.2\r\n", + "pyobjc-framework-ExceptionHandling==5.2\r\n", + "pyobjc-framework-ExternalAccessory==5.2\r\n", + "pyobjc-framework-FinderSync==5.2\r\n", + "pyobjc-framework-FSEvents==5.2\r\n", + "pyobjc-framework-GameCenter==5.2\r\n", + "pyobjc-framework-GameController==5.2\r\n", + "pyobjc-framework-GameKit==5.2\r\n", + "pyobjc-framework-GameplayKit==5.2\r\n", + "pyobjc-framework-ImageCaptureCore==5.2\r\n", + "pyobjc-framework-IMServicePlugIn==5.2\r\n", + "pyobjc-framework-InputMethodKit==5.2\r\n", + "pyobjc-framework-InstallerPlugins==5.2\r\n", + "pyobjc-framework-InstantMessage==5.2\r\n", + "pyobjc-framework-Intents==5.2\r\n", + "pyobjc-framework-IOSurface==5.2\r\n", + "pyobjc-framework-iTunesLibrary==5.2\r\n", + "pyobjc-framework-LatentSemanticMapping==5.2\r\n", + "pyobjc-framework-LaunchServices==5.2\r\n", + "pyobjc-framework-libdispatch==5.2\r\n", + "pyobjc-framework-LocalAuthentication==5.2\r\n", + "pyobjc-framework-MapKit==5.2\r\n", + "pyobjc-framework-MediaAccessibility==5.2\r\n", + "pyobjc-framework-MediaLibrary==5.2\r\n", + "pyobjc-framework-MediaPlayer==5.2\r\n", + "pyobjc-framework-MediaToolbox==5.2\r\n", + "pyobjc-framework-ModelIO==5.2\r\n", + "pyobjc-framework-MultipeerConnectivity==5.2\r\n", + "pyobjc-framework-NaturalLanguage==5.2\r\n", + "pyobjc-framework-NetFS==5.2\r\n", + "pyobjc-framework-Network==5.2\r\n", + "pyobjc-framework-NetworkExtension==5.2\r\n", + "pyobjc-framework-NotificationCenter==5.2\r\n", + "pyobjc-framework-OpenDirectory==5.2\r\n", + "pyobjc-framework-OSAKit==5.2\r\n", + "pyobjc-framework-Photos==5.2\r\n", + "pyobjc-framework-PhotosUI==5.2\r\n", + "pyobjc-framework-PreferencePanes==5.2\r\n", + "pyobjc-framework-PubSub==5.2\r\n", + "pyobjc-framework-QTKit==5.2\r\n", + "pyobjc-framework-Quartz==5.2\r\n", + "pyobjc-framework-SafariServices==5.2\r\n", + "pyobjc-framework-SceneKit==5.2\r\n", + "pyobjc-framework-ScreenSaver==5.2\r\n", + "pyobjc-framework-ScriptingBridge==5.2\r\n", + "pyobjc-framework-SearchKit==5.2\r\n", + "pyobjc-framework-Security==5.2\r\n", + "pyobjc-framework-SecurityFoundation==5.2\r\n", + "pyobjc-framework-SecurityInterface==5.2\r\n", + "pyobjc-framework-ServiceManagement==5.2\r\n", + "pyobjc-framework-Social==5.2\r\n", + "pyobjc-framework-SpriteKit==5.2\r\n", + "pyobjc-framework-StoreKit==5.2\r\n", + "pyobjc-framework-SyncServices==5.2\r\n", + "pyobjc-framework-SystemConfiguration==5.2\r\n", + "pyobjc-framework-UserNotifications==5.2\r\n", + "pyobjc-framework-VideoSubscriberAccount==5.2\r\n", + "pyobjc-framework-VideoToolbox==5.2\r\n", + "pyobjc-framework-Vision==5.2\r\n", + "pyobjc-framework-WebKit==5.2\r\n", + "pyOpenSSL==19.0.0\r\n", + "pyparsing==2.4.0\r\n", + "pyproj==2.2.0\r\n", + "pyrsistent==0.14.11\r\n", + "PySocks==1.7.0\r\n", + "pytesseract==0.2.7\r\n", + "pytest==5.4.3\r\n", + "python-dateutil==2.8.0\r\n", + "pytz==2019.1\r\n", + "PyWavelets==1.1.1\r\n", + "PyYAML==5.1.1\r\n", + "pyzmq==18.0.0\r\n", + "qtconsole==4.5.3\r\n", + "QtPy==1.8.0\r\n", + "queuelib==1.5.0\r\n", + "rasterio==1.1.2\r\n", + "readability-lxml==0.7.1\r\n", + "regex==2019.6.8\r\n", + "requests==2.23.0\r\n", + "requests-file==1.4.3\r\n", + "requests-oauthlib==1.3.0\r\n", + "retrying==1.3.3\r\n", + "rio-cogeo==1.1.9\r\n", + "rsa==3.4.2\r\n", + "ruamel-yaml==0.15.46\r\n", + "s3transfer==0.2.1\r\n", + "scikit-learn==0.21.3\r\n", + "scipy==1.4.1\r\n", + "Scrapy==1.7.3\r\n", + "seaborn==0.10.1\r\n", + "Send2Trash==1.5.0\r\n", + "sentinelhub==2.5.3\r\n", + "seqeval==0.0.12\r\n", + "service-identity==18.1.0\r\n", + "Shapely==1.6.4.post2\r\n", + "six==1.12.0\r\n", + "snorkel==0.7.0b0\r\n", + "snorkel-metal==0.5.0\r\n", + "snuggs==1.4.7\r\n", + "soupsieve==1.8\r\n", + "spacy==2.1.6\r\n", + "SQLAlchemy==1.3.6\r\n", + "srsly==0.0.7\r\n", + "supermercado==0.0.5\r\n", + "tabula-py==2.1.1\r\n", + "tangled-up-in-unicode==0.0.6\r\n", + "tensorboard==1.14.0\r\n", + "tensorboardX==1.8\r\n", + "tensorflow==1.14.0\r\n", + "tensorflow-estimator==1.14.0\r\n", + "termcolor==1.1.0\r\n", + "terminado==0.8.2\r\n", + "testpath==0.4.2\r\n", + "thinc==7.0.8\r\n", + "tifffile==2019.5.30\r\n", + "tinysegmenter==0.3\r\n", + "tldextract==2.2.1\r\n", + "torch==1.1.0.post2\r\n", + "tornado==6.0.3\r\n", + "tqdm==4.46.1\r\n", + "traitlets==4.3.2\r\n", + "treedlib==0.1.1\r\n", + "Twisted==19.7.0\r\n", + "typing==3.7.4.1\r\n", + "typing-extensions==3.7.4.2\r\n", + "Unidecode==1.1.1\r\n", + "urllib3==1.24.2\r\n", + "utm==0.4.2\r\n", + "virtualenv==16.6.0\r\n", + "visions==0.4.4\r\n", + "w3lib==1.21.0\r\n", + "Wand==0.5.5\r\n", + "warcio==1.7.1\r\n", + "wasabi==0.2.2\r\n", + "wcwidth==0.1.7\r\n", + "webencodings==0.5.1\r\n", + "Werkzeug==0.15.5\r\n", + "whichcraft==0.5.2\r\n", + "widgetsnbextension==3.5.1\r\n", + "wrapt==1.11.2\r\n", + "xmltodict==0.12.0\r\n", + "zipp==3.1.0\r\n", + "zope.interface==4.6.0\r\n" + ] + } + ], + "source": [ + "!pip freeze" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " % Total % Received % Xferd Average Speed Time Time Time Current\n", + " Dload Upload Total Spent Left Speed\n", + "100 374M 100 374M 0 0 11.4M 0 0:00:32 0:00:32 --:--:-- 13.0M 0 0 13.0M 0 0:00:28 0:00:06 0:00:22 13.3M 0 0:00:27 0:00:10 0:00:17 14.0M 0 0 11.2M 0 0:00:33 0:00:28 0:00:05 5573k\n" + ] + } + ], + "source": [ + "!curl https://restoration-monitoring-external.s3.amazonaws.com/restoration-mapper/example/example-raw.zip \\\n", + " -o ../data/data.zip" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/tests/test_download.py b/tests/do_not_run2.py similarity index 100% rename from tests/test_download.py rename to tests/do_not_run2.py