From fbc7db2884b1c59c5e3edbe5b043d03ab46f589c Mon Sep 17 00:00:00 2001 From: gbrener Date: Fri, 1 Dec 2017 14:07:28 -0600 Subject: [PATCH 1/3] Fix NLDAS data exploration notebook Add instructions on how to setup environment at the top of the notebook. --- examples/NLDAS_Data_Exploration.ipynb | 18 ++++++--- examples/example_utils.py | 56 +++++++++++++++++---------- 2 files changed, 49 insertions(+), 25 deletions(-) diff --git a/examples/NLDAS_Data_Exploration.ipynb b/examples/NLDAS_Data_Exploration.ipynb index 56a143b..6ae693b 100644 --- a/examples/NLDAS_Data_Exploration.ipynb +++ b/examples/NLDAS_Data_Exploration.ipynb @@ -11,7 +11,17 @@ "- Downloads data file(s) from NASA\n", "- Show attribute statistics and visualizations\n", "- Do viz-related data cleaning\n", - "- Show (corrected) attribute statistics and visualizations" + "- Show (corrected) attribute statistics and visualizations\n", + "\n", + "### Setup Instructions:\n", + "1. Create *.netrc* file in home dir according to [GES DISC site instructions](https://disc.gsfc.nasa.gov/information/howto?title=How%20to%20Download%20Data%20Files%20from%20HTTP%20Service%20with%20wget)\n", + "2. Create environment, install notebook pkgs, enable extension:\n", + "```\n", + "conda env create -n elm python=2.7\n", + "source activate elm\n", + "conda install -c conda-forge pycurl lxml holoviews\n", + "jupyter nbextension enable --py widgetsnbextension` # This should report \"OK\"\n", + "```" ] }, { @@ -179,9 +189,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "collapsed": true - }, + "metadata": {}, "outputs": [], "source": [] } @@ -202,7 +210,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", - "version": "2.7.13" + "version": "2.7.14" } }, "nbformat": 4, diff --git a/examples/example_utils.py b/examples/example_utils.py index a4be88b..8a357bd 100644 --- a/examples/example_utils.py +++ b/examples/example_utils.py @@ -12,38 +12,54 @@ import requests from six.moves.urllib.parse import urlparse -from six.moves import range +from six.moves import range, input from lxml import etree, html from ipywidgets import widgets, Layout from IPython.display import display, Javascript -from pydap.cas.urs import setup_session -session = setup_session( - os.environ.get('NLDAS_USERNAME') or raw_input('NLDAS Username: '), - os.environ.get('NLDAS_PASSWORD') or getpass.getpass('Password: ') -) - -def get_request(url): - import pycurl - from io import BytesIO - buffer = BytesIO() - c = pycurl.Curl() - c.setopt(c.URL, url) - c.setopt(c.WRITEDATA, buffer) - c.perform() - c.close() - return buffer.getvalue() +PYCURL = True + +if not PYCURL: + from pydap.cas.urs import setup_session + session = setup_session( + os.environ.get('NLDAS_USERNAME') or input('NLDAS Username: '), + os.environ.get('NLDAS_PASSWORD') or getpass.getpass('Password: ') + ) + +def get_request(url, outfpath=None): + global PYCURL + if PYCURL: + # outfpath must be set + import pycurl + from io import BytesIO + buffer = BytesIO() + c = pycurl.Curl() + c.setopt(c.URL, url) + c.setopt(c.WRITEDATA, buffer) + c.setopt(c.COOKIEJAR, '/tmp/cookie.jar') + c.setopt(c.NETRC, True) + c.setopt(c.FOLLOWLOCATION, True) + #c.setopt(c.REMOTE_NAME, outfpath) + c.perform() + c.close() + return buffer.getvalue() + resp = requests.get(url) + return resp.text def dl_file(url): data_fpath = urlparse(url).path.lstrip(os.sep) data_dpath = os.path.dirname(data_fpath) if not os.path.exists(data_fpath): - resp = session.get(url) if not os.path.isdir(data_dpath): os.makedirs(data_dpath) - with open(data_fpath, 'w') as outfp: - outfp.write(resp.content) + if PYCURL: + with open(data_fpath, 'w') as outfp: + outfp.write(get_request(url)) + else: + resp = session.get(url) + with open(data_fpath, 'w') as outfp: + outfp.write(resp.content) return data_fpath def dups_to_indexes(field_names): From 1d15f642c8231f0182ca99f177f4b6f7f41f5e6f Mon Sep 17 00:00:00 2001 From: gbrener Date: Fri, 1 Dec 2017 14:16:07 -0600 Subject: [PATCH 2/3] Add comments --- examples/NLDAS_Data_Exploration.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/NLDAS_Data_Exploration.ipynb b/examples/NLDAS_Data_Exploration.ipynb index 6ae693b..024483d 100644 --- a/examples/NLDAS_Data_Exploration.ipynb +++ b/examples/NLDAS_Data_Exploration.ipynb @@ -17,10 +17,10 @@ "1. Create *.netrc* file in home dir according to [GES DISC site instructions](https://disc.gsfc.nasa.gov/information/howto?title=How%20to%20Download%20Data%20Files%20from%20HTTP%20Service%20with%20wget)\n", "2. Create environment, install notebook pkgs, enable extension:\n", "```\n", - "conda env create -n elm python=2.7\n", + "conda env create -n elm python=2.7\n", # 2.7 needed for pynio "source activate elm\n", "conda install -c conda-forge pycurl lxml holoviews\n", - "jupyter nbextension enable --py widgetsnbextension` # This should report \"OK\"\n", + "jupyter nbextension enable --py widgetsnbextension # This should report \"OK\"\n", "```" ] }, From 02f895072cbaab601ab37913a484c001c70a0673 Mon Sep 17 00:00:00 2001 From: gbrener Date: Fri, 1 Dec 2017 14:17:31 -0600 Subject: [PATCH 3/3] Fix typo --- examples/NLDAS_Data_Exploration.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/NLDAS_Data_Exploration.ipynb b/examples/NLDAS_Data_Exploration.ipynb index 024483d..db762d4 100644 --- a/examples/NLDAS_Data_Exploration.ipynb +++ b/examples/NLDAS_Data_Exploration.ipynb @@ -17,7 +17,7 @@ "1. Create *.netrc* file in home dir according to [GES DISC site instructions](https://disc.gsfc.nasa.gov/information/howto?title=How%20to%20Download%20Data%20Files%20from%20HTTP%20Service%20with%20wget)\n", "2. Create environment, install notebook pkgs, enable extension:\n", "```\n", - "conda env create -n elm python=2.7\n", # 2.7 needed for pynio + "conda env create -n elm python=2.7 # 2.7 needed for pynio\n", "source activate elm\n", "conda install -c conda-forge pycurl lxml holoviews\n", "jupyter nbextension enable --py widgetsnbextension # This should report \"OK\"\n",