diff --git a/PythonSerialize/Coffee-Code-Scripts.ipynb b/PythonSerialize/Coffee-Code-Scripts.ipynb
new file mode 100644
index 0000000..cca3992
--- /dev/null
+++ b/PythonSerialize/Coffee-Code-Scripts.ipynb
@@ -0,0 +1,918 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "%matplotlib inline\n",
+ "import os\n",
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "import seaborn\n",
+ "\n",
+ "from IPython.display import HTML, Image\n",
+ "\n",
+ "plt.style.use('seaborn-darkgrid')"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "# Python Data Structures\n",
+ "\n",
+ "Different types and ways of holding data. Essentially variables.\n",
+ "\n",
+ "* Built-in common types:\n",
+ " * Scalar (single-value): \n",
+ " `str`, `int`, `float`, `bool`, `complex`, ...\n",
+ " \n",
+ " * Compound (multi-value): \n",
+ " `list`, `dict`, `tuple`, `set`, ... \n",
+ " \n",
+ " * Fancy things: \n",
+ " `datetime` \n",
+ " [`collections`](https://docs.python.org/3/library/collections.html): \n",
+ " `namedtuple`, `defaultdict`, `deque`, `Counter`, ...\n",
+ " \n",
+ " * `numpy`: \n",
+ " `numpy.array`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "source": [
+ "## Scalar\n",
+ "\n",
+ "Most familiar with these types."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "foo = 42 # int\n",
+ "bar = 'meaning of life' # str\n",
+ "baz = 3.14158 # float\n",
+ "bam = False # bool\n",
+ "moo = 42j # complex uses 'j'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "fragment"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "print(foo * baz)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "fragment"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "print(foo * bar)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "fragment"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "print(foo * bam)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## Compound\n",
+ "\n",
+ "Mostly familiar with these types, especially `list` and `dict`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "foo = [1, 2, 3, 4, 5]\n",
+ "\n",
+ "bar = {\n",
+ " 'stars': 1e12,\n",
+ " 'galaxies': 1000,\n",
+ " 'black_holes': 1\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "print(f\"There are approx {bar['stars']} in a monolith.\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "Image('https://s.hdnux.com/photos/63/02/07/13377985/3/920x920.jpg')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "for i in foo:\n",
+ " print(i)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for obj, num in bar.items():\n",
+ " print(f\"There are {num} {obj} in the Milky Way\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## More on dictionaries\n",
+ "\n",
+ "Wilfred says, \"The most useful of data types\"\n",
+ "\n",
+ "Think of a dictionary as a way to hold and organize all of your other variables but with convenient semantic labelling names."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "location = {\n",
+ " 'name': \"Macquarie University\",\n",
+ " 'latitude': 33.7771, # degrees\n",
+ " 'longitude': 151.1180, # degrees\n",
+ " 'elevation': 100, # meters\n",
+ " 'pressure': 1000, # mbar \n",
+ " 'horizon': 30, # degrees\n",
+ " 'timezone': 'Australia/Sydney', \n",
+ "}\n",
+ "\n",
+ "location"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "We can then use this dictonary in various ways:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from astroplan import Observer\n",
+ "from astropy import units as u\n",
+ "\n",
+ "mqu = Observer(\n",
+ " longitude=location['longitude'] * u.deg,\n",
+ " latitude=location['latitude'] * u.deg,\n",
+ " elevation=location['elevation'] * u.meter,\n",
+ " timezone=location['timezone'],\n",
+ " name=location['name']\n",
+ ")\n",
+ "\n",
+ "mqu"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "from astropy.coordinates import SkyCoord\n",
+ "from astroplan import FixedTarget\n",
+ "from astropy.time import Time\n",
+ "from astroplan.plots import plot_airmass \n",
+ "\n",
+ "alpha_centauri = FixedTarget.from_name('Arcturus')\n",
+ "\n",
+ "plot_airmass(alpha_centauri, mqu, Time.now() + 1 * u.hour);"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## But...\n",
+ "\n",
+ "What happens when we need to use this information a number of different scripts or programs?\n",
+ "\n",
+ "* Copy and paste `location` into every script we run?\n",
+ "* What if our `elevation` were incorrect? (Which we only discover after copying-and-pasting 1000 times)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## Serialization\n",
+ "\n",
+ "> ...serialization (or serialisation) is the process of translating data structures or object state into a format that can be stored (for example, in a file or memory buffer) or transmitted (for example, across a network connection link) and reconstructed later (possibly in a different computer environment). \n",
+ "\n",
+ "> When the resulting series of bits is reread according to the serialization format, it can be used to create a semantically identical clone of the original object. \n",
+ "\n",
+ "https://en.wikipedia.org/wiki/Serialization"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "fragment"
+ }
+ },
+ "source": [
+ "Save stuff. Use it later."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## Pickle\n",
+ "\n",
+ "Built-in Python serialiation format.\n",
+ "\n",
+ "* Python only.\n",
+ "* Not human readable.\n",
+ "* They say it's slow."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "## JSON and YAML\n",
+ "\n",
+ "Easy-peasy serialization.\n",
+ "\n",
+ "#### JAY-sun\n",
+ "JavaScript Object Notation\n",
+ "\n",
+ "\n",
+ "#### /ˈjæməl/ (think camel)\n",
+ "Yet Another Markup Language"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "source": [
+ "## Advantages\n",
+ "\n",
+ "* Any language\n",
+ "* Human-readable*\n",
+ "* Faster\n",
+ "\n",
+ "*JSON isn't super friendly"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The basic idea with serialization is to convert your object into a text string*, which can be saved to a regular file.\n",
+ "\n",
+ "When you want to use the object you deserialize the string and have the same object.\n",
+ "\n",
+ "Serialize: `object` -> `string` \n",
+ "Deseriialize: `string` -> `object`\n",
+ "\n",
+ "\n",
+ "*Can be converted to bytes (i.e. jsonb), not discussed here."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Differences between YAML and JSON\n",
+ "\n",
+ "[Stolen from here](https://www.json2yaml.com/yaml-vs-json)\n",
+ "\n",
+ "\n",
+ "#### YAML vs JSON\n",
+ "YAML is best suited for configuration where JSON is better as a serialization format or serving up data for your APIs.\n",
+ "\n",
+ "In some cases, YAML has a couple of big advantages over JSON, including:\n",
+ "\n",
+ "* **comments**\n",
+ "* ability to self reference\n",
+ "* support for complex datatypes\n",
+ "* more...\n",
+ "\n",
+ "**Write your configuration files in YAML format where you have the opportunity - it is designed to be readable and editable by humans.**"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### JSON vs YAML\n",
+ "JSON wins as a serialization format. \n",
+ "\n",
+ "* It is more explicit and more suitable for data interchange between your apis.\n",
+ "* YAML is a superset of JSON, which means you can parse JSON with a YAML parser. (Wilfred says \"Don't do this\")\n",
+ "* Try mixing JSON and YAML in the same document: `[..., ..]` for annotating arrays and `{ \"foo\" : \"bar\"}` for objects. (Wilfred says \"Don't do this either.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "source": [
+ "### JSON\n",
+ "\n",
+ "Machines talking to machines\n",
+ "\n",
+ "### YAML\n",
+ "\n",
+ "Humans talking to machines (and vice versa)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "Let's take another look at our `location` object:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "location"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import json # Part of Python standard library\n",
+ "\n",
+ "json.dumps(location)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# It is just a string\n",
+ "type(json.dumps(location))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# We can save to a file\n",
+ "with open('my_location.json', 'w') as f:\n",
+ " f.write(json.dumps(location))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Using shell commands from Jupyter with exclamation point\n",
+ "!ls -l"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!cat my_location.json"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "import yaml # pip install pyyaml - or pip install ruamel.yaml\n",
+ "\n",
+ "yaml.dump(location)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "type(yaml.dump(location))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# We can save to a file\n",
+ "with open('my_location.yaml', 'w') as f:\n",
+ " f.write(yaml.dump(location))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!ls -l"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!cat my_location.yaml"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "### YAML as super-fancy persistent dictionary\n",
+ "\n",
+ "YAML (and JSON) handle all built-in types.*\n",
+ "\n",
+ "This means it can handle an array in a dict in a dict, etc.\n",
+ "\n",
+ "\n",
+ "*You can also handle fancy types such as `datetime`, `numpy.array`, etc. Only slightly more involved."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "foobar = {\n",
+ " 'baz': {\n",
+ " 'bar': [1, 2, 3, 4, 5],\n",
+ " 'bam': [\n",
+ " {'one': 1},\n",
+ " {'two': 2}\n",
+ " ]\n",
+ " },\n",
+ " 'moo': 42,\n",
+ " 'boo': \"Hello World\"\n",
+ "}\n",
+ "\n",
+ "foobar"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "json.dumps(foobar)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "yaml.dump(foobar)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "### A real configuation file\n",
+ "\n",
+ "Here we write out all of our configuration details in a big YAML file called `config.yaml`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!cat config.yaml"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "Accessing this file is as simple as loading it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Note the use of `safe_load`\n",
+ "with open('config.yaml', 'r') as f:\n",
+ " my_config = yaml.safe_load(f.read())"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "my_config"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "location = my_config['location']\n",
+ "\n",
+ "location"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "Let's make ourselves a quick load function:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "-"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "def load_config(filename):\n",
+ " with open(filename, 'r') as f:\n",
+ " my_config = yaml.safe_load(f.read())\n",
+ " \n",
+ " return my_config"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "source": [
+ "### Change file externally\n",
+ "\n",
+ "Change the file outside the script, then re-read"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "my_config = load_config('config.yaml')"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "my_config"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "slide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "planet_names = [\n",
+ " 'Alpha',\n",
+ " 'Beta',\n",
+ " 'Gamma',\n",
+ " 'Bob'\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "planets = dict()\n",
+ "\n",
+ "# Generate a random number of fake light-curves\n",
+ "for i, name in enumerate(planet_names):\n",
+ " # Random number between 1 - 5\n",
+ " light_curves = list()\n",
+ " for j in range(np.random.randint(1, 5)):\n",
+ " \n",
+ " # Generate a light curve\n",
+ " lc0 = np.random.normal(1, 0.1, size=100)\n",
+ " \n",
+ " # Set up file names\n",
+ " planet_directory_name = f'data/planet_{i:003d}'\n",
+ " # Create our directory\n",
+ " os.makedirs(planet_directory_name, exist_ok=True)\n",
+ " \n",
+ " # Set up json filename\n",
+ " lc_filename = f'{planet_directory_name}/lc_{j:003d}.json'\n",
+ " \n",
+ " # Save light curve to a json file (probably not ideal)\n",
+ " with open(lc_filename, 'w') as f:\n",
+ " f.write(json.dumps(list(lc0)))\n",
+ " \n",
+ " # Add the name of our file to the list\n",
+ " light_curves.append(lc_filename)\n",
+ " \n",
+ " # Save to our big list\n",
+ " planets[name] = light_curves"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "slideshow": {
+ "slide_type": "subslide"
+ }
+ },
+ "outputs": [],
+ "source": [
+ "# Look at our config\n",
+ "planets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Save our config to a yaml file\n",
+ "with open('planet_config.yaml', 'w') as f:\n",
+ " f.write(yaml.dump(planets))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!cat planet_config.yaml"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### A different script\n",
+ "\n",
+ "Now we go to an entirely different script: [Use Planet Config](Use-Planet-Config.ipynb)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Logging\n",
+ "\n",
+ "Logging can allow for flexible output, which allows you to easily separate information from warnings from debugging."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "display(HTML(''))"
+ ]
+ }
+ ],
+ "metadata": {
+ "celltoolbar": "Slideshow",
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/PythonSerialize/README.md b/PythonSerialize/README.md
new file mode 100644
index 0000000..dac4d0d
--- /dev/null
+++ b/PythonSerialize/README.md
@@ -0,0 +1,5 @@
+# Python Serialization (and other script fun)
+
+## Binder
+
+[](https://mybinder.org/v2/gh/OZAstroComputingResources/MQCoffee-CodeResources/python-serialize?urlpath=lab)
diff --git a/PythonSerialize/Use-Planet-Config.ipynb b/PythonSerialize/Use-Planet-Config.ipynb
new file mode 100644
index 0000000..4eac5f3
--- /dev/null
+++ b/PythonSerialize/Use-Planet-Config.ipynb
@@ -0,0 +1,148 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import yaml\n",
+ "import json\n",
+ "import numpy as np\n",
+ "from matplotlib import pyplot as plt\n",
+ "%matplotlib inline"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "planet_config = 'planet_config.yaml'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Our convenience function (could be loaded rather than copied):\n",
+ "def load_config(filename):\n",
+ " with open(filename, 'r') as f:\n",
+ " my_config = yaml.safe_load(f.read())\n",
+ " \n",
+ " return my_config"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "planets = load_config(planet_config)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "planets"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## What about Bob?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "name = 'Bob'"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "planets[name]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "for lc_json_file in planets[name]:\n",
+ " # Load the json data\n",
+ " with open(lc_json_file, 'r') as f:\n",
+ " lc0 = json.loads(f.read())\n",
+ " \n",
+ " plt.plot(np.array(lc0))\n",
+ " plt.title(name)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### All planets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "fig, ax = plt.subplots(1)\n",
+ "fig.set_size_inches(12, 9)\n",
+ "\n",
+ "\n",
+ "for name, files in planets.items():\n",
+ " for lc_json_file in files:\n",
+ " # Load the json data\n",
+ " with open(lc_json_file, 'r') as f:\n",
+ " lc0 = json.loads(f.read())\n",
+ "\n",
+ " ax.plot(np.array(lc0), label=f'{name} - {lc_json_file}')\n",
+ " ax.set_title(name)\n",
+ " ax.legend()\n",
+ " "
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.7.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/PythonSerialize/config.yaml b/PythonSerialize/config.yaml
new file mode 100644
index 0000000..2ae3f37
--- /dev/null
+++ b/PythonSerialize/config.yaml
@@ -0,0 +1,117 @@
+---
+######################### PANOPTES UNIT ########################################
+# name: Can be anything you want it to be. This name is displayed in several
+# places and should be a "personal" name for the unit.
+#
+# pan_id: This is an identification number assigned by the PANOPTES team and is
+# the official designator for your unit. This id is used to store image
+# files and communicate with the Google Cloud network.
+#
+# Leave the pan_id at `PAN000` for testing until you have been assigned
+# an official id. Update pocs_local.yaml with offical name once received.
+################################################################################
+name: Generic PANOPTES Unit
+pan_id: PAN000
+
+location:
+ name: Mauna Loa Observatory
+ latitude: 19.54 # Degrees
+ longitude: -155.58 # Degrees
+ elevation: 3400.0 # Meters
+ horizon: 30 # Degrees; targets must be above this to be considered valid.
+ flat_horizon: -6 # Degrees - Flats when sun between this and focus horizon.
+ focus_horizon: -12 # Degrees - Dark enough to focus on stars.
+ observe_horizon: -18 # Degrees - Sun below this limit to observe.
+ timezone: US/Hawaii
+ gmt_offset: -600 # Offset in minutes from GMT during.
+ # standard time (not daylight saving).
+directories:
+ base: /var/panoptes
+ images: images
+ data: data
+ resources: POCS/resources/
+ targets: POCS/resources/targets
+ mounts: POCS/resources/mounts
+db:
+ name: panoptes
+ type: file
+scheduler:
+ type: dispatch
+ fields_file: simple.yaml
+ check_file: False
+mount:
+ brand: ioptron
+ model: 30
+ driver: ioptron
+ serial:
+ port: /dev/ttyUSB0
+ timeout: 0.
+ baudrate: 9600
+ non_sidereal_available: True
+pointing:
+ auto_correct: False
+ threshold: 500 # arcseconds ~ 50 pixels
+ exptime: 30 # seconds
+ max_iterations: 3
+cameras:
+ auto_detect: True
+ primary: 14d3bd
+ devices:
+ -
+ model: canon_gphoto2
+ -
+ model: canon_gphoto2
+messaging:
+ # Must match ports in peas.yaml.
+ cmd_port: 6500
+ msg_port: 6510
+
+########################## Observations ########################################
+# An observation folder contains a contiguous sequence of images of a target/field
+# recorded by a single camera, with no slewing of the mount during the sequence;
+# there may be tracking adjustments during the observation.
+#
+# An example folder structure would be:
+#
+# $PANDIR/images/fields/Hd189733/14d3bd/20180901T120001/
+#
+# In this folder will be stored JPG and FITS images. A timelapse of the
+# observation can be made (one per camera) and the JPGs optionally removed
+# afterward.
+#
+# TODO: Add options for cleaning up old data (e.g. >30 days)
+################################################################################
+observations:
+ make_timelapse: True
+ keep_jpgs: True
+
+######################## Google Network ########################################
+# By default all images are stored on googlecloud servers and we also
+# use a few google services to store metadata, communicate with servers, etc.
+#
+# See $POCS/pocs/utils/google/README.md for details about authentication.
+#
+# Options to change:
+# image_storage: If images should be uploaded to Google Cloud Storage.
+# service_account_key: Location of the JSON service account key.
+################################################################################
+panoptes_network:
+ image_storage: True
+ service_account_key: # Location of JSON account key
+ project_id: panoptes-survey
+ buckets:
+ images: panoptes-survey
+
+#Enable to output POCS messages to social accounts
+# social_accounts:
+# twitter:
+# consumer_key: [your_consumer_key]
+# consumer_secret: [your_consumer_secret]
+# access_token: [your_access_token]
+# access_token_secret: [your_access_token_secret]
+# slack:
+# webhook_url: [your_webhook_url]
+# output_timestamp: False
+
+state_machine: simple_state_table
+
diff --git a/PythonSerialize/scripts/argparse-lc-plotter.py b/PythonSerialize/scripts/argparse-lc-plotter.py
new file mode 100755
index 0000000..522b04e
--- /dev/null
+++ b/PythonSerialize/scripts/argparse-lc-plotter.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+
+import os
+import yaml
+import json
+import numpy as np
+from matplotlib import pyplot as plt
+
+def main(filename, planet_name, overwrite=False, verbose=False):
+ if verbose:
+ print(f'Making plot for {planet_name}')
+
+ # Our convenience function (could be loaded rather than copied):
+ def load_config(filename):
+ if verbose:
+ print(f'Loading {filename}')
+
+ with open(filename, 'r') as f:
+ my_config = yaml.safe_load(f.read())
+
+ return my_config
+
+ # Load our planet info
+ planets = load_config(filename)
+
+ fig, ax = plt.subplots(1)
+ fig.set_size_inches(12, 9)
+
+ for lc_json_file in planets[planet_name]:
+ # Load the json data
+ with open(lc_json_file, 'r') as f:
+ lc0 = json.loads(f.read())
+
+ ax.plot(np.array(lc0))
+ ax.set_title(planet_name)
+
+ plot_filename = f'plots/{planet_name.lower()}-light-curve.png'
+
+ if os.path.exists(plot_filename) is False or overwrite is True:
+ fig.savefig(plot_filename)
+ if verbose:
+ print(f'Plot created for {planet_name} at {plot_filename}')
+ else:
+ print(f'Plot already exists for {planet_name}, use --overwrite')
+
+ plt.close(fig)
+
+if __name__ == '__main__':
+
+ import argparse
+
+ parser = argparse.ArgumentParser(description="Make a plot for a planet")
+ parser.add_argument('--config', default='planet_config.yaml',
+ help='Config file to use')
+ parser.add_argument('--planet-name', required=True,
+ help='Planet to plot')
+ parser.add_argument('--overwrite', action='store_true', default=False,
+ help='Overwrite any existing files, default False.')
+ parser.add_argument('--verbose', action='store_true', default=False, help='Verbose.')
+
+ # Load the arguments
+ args = parser.parse_args()
+
+ if not os.path.exists(args.config):
+ print("Config file does not exist:", args.config)
+
+ clean_dir = main(
+ filename=args.config,
+ planet_name=args.planet_name,
+ overwrite=args.overwrite,
+ verbose=args.verbose
+ )
+ if args.verbose:
+ print("Done making plot for", args.planet_name)
diff --git a/PythonSerialize/scripts/lc-plotter.py b/PythonSerialize/scripts/lc-plotter.py
new file mode 100755
index 0000000..201867e
--- /dev/null
+++ b/PythonSerialize/scripts/lc-plotter.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+
+import yaml
+import json
+import numpy as np
+from matplotlib import pyplot as plt
+
+planet_config = 'planet_config.yaml'
+planet_name = 'Bob'
+
+# Our convenience function (could be loaded rather than copied):
+def load_config(filename):
+ with open(filename, 'r') as f:
+ my_config = yaml.safe_load(f.read())
+
+ return my_config
+
+# Load our planet info
+planets = load_config(planet_config)
+
+fig, ax = plt.subplots(1)
+fig.set_size_inches(12, 9)
+
+for lc_json_file in planets[planet_name]:
+ # Load the json data
+ with open(lc_json_file, 'r') as f:
+ lc0 = json.loads(f.read())
+
+ ax.plot(np.array(lc0))
+ ax.set_title(planet_name)
+
+fig.savefig(f'plots/{planet_name.lower()}-light-curve.png')
+plt.close(fig)
diff --git a/PythonSerialize/scripts/logger-argparse-lc-plotter.py b/PythonSerialize/scripts/logger-argparse-lc-plotter.py
new file mode 100755
index 0000000..6ecab11
--- /dev/null
+++ b/PythonSerialize/scripts/logger-argparse-lc-plotter.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import yaml
+import json
+import numpy as np
+from matplotlib import pyplot as plt
+
+# Set up our logging. A name is not required but is nice.
+import logging
+logger = logging.getLogger('planet_plotter')
+# Lowest possible log level you want
+logger.setLevel(logging.DEBUG)
+logger.propagate = False
+
+
+def main(filename, planet_name=None, overwrite=False):
+ logging.debug(f'Entering main')
+
+ # Our convenience function (could be loaded rather than copied):
+ def load_config(filename):
+ logger.debug(f'Loading {filename}')
+
+ with open(filename, 'r') as f:
+ my_config = yaml.safe_load(f.read())
+
+ return my_config
+
+ # Load our planet info
+ planets = load_config(filename)
+
+ if planet_name is not None:
+ # Check for planet
+ if planet_name not in planets:
+ logger.warning(f'{planet_name} not in list of planets, exiting.')
+ return
+
+ # Otherwise use only that planet
+ planets = {planet_name: planets[planet_name]}
+ else:
+ planet_name = 'All Planets'
+
+ logger.info(f'Making plot for {planet_name}')
+
+ fig, ax = plt.subplots(1)
+ fig.set_size_inches(12, 9)
+
+ for name, json_files in planets.items():
+ for lc_json_file in json_files:
+ logger.debug(f'Getting {lc_json_file} for {name}')
+ # Load the json data
+ with open(lc_json_file, 'r') as f:
+ lc0 = json.loads(f.read())
+
+ ax.plot(np.array(lc0))
+ ax.set_title(planet_name)
+
+ plot_fn = planet_name.lower().replace(' ', '-')
+
+ logger.debug(f'Using {plot_fn}')
+ plot_path = f'plots/{plot_fn}-light-curve.png'
+
+ if os.path.exists(plot_path) is False or overwrite is True:
+ fig.savefig(plot_path)
+ logger.info(f'Plot created for {planet_name} at {plot_path}')
+ else:
+ logger.warning(f'Plot already exists for {planet_name}, use --overwrite')
+ sys.exit(1)
+
+ plt.close(fig)
+
+ return plot_fn
+
+
+if __name__ == '__main__':
+
+ import argparse
+
+ parser = argparse.ArgumentParser(description="Make a plot for a planet")
+ parser.add_argument('--config', default='planet_config.yaml',
+ help='Config file to use')
+ parser.add_argument('--planet-name', help='Planet to plot, otherwise plot all planets')
+ parser.add_argument('--overwrite', action='store_true', default=False,
+ help='Overwrite any existing files, default False.')
+ # See logging: https://docs.python.org/3.7/howto/logging.html#logging-advanced-tutorial
+ parser.add_argument('--log-file', default='planet-plotter.log', help='Filename to log to')
+ parser.add_argument('--log-level', default='info', help='Log level, default INFO')
+
+ # Load the arguments
+ args = parser.parse_args()
+
+ # Set the log level
+ log_levels = {
+ 'info': logging.INFO,
+ 'debug': logging.DEBUG,
+ }
+
+ # create file handler which logs even debug messages
+ fh = logging.FileHandler(args.log_file)
+ # File always has debug
+ fh.setLevel(logging.DEBUG)
+
+ # create console handler with custom log level
+ ch = logging.StreamHandler()
+ ch.setLevel(log_levels[args.log_level])
+
+ # create formatter and add it to the handlers
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ fh.setFormatter(formatter)
+ ch.setFormatter(formatter)
+
+ # add the handlers to the logger
+ logger.addHandler(ch)
+ logger.addHandler(fh)
+
+ logger.info(f'******************************')
+
+ if not os.path.exists(args.config):
+ logger.warning("Config file does not exist:", args.config)
+
+ planet_name = main(
+ filename=args.config,
+ planet_name=args.planet_name,
+ overwrite=args.overwrite,
+ )
+
+ if planet_name is not None:
+ logger.info(f"Done making plot for {planet_name}")
diff --git a/python_seriz.tgz b/python_seriz.tgz
new file mode 100644
index 0000000..a93d751
Binary files /dev/null and b/python_seriz.tgz differ
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..519fe64
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,8 @@
+pyyaml
+numpy
+matplotlib
+seaborn
+astropy
+astroplan
+rise
+Faker