fpl demo

.github/workflows/CI.yaml

@@ -5,25 +5,49 @@ on:
     branches:
       - main
       - dev
+      - redis
   push:
     branches:
       - main
       - dev
+      - redis
 
 jobs:
   test:
 
     runs-on: ${{matrix.os}}
+    if: ${{ !github.event.pull_request.draft }}
     strategy:
+      max-parallel: 1
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10"]
         os: [ubuntu-latest, macos-latest] # [ubuntu-latest, macos-latest, windows-latest]
 
     steps:
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
+      - name: Update Ubuntu and Install Dependencies
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y lsb-release curl gpg
+      - name: Fetch Redis gpg (Ubuntu)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          curl -fsSL https://packages.redis.io/gpg | sudo gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg
+      - name: Configure Redis gpg (Ubuntu)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/redis.list
+      - name: Install Redis (Ubuntu)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update && sudo apt-get upgrade -y && sudo apt-get install -y redis && sleep 5 && sudo systemctl stop redis-server
+      - name: Install Redis (macOS)
+        if: startsWith(matrix.os, 'macos')
+        run: |
+          brew install redis
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -34,7 +58,7 @@ jobs:
       - name: Install package (Ubuntu)
         if: startsWith(matrix.os, 'ubuntu')
         run: |
-          pip install -e .[tests,lint] --no-binary pyzmq
+          pip install -e .[tests,lint] 
       - name: Install package (Mac)
         if: startsWith(matrix.os, 'macos')
         run: |
@@ -64,7 +88,7 @@ jobs:
 
       - name: Install package (Ubuntu)
         run: |
-          pip install -e .[lint] --no-binary pyzmq
+          pip install -e .[lint] 
 
       - name: Check format with Black
         uses: psf/black@stable
@@ -81,4 +105,4 @@ jobs:
       - name: Close parallel build
         uses: coverallsapp/github-action@v1
         with:
-          parallel-finished: true
+          parallel-finished: true

.github/workflows/pypi.yaml

@@ -5,6 +5,8 @@ on:
   push:
     tags: 
       - 'v[0-9]+.[0-9]+.[0-9]+'
+    branches:
+      - 'main'
 
 jobs:
   build:

.github/workflows/test-pypi.yaml

@@ -2,7 +2,11 @@
 # based on https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
 name: Publish Python 🐍 distribution 📦 to TestPyPI
 
-on: push
+on: 
+  push:
+    branches:
+      - 'main'
+      - 'dev'
 
 jobs:
   build:

.gitignore

@@ -124,12 +124,6 @@ dmypy.json
 # Pyre type checker
 .pyre/
 
-# LMDB
-*.mdb
-*.xml
-*.hdf5
-*.iml
-
 *.tif
 arrow
 

README.md

@@ -1,30 +1,39 @@
 # improv
+[![PyPI](https://img.shields.io/pypi/v/improv?style=flat-square?style=flat-square)](https://pypi.org/project/improv)
+[![PyPI - Python Version](https://img.shields.io/pypi/pyversions/improv?style=flat-square)](https://pypi.org/project/improv)
+[![docs](https://github.com/project-improv/improv/actions/workflows/docs.yaml/badge.svg?style=flat-square)](https://project-improv.github.io/)
+[![tests](https://github.com/project-improv/improv/actions/workflows/CI.yaml/badge.svg?style=flat-square)](https://project-improv.github.io/)
+[![Coverage Status](https://coveralls.io/repos/github/project-improv/improv/badge.svg?branch=main)](https://coveralls.io/github/project-improv/improv?branch=main)
+[![PyPI - License](https://img.shields.io/pypi/l/improv?style=flat-square)](https://opensource.org/licenses/MIT)
+[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg?style=flat-square)](https://github.com/psf/black)
+
 A flexible software platform for real-time and adaptive neuroscience experiments.
 
-improv is a streaming software platform designed to enable adaptive experiments. By analyzing data, such as 2-photon calcium images, as it comes in, we can obtain information about the current brain state in real time and use it to adaptively modify an experiment as data collection is ongoing. 
+_improv_ is a streaming software platform designed to enable adaptive experiments. By analyzing data, such as 2-photon calcium images, as it comes in, we can obtain information about the current brain state in real time and use it to adaptively modify an experiment as data collection is ongoing. 
 
-![](https://dibs-web01.vm.duke.edu/pearson/assets/videos/zebrafish/improvGif.gif)
+![](https://dibs.duke.edu/sites/default/files/improvGif.gif)
 
 This video shows raw 2-photon calcium imaging data in zebrafish, with cells detected in real time by [CaImAn](https://github.com/flatironinstitute/CaImAn), and directional tuning curves (shown as colored neurons) and functional connectivity (lines) estimated online, during a live experiment. Here only a few minutes of data have been acquired, and neurons are colored by their strongest response to visual simuli shown so far.
 We also provide up-to-the-moment estimates of the functional connectivity by fitting linear-nonlinear-Poisson models online, as each new piece of data is acquired. Simple visualizations offer real-time insights, allowing for adaptive experiments that change in response to the current state of the brain.
 
 
 ### How improv works
 
-<img src="figures/improv_design.png" width=85%>
+<img src="https://dibs.duke.edu/sites/default/files/improv_design.png" width=85%>
+
 improv allows users to flexibly specify and manage adaptive experiments to integrate data collection, preprocessing, visualization, and user-defined analytics. All kinds of behavioral, neural, or modeling data can be incorporated, and input and output data streams are managed independently and asynchronously. With this design, streaming analyses and real-time interventions can be easily integrated into various experimental setups. improv manages the backend engineering of data flow and task execution for all steps in an experimental pipeline in real time, without requiring user oversight. Users need only define their particular processing pipeline with simple text files and are free to define their own streaming analyses via Python classes, allowing for rapid prototyping of adaptive experiments.  
   <br />
   <br />
 
-<img src="https://dibs-web01.vm.duke.edu/pearson/assets/images/zebrafish/actor_model.png" width=60%>
+<img src="https://dibs.duke.edu/sites/default/files/actor_model.png" width=60%>
 
-improv's design is based on a steamlined version of the actor model for concurrent computation. Each component of the system (experimental pipeline) is considered an 'actor' and has a unique role. They interact via message passing, without the need for a central broker. Actors are implemented as user-defined classes that inherit from improv's Actor class, which supplies all queues for message passing and orchestrates process execution and error handling. Messages between actors are composed of keys that correspond to items in a shared, in-memory data store. This both minimizes communication overhead and data copying between processes. 
+_improv_'s design is based on a streamlined version of the actor model for concurrent computation. Each component of the system (experimental pipeline) is considered an 'actor' and has a unique role. They interact via message passing, without the need for a central broker. Actors are implemented as user-defined classes that inherit from _improv_'s `Actor` class, which supplies all queues for message passing and orchestrates process execution and error handling. Messages between actors are composed of keys that correspond to items in a shared, in-memory data store. This both minimizes communication overhead and data copying between processes. 
 
 
 
 ## Installation
 
-For installation instructions, please consult the [wiki](https://github.com/project-improv/improv/wiki/Installation) on our github.
+For installation instructions, please consult the [docs](https://project-improv.github.io/improv/installation.html) on our github.
 
 ### Contact
-To get in touch, feel free to reach out on Twitter <a href="http://twitter.com/annedraelos" target="_blank">@annedraelos</a>. 
+To get in touch, feel free to reach out on Twitter <a href="http://twitter.com/annedraelos" target="_blank">@annedraelos</a> or <a href="http://twitter.com/jmxpearson" target="_blank">@jmxpearson</a>. 

demos/bubblewrap/README.md

@@ -1,6 +1,6 @@
 # Running Bubblewrap demo
 ## Ubuntu users
-After installing improv, install additional dependencies (JAX on CPU by default) navigating to `demos/bubblewrap` and doing
+After installing _improv_, install additional dependencies (JAX on CPU by default) navigating to `demos/bubblewrap` and doing
 
 - `pip install -r requirements.txt`
 

demos/minimal/README.md

@@ -0,0 +1,67 @@
+# minimal demo
+
+This folder contains a minimal demo for running improv. In this demo, data generated by the **generator** `actor` is 
+stored in a data store and a key to the generated data is sent via a queue to the **processor** `actor` that accesses 
+the data in the store and processes it. 
+
+Usage:
+
+```bash
+# cd to this dir
+cd .../improv/demos/minimal
+
+# start improv
+improv run ./minimal.yaml
+
+# call `setup` in the improv TUI
+setup
+
+# call `run` in the improv TUI
+run
+
+# when you are ready to stop the process, call `stop` in the improv TUI
+stop
+```
+
+## Visualization using `fastplotlib`
+
+You can also run the minimal demo and visualize the generated data using `fastplotlib`.
+
+As before, data is generated by the **generator** `actor` is stored in a data store and a key to the generated data is 
+sent via a queue to the **processor** `actor` that accesses the data in the store and processes it. Additionally, the
+`fastplotlib.ipynb` notebook then receives the most recent data via `zmq` and displays it using 
+[`fastplotlib`](https://github.com/fastplotlib/fastplotlib).
+
+### Instructions
+
+1. Swap the **processor** `actor` found in `/improv/demos/sample_actors/visual/sample_processor.py` for the one in 
+the minimal demo
+
+2. Run `pip install -r requirements.txt` in this directory.
+
+Usage:
+
+```bash
+# cd to this dir
+cd .../improv/demos/minimal
+
+# start improv
+improv run ./minimal.yaml
+
+# call `setup` in the improv TUI
+setup
+
+# Run the cells in the jupyter notebook until you receive
+# a plot that has a white cosine wave
+
+# once the plot is ready call `run` in the improv TUI
+run
+
+# You should see the plot updating between a cosine and sine wave with different colors depending on 
+# whether the frame number is even or odd
+
+# when you are ready to stop the process, call `stop` in the improv TUI
+stop
+```
+
+#### Note: The `fastplotlib.ipynb` can only be run in `jupyter lab`

demos/minimal/actors/sample_generator.py

@@ -1,5 +1,4 @@
-from improv.actor import Actor, RunManager
-from datetime import date  # used for saving
+from improv.actor import Actor
 import numpy as np
 import logging
 
@@ -50,13 +49,20 @@ def runStep(self):
         """
 
         if self.frame_num < np.shape(self.data)[0]:
-            data_id = self.client.put(
-                self.data[self.frame_num], str(f"Gen_raw: {self.frame_num}")
-            )
+            if self.store_loc:
+                data_id = self.client.put(
+                    self.data[self.frame_num], str(f"Gen_raw: {self.frame_num}")
+                )
+            else:
+                data_id = self.client.put(self.data[self.frame_num])
             # logger.info('Put data in store')
             try:
-                self.q_out.put([[data_id, str(self.frame_num)]])
-                logger.info("Sent message on")
+                if self.store_loc:
+                    self.q_out.put([[data_id, str(self.frame_num)]])
+                else:
+                    self.q_out.put(data_id)
+                # logger.info("Sent message on")
+
                 self.frame_num += 1
             except Exception as e:
                 logger.error(

demos/minimal/actors/sample_processor.py

@@ -45,19 +45,23 @@ def runStep(self):
 
         frame = None
         try:
-            frame = self.q_in.get(timeout=0.001)
+            frame = self.q_in.get(timeout=0.05)
 
-        except:
+        except Exception:
             logger.error("Could not get frame!")
             pass
 
         if frame is not None and self.frame_num is not None:
             self.done = False
-            self.frame = self.client.getID(frame[0][0])
+            if self.store_loc:
+                self.frame = self.client.getID(frame[0][0])
+            else:
+                self.frame = self.client.get(frame)
             avg = np.mean(self.frame[0])
 
-            # print(f"Average: {avg}")
+            logger.info(f"Average: {avg}")
             self.avg_list.append(avg)
-            # print(f"Overall Average: {np.mean(self.avg_list)}")
-            # print(f"Frame number: {self.frame_num}")
-            self.frame_num += 1
+            logger.info(f"Overall Average: {np.mean(self.avg_list)}")
+            logger.info(f"Frame number: {self.frame_num}")
+
+            self.frame_num += 1

demos/minimal/actors/sample_spawn_processor.py

@@ -1,4 +1,4 @@
-from improv.actor import Actor, RunManager
+from improv.actor import Actor
 import numpy as np
 from queue import Empty
 import logging
@@ -39,17 +39,6 @@ def stop(self):
         logger.info("Processor stopping")
         return 0
 
-    # def run(self):
-    #     """ Send array into the store.
-    #     """
-    #     self.fcns = {}
-    #     self.fcns['setup'] = self.setup
-    #     self.fcns['run'] = self.runStep
-    #     self.fcns['stop'] = self.stop
-
-    #     with RunManager(self.name, self.fcns, self.links) as rm:
-    #         logger.info(rm)
-
     def runStep(self):
         """Gets from the input queue and calculates the average.
 
@@ -63,16 +52,21 @@ def runStep(self):
             frame = self.q_in.get(timeout=0.05)
         except Empty:
             pass
-        except:
+        except Exception:
             logger.error("Could not get frame!")
             pass
 
         if frame is not None and self.frame_num is not None:
             self.done = False
-            self.frame = self.client.getID(frame[0][0])
+            if self.store_loc:
+                self.frame = self.client.getID(frame[0][0])
+            else:
+                self.frame = self.client.get(frame)
             avg = np.mean(self.frame[0])
-            print(f"Average: {avg}")
+
+            logger.info(f"Average: {avg}")
             self.avg_list.append(avg)
-            print(f"Overall Average: {np.mean(self.avg_list)}")
-            print(f"Frame number: {self.frame_num}")
+            logger.info(f"Overall Average: {np.mean(self.avg_list)}")
+            logger.info(f"Frame number: {self.frame_num}")
+
             self.frame_num += 1

demos/minimal/minimal.yaml

@@ -8,4 +8,7 @@ actors:
     class: Processor
 
 connections:
-  Generator.q_out: [Processor.q_in]
+  Generator.q_out: [Processor.q_in]
+
+redis_config:
+  port: 6379

demos/minimal/minimal_plasma.yaml

@@ -0,0 +1,13 @@
+actors:
+  Generator:
+    package: actors.sample_generator
+    class: Generator
+
+  Processor:
+    package: actors.sample_processor
+    class: Processor
+
+connections:
+  Generator.q_out: [Processor.q_in]
+
+plasma_config:

demos/minimal/minimal_spawn.yaml

@@ -10,3 +10,6 @@ actors:
 
 connections:
   Generator.q_out: [Processor.q_in]
+
+redis_config:
+  port: 6378

demos/minimal/requirements.txt

@@ -0,0 +1,2 @@
+fastplotlib[notebook]
+simplejpeg

demos/sample_actors/analysis.py

@@ -16,8 +16,8 @@
 class MeanAnalysis(Actor):
     # TODO: Add additional error handling
     # TODO: this is too complex for a sample actor?
-    def __init__(self, *args):
-        super().__init__(*args)
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
 
     def setup(self, param_file=None):
         """Set custom parameters here