From 6f06eca4e1e185abdf98c5791481bbb8d79a4999 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 30 Aug 2023 15:45:28 +0100 Subject: [PATCH 01/21] WIP: Implementing dynamic retrieval of data --- src/Dashboard.js | 163 ++++++++++++++++++++++++++++++++++++++++---- src/util/helpers.js | 1 - 2 files changed, 148 insertions(+), 16 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 820b182..8ae1cb9 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -1,6 +1,6 @@ import React from "react"; import { Routes, Route, Link, HashRouter } from "react-router-dom"; -import { cloneDeep, has, set} from "lodash"; +import { cloneDeep, has, set } from "lodash"; import ControlPanel from "./components/ControlPanel/ControlPanel"; import OverlayContainer from "./components/OverlayContainer/OverlayContainer"; @@ -17,7 +17,7 @@ import styles from "./Dashboard.module.css"; import siteInfoJSON from "./data/siteInfo.json"; import deccMeasData from "./data/decc_example.json"; import { Button } from "@mui/material"; -import LaunchIcon from '@mui/icons-material/Launch'; +import LaunchIcon from "@mui/icons-material/Launch"; class Dashboard extends React.Component { constructor(props) { @@ -27,19 +27,18 @@ class Dashboard extends React.Component { error: null, isLoaded: false, showSidebar: false, - selectedDate: 0, processedData: {}, dataKeys: {}, selectedKeys: {}, - footprintView: true, emptySelection: true, overlayOpen: false, overlay: null, - plotType: "footprint", layoutMode: "dashboard", colours: {}, }; + this.dataRepoURL = "https://github.com/openghg/decc_dashboard_data/main/raw/"; + // Build the site info for the overlays this.buildSiteInfo(); @@ -75,6 +74,11 @@ class Dashboard extends React.Component { /* eslint-enable react/no-direct-mutation-state */ } + /** + * Selects a data source - a specific species at a site at an inlet + * + * @param {string} selection - Key for a data source + */ sourceSelector(selection) { let selectedSourcesSet = new Set(); @@ -98,10 +102,19 @@ class Dashboard extends React.Component { this.setState({ selectedSources: selectedSources }); } + /** + * Clear the currently selected data sources + */ clearSources() { this.setState({ selectedSources: new Set() }); } + + /** + * Selects a species + * + * @param {string} species - Species name + */ speciesSelector(species) { const speciesLower = species.toLowerCase(); @@ -146,7 +159,120 @@ class Dashboard extends React.Component { this.setState({ showSidebar: !this.state.showSidebar }); } - processData(rawData) { + /** + * Convert the data to a format recognised by Plotly + * + * @param {object} data - JSON data from pandas + * + * @returns {object} + * + */ + to_plotly(data) { + const x_timestamps = Object.keys(data); + const x_values = x_timestamps.map((d) => new Date(parseInt(d))); + const y_values = Object.values(data); + + const graphData = { + x_values: x_values, + y_values: y_values, + }; + + return graphData; + } + + /** + * Retrieves data from the given URL and processes it into a format + * plotly can read + * + * @param {string} url - URl of JSON file (could be gzipped) + * @param {boolean} compressed - is the file compressed + * + */ + retrieveData(url, compressed = false) {} + + /** + * Create the data structure for the retrieval of the separated + * out data + * + * @param {object} metadata - metadata object holding filenames for each chunk + * + */ + createDataStructure(metadata) { + // Create the datastructure from the file metadata object and populate it + // with the data for the default site, species, inlet + let dataKeys = {}; + let processedData = {}; + let siteStructure = {}; + + let defaultSpecies = null; + let defaultSourceKey = null; + let defaultInlet = null; + + let retrievedDefault = false; + + try { + for (const [species, networkData] of Object.entries(metadata)) { + if (!defaultSpecies) { + defaultSpecies = species; + } + for (const [network, siteData] of Object.entries(networkData)) { + for (const [site, inletData] of Object.entries(siteData)) { + for (const [inlet, instrumentData] of Object.entries(inletData)) { + if (!defaultInlet) { + defaultInlet = inlet; + } + for (const [instrument, fileInfo] of Object.entries(instrumentData)) { + // Data key + // TODO - why use underscores here? + const sourceKey = `${network}_${site}_${inlet}_${instrument}`; + // This is for the data dictionary that by default is only populated with one dataset + const dataKey = `${species}.${network}.${site}.${inlet}.${instrument}`; + // This uses the site info JSON so we can dynamically create the interface + // const sourceKey = `${network}.${site}.${inlet}.${instrument}` + + // This is the default plot we'll show when the site loads? + if (!defaultSourceKey) { + defaultSourceKey = sourceKey; + } + + // We create a nested object for easy automated creation of the interface + set(siteStructure, dataKey, sourceKey); + + // Let's retrieve the default data if we haven't already + if (!retrievedDefault) { + this.defaultDataKey = dataKey; + // Retrieve the default data + const url = this.dataRepoURL + fileInfo["filename"]; + const retrievedData = this.retrieveData(url); + set(processedData, dataKey, retrievedData); + retrievedDefault = true; + } else { + set(processedData, dataKey, null); + } + } + } + } + } + } + } catch (error) { + console.error(`Error processing raw data - ${error}`); + } + + // Disabled the no direct mutation rule here as this only gets called from the constructor + /* eslint-disable react/no-direct-mutation-state */ + // Give each site a colour + this.state.defaultSpecies = defaultSpecies; + this.state.defaultSourceKey = defaultSourceKey; + this.state.selectedSources = new Set([defaultSourceKey]); + this.state.selectedSpecies = defaultSpecies; + this.state.processedData = processedData; + this.state.selectedKeys = dataKeys; + this.state.isLoaded = true; + this.state.siteStructure = siteStructure; + /* eslint-enable react/no-direct-mutation-state */ + } + + processRawData(rawData) { // Process the data and create the correct Javascript time objects // expected by plotly @@ -196,17 +322,14 @@ class Dashboard extends React.Component { y_values: y_values, }; - const dataKey = `${species}.${sourceKey}`; - - - const combinedData = { data: graphData, metadata: metadata}; + const combinedData = { data: graphData, metadata: metadata }; + const dataKey = `${species}.${sourceKey}`; + // use lodash set set(processedData, dataKey, combinedData); -; } } } - } } } catch (error) { @@ -233,14 +356,16 @@ class Dashboard extends React.Component { } componentDidMount() { - this.processData(deccMeasData); + this.processRawData(deccMeasData); + // Retrieve the default data - we can just save the key for the default data so we don't have to loop + // through the whole structure to find it this.setState({ isLoaded: true }); // const apiURL = "https://raw.githubusercontent.com/openghg/dashboard_data/main/combined_data.json"; // fetch(apiURL) // .then((res) => res.json()) // .then( // (result) => { - // this.processData(result); + // this.processRawData(result); // this.setState({ // isLoaded: true, // }); @@ -324,7 +449,15 @@ class Dashboard extends React.Component {
- +
☰ diff --git a/src/util/helpers.js b/src/util/helpers.js index 19d2482..98abd36 100644 --- a/src/util/helpers.js +++ b/src/util/helpers.js @@ -90,7 +90,6 @@ export function importSiteImages() { } for (const path of paths) { - // Here we need to read the filename and convert it to a UNIX timestamp const filename = String(path).split("./")[1]; const sansExtension = String(filename).split(".")[0].toUpperCase(); From 1202f8c2e9f7438bc91d69a6b5b92ea80d462320 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 31 Aug 2023 10:08:08 +0100 Subject: [PATCH 02/21] WIP: Creating data structure with only the default data --- src/Dashboard.js | 70 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 8ae1cb9..8777a8e 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -15,7 +15,7 @@ import styles from "./Dashboard.module.css"; // Site description information import siteInfoJSON from "./data/siteInfo.json"; -import deccMeasData from "./data/decc_example.json"; +import completeMetadata from "./deccoutput/metadata_complete.json"; import { Button } from "@mui/material"; import LaunchIcon from "@mui/icons-material/Launch"; @@ -109,7 +109,6 @@ class Dashboard extends React.Component { this.setState({ selectedSources: new Set() }); } - /** * Selects a species * @@ -188,7 +187,71 @@ class Dashboard extends React.Component { * @param {boolean} compressed - is the file compressed * */ - retrieveData(url, compressed = false) {} + retrieveData(url, compressed = false) { + + } + + /** + * Create the data structure used to create the plots Plotly can read + */ + createDataStructure() { + // Loop over the metadata dictionary + // Create the + // This should aleady be in the right shape + this.completeMetadata = completeMetadata; + let defaultSpecies = null; + let defaultSite = null; + let defaultInlet = null; + // Not sure if we need default instrument but + let defaultInstrument = null; + let defaultNetwork = null; + // We just need to pull out the initial data + // Key format: metadata_complete[species][network][site][inlet][instrument] + // const defaultSpecies = Object.keys(completeMetadata)[0]; + // const defaultNetwork = Object.keys(completeMetadata[defaultSpecies])[0]; + // const defaultSite = Object.keys(completeMetadata[defaultSpecies][defaultNetwork])[0]; + // const defaultInlet = Object.keys(completeMetadata[defaultSpecies][defaultNetwork][defaultSite])[0]; + // const defaultInstrument = Object.keys(completeMetadata[defaultSpecies][defaultNetwork][defaultSite][defaultInlet])[0]; + + // This will hold the data itself + // It's structure is + // dataStore = { + // "species": { + // "network_site_inlet_instrument": {"x_values": [1,2,3], "y_values": [1,2,3]} + // } + // } + // We retrieve only the first dataset and then populate the other data values with nulls + // When this data is selected the app will retrieve the data + + let dataStore = {}; + + try { + for (const [species, networkData] of Object.entries(completeMetadata)) { + if (!defaultSpecies) defaultSpecies = species; + for (const [network, siteData] of Object.entries(networkData)) { + if (!defaultNetwork) defaultNetwork = network; + for (const [site, inletData] of Object.entries(siteData)) { + if (!defaultSite) defaultSite = site; + for (const [inlet, instrumentData] of Object.entries(inletData)) { + if (!defaultInlet) defaultInlet = inlet; + for (const [instrument, fileMetadata] of Object.entries(instrumentData)) { + const sourceKey = `${species}.${network}_${site}_${inlet}_${instrument}`; + let measurementData = null; + if (!defaultInstrument) { + defaultInstrument = instrument; + // const measurementData = retrieve_data + } + + set(dataStore, sourceKey, measurementData); + } + } + } + } + } + } catch (error) { + console.error(`Error processing raw data - ${error}`); + } + } /** * Create the data structure for the retrieval of the separated @@ -431,7 +494,6 @@ class Dashboard extends React.Component { } else { const liveData = ( Date: Thu, 31 Aug 2023 13:01:57 +0100 Subject: [PATCH 03/21] Added in async file retrieval and now adding metadata retrieval --- src/Dashboard.js | 235 +++++------------------- src/components/LeafletMap/LeafletMap.js | 4 +- src/components/LiveData/LiveData.js | 6 +- src/components/ObsBox/ObsBox.js | 8 +- 4 files changed, 58 insertions(+), 195 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 8777a8e..51a49ca 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -1,6 +1,6 @@ import React from "react"; import { Routes, Route, Link, HashRouter } from "react-router-dom"; -import { cloneDeep, has, set } from "lodash"; +import { cloneDeep, has, set, get } from "lodash"; import ControlPanel from "./components/ControlPanel/ControlPanel"; import OverlayContainer from "./components/OverlayContainer/OverlayContainer"; @@ -27,7 +27,6 @@ class Dashboard extends React.Component { error: null, isLoaded: false, showSidebar: false, - processedData: {}, dataKeys: {}, selectedKeys: {}, emptySelection: true, @@ -35,6 +34,7 @@ class Dashboard extends React.Component { overlay: null, layoutMode: "dashboard", colours: {}, + dataStore: {}, }; this.dataRepoURL = "https://github.com/openghg/decc_dashboard_data/main/raw/"; @@ -127,8 +127,8 @@ class Dashboard extends React.Component { } sourceSpeciesChange(species, oldSelectedSources) { - const processedData = this.state.processedData; - const speciesData = processedData[species]; + const dataStore = this.state.dataStore; + const speciesData = dataStore[species]; let newSources = new Set(); @@ -159,14 +159,33 @@ class Dashboard extends React.Component { } /** - * Convert the data to a format recognised by Plotly - * - * @param {object} data - JSON data from pandas + * Retrieves data from the given URL and processes it into a format + * plotly can read * - * @returns {object} + * @param {string} filename - Name of file to be retrieved from data store + * @param {string} species - Species + * @param {string} sourceKey - Source key + * @param {boolean} compressed - is the file compressed * */ - to_plotly(data) { + retrieveData(filename, species, sourceKey, compressed = false) { + const key = `${species}.${sourceKey}`; + const currentVal = get(this.state.dataStore, key); + if (currentVal !== null) { + console.log(`We already have data for ${species}.${sourceKey}`); + return; + } + // TODO - add quick check to see if we have the correct filename? + // Base URLs: + const url = new URL(filename, this.dataRepoURL).href; + + async function retrieveData(url) { + const res = await fetch(url); + return await res.json(); + } + + const data = retrieveData(url); + const x_timestamps = Object.keys(data); const x_values = x_timestamps.map((d) => new Date(parseInt(d))); const y_values = Object.values(data); @@ -176,19 +195,8 @@ class Dashboard extends React.Component { y_values: y_values, }; - return graphData; - } - - /** - * Retrieves data from the given URL and processes it into a format - * plotly can read - * - * @param {string} url - URl of JSON file (could be gzipped) - * @param {boolean} compressed - is the file compressed - * - */ - retrieveData(url, compressed = false) { - + // Add the data to the dataStore object + set(this.state.dataStore, key, graphData); } /** @@ -198,20 +206,15 @@ class Dashboard extends React.Component { // Loop over the metadata dictionary // Create the // This should aleady be in the right shape - this.completeMetadata = completeMetadata; + this.state.completeMetadata = completeMetadata; let defaultSpecies = null; let defaultSite = null; let defaultInlet = null; // Not sure if we need default instrument but let defaultInstrument = null; let defaultNetwork = null; + let defaultSourceKey = null; // We just need to pull out the initial data - // Key format: metadata_complete[species][network][site][inlet][instrument] - // const defaultSpecies = Object.keys(completeMetadata)[0]; - // const defaultNetwork = Object.keys(completeMetadata[defaultSpecies])[0]; - // const defaultSite = Object.keys(completeMetadata[defaultSpecies][defaultNetwork])[0]; - // const defaultInlet = Object.keys(completeMetadata[defaultSpecies][defaultNetwork][defaultSite])[0]; - // const defaultInstrument = Object.keys(completeMetadata[defaultSpecies][defaultNetwork][defaultSite][defaultInlet])[0]; // This will hold the data itself // It's structure is @@ -227,169 +230,28 @@ class Dashboard extends React.Component { try { for (const [species, networkData] of Object.entries(completeMetadata)) { - if (!defaultSpecies) defaultSpecies = species; + if (defaultSpecies === null) defaultSpecies = species; for (const [network, siteData] of Object.entries(networkData)) { - if (!defaultNetwork) defaultNetwork = network; + if (defaultNetwork === null) defaultNetwork = network; for (const [site, inletData] of Object.entries(siteData)) { - if (!defaultSite) defaultSite = site; + if (defaultSite === null) defaultSite = site; for (const [inlet, instrumentData] of Object.entries(inletData)) { - if (!defaultInlet) defaultInlet = inlet; + if (defaultInlet === null) defaultInlet = inlet; for (const [instrument, fileMetadata] of Object.entries(instrumentData)) { const sourceKey = `${species}.${network}_${site}_${inlet}_${instrument}`; - let measurementData = null; - if (!defaultInstrument) { - defaultInstrument = instrument; - // const measurementData = retrieve_data - } - - set(dataStore, sourceKey, measurementData); - } - } - } - } - } - } catch (error) { - console.error(`Error processing raw data - ${error}`); - } - } - - /** - * Create the data structure for the retrieval of the separated - * out data - * - * @param {object} metadata - metadata object holding filenames for each chunk - * - */ - createDataStructure(metadata) { - // Create the datastructure from the file metadata object and populate it - // with the data for the default site, species, inlet - let dataKeys = {}; - let processedData = {}; - let siteStructure = {}; - - let defaultSpecies = null; - let defaultSourceKey = null; - let defaultInlet = null; - - let retrievedDefault = false; - - try { - for (const [species, networkData] of Object.entries(metadata)) { - if (!defaultSpecies) { - defaultSpecies = species; - } - for (const [network, siteData] of Object.entries(networkData)) { - for (const [site, inletData] of Object.entries(siteData)) { - for (const [inlet, instrumentData] of Object.entries(inletData)) { - if (!defaultInlet) { - defaultInlet = inlet; - } - for (const [instrument, fileInfo] of Object.entries(instrumentData)) { - // Data key - // TODO - why use underscores here? - const sourceKey = `${network}_${site}_${inlet}_${instrument}`; - // This is for the data dictionary that by default is only populated with one dataset - const dataKey = `${species}.${network}.${site}.${inlet}.${instrument}`; - // This uses the site info JSON so we can dynamically create the interface - // const sourceKey = `${network}.${site}.${inlet}.${instrument}` - - // This is the default plot we'll show when the site loads? - if (!defaultSourceKey) { - defaultSourceKey = sourceKey; - } - // We create a nested object for easy automated creation of the interface - set(siteStructure, dataKey, sourceKey); - - // Let's retrieve the default data if we haven't already - if (!retrievedDefault) { - this.defaultDataKey = dataKey; - // Retrieve the default data - const url = this.dataRepoURL + fileInfo["filename"]; - const retrievedData = this.retrieveData(url); - set(processedData, dataKey, retrievedData); - retrievedDefault = true; - } else { - set(processedData, dataKey, null); - } - } - } - } - } - } - } catch (error) { - console.error(`Error processing raw data - ${error}`); - } + if (defaultSourceKey === null) defaultSourceKey = sourceKey; - // Disabled the no direct mutation rule here as this only gets called from the constructor - /* eslint-disable react/no-direct-mutation-state */ - // Give each site a colour - this.state.defaultSpecies = defaultSpecies; - this.state.defaultSourceKey = defaultSourceKey; - this.state.selectedSources = new Set([defaultSourceKey]); - this.state.selectedSpecies = defaultSpecies; - this.state.processedData = processedData; - this.state.selectedKeys = dataKeys; - this.state.isLoaded = true; - this.state.siteStructure = siteStructure; - /* eslint-enable react/no-direct-mutation-state */ - } - - processRawData(rawData) { - // Process the data and create the correct Javascript time objects - // expected by plotly - - // NOTE - I use data source here, please don't confuse with an OpenGHG Datasource, - let dataKeys = {}; - let processedData = {}; - let siteStructure = {}; - - let defaultSpecies = null; - let defaultSourceKey = null; - let defaultNetwork = null; + let measurementData = null; - try { - for (const [species, networkData] of Object.entries(rawData)) { - if (!defaultSpecies) { - defaultSpecies = species; - } - for (const [network, siteData] of Object.entries(networkData)) { - if (!defaultNetwork) { - defaultNetwork = network; - } - for (const [site, inletData] of Object.entries(siteData)) { - for (const [inlet, instrumentData] of Object.entries(inletData)) { - for (const [instrument, measurementData] of Object.entries(instrumentData)) { - // Data key - const sourceKey = `${network}_${site}_${inlet}_${instrument}`; - const nestedPath = `${species}.${network}.${site}.${inlet}.${instrument}`; + if (!defaultInstrument) { + defaultInstrument = instrument; - if (!defaultSourceKey) { - defaultSourceKey = sourceKey; + const filename = fileMetadata["filename"]; + this.retrieveData(filename, species, sourceKey); } - // We create a nested object for easy automated creation of the interface - set(siteStructure, nestedPath, sourceKey); - - // Create the data structures expected by plotly - const timeseriesData = measurementData["data"]; - const metadata = measurementData["metadata"]; - const rawData = timeseriesData[species]; - - const x_timestamps = Object.keys(rawData); - const x_values = x_timestamps.map((d) => new Date(parseInt(d))); - const y_values = Object.values(rawData); - - const graphData = { - x_values: x_values, - y_values: y_values, - }; - - const combinedData = { data: graphData, metadata: metadata }; - - const dataKey = `${species}.${sourceKey}`; - // use lodash set - set(processedData, dataKey, combinedData); + set(dataStore, sourceKey, measurementData); } } } @@ -399,18 +261,17 @@ class Dashboard extends React.Component { console.error(`Error processing raw data - ${error}`); } + // Should we use setState here? Does that work properly now? // Disabled the no direct mutation rule here as this only gets called from the constructor /* eslint-disable react/no-direct-mutation-state */ // Give each site a colour + this.state.dataStore = dataStore; this.state.defaultSpecies = defaultSpecies; - this.state.defaultNetwork = defaultNetwork; this.state.defaultSourceKey = defaultSourceKey; this.state.selectedSources = new Set([defaultSourceKey]); this.state.selectedSpecies = defaultSpecies; - this.state.processedData = processedData; this.state.selectedKeys = dataKeys; this.state.isLoaded = true; - this.state.siteStructure = siteStructure; /* eslint-enable react/no-direct-mutation-state */ } @@ -419,7 +280,9 @@ class Dashboard extends React.Component { } componentDidMount() { - this.processRawData(deccMeasData); + // Retrieve the metadata + const metadataURL = this. + // Retrieve the default data - we can just save the key for the default data so we don't have to loop // through the whole structure to find it this.setState({ isLoaded: true }); @@ -497,7 +360,7 @@ class Dashboard extends React.Component { clearSources={this.clearSources} speciesSelector={this.speciesSelector} sourceSelector={this.sourceSelector} - processedData={this.state.processedData} + dataStore={this.state.dataStore} selectedSources={this.state.selectedSources} selectedKeys={this.state.selectedKeys} selectedSpecies={this.state.selectedSpecies} diff --git a/src/components/LeafletMap/LeafletMap.js b/src/components/LeafletMap/LeafletMap.js index f419a90..c6f1c88 100644 --- a/src/components/LeafletMap/LeafletMap.js +++ b/src/components/LeafletMap/LeafletMap.js @@ -22,12 +22,12 @@ class LeafletMap extends React.Component { } createMarkers() { - const processedData = this.props.processedData; + const dataStore = this.props.dataStore; const siteStructure = this.props.siteStructure; const selectedSpecies = this.props.selectedSpecies; const speciesStructure = siteStructure[selectedSpecies]; - const speciesData = processedData[selectedSpecies]; + const speciesData = dataStore[selectedSpecies]; let markers = []; diff --git a/src/components/LiveData/LiveData.js b/src/components/LiveData/LiveData.js index 0a132a2..e8572a5 100644 --- a/src/components/LiveData/LiveData.js +++ b/src/components/LiveData/LiveData.js @@ -28,7 +28,7 @@ class LiveData extends React.Component { clearSources={this.props.clearSources} speciesSelector={this.props.speciesSelector} selectedSources={this.props.selectedSources} - processedData={this.props.processedData} + dataStore={this.props.dataStore} selectedSpecies={this.props.selectedSpecies} defaultSpecies={this.props.defaultSpecies} /> @@ -42,7 +42,7 @@ class LiveData extends React.Component { selectedSpecies={this.props.selectedSpecies} centre={mapCentre} zoom={5} - processedData={this.props.processedData} + dataStore={this.props.dataStore} siteInfoOverlay={this.props.setSiteOverlay} siteStructure={this.props.siteStructure} /> @@ -55,7 +55,7 @@ class LiveData extends React.Component { LiveData.propTypes = { clearSources: PropTypes.func.isRequired, defaultSpecies: PropTypes.string.isRequired, - processedData: PropTypes.object.isRequired, + dataStore: PropTypes.object.isRequired, selectedSources: PropTypes.object.isRequired, selectedSpecies: PropTypes.string.isRequired, setSiteOverlay: PropTypes.func.isRequired, diff --git a/src/components/ObsBox/ObsBox.js b/src/components/ObsBox/ObsBox.js index 1394ff4..2422f00 100644 --- a/src/components/ObsBox/ObsBox.js +++ b/src/components/ObsBox/ObsBox.js @@ -11,7 +11,7 @@ import { Button } from "@mui/material"; class ObsBox extends React.Component { createEmissionsGraphs() { - const processedData = this.props.processedData; + const dataStore = this.props.dataStore; const selectedSources = this.props.selectedSources; const selectedSpecies = this.props.selectedSpecies; @@ -25,7 +25,7 @@ class ObsBox extends React.Component { let multiUnits = []; if (selectedSources) { - const speciesData = processedData[selectedSpecies]; + const speciesData = dataStore[selectedSpecies]; for (const key of selectedSources) { dataToPlot[key] = speciesData[key]; @@ -88,7 +88,7 @@ class ObsBox extends React.Component { clearButton = ; } - const availableSpecies = Object.keys(this.props.processedData); + const availableSpecies = Object.keys(this.props.dataStore); return (
@@ -108,7 +108,7 @@ class ObsBox extends React.Component { ObsBox.propTypes = { clearSources: PropTypes.func.isRequired, - processedData: PropTypes.object.isRequired, + dataStore: PropTypes.object.isRequired, selectedSources: PropTypes.object.isRequired, selectedSpecies: PropTypes.string.isRequired, speciesSelector: PropTypes.func.isRequired, From 38e3023916c7ffcdbbb68ab78bc8446b75545560 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 31 Aug 2023 13:10:10 +0100 Subject: [PATCH 04/21] Tidy code layout --- src/Dashboard.js | 217 ++++++++++++++++++++++++----------------------- 1 file changed, 111 insertions(+), 106 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 51a49ca..807e2a9 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -54,111 +54,9 @@ class Dashboard extends React.Component { this.setSiteOverlay = this.setSiteOverlay.bind(this); } - buildSiteInfo() { - const siteImages = importSiteImages(); - - let siteData = {}; - for (const site of Object.keys(siteInfoJSON)) { - try { - siteData[site] = {}; - siteData[site]["image"] = siteImages[site]; - siteData[site]["description"] = siteInfoJSON[site]["description"]; - } catch (error) { - console.error(error); - } - } - - // Disabled the no direct mutation rule here as this only gets called from the constructor - /* eslint-disable react/no-direct-mutation-state */ - this.state.siteInfo = siteData; - /* eslint-enable react/no-direct-mutation-state */ - } - - /** - * Selects a data source - a specific species at a site at an inlet - * - * @param {string} selection - Key for a data source - */ - sourceSelector(selection) { - let selectedSourcesSet = new Set(); + // These handle the initial setup of the app - if (selection instanceof Set) { - selectedSourcesSet = selection; - } else { - selectedSourcesSet.add(selection); - } - - // Here we change all the sites and select all species / sectors at that site - let selectedSources = cloneDeep(this.state.selectedSources); - - for (const source of selectedSourcesSet) { - if (selectedSources.has(source)) { - selectedSources.delete(source); - } else { - selectedSources.add(source); - } - } - - this.setState({ selectedSources: selectedSources }); - } - - /** - * Clear the currently selected data sources - */ - clearSources() { - this.setState({ selectedSources: new Set() }); - } - - /** - * Selects a species - * - * @param {string} species - Species name - */ - speciesSelector(species) { - const speciesLower = species.toLowerCase(); - - const selectedSourcesClone = cloneDeep(this.state.selectedSources); - - this.setState({ selectedSources: new Set() }, () => { - this.sourceSpeciesChange(species, selectedSourcesClone); - }); - - this.setState({ selectedSpecies: speciesLower }); - } - - sourceSpeciesChange(species, oldSelectedSources) { - const dataStore = this.state.dataStore; - const speciesData = dataStore[species]; - - let newSources = new Set(); - - for (const sourceKey of oldSelectedSources) { - if (has(speciesData, sourceKey)) { - newSources.add(sourceKey); - } - } - - if (newSources.size === 0) { - const defaultSource = Object.keys(speciesData)[0]; - newSources.add(defaultSource); - } - - this.sourceSelector(newSources); - } - - toggleOverlay() { - this.setState({ overlayOpen: !this.state.overlayOpen }); - } - - setOverlay(overlay) { - this.setState({ overlayOpen: true, overlay: overlay }); - } - - toggleSidebar() { - this.setState({ showSidebar: !this.state.showSidebar }); - } - - /** + /** * Retrieves data from the given URL and processes it into a format * plotly can read * @@ -168,7 +66,7 @@ class Dashboard extends React.Component { * @param {boolean} compressed - is the file compressed * */ - retrieveData(filename, species, sourceKey, compressed = false) { + retrieveData(filename, species, sourceKey, compressed = false) { const key = `${species}.${sourceKey}`; const currentVal = get(this.state.dataStore, key); if (currentVal !== null) { @@ -267,7 +165,6 @@ class Dashboard extends React.Component { // Give each site a colour this.state.dataStore = dataStore; this.state.defaultSpecies = defaultSpecies; - this.state.defaultSourceKey = defaultSourceKey; this.state.selectedSources = new Set([defaultSourceKey]); this.state.selectedSpecies = defaultSpecies; this.state.selectedKeys = dataKeys; @@ -275,6 +172,114 @@ class Dashboard extends React.Component { /* eslint-enable react/no-direct-mutation-state */ } + buildSiteInfo() { + const siteImages = importSiteImages(); + + let siteData = {}; + for (const site of Object.keys(siteInfoJSON)) { + try { + siteData[site] = {}; + siteData[site]["image"] = siteImages[site]; + siteData[site]["description"] = siteInfoJSON[site]["description"]; + } catch (error) { + console.error(error); + } + } + + // Disabled the no direct mutation rule here as this only gets called from the constructor + /* eslint-disable react/no-direct-mutation-state */ + this.state.siteInfo = siteData; + /* eslint-enable react/no-direct-mutation-state */ + } + + // These handle app control by the components + + /** + * Selects a data source - a specific species at a site at an inlet + * + * @param {string} selection - Key for a data source + */ + sourceSelector(selection) { + let selectedSourcesSet = new Set(); + + if (selection instanceof Set) { + selectedSourcesSet = selection; + } else { + selectedSourcesSet.add(selection); + } + + // Here we change all the sites and select all species / sectors at that site + let selectedSources = cloneDeep(this.state.selectedSources); + + for (const source of selectedSourcesSet) { + if (selectedSources.has(source)) { + selectedSources.delete(source); + } else { + selectedSources.add(source); + } + } + + this.setState({ selectedSources: selectedSources }); + } + + /** + * Clear the currently selected data sources + */ + clearSources() { + this.setState({ selectedSources: new Set() }); + } + + /** + * Selects a species + * + * @param {string} species - Species name + */ + speciesSelector(species) { + const speciesLower = species.toLowerCase(); + + const selectedSourcesClone = cloneDeep(this.state.selectedSources); + + this.setState({ selectedSources: new Set() }, () => { + this.sourceSpeciesChange(species, selectedSourcesClone); + }); + + this.setState({ selectedSpecies: speciesLower }); + } + + sourceSpeciesChange(species, oldSelectedSources) { + const dataStore = this.state.dataStore; + const speciesData = dataStore[species]; + + let newSources = new Set(); + + for (const sourceKey of oldSelectedSources) { + if (has(speciesData, sourceKey)) { + newSources.add(sourceKey); + } + } + + if (newSources.size === 0) { + const defaultSource = Object.keys(speciesData)[0]; + newSources.add(defaultSource); + } + + this.sourceSelector(newSources); + } + + toggleOverlay() { + this.setState({ overlayOpen: !this.state.overlayOpen }); + } + + setOverlay(overlay) { + this.setState({ overlayOpen: true, overlay: overlay }); + } + + toggleSidebar() { + this.setState({ showSidebar: !this.state.showSidebar }); + } + + + dataSelector(dataKeys) { this.setState({ selectedKeys: dataKeys }); } From 4eddfe0ca2d885464584d10fe39458538d2d98d4 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 31 Aug 2023 15:10:57 +0100 Subject: [PATCH 05/21] WIP: Adding async data stuff --- src/Dashboard.js | 124 ++++++++++++++-------------- src/components/LiveData/LiveData.js | 2 +- 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 807e2a9..d59dc58 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -19,6 +19,11 @@ import completeMetadata from "./deccoutput/metadata_complete.json"; import { Button } from "@mui/material"; import LaunchIcon from "@mui/icons-material/Launch"; +async function fetchData(url) { + const res = await fetch(url); + return await res.json(); +} + class Dashboard extends React.Component { constructor(props) { super(props); @@ -27,7 +32,6 @@ class Dashboard extends React.Component { error: null, isLoaded: false, showSidebar: false, - dataKeys: {}, selectedKeys: {}, emptySelection: true, overlayOpen: false, @@ -37,26 +41,28 @@ class Dashboard extends React.Component { dataStore: {}, }; - this.dataRepoURL = "https://github.com/openghg/decc_dashboard_data/main/raw/"; - - // Build the site info for the overlays - this.buildSiteInfo(); + this.dataRepoURL = "https://raw.githubusercontent.com/openghg/decc_dashaboard_data/main"; - // Select the data - this.dataSelector = this.dataSelector.bind(this); - // Selects the dates this.sourceSelector = this.sourceSelector.bind(this); this.toggleOverlay = this.toggleOverlay.bind(this); this.setOverlay = this.setOverlay.bind(this); this.speciesSelector = this.speciesSelector.bind(this); this.clearSources = this.clearSources.bind(this); this.toggleSidebar = this.toggleSidebar.bind(this); - this.setSiteOverlay = this.setSiteOverlay.bind(this); + // this.setSiteOverlay = this.setSiteOverlay.bind(this); } // These handle the initial setup of the app - /** + storeData(species, sourceKey, data) { + this.setState(prevState => { + let previous = Object.assign({}, prevState[species]); + previous[sourceKey] = data; + return { previous }; + }) + } + + /** * Retrieves data from the given URL and processes it into a format * plotly can read * @@ -66,7 +72,7 @@ class Dashboard extends React.Component { * @param {boolean} compressed - is the file compressed * */ - retrieveData(filename, species, sourceKey, compressed = false) { + retrieveData(filename, species, sourceKey, compressed = false) { const key = `${species}.${sourceKey}`; const currentVal = get(this.state.dataStore, key); if (currentVal !== null) { @@ -77,12 +83,10 @@ class Dashboard extends React.Component { // Base URLs: const url = new URL(filename, this.dataRepoURL).href; - async function retrieveData(url) { - const res = await fetch(url); - return await res.json(); - } + // Fetch the data and store it + const data = fetchData(url).then((result) => { - const data = retrieveData(url); + }); const x_timestamps = Object.keys(data); const x_values = x_timestamps.map((d) => new Date(parseInt(d))); @@ -167,12 +171,45 @@ class Dashboard extends React.Component { this.state.defaultSpecies = defaultSpecies; this.state.selectedSources = new Set([defaultSourceKey]); this.state.selectedSpecies = defaultSpecies; - this.state.selectedKeys = dataKeys; this.state.isLoaded = true; /* eslint-enable react/no-direct-mutation-state */ } + componentDidMount() { + // Retrieve the metadata + + const metadataURL = new URL("fileMetadata.json", this.dataRepoURL).href + + // const + + // this// Retrieve the default data - we can just save the key for the default data so we don't have to loop + // // through the whole structure to find it + // .this + // .setState({ isLoaded: true }); + // const apiURL = "https://raw.githubusercontent.com/openghg/dashboard_data/main/combined_data.json"; + // fetch(apiURL) + // .then((res) => res.json()) + // .then( + // (result) => { + // this.processRawData(result); + // this.setState({ + // isLoaded: true, + // }); + // }, + // (error) => { + // this.setState({ + // isLoaded: true, + // error, + // }); + // } + // ); + } + + /** + * @deprecated Deprecated in the DECC Dashboard + */ buildSiteInfo() { + console.warn("This function may be removed as it unused in this version of the dashboard."); const siteImages = importSiteImages(); let siteData = {}; @@ -246,9 +283,14 @@ class Dashboard extends React.Component { this.setState({ selectedSpecies: speciesLower }); } + /** + * Selects a species + * + * @param {string} species - Species name + * @param {Set} oldSelectedSources - Set of previously selected sources + */ sourceSpeciesChange(species, oldSelectedSources) { - const dataStore = this.state.dataStore; - const speciesData = dataStore[species]; + const speciesData = this.state.dataStore[species]; let newSources = new Set(); @@ -278,49 +320,7 @@ class Dashboard extends React.Component { this.setState({ showSidebar: !this.state.showSidebar }); } - - - dataSelector(dataKeys) { - this.setState({ selectedKeys: dataKeys }); - } - - componentDidMount() { - // Retrieve the metadata - const metadataURL = this. - - // Retrieve the default data - we can just save the key for the default data so we don't have to loop - // through the whole structure to find it - this.setState({ isLoaded: true }); - // const apiURL = "https://raw.githubusercontent.com/openghg/dashboard_data/main/combined_data.json"; - // fetch(apiURL) - // .then((res) => res.json()) - // .then( - // (result) => { - // this.processRawData(result); - // this.setState({ - // isLoaded: true, - // }); - // }, - // (error) => { - // this.setState({ - // isLoaded: true, - // error, - // }); - // } - // ); - } - - anySelected() { - for (const subdict of Object.values(this.state.selectedKeys)) { - for (const value of Object.values(subdict)) { - if (value === true) { - return true; - } - } - } - - return false; - } + setSiteOverlay(e) { const siteCode = String(e.target.dataset.onclickparam).toUpperCase(); diff --git a/src/components/LiveData/LiveData.js b/src/components/LiveData/LiveData.js index e8572a5..eaee4fc 100644 --- a/src/components/LiveData/LiveData.js +++ b/src/components/LiveData/LiveData.js @@ -43,7 +43,7 @@ class LiveData extends React.Component { centre={mapCentre} zoom={5} dataStore={this.props.dataStore} - siteInfoOverlay={this.props.setSiteOverlay} + // siteInfoOverlay={this.props.setSiteOverlay} siteStructure={this.props.siteStructure} />
From 1f94e14f2383cdfed3ff85fa506de79ee7f00ceb Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Fri, 1 Sep 2023 14:20:17 +0100 Subject: [PATCH 06/21] Fix async retrieve --- src/Dashboard.js | 74 +++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 42 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index d59dc58..488d43c 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -19,9 +19,8 @@ import completeMetadata from "./deccoutput/metadata_complete.json"; import { Button } from "@mui/material"; import LaunchIcon from "@mui/icons-material/Launch"; -async function fetchData(url) { - const res = await fetch(url); - return await res.json(); +async function retrieveJSON(url) { + return await (await fetch(url)).json(); } class Dashboard extends React.Component { @@ -53,13 +52,12 @@ class Dashboard extends React.Component { } // These handle the initial setup of the app - - storeData(species, sourceKey, data) { - this.setState(prevState => { + addDataToStore(species, sourceKey, data) { + this.setState((prevState) => { let previous = Object.assign({}, prevState[species]); - previous[sourceKey] = data; + previous[sourceKey] = data; return { previous }; - }) + }); } /** @@ -84,9 +82,7 @@ class Dashboard extends React.Component { const url = new URL(filename, this.dataRepoURL).href; // Fetch the data and store it - const data = fetchData(url).then((result) => { - - }); + const data = fetchData(url).then((result) => {}); const x_timestamps = Object.keys(data); const x_values = x_timestamps.map((d) => new Date(parseInt(d))); @@ -104,11 +100,11 @@ class Dashboard extends React.Component { /** * Create the data structure used to create the plots Plotly can read */ - createDataStructure() { + populateAndRetrieve(metadata) { // Loop over the metadata dictionary // Create the // This should aleady be in the right shape - this.state.completeMetadata = completeMetadata; + this.state.completeMetadata = metadata; let defaultSpecies = null; let defaultSite = null; let defaultInlet = null; @@ -150,7 +146,10 @@ class Dashboard extends React.Component { defaultInstrument = instrument; const filename = fileMetadata["filename"]; - this.retrieveData(filename, species, sourceKey); + const url = new URL(filename, this.dataRepoURL) + retrieveJSON(url).then((result) => { + this.addDataToStore(species, sourceKey, result) + }) } set(dataStore, sourceKey, measurementData); @@ -177,32 +176,25 @@ class Dashboard extends React.Component { componentDidMount() { // Retrieve the metadata - - const metadataURL = new URL("fileMetadata.json", this.dataRepoURL).href - - // const - - // this// Retrieve the default data - we can just save the key for the default data so we don't have to loop - // // through the whole structure to find it - // .this - // .setState({ isLoaded: true }); - // const apiURL = "https://raw.githubusercontent.com/openghg/dashboard_data/main/combined_data.json"; - // fetch(apiURL) - // .then((res) => res.json()) - // .then( - // (result) => { - // this.processRawData(result); - // this.setState({ - // isLoaded: true, - // }); - // }, - // (error) => { - // this.setState({ - // isLoaded: true, - // error, - // }); - // } - // ); + // const metadataURL = new URL("fileMetadata.json", this.dataRepoURL).href + + const metadataURL = + "https://gist.githubusercontent.com/gareth-j/328fa8a1b5d61ed3a543710b10de4ddc/raw/6f616b8046b6bc266b82c3bdfc0ceaa198f0bbb5/metadata_complete.json"; + + retrieveJSON(metadataURL).then( + (result) => { + this.populateAndRetrieve(result); + this.setState({ + isLoaded: true, + }); + }, + (error) => { + this.setState({ + isLoaded: true, + error, + }); + } + ); } /** @@ -320,8 +312,6 @@ class Dashboard extends React.Component { this.setState({ showSidebar: !this.state.showSidebar }); } - - setSiteOverlay(e) { const siteCode = String(e.target.dataset.onclickparam).toUpperCase(); const siteInfo = this.state.siteInfo[siteCode]; From 20529b1929f8b0009392763c6603a765e19bb4cc Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Fri, 1 Sep 2023 14:50:20 +0100 Subject: [PATCH 07/21] Reduce number of functions --- src/Dashboard.js | 50 ++++++++++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 488d43c..d896b1f 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -52,49 +52,57 @@ class Dashboard extends React.Component { } // These handle the initial setup of the app + + /** + * Adds data to the data store, converting it to the structure required by Plotly + * + * @param {string} species - Species + * @param {string} sourceKey - Source key + * @param {string} data - Data that's been passed through the to_plotly function + * + */ addDataToStore(species, sourceKey, data) { + const x_timestamps = Object.keys(data); + const x_values = x_timestamps.map((d) => new Date(parseInt(d))); + const y_values = Object.values(data); + + const forPlotly = { + x_values: x_values, + y_values: y_values, + }; + this.setState((prevState) => { let previous = Object.assign({}, prevState[species]); - previous[sourceKey] = data; + previous[sourceKey] = forPlotly; return { previous }; }); } + /** * Retrieves data from the given URL and processes it into a format * plotly can read * - * @param {string} filename - Name of file to be retrieved from data store + * @param {string} filename - Name of file to be retrieved from remote data store * @param {string} species - Species * @param {string} sourceKey - Source key - * @param {boolean} compressed - is the file compressed * */ - retrieveData(filename, species, sourceKey, compressed = false) { + retrieveData(filename, species, sourceKey) { const key = `${species}.${sourceKey}`; const currentVal = get(this.state.dataStore, key); if (currentVal !== null) { console.log(`We already have data for ${species}.${sourceKey}`); return; } + // TODO - add quick check to see if we have the correct filename? // Base URLs: const url = new URL(filename, this.dataRepoURL).href; - // Fetch the data and store it - const data = fetchData(url).then((result) => {}); - - const x_timestamps = Object.keys(data); - const x_values = x_timestamps.map((d) => new Date(parseInt(d))); - const y_values = Object.values(data); - - const graphData = { - x_values: x_values, - y_values: y_values, - }; - - // Add the data to the dataStore object - set(this.state.dataStore, key, graphData); + retrieveJSON(url).then((result) => { + + }) } /** @@ -146,10 +154,10 @@ class Dashboard extends React.Component { defaultInstrument = instrument; const filename = fileMetadata["filename"]; - const url = new URL(filename, this.dataRepoURL) + const url = new URL(filename, this.dataRepoURL); retrieveJSON(url).then((result) => { - this.addDataToStore(species, sourceKey, result) - }) + this.addDataToStore(species, sourceKey, result); + }); } set(dataStore, sourceKey, measurementData); From 8af92367a0e5f12249cb2e6958d6a4628bd8704b Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Fri, 1 Sep 2023 14:59:01 +0100 Subject: [PATCH 08/21] Add data to store WIP.. --- src/Dashboard.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index d896b1f..81f5fb6 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -78,7 +78,6 @@ class Dashboard extends React.Component { }); } - /** * Retrieves data from the given URL and processes it into a format * plotly can read @@ -96,13 +95,11 @@ class Dashboard extends React.Component { return; } - // TODO - add quick check to see if we have the correct filename? - // Base URLs: const url = new URL(filename, this.dataRepoURL).href; retrieveJSON(url).then((result) => { - - }) + this.addDataToStore(species, sourceKey, result); + }); } /** From 23588f70fa7d596247b0928b63a2fac9ba7d7feb Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Fri, 1 Sep 2023 15:37:58 +0100 Subject: [PATCH 09/21] WIP: Adding partial update of state --- src/Dashboard.js | 57 +++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 20 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 81f5fb6..73e1a47 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -61,7 +61,7 @@ class Dashboard extends React.Component { * @param {string} data - Data that's been passed through the to_plotly function * */ - addDataToStore(species, sourceKey, data) { + addDataToStore(sourceKey, data) { const x_timestamps = Object.keys(data); const x_values = x_timestamps.map((d) => new Date(parseInt(d))); const y_values = Object.values(data); @@ -71,9 +71,11 @@ class Dashboard extends React.Component { y_values: y_values, }; + // TODO - can we do this better so we don't end up copying all the data each time? + // Will this work? this.setState((prevState) => { - let previous = Object.assign({}, prevState[species]); - previous[sourceKey] = forPlotly; + let previous = {...prevState.dataSource.sourceKey} + previous = forPlotly return { previous }; }); } @@ -87,28 +89,37 @@ class Dashboard extends React.Component { * @param {string} sourceKey - Source key * */ - retrieveData(filename, species, sourceKey) { - const key = `${species}.${sourceKey}`; - const currentVal = get(this.state.dataStore, key); + retrieveData(filename, sourceKey) { + const currentVal = get(this.state.dataStore, sourceKey); if (currentVal !== null) { - console.log(`We already have data for ${species}.${sourceKey}`); + console.log(`We already have data for ${sourceKey}`); return; } const url = new URL(filename, this.dataRepoURL).href; retrieveJSON(url).then((result) => { - this.addDataToStore(species, sourceKey, result); + this.addDataToStore(sourceKey, result); }); } /** * Create the data structure used to create the plots Plotly can read + * We create the dataStore object which has the following structure + * + * dataStore = { + * "species": { + * "network_site_inlet_instrument": {"x_values": [1,2,3], "y_values": [1,2,3]} + * } + * } + * */ populateAndRetrieve(metadata) { // Loop over the metadata dictionary // Create the - // This should aleady be in the right shape + // This should aleady be in the right shape, we use the nested aspect of + // the dictionary to make it easy to populate the interface, creating + // some lookup tables for filenames etc below this.state.completeMetadata = metadata; let defaultSpecies = null; let defaultSite = null; @@ -121,15 +132,12 @@ class Dashboard extends React.Component { // This will hold the data itself // It's structure is - // dataStore = { - // "species": { - // "network_site_inlet_instrument": {"x_values": [1,2,3], "y_values": [1,2,3]} - // } - // } + // We retrieve only the first dataset and then populate the other data values with nulls // When this data is selected the app will retrieve the data let dataStore = {}; + let filenameLookup = {}; try { for (const [species, networkData] of Object.entries(completeMetadata)) { @@ -141,23 +149,24 @@ class Dashboard extends React.Component { for (const [inlet, instrumentData] of Object.entries(inletData)) { if (defaultInlet === null) defaultInlet = inlet; for (const [instrument, fileMetadata] of Object.entries(instrumentData)) { - const sourceKey = `${species}.${network}_${site}_${inlet}_${instrument}`; - - if (defaultSourceKey === null) defaultSourceKey = sourceKey; + // The complete source key should always have the species + const completeSourceKey = `${species}.${network}_${site}_${inlet}_${instrument}`; + if (defaultSourceKey === null) defaultSourceKey = completeSourceKey; let measurementData = null; + const filename = fileMetadata["filename"]; if (!defaultInstrument) { defaultInstrument = instrument; - const filename = fileMetadata["filename"]; const url = new URL(filename, this.dataRepoURL); retrieveJSON(url).then((result) => { - this.addDataToStore(species, sourceKey, result); + this.addDataToStore(species, completeSourceKey, result); }); } - set(dataStore, sourceKey, measurementData); + set(dataStore, completeSourceKey, measurementData); + set(filenameLookup, completeSourceKey, filename); } } } @@ -253,6 +262,14 @@ class Dashboard extends React.Component { } } + // Now let's make sure we have all the data for these new selections + // the source will be the key in the + for (const source of selectedSources) { + const key = `${this.state.selectedSpecies}.${source}`; + const filename = this.state.filenameLookup[key] + this.retrieveData(); + } + this.setState({ selectedSources: selectedSources }); } From b0d8bb8af572d5a0155415062b0c3ded674a01a8 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Sep 2023 10:26:28 +0100 Subject: [PATCH 10/21] Update only specified key and copy --- src/Dashboard.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 73e1a47..e080ab0 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -53,6 +53,13 @@ class Dashboard extends React.Component { // These handle the initial setup of the app + /** + * Read the config file to setup the default site, species etc + */ + setupDefaults() { + // this.state... + } + /** * Adds data to the data store, converting it to the structure required by Plotly * @@ -74,8 +81,8 @@ class Dashboard extends React.Component { // TODO - can we do this better so we don't end up copying all the data each time? // Will this work? this.setState((prevState) => { - let previous = {...prevState.dataSource.sourceKey} - previous = forPlotly + let previous = { ...prevState.dataSource.sourceKey }; + previous = forPlotly; return { previous }; }); } @@ -129,10 +136,6 @@ class Dashboard extends React.Component { let defaultNetwork = null; let defaultSourceKey = null; // We just need to pull out the initial data - - // This will hold the data itself - // It's structure is - // We retrieve only the first dataset and then populate the other data values with nulls // When this data is selected the app will retrieve the data @@ -266,7 +269,7 @@ class Dashboard extends React.Component { // the source will be the key in the for (const source of selectedSources) { const key = `${this.state.selectedSpecies}.${source}`; - const filename = this.state.filenameLookup[key] + const filename = this.state.filenameLookup[key]; this.retrieveData(); } From c1c36cee8e5dfab4f003e25b05824bd4e53d30ba Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Sep 2023 14:04:31 +0100 Subject: [PATCH 11/21] Tidy to correct retrieval of metadata and storing of nulls --- src/Dashboard.js | 57 +++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 09885b5..fa09c21 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -17,7 +17,6 @@ import styles from "./Dashboard.module.css"; import siteInfoJSON from "./data/siteInfo.json"; import { Button, MenuItem } from "@mui/material"; -import completeMetadata from "./deccoutput/metadata_complete.json"; import LaunchIcon from "@mui/icons-material/Launch"; async function retrieveJSON(url) { @@ -40,8 +39,8 @@ class Dashboard extends React.Component { colours: {}, dataStore: {}, }; - - this.dataRepoURL = "https://raw.githubusercontent.com/openghg/decc_dashaboard_data/main"; + + this.dataRepoURL = "https://raw.githubusercontent.com/openghg/temp_data_dashboard/main/" this.sourceSelector = this.sourceSelector.bind(this); this.toggleOverlay = this.toggleOverlay.bind(this); @@ -58,6 +57,7 @@ class Dashboard extends React.Component { * Read the config file to setup the default site, species etc */ setupDefaults() { + throw new Error("Not implemented."); // this.state... } @@ -79,12 +79,10 @@ class Dashboard extends React.Component { y_values: y_values, }; - // TODO - can we do this better so we don't end up copying all the data each time? - // Will this work? + // Now update the current dataStore with the new data this.setState((prevState) => { - let previous = { ...prevState.dataSource.sourceKey }; - previous = forPlotly; - return { previous }; + set(prevState.dataStore, sourceKey, forPlotly); + return prevState; }); } @@ -144,7 +142,7 @@ class Dashboard extends React.Component { let filenameLookup = {}; try { - for (const [species, networkData] of Object.entries(completeMetadata)) { + for (const [species, networkData] of Object.entries(metadata)) { if (defaultSpecies === null) defaultSpecies = species; for (const [network, siteData] of Object.entries(networkData)) { if (defaultNetwork === null) defaultNetwork = network; @@ -153,24 +151,28 @@ class Dashboard extends React.Component { for (const [inlet, instrumentData] of Object.entries(inletData)) { if (defaultInlet === null) defaultInlet = inlet; for (const [instrument, fileMetadata] of Object.entries(instrumentData)) { - // The complete source key should always have the species + // The complete source key should always have the species at the start + // Then we use the lodash set, get etc commands to easily access the objects const completeSourceKey = `${species}.${network}_${site}_${inlet}_${instrument}`; if (defaultSourceKey === null) defaultSourceKey = completeSourceKey; - let measurementData = null; - const filename = fileMetadata["filename"]; - + const filepath = fileMetadata["filepath"]; + + // We retrieve the data for the default source + // and store a null in all the sources we don't retrieve if (!defaultInstrument) { defaultInstrument = instrument; - const url = new URL(filename, this.dataRepoURL); + const url = new URL(filepath, this.dataRepoURL).href; retrieveJSON(url).then((result) => { - this.addDataToStore(species, completeSourceKey, result); + this.addDataToStore(completeSourceKey, result); }); + } else { + set(dataStore, completeSourceKey, null); } - - set(dataStore, completeSourceKey, measurementData); - set(filenameLookup, completeSourceKey, filename); + + // Save the filepath in the data repository for each lookup using the source key + set(filenameLookup, completeSourceKey, filepath); } } } @@ -194,14 +196,12 @@ class Dashboard extends React.Component { componentDidMount() { // Retrieve the metadata - // const metadataURL = new URL("fileMetadata.json", this.dataRepoURL).href - - const metadataURL = - "https://gist.githubusercontent.com/gareth-j/328fa8a1b5d61ed3a543710b10de4ddc/raw/6f616b8046b6bc266b82c3bdfc0ceaa198f0bbb5/metadata_complete.json"; + const metadata_filename = "metadata_complete.json"; + const metadataURL = new URL(metadata_filename, this.dataRepoURL); retrieveJSON(metadataURL).then( - (result) => { - this.populateAndRetrieve(result); + (metadata) => { + this.populateAndRetrieve(metadata); this.setState({ isLoaded: true, }); @@ -376,6 +376,13 @@ class Dashboard extends React.Component {
); } else { + // return ( + //
+ //

{Object.keys(this.state.dataStore)}

+ //
+ // ); + // } + // Remove the brack above and uncomment const liveData = ( } /> - }/> + } /> {overlay}
From 8dc11ed28d2d4605cd57465fd1474e73d7a524a4 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Sep 2023 15:39:25 +0100 Subject: [PATCH 12/21] WIP: Mostly working, need metadata for line plot --- src/Dashboard.js | 6 +- src/components/LeafletMap/LeafletMap.js | 154 +++++++++--------- src/components/LiveData/LiveData.js | 3 +- .../MultiSiteLineChart/MultiSiteLineChart.js | 86 +++++----- src/util/helpers.js | 28 +++- 5 files changed, 146 insertions(+), 131 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index fa09c21..6a8193c 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -10,7 +10,7 @@ import FAQ from "./components/FAQ/FAQ"; import LiveData from "./components/LiveData/LiveData"; import Explainer from "./components/Explainer/Explainer"; -import { importSiteImages } from "./util/helpers"; +import { importSiteImages, createSourceKey } from "./util/helpers"; import styles from "./Dashboard.module.css"; // Site description information @@ -153,7 +153,7 @@ class Dashboard extends React.Component { for (const [instrument, fileMetadata] of Object.entries(instrumentData)) { // The complete source key should always have the species at the start // Then we use the lodash set, get etc commands to easily access the objects - const completeSourceKey = `${species}.${network}_${site}_${inlet}_${instrument}`; + const completeSourceKey = createSourceKey(species, network, site, inlet, instrument) if (defaultSourceKey === null) defaultSourceKey = completeSourceKey; const filepath = fileMetadata["filepath"]; @@ -394,7 +394,7 @@ class Dashboard extends React.Component { selectedSpecies={this.state.selectedSpecies} defaultSpecies={this.state.defaultSpecies} setSiteOverlay={this.state.setSiteOverlay} - siteStructure={this.state.siteStructure} + siteMetadata={this.state.completeMetadata} /> ); diff --git a/src/components/LeafletMap/LeafletMap.js b/src/components/LeafletMap/LeafletMap.js index 47efc41..e4587da 100644 --- a/src/components/LeafletMap/LeafletMap.js +++ b/src/components/LeafletMap/LeafletMap.js @@ -1,6 +1,7 @@ import PropTypes from "prop-types"; import React from "react"; import { LayerGroup, MapContainer, ImageOverlay, TileLayer, CircleMarker, Popup } from "react-leaflet"; +import { createSourceKey } from "../../util/helpers"; // import TextButton from "../TextButton/TextButton"; // import "./LeafletMapResponsive.css"; @@ -22,94 +23,91 @@ class LeafletMap extends React.Component { createMarkers() { const dataStore = this.props.dataStore; - const siteStructure = this.props.siteStructure; const selectedSpecies = this.props.selectedSpecies; + const siteMetadata = this.props.siteMetadata[selectedSpecies]; let markers = []; - if(siteStructure !== undefined && dataStore !== undefined){ - const speciesStructure = siteStructure[selectedSpecies]; - const speciesData = dataStore[selectedSpecies]; - - // We want a marker for each site, with selection buttons within the popup - for (const siteData of Object.values(speciesStructure)) { - for (const inletData of Object.values(siteData)) { - let marker = null; - let sourceButtons = []; - // The site metadata we require will be the same for each inlet / instrument - let siteMetadata = null; - - const buttonStyling = { fontSize: "0.8em", width:"0.8em" }; - - for (const [inlet, instrumentData] of Object.entries(inletData)) { - for (const sourceKey of Object.values(instrumentData)) { - const button = ( - - {inlet} - - ); - - sourceButtons.push(button); - if (!siteMetadata) { - siteMetadata = speciesData[sourceKey]["metadata"]; + if (siteMetadata !== undefined && dataStore !== undefined) { + const speciesData = dataStore[selectedSpecies]; + + // We want a marker for each site, with selection buttons within the popup + for (const [network, siteData] of Object.entries(siteMetadata)) { + for (const [site, inletData] of Object.entries(siteData)) { + let marker = null; + let sourceButtons = []; + // The site metadata we require will be the same for each inlet / instrument + let siteSpecificMetadata = null; + + const buttonStyling = { fontSize: "0.8em", width: "0.8em" }; + + for (const [inlet, instrumentData] of Object.entries(inletData)) { + let inletDone = false; + for (const [instrument, inletSpecificMetadata] of Object.entries(instrumentData)) { + // TODO - do we want separate buttons for the different instruments? + if (!inletDone) { + siteSpecificMetadata = inletSpecificMetadata["metadata"] + const sourceKey = createSourceKey(selectedSpecies, network, site, inlet, instrument); + const button = ( + + {inlet} + + ); + + sourceButtons.push(button); + inletDone = true;; + } + } - } - } - try { - const station_latitude = siteMetadata["station_latitude"]; - const station_longitude = siteMetadata["station_longitude"]; - - const locationStr = `${station_latitude}, ${station_longitude}`; - const location = [station_latitude, station_longitude]; - const siteName = siteMetadata["station_long_name"]; - - marker = ( - - -
-
-
{siteName.toUpperCase()}
-
- Select inlet: - {sourceButtons}
-
- For more information please visit the  - - the DECC network website. - + try { + const station_latitude = siteSpecificMetadata["station_latitude"]; + const station_longitude = siteSpecificMetadata["station_longitude"]; + + const locationStr = `${station_latitude}, ${station_longitude}`; + const location = [station_latitude, station_longitude]; + const siteName = siteSpecificMetadata["station_long_name"]; + + marker = ( + + +
+
+
{siteName.toUpperCase()}
+
+ Select inlet: + {sourceButtons} +
+
+ For more information please visit the  + + the DECC network website. + +
+
Location: {locationStr}
-
Location: {locationStr}
-
- - - ); - - markers.push(marker); - } catch (error) { - console.log(error); - continue; + + + ); + + markers.push(marker); + } catch (error) { + console.log(error); + continue; + } } } } - - } return markers; } @@ -135,7 +133,7 @@ class LeafletMap extends React.Component { const markers = this.createMarkers(); const zoom = this.props.zoom ? this.props.zoom : 5; - const style = { width: "90%"}; + const style = { width: "90%" }; return (
diff --git a/src/components/LiveData/LiveData.js b/src/components/LiveData/LiveData.js index eaee4fc..456f85a 100644 --- a/src/components/LiveData/LiveData.js +++ b/src/components/LiveData/LiveData.js @@ -43,8 +43,7 @@ class LiveData extends React.Component { centre={mapCentre} zoom={5} dataStore={this.props.dataStore} - // siteInfoOverlay={this.props.setSiteOverlay} - siteStructure={this.props.siteStructure} + siteMetadata={this.props.siteMetadata} />
diff --git a/src/components/MultiSiteLineChart/MultiSiteLineChart.js b/src/components/MultiSiteLineChart/MultiSiteLineChart.js index b15c2d4..779a002 100644 --- a/src/components/MultiSiteLineChart/MultiSiteLineChart.js +++ b/src/components/MultiSiteLineChart/MultiSiteLineChart.js @@ -5,26 +5,26 @@ import { toTitleCase } from "../../util/helpers"; import styles from "./MultiSiteLineChart.module.css"; import jsPDF from "jspdf"; import html2canvas from "html2canvas"; -import FileDownloadOutlinedIcon from '@mui/icons-material/FileDownloadOutlined'; +import FileDownloadOutlinedIcon from "@mui/icons-material/FileDownloadOutlined"; import { Button } from "@mui/material"; -import { createImage } from "../../util/helpers" +import { createImage } from "../../util/helpers"; import colours from "../../data/colours.json"; class MultiSiteLineChart extends React.Component { - /* + /* This method takes care of downloading the Plot on the website in format of PNG It fetches the html tag for plot and converts to PNG */ handleDownloadPNG = (species, sites) => { const chartContainer = document.getElementById("chart-container"); - let filenames = [species, ...sites].join('_'); - + let filenames = [species, ...sites].join("_"); + if (chartContainer) { html2canvas(chartContainer).then((canvas) => { const imgData = canvas.toDataURL("image/png"); - + const link = document.createElement("a"); link.href = imgData; link.download = `${filenames}.png`; @@ -34,8 +34,8 @@ class MultiSiteLineChart extends React.Component { console.error("Chart container not found."); } }; - - /* + + /* This method takes care of downloading the Plot on the website in format of PDF It fetches the html tag for plot and converts to PDF @@ -43,7 +43,7 @@ class MultiSiteLineChart extends React.Component { handleDownloadPDF = (species, sites) => { // Here we fetch the html element of chart-container that needs to be downloaded by id. const chartContainer = document.getElementById("chart-container"); - let filenames = [species, ...sites].join('_'); + let filenames = [species, ...sites].join("_"); if (chartContainer) { html2canvas(chartContainer).then((canvas) => { @@ -51,21 +51,22 @@ class MultiSiteLineChart extends React.Component { const chartWidth = chartContainer.offsetWidth; const chartHeight = chartContainer.offsetHeight; - + const pdf = new jsPDF({ orientation: chartWidth > chartHeight ? "landscape" : "portrait", unit: "mm", format: [chartWidth, chartHeight], }); - + pdf.addImage(imgData, "JPEG", 0, 0, chartWidth, chartHeight); - + pdf.save(filenames); }); } else { console.error("Chart container not found."); } }; + render() { let plotData = []; let maxY = 0; @@ -106,18 +107,18 @@ class MultiSiteLineChart extends React.Component { const colour = colours["pastelColours"]; units = metadata["units"]; - species = metadata["species"] - + species = metadata["species"]; + if (units === undefined) { - if (species === 'ch4' || species === 'co' || species === 'n2o') { - units = 'ppb'; - } else if (species === 'co2') { - units = 'ppm'; + if (species === "ch4" || species === "co" || species === "n2o") { + units = "ppb"; + } else if (species === "co2") { + units = "ppm"; } else { units = metadata["units"]; } } - + const trace = { x: xValues, y: yValues, @@ -138,8 +139,8 @@ class MultiSiteLineChart extends React.Component { using regex to remove and "-" within the name */ let sites = []; - sites = plotData.map(item => item.name); - sites = sites.map(item => item.replace(/<\/?b>/g, '').replace(/\s*-\s*/g, '')); + sites = plotData.map((item) => item.name); + sites = sites.map((item) => item.replace(/<\/?b>/g, "").replace(/\s*-\s*/g, "")); let dateMarkObject = null; const selectedDate = this.props.selectedDate; @@ -165,7 +166,7 @@ class MultiSiteLineChart extends React.Component { const metOffice = require(`../../images/Metoffice.png`); const ncas = require(`../../images/ncas.png`); const openghg = require(`../../images/OpenGHG_Logo_Landscape.png`); - + const layout = { title: { text: this.props.title ? this.props.title : null, @@ -188,7 +189,6 @@ class MultiSiteLineChart extends React.Component { linecolor: "black", autotick: true, ticks: "outside", - }, yaxis: { automargin: true, @@ -196,8 +196,8 @@ class MultiSiteLineChart extends React.Component { text: `${species.toUpperCase()} (${units})`, standoff: 10, font: { - size:16, - } + size: 16, + }, }, range: this.props.yRange ? this.props.yRange : null, showgrid: false, @@ -229,26 +229,26 @@ class MultiSiteLineChart extends React.Component {
- + size="small" + variant="contained" + color="primary" + startIcon={} + onClick={() => this.handleDownloadPNG(species, sites)} + style={{ width: "20px", height: "20px" }} + > + PNG +
); diff --git a/src/util/helpers.js b/src/util/helpers.js index e693292..c735700 100644 --- a/src/util/helpers.js +++ b/src/util/helpers.js @@ -19,16 +19,17 @@ export function isEmpty(obj) { export function createImage(source, x, opacity = 0.6) { return { source: source, - xref: 'paper', - yref: 'paper', + xref: "paper", + yref: "paper", x: x, y: 0.89, sizex: 0.09, sizey: 0.09, opacity: opacity, - xanchor: 'center', - yanchor: 'middle', - }}; + xanchor: "center", + yanchor: "middle", + }; +} // export function importSVGs() { // let footprints = {}; @@ -111,3 +112,20 @@ export function toTitleCase(str) { return txt.charAt(0).toUpperCase() + txt.substr(1).toLowerCase(); }); } + +/** + * Creates an instrument key to be used in the lookup tables + * for data and filenames + * + * @param {string} species - Species + * @param {string} network- Network code + * @param {string} site - Site code + * @param {string} inlet - Inlet height + * @param {string} instrument - Instrument name + + * @param {string} sourceKey - Source key + * + */ +export function createSourceKey(species, network, site, inlet, instrument) { + return `${species}.${network}_${site}_${inlet}_${instrument}`; +} From ba0fc843a2fed2032ff9782bfa884c5fb6d69393 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Sep 2023 17:36:50 +0100 Subject: [PATCH 13/21] Adding metastore --- src/Dashboard.js | 33 ++++++++++++++++++------- src/components/LeafletMap/LeafletMap.js | 21 ++++++++-------- src/components/LiveData/LiveData.js | 9 ++++--- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 6a8193c..8ba056a 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -39,8 +39,8 @@ class Dashboard extends React.Component { colours: {}, dataStore: {}, }; - - this.dataRepoURL = "https://raw.githubusercontent.com/openghg/temp_data_dashboard/main/" + + this.dataRepoURL = "https://raw.githubusercontent.com/openghg/temp_data_dashboard/main/"; this.sourceSelector = this.sourceSelector.bind(this); this.toggleOverlay = this.toggleOverlay.bind(this); @@ -126,7 +126,7 @@ class Dashboard extends React.Component { // This should aleady be in the right shape, we use the nested aspect of // the dictionary to make it easy to populate the interface, creating // some lookup tables for filenames etc below - this.state.completeMetadata = metadata; + // this.state.completeMetadata = metadata; let defaultSpecies = null; let defaultSite = null; let defaultInlet = null; @@ -138,7 +138,13 @@ class Dashboard extends React.Component { // We retrieve only the first dataset and then populate the other data values with nulls // When this data is selected the app will retrieve the data + // Store the data itself let dataStore = {}; + // Store the metadata for each source + let metaStore = {}; + // Store the structure to allow easy building of the interface dynamically + let siteStructure = {}; + // Do we need filename lookup? let filenameLookup = {}; try { @@ -153,11 +159,13 @@ class Dashboard extends React.Component { for (const [instrument, fileMetadata] of Object.entries(instrumentData)) { // The complete source key should always have the species at the start // Then we use the lodash set, get etc commands to easily access the objects - const completeSourceKey = createSourceKey(species, network, site, inlet, instrument) + const completeSourceKey = createSourceKey(species, network, site, inlet, instrument); + // We'll use this to create a lightweight structure for the creation of the interface + const nestedSourceKey = `${species}.${network}.${site}.${inlet}.${instrument}`; if (defaultSourceKey === null) defaultSourceKey = completeSourceKey; const filepath = fileMetadata["filepath"]; - + // We retrieve the data for the default source // and store a null in all the sources we don't retrieve if (!defaultInstrument) { @@ -170,9 +178,13 @@ class Dashboard extends React.Component { } else { set(dataStore, completeSourceKey, null); } - + // Save the filepath in the data repository for each lookup using the source key set(filenameLookup, completeSourceKey, filepath); + + const sourceMetadata = fileMetadata["metadata"]; + set(metaStore, completeSourceKey, sourceMetadata); + set(siteStructure, nestedSourceKey, completeSourceKey); } } } @@ -187,6 +199,8 @@ class Dashboard extends React.Component { /* eslint-disable react/no-direct-mutation-state */ // Give each site a colour this.state.dataStore = dataStore; + this.state.metaStore = metaStore; + this.state.siteStructure = siteStructure; this.state.defaultSpecies = defaultSpecies; this.state.selectedSources = new Set([defaultSourceKey]); this.state.selectedSpecies = defaultSpecies; @@ -381,20 +395,21 @@ class Dashboard extends React.Component { //

{Object.keys(this.state.dataStore)}

// // ); - // } - // Remove the brack above and uncomment + // } + // Remove the brack above and uncomment const liveData = ( ); diff --git a/src/components/LeafletMap/LeafletMap.js b/src/components/LeafletMap/LeafletMap.js index e4587da..00b1c7d 100644 --- a/src/components/LeafletMap/LeafletMap.js +++ b/src/components/LeafletMap/LeafletMap.js @@ -2,6 +2,7 @@ import PropTypes from "prop-types"; import React from "react"; import { LayerGroup, MapContainer, ImageOverlay, TileLayer, CircleMarker, Popup } from "react-leaflet"; import { createSourceKey } from "../../util/helpers"; +import { get } from "lodash" // import TextButton from "../TextButton/TextButton"; // import "./LeafletMapResponsive.css"; @@ -24,15 +25,15 @@ class LeafletMap extends React.Component { createMarkers() { const dataStore = this.props.dataStore; const selectedSpecies = this.props.selectedSpecies; - const siteMetadata = this.props.siteMetadata[selectedSpecies]; + const metaStore = this.props.metaStore; + const siteStructure = this.props.siteStructure; let markers = []; - if (siteMetadata !== undefined && dataStore !== undefined) { - const speciesData = dataStore[selectedSpecies]; - + // TODO - Are props always defined? Are we making the right comparison here? + if (metaStore !== undefined && dataStore !== undefined) { // We want a marker for each site, with selection buttons within the popup - for (const [network, siteData] of Object.entries(siteMetadata)) { + for (const [network, siteData] of Object.entries(siteStructure)) { for (const [site, inletData] of Object.entries(siteData)) { let marker = null; let sourceButtons = []; @@ -43,11 +44,10 @@ class LeafletMap extends React.Component { for (const [inlet, instrumentData] of Object.entries(inletData)) { let inletDone = false; - for (const [instrument, inletSpecificMetadata] of Object.entries(instrumentData)) { + for (const sourceKey of Object.values(instrumentData)) { // TODO - do we want separate buttons for the different instruments? if (!inletDone) { - siteSpecificMetadata = inletSpecificMetadata["metadata"] - const sourceKey = createSourceKey(selectedSpecies, network, site, inlet, instrument); + siteSpecificMetadata = get(metaStore, sourceKey) const button = (
{this.createIntro()}
- + /> */}
@@ -43,7 +45,8 @@ class LiveData extends React.Component { centre={mapCentre} zoom={5} dataStore={this.props.dataStore} - siteMetadata={this.props.siteMetadata} + metaStore={this.props.metaStore} + siteStructure={this.props.siteStructure} />
From dc5fa51964c7a53a797b5bdc5afe6a2172c3aa6d Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Tue, 5 Sep 2023 18:26:01 +0100 Subject: [PATCH 14/21] WIP: Site structure build working, default source not being set correctly --- src/components/LeafletMap/LeafletMap.js | 41 +++++++++---------- src/components/LiveData/LiveData.js | 4 +- .../MultiSiteLineChart/MultiSiteLineChart.js | 31 ++++++++------ src/components/ObsBox/ObsBox.js | 36 ++++++++-------- 4 files changed, 60 insertions(+), 52 deletions(-) diff --git a/src/components/LeafletMap/LeafletMap.js b/src/components/LeafletMap/LeafletMap.js index 00b1c7d..54c4d4a 100644 --- a/src/components/LeafletMap/LeafletMap.js +++ b/src/components/LeafletMap/LeafletMap.js @@ -2,7 +2,7 @@ import PropTypes from "prop-types"; import React from "react"; import { LayerGroup, MapContainer, ImageOverlay, TileLayer, CircleMarker, Popup } from "react-leaflet"; import { createSourceKey } from "../../util/helpers"; -import { get } from "lodash" +import { get } from "lodash"; // import TextButton from "../TextButton/TextButton"; // import "./LeafletMapResponsive.css"; @@ -26,15 +26,18 @@ class LeafletMap extends React.Component { const dataStore = this.props.dataStore; const selectedSpecies = this.props.selectedSpecies; const metaStore = this.props.metaStore; - const siteStructure = this.props.siteStructure; + // The only site structure we want here are sites that have provided data + // for the species we're interested in + const siteStructure = this.props.siteStructure[selectedSpecies]; + // We'll make a marker for each site let markers = []; // TODO - Are props always defined? Are we making the right comparison here? if (metaStore !== undefined && dataStore !== undefined) { // We want a marker for each site, with selection buttons within the popup - for (const [network, siteData] of Object.entries(siteStructure)) { - for (const [site, inletData] of Object.entries(siteData)) { + for (const siteData of Object.values(siteStructure)) { + for (const inletData of Object.values(siteData)) { let marker = null; let sourceButtons = []; // The site metadata we require will be the same for each inlet / instrument @@ -43,25 +46,21 @@ class LeafletMap extends React.Component { const buttonStyling = { fontSize: "0.8em", width: "0.8em" }; for (const [inlet, instrumentData] of Object.entries(inletData)) { - let inletDone = false; for (const sourceKey of Object.values(instrumentData)) { // TODO - do we want separate buttons for the different instruments? - if (!inletDone) { - siteSpecificMetadata = get(metaStore, sourceKey) - const button = ( - - {inlet} - - ); - - sourceButtons.push(button); - inletDone = true; - } + siteSpecificMetadata = get(metaStore, sourceKey); + const button = ( + + {inlet} + + ); + + sourceButtons.push(button); } } diff --git a/src/components/LiveData/LiveData.js b/src/components/LiveData/LiveData.js index 62cd91f..7ac2db2 100644 --- a/src/components/LiveData/LiveData.js +++ b/src/components/LiveData/LiveData.js @@ -24,7 +24,7 @@ class LiveData extends React.Component {
{this.createIntro()}
- {/* */} + />
diff --git a/src/components/MultiSiteLineChart/MultiSiteLineChart.js b/src/components/MultiSiteLineChart/MultiSiteLineChart.js index 779a002..944ce19 100644 --- a/src/components/MultiSiteLineChart/MultiSiteLineChart.js +++ b/src/components/MultiSiteLineChart/MultiSiteLineChart.js @@ -10,6 +10,7 @@ import { Button } from "@mui/material"; import { createImage } from "../../util/helpers"; import colours from "../../data/colours.json"; +import { get } from "lodash"; class MultiSiteLineChart extends React.Component { /* @@ -72,19 +73,23 @@ class MultiSiteLineChart extends React.Component { let maxY = 0; let minY = Infinity; - const data = this.props.data; + const measurementData = this.props.measurementData; + const metaStore = this.props.metaStore; + let species = null; let units = null; + let sites = []; - for (const sourceData of Object.values(data)) { - const metadata = sourceData["metadata"]; - const measurementData = sourceData["data"]; + for (const [sourceKey, data] of Object.entries(measurementData)) { + const metadata = get(metaStore, sourceKey); - const xValues = measurementData["x_values"]; - const yValues = measurementData["y_values"]; + const xValues = data["x_values"]; + const yValues = data["y_values"]; - const max = Math.max(...yValues); - const min = Math.min(...yValues); + // const max = Math.max(...yValues); + // const min = Math.min(...yValues); + + return

That's all folks.

; if (max > maxY) { maxY = max; @@ -98,6 +103,8 @@ class MultiSiteLineChart extends React.Component { let name = null; try { const siteName = metadata["station_long_name"]; + // We'll save these in case we want to write to file + sites.push(metadata["site"]); const inlet = metadata["inlet"]; name = `${toTitleCase(siteName)} - ${inlet}`; @@ -114,8 +121,6 @@ class MultiSiteLineChart extends React.Component { units = "ppb"; } else if (species === "co2") { units = "ppm"; - } else { - units = metadata["units"]; } } @@ -138,9 +143,9 @@ class MultiSiteLineChart extends React.Component { /*fetching all the site names to pass them as filename using regex to remove and "-" within the name */ - let sites = []; - sites = plotData.map((item) => item.name); - sites = sites.map((item) => item.replace(/<\/?b>/g, "").replace(/\s*-\s*/g, "")); + // let sites = []; + // sites = plotData.map((item) => item.name); + // sites = sites.map((item) => item.replace(/<\/?b>/g, "").replace(/\s*-\s*/g, "")); let dateMarkObject = null; const selectedDate = this.props.selectedDate; diff --git a/src/components/ObsBox/ObsBox.js b/src/components/ObsBox/ObsBox.js index 0941673..695a532 100644 --- a/src/components/ObsBox/ObsBox.js +++ b/src/components/ObsBox/ObsBox.js @@ -6,12 +6,14 @@ import MultiSiteLineChart from "../MultiSiteLineChart/MultiSiteLineChart"; import { isEmpty, getVisID } from "../../util/helpers"; import styles from "./ObsBox.module.css"; -import SelectOptions from "../SelectOptions/SelectOptions" +import SelectOptions from "../SelectOptions/SelectOptions"; import { Button } from "@mui/material"; +import { get, set } from "lodash"; class ObsBox extends React.Component { createEmissionsGraphs() { const dataStore = this.props.dataStore; + const metaStore = this.props.metaStore; const selectedSources = this.props.selectedSources; const selectedSpecies = this.props.selectedSpecies; @@ -25,16 +27,13 @@ class ObsBox extends React.Component { let multiUnits = []; if (selectedSources) { - const speciesData = dataStore[selectedSpecies]; - for (const key of selectedSources) { - dataToPlot[key] = speciesData[key]; - - try { - const units = speciesData[key]["metadata"]["units"]; + set(dataToPlot, key, get(dataStore, key)); + const units = get(metaStore, key); + if (units) { multiUnits.push(units); - } catch (error) { - console.error(`Error reading units - ${error}`); + } else { + console.error(`Error reading units from ${key}.`); } } @@ -62,7 +61,8 @@ class ObsBox extends React.Component { Clear; + clearButton = ( + + ); } const availableSpecies = Object.keys(this.props.dataStore); @@ -95,11 +99,11 @@ class ObsBox extends React.Component {
{this.createEmissionsGraphs()}
{clearButton}
- +
); From 5720ba29e718d3e8d5a14ac76e390a8aafb696a8 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 6 Sep 2023 11:28:42 +0100 Subject: [PATCH 15/21] Retrieval of data working, some tidying of interface and loading/downloading message/modal required --- src/Dashboard.js | 89 +++++++++++-------- .../MultiSiteLineChart/MultiSiteLineChart.js | 88 ++++++++---------- src/components/ObsBox/ObsBox.js | 85 +++++------------- 3 files changed, 113 insertions(+), 149 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 8ba056a..3c1aaaf 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -48,7 +48,6 @@ class Dashboard extends React.Component { this.speciesSelector = this.speciesSelector.bind(this); this.clearSources = this.clearSources.bind(this); this.toggleSidebar = this.toggleSidebar.bind(this); - // this.setSiteOverlay = this.setSiteOverlay.bind(this); } // These handle the initial setup of the app @@ -62,23 +61,32 @@ class Dashboard extends React.Component { } /** - * Adds data to the data store, converting it to the structure required by Plotly + * Converts data to the format required by Plotly * - * @param {string} species - Species - * @param {string} sourceKey - Source key - * @param {string} data - Data that's been passed through the to_plotly function + * @param {object} data - Data object * */ - addDataToStore(sourceKey, data) { + toPlotly(data) { const x_timestamps = Object.keys(data); const x_values = x_timestamps.map((d) => new Date(parseInt(d))); const y_values = Object.values(data); - const forPlotly = { + return { x_values: x_values, y_values: y_values, }; + } + /** + * Adds data to the data store, converting it to the structure required by Plotly + * + * @param {string} species - Species + * @param {string} sourceKey - Source key + * @param {string} data - Data that's been passed through the to_plotly function + * + */ + addDataToStore(sourceKey, data) { + const forPlotly = this.toPlotly(data); // Now update the current dataStore with the new data this.setState((prevState) => { set(prevState.dataStore, sourceKey, forPlotly); @@ -90,12 +98,16 @@ class Dashboard extends React.Component { * Retrieves data from the given URL and processes it into a format * plotly can read * - * @param {string} filename - Name of file to be retrieved from remote data store - * @param {string} species - Species * @param {string} sourceKey - Source key * */ - retrieveData(filename, sourceKey) { + retrieveData(sourceKey) { + const filename = get(this.state.filenameLookup, sourceKey, null); + if (filename === null) { + console.error(`No filename available for ${sourceKey}`); + return; + } + const currentVal = get(this.state.dataStore, sourceKey); if (currentVal !== null) { console.log(`We already have data for ${sourceKey}`); @@ -104,6 +116,7 @@ class Dashboard extends React.Component { const url = new URL(filename, this.dataRepoURL).href; + console.log(`Retrieving data from ${url}`); retrieveJSON(url).then((result) => { this.addDataToStore(sourceKey, result); }); @@ -131,7 +144,7 @@ class Dashboard extends React.Component { let defaultSite = null; let defaultInlet = null; // Not sure if we need default instrument but - let defaultInstrument = null; + // let defaultInstrument = null; let defaultNetwork = null; let defaultSourceKey = null; // We just need to pull out the initial data @@ -163,28 +176,28 @@ class Dashboard extends React.Component { // We'll use this to create a lightweight structure for the creation of the interface const nestedSourceKey = `${species}.${network}.${site}.${inlet}.${instrument}`; - if (defaultSourceKey === null) defaultSourceKey = completeSourceKey; const filepath = fileMetadata["filepath"]; // We retrieve the data for the default source // and store a null in all the sources we don't retrieve - if (!defaultInstrument) { - defaultInstrument = instrument; - + if (defaultSourceKey === null) { + defaultSourceKey = completeSourceKey; const url = new URL(filepath, this.dataRepoURL).href; + // Here we add the data directly as this is on first load retrieveJSON(url).then((result) => { - this.addDataToStore(completeSourceKey, result); + console.log(`Retrieving data from ${url}`); + const forPlotly = this.toPlotly(result); + set(dataStore, completeSourceKey, forPlotly); }); } else { set(dataStore, completeSourceKey, null); } - // Save the filepath in the data repository for each lookup using the source key - set(filenameLookup, completeSourceKey, filepath); - const sourceMetadata = fileMetadata["metadata"]; set(metaStore, completeSourceKey, sourceMetadata); set(siteStructure, nestedSourceKey, completeSourceKey); + // Save the filepath in the data repository for each lookup using the source key + set(filenameLookup, completeSourceKey, filepath); } } } @@ -201,6 +214,7 @@ class Dashboard extends React.Component { this.state.dataStore = dataStore; this.state.metaStore = metaStore; this.state.siteStructure = siteStructure; + this.state.filenameLookup = filenameLookup; this.state.defaultSpecies = defaultSpecies; this.state.selectedSources = new Set([defaultSourceKey]); this.state.selectedSpecies = defaultSpecies; @@ -256,11 +270,12 @@ class Dashboard extends React.Component { // These handle app control by the components /** - * Selects a data source - a specific species at a site at an inlet + * Selects a data source using a selected sourceKey * * @param {string} selection - Key for a data source */ sourceSelector(selection) { + console.log(selection); let selectedSourcesSet = new Set(); if (selection instanceof Set) { @@ -272,20 +287,18 @@ class Dashboard extends React.Component { // Here we change all the sites and select all species / sectors at that site let selectedSources = cloneDeep(this.state.selectedSources); - for (const source of selectedSourcesSet) { - if (selectedSources.has(source)) { - selectedSources.delete(source); + for (const sourceKey of selectedSourcesSet) { + if (selectedSources.has(sourceKey)) { + selectedSources.delete(sourceKey); } else { - selectedSources.add(source); + selectedSources.add(sourceKey); } } // Now let's make sure we have all the data for these new selections // the source will be the key in the - for (const source of selectedSources) { - const key = `${this.state.selectedSpecies}.${source}`; - const filename = this.state.filenameLookup[key]; - this.retrieveData(); + for (const sourceKey of selectedSources) { + this.retrieveData(sourceKey); } this.setState({ selectedSources: selectedSources }); @@ -332,9 +345,13 @@ class Dashboard extends React.Component { } } + // Here we want the default key of another species if (newSources.size === 0) { const defaultSource = Object.keys(speciesData)[0]; - newSources.add(defaultSource); + // This just gives us a partial key, we need to add in the species to get + // a complete sourceKey + const sourceKey = `${species}.${defaultSource}` + newSources.add(sourceKey); } this.sourceSelector(newSources); @@ -348,11 +365,17 @@ class Dashboard extends React.Component { this.setState({ overlayOpen: true, overlay: overlay }); } + /** Toggles the sidebar */ toggleSidebar() { this.setState({ showSidebar: !this.state.showSidebar }); } + /** Was used to set an overlay of the site and show an image, currently unused + * @deprecated + * + */ setSiteOverlay(e) { + console.warn("Deprecated function. May be removed."); const siteCode = String(e.target.dataset.onclickparam).toUpperCase(); const siteInfo = this.state.siteInfo[siteCode]; @@ -390,13 +413,6 @@ class Dashboard extends React.Component {
); } else { - // return ( - //
- //

{Object.keys(this.state.dataStore)}

- //
- // ); - // } - // Remove the brack above and uncomment const liveData = ( ); diff --git a/src/components/MultiSiteLineChart/MultiSiteLineChart.js b/src/components/MultiSiteLineChart/MultiSiteLineChart.js index 944ce19..b024833 100644 --- a/src/components/MultiSiteLineChart/MultiSiteLineChart.js +++ b/src/components/MultiSiteLineChart/MultiSiteLineChart.js @@ -73,39 +73,36 @@ class MultiSiteLineChart extends React.Component { let maxY = 0; let minY = Infinity; - const measurementData = this.props.measurementData; - const metaStore = this.props.metaStore; - let species = null; let units = null; let sites = []; - for (const [sourceKey, data] of Object.entries(measurementData)) { - const metadata = get(metaStore, sourceKey); - - const xValues = data["x_values"]; - const yValues = data["y_values"]; + for (const sourceKey of this.props.selectedSources) { + const metadata = get(this.props.metaStore, sourceKey); - // const max = Math.max(...yValues); - // const min = Math.min(...yValues); + const timeseriesData = get(this.props.dataStore, sourceKey, null); + // TODO - how to handle this error cleanly? + if (timeseriesData === null) { + console.error(`No data available for ${sourceKey}`); + break; + } - return

That's all folks.

; + const xValues = timeseriesData["x_values"]; + const yValues = timeseriesData["y_values"]; - if (max > maxY) { - maxY = max; - } + const max = Math.max(...yValues); + const min = Math.min(...yValues); - if (min < minY) { - minY = min; - } + if (max > maxY) maxY = max; + if (min < minY) minY = min; // Set the name for the legend let name = null; try { const siteName = metadata["station_long_name"]; + const inlet = metadata["inlet"]; // We'll save these in case we want to write to file sites.push(metadata["site"]); - const inlet = metadata["inlet"]; name = `${toTitleCase(siteName)} - ${inlet}`; } catch (error) { @@ -147,31 +144,19 @@ class MultiSiteLineChart extends React.Component { // sites = plotData.map((item) => item.name); // sites = sites.map((item) => item.replace(/<\/?b>/g, "").replace(/\s*-\s*/g, "")); - let dateMarkObject = null; - const selectedDate = this.props.selectedDate; - - if (selectedDate) { - const date = new Date(parseInt(selectedDate)); - - dateMarkObject = { - type: "line", - x0: date, - y0: minY, - x1: date, - y1: maxY, - line: { - color: "black", - width: 1, - }, - }; - } - const widthScaleFactor = 0.925; const uniOfBristol = require(`../../images/UniOfBristolLogo.png`); const metOffice = require(`../../images/Metoffice.png`); const ncas = require(`../../images/ncas.png`); const openghg = require(`../../images/OpenGHG_Logo_Landscape.png`); + let yLabel = null; + if (species !== null) { + yLabel = `${species.toUpperCase()} (${units})`; + } else { + console.error(`species is null`); + } + const layout = { title: { text: this.props.title ? this.props.title : null, @@ -198,7 +183,7 @@ class MultiSiteLineChart extends React.Component { yaxis: { automargin: true, title: { - text: `${species.toUpperCase()} (${units})`, + text: yLabel, standoff: 10, font: { size: 16, @@ -226,15 +211,19 @@ class MultiSiteLineChart extends React.Component { t: 20, pad: 5, }, - shapes: [dateMarkObject], }; - return ( -
-
- -
-
-
; + } else { + return ( +
+
+ +
+
+ {/* + */} +
-
- ); + ); + } } } diff --git a/src/components/ObsBox/ObsBox.js b/src/components/ObsBox/ObsBox.js index 695a532..dabf08b 100644 --- a/src/components/ObsBox/ObsBox.js +++ b/src/components/ObsBox/ObsBox.js @@ -2,81 +2,40 @@ import PropTypes from "prop-types"; import React from "react"; import GraphContainer from "../GraphContainer/GraphContainer"; import MultiSiteLineChart from "../MultiSiteLineChart/MultiSiteLineChart"; - -import { isEmpty, getVisID } from "../../util/helpers"; - -import styles from "./ObsBox.module.css"; import SelectOptions from "../SelectOptions/SelectOptions"; +import { getVisID } from "../../util/helpers"; import { Button } from "@mui/material"; -import { get, set } from "lodash"; + +import styles from "./ObsBox.module.css"; class ObsBox extends React.Component { createEmissionsGraphs() { - const dataStore = this.props.dataStore; - const metaStore = this.props.metaStore; const selectedSources = this.props.selectedSources; - const selectedSpecies = this.props.selectedSpecies; - - const noSiteSelected = selectedSources.size === 0; - if (noSiteSelected) { + if (selectedSources.size === 0) { return
Please select a site
; } - let dataToPlot = {}; - let multiUnits = []; - if (selectedSources) { - for (const key of selectedSources) { - set(dataToPlot, key, get(dataStore, key)); - const units = get(metaStore, key); - if (units) { - multiUnits.push(units); - } else { - console.error(`Error reading units from ${key}.`); - } - } - - if (!isEmpty(dataToPlot)) { - // Do a quick check to make sure all the units are the same - let units = ""; - if (new Set(multiUnits).size === 1) { - units = ` (${multiUnits[0]})`; - } else { - console.error(`Multiple units for same species - ${multiUnits}`); - } - - const key = Object.keys(dataToPlot).join("-"); - - const widthScale = 0.9; - const heightScale = 0.9; - - // We only set the title of the graph if there's one site selected - let title = null; - const xLabel = "Date"; - const yLabel = `Concentration${units}`; - - const vis = ( - - - - ); + const key = Object.keys(selectedSources).join("-"); + + const widthScale = 0.9; + const heightScale = 0.9; + + const vis = ( + + + + ); - return vis; - } else { - console.error("No data to plot."); - return null; - } + return vis; } } From 9c3731b8c7b07402eef6445eca321e8ba6d9b5b1 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 6 Sep 2023 14:25:37 +0100 Subject: [PATCH 16/21] Working except initial load doesn't retrieve data on time --- src/Dashboard.js | 3 +- .../MultiSiteLineChart/MultiSiteLineChart.js | 49 +++++++++---------- 2 files changed, 25 insertions(+), 27 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 3c1aaaf..6d01b4c 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -175,7 +175,6 @@ class Dashboard extends React.Component { const completeSourceKey = createSourceKey(species, network, site, inlet, instrument); // We'll use this to create a lightweight structure for the creation of the interface const nestedSourceKey = `${species}.${network}.${site}.${inlet}.${instrument}`; - const filepath = fileMetadata["filepath"]; // We retrieve the data for the default source @@ -185,7 +184,7 @@ class Dashboard extends React.Component { const url = new URL(filepath, this.dataRepoURL).href; // Here we add the data directly as this is on first load retrieveJSON(url).then((result) => { - console.log(`Retrieving data from ${url}`); + console.log(`Retrieving data from ${url} for initial setup.`); const forPlotly = this.toPlotly(result); set(dataStore, completeSourceKey, forPlotly); }); diff --git a/src/components/MultiSiteLineChart/MultiSiteLineChart.js b/src/components/MultiSiteLineChart/MultiSiteLineChart.js index b024833..571a8ff 100644 --- a/src/components/MultiSiteLineChart/MultiSiteLineChart.js +++ b/src/components/MultiSiteLineChart/MultiSiteLineChart.js @@ -81,10 +81,10 @@ class MultiSiteLineChart extends React.Component { const metadata = get(this.props.metaStore, sourceKey); const timeseriesData = get(this.props.dataStore, sourceKey, null); - // TODO - how to handle this error cleanly? + // // TODO - how to handle this error cleanly? if (timeseriesData === null) { - console.error(`No data available for ${sourceKey}`); - break; + console.log("No timeseries data available for this source.") + continue; } const xValues = timeseriesData["x_values"]; @@ -214,8 +214,7 @@ class MultiSiteLineChart extends React.Component { }; if (plotData.length === 0) { - const keys = this.props.selectedSources; - return
No data for {keys}
; + return
Retrieving data...
; } else { return (
@@ -223,26 +222,26 @@ class MultiSiteLineChart extends React.Component {
- {/* - */} + +
); From c816016c95a4a888c31f9b72deb8f728d83c983d Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 7 Sep 2023 09:23:38 +0100 Subject: [PATCH 17/21] Retrieval and initial plot working, added defaults.json to set defaults --- src/Dashboard.js | 175 +++++++++--------------- src/components/LeafletMap/LeafletMap.js | 4 - src/data/defaults.json | 7 + 3 files changed, 71 insertions(+), 115 deletions(-) create mode 100644 src/data/defaults.json diff --git a/src/Dashboard.js b/src/Dashboard.js index 6d01b4c..6027597 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -1,23 +1,17 @@ import React from "react"; import { Routes, Route, Link, HashRouter } from "react-router-dom"; import { cloneDeep, has, set, get } from "lodash"; +import { Button, MenuItem } from "@mui/material"; +import LaunchIcon from "@mui/icons-material/Launch"; import ControlPanel from "./components/ControlPanel/ControlPanel"; -import OverlayContainer from "./components/OverlayContainer/OverlayContainer"; - -import Overlay from "./components/Overlay/Overlay"; import FAQ from "./components/FAQ/FAQ"; import LiveData from "./components/LiveData/LiveData"; import Explainer from "./components/Explainer/Explainer"; - -import { importSiteImages, createSourceKey } from "./util/helpers"; +import { createSourceKey } from "./util/helpers"; import styles from "./Dashboard.module.css"; -// Site description information -import siteInfoJSON from "./data/siteInfo.json"; -import { Button, MenuItem } from "@mui/material"; - -import LaunchIcon from "@mui/icons-material/Launch"; +import siteDefaults from "./data/defaults.json"; async function retrieveJSON(url) { return await (await fetch(url)).json(); @@ -27,6 +21,24 @@ class Dashboard extends React.Component { constructor(props) { super(props); + let defaultSite = null; + let defaultSpecies = null; + let defaultInlet = null; + let defaultInstrument = null; + let defaultNetwork = null; + let defaultSourceKey = null; + + try { + defaultSite = siteDefaults["site"]; + defaultSpecies = siteDefaults["species"]; + defaultInlet = siteDefaults["inlet"]; + defaultInstrument = siteDefaults["instrument"]; + defaultNetwork = siteDefaults["network"]; + defaultSourceKey = createSourceKey(defaultSpecies, defaultNetwork, defaultSite, defaultInlet, defaultInstrument); + } catch (error) { + console.error("Unable to set defaults."); + } + this.state = { error: null, isLoaded: false, @@ -38,6 +50,12 @@ class Dashboard extends React.Component { layoutMode: "dashboard", colours: {}, dataStore: {}, + defaultSite: defaultSite, + defaultSpecies: defaultSpecies, + defaultInlet: defaultInlet, + defaultInstrument: defaultInstrument, + defaultNetwork: defaultNetwork, + defaultSourceKey: defaultSourceKey, }; this.dataRepoURL = "https://raw.githubusercontent.com/openghg/temp_data_dashboard/main/"; @@ -52,14 +70,6 @@ class Dashboard extends React.Component { // These handle the initial setup of the app - /** - * Read the config file to setup the default site, species etc - */ - setupDefaults() { - throw new Error("Not implemented."); - // this.state... - } - /** * Converts data to the format required by Plotly * @@ -67,6 +77,7 @@ class Dashboard extends React.Component { * */ toPlotly(data) { + console.log("Processing datas for plotly"); const x_timestamps = Object.keys(data); const x_values = x_timestamps.map((d) => new Date(parseInt(d))); const y_values = Object.values(data); @@ -131,25 +142,17 @@ class Dashboard extends React.Component { * "network_site_inlet_instrument": {"x_values": [1,2,3], "y_values": [1,2,3]} * } * } - * + * Only the default source's data will be retrieved, all others will be assigned a null */ populateAndRetrieve(metadata) { - // Loop over the metadata dictionary - // Create the - // This should aleady be in the right shape, we use the nested aspect of - // the dictionary to make it easy to populate the interface, creating - // some lookup tables for filenames etc below - // this.state.completeMetadata = metadata; let defaultSpecies = null; let defaultSite = null; let defaultInlet = null; // Not sure if we need default instrument but - // let defaultInstrument = null; let defaultNetwork = null; - let defaultSourceKey = null; - // We just need to pull out the initial data - // We retrieve only the first dataset and then populate the other data values with nulls - // When this data is selected the app will retrieve the data + // Don't change this, it's handled in the constructor by reading defaults.json + // otherwise a default is selected from the first + let defaultSourceKey = this.state.defaultSourceKey; // Store the data itself let dataStore = {}; @@ -157,7 +160,7 @@ class Dashboard extends React.Component { let metaStore = {}; // Store the structure to allow easy building of the interface dynamically let siteStructure = {}; - // Do we need filename lookup? + // Filename lookup for dynamic retrieval let filenameLookup = {}; try { @@ -176,23 +179,16 @@ class Dashboard extends React.Component { // We'll use this to create a lightweight structure for the creation of the interface const nestedSourceKey = `${species}.${network}.${site}.${inlet}.${instrument}`; const filepath = fileMetadata["filepath"]; + const sourceMetadata = fileMetadata["metadata"]; - // We retrieve the data for the default source - // and store a null in all the sources we don't retrieve + // We don't setState here as we need to use defaultSourceKey below + // and setState runs asynchronously so may not update the state value by the time + // we get to where we need it if (defaultSourceKey === null) { defaultSourceKey = completeSourceKey; - const url = new URL(filepath, this.dataRepoURL).href; - // Here we add the data directly as this is on first load - retrieveJSON(url).then((result) => { - console.log(`Retrieving data from ${url} for initial setup.`); - const forPlotly = this.toPlotly(result); - set(dataStore, completeSourceKey, forPlotly); - }); - } else { - set(dataStore, completeSourceKey, null); } - const sourceMetadata = fileMetadata["metadata"]; + set(dataStore, completeSourceKey, null); set(metaStore, completeSourceKey, sourceMetadata); set(siteStructure, nestedSourceKey, completeSourceKey); // Save the filepath in the data repository for each lookup using the source key @@ -206,19 +202,30 @@ class Dashboard extends React.Component { console.error(`Error processing raw data - ${error}`); } - // Should we use setState here? Does that work properly now? - // Disabled the no direct mutation rule here as this only gets called from the constructor - /* eslint-disable react/no-direct-mutation-state */ - // Give each site a colour - this.state.dataStore = dataStore; - this.state.metaStore = metaStore; - this.state.siteStructure = siteStructure; - this.state.filenameLookup = filenameLookup; - this.state.defaultSpecies = defaultSpecies; - this.state.selectedSources = new Set([defaultSourceKey]); - this.state.selectedSpecies = defaultSpecies; - this.state.isLoaded = true; - /* eslint-enable react/no-direct-mutation-state */ + // Here we add the data directly as this is on first load + // We retrieve the data for the default source + const filepath = get(filenameLookup, defaultSourceKey); + const url = new URL(filepath, this.dataRepoURL).href; + console.log("Retrieving default source data from ", url); + retrieveJSON(url) + .then((result) => { + const forPlotly = this.toPlotly(result); + set(dataStore, defaultSourceKey, forPlotly); + }) + .then(() => { + this.setState({ isLoaded: true }); + }); + + this.setState({ + dataStore: dataStore, + metaStore: metaStore, + siteStructure: siteStructure, + filenameLookup: filenameLookup, + defaultSpecies: defaultSpecies, + selectedSources: new Set([defaultSourceKey]), + selectedSpecies: defaultSpecies, + defaultSourceKey: defaultSourceKey, + }); } componentDidMount() { @@ -229,9 +236,6 @@ class Dashboard extends React.Component { retrieveJSON(metadataURL).then( (metadata) => { this.populateAndRetrieve(metadata); - this.setState({ - isLoaded: true, - }); }, (error) => { this.setState({ @@ -242,30 +246,6 @@ class Dashboard extends React.Component { ); } - /** - * @deprecated Deprecated in the DECC Dashboard - */ - buildSiteInfo() { - console.warn("This function may be removed as it unused in this version of the dashboard."); - const siteImages = importSiteImages(); - - let siteData = {}; - for (const site of Object.keys(siteInfoJSON)) { - try { - siteData[site] = {}; - siteData[site]["image"] = siteImages[site]; - siteData[site]["description"] = siteInfoJSON[site]["description"]; - } catch (error) { - console.error(error); - } - } - - // Disabled the no direct mutation rule here as this only gets called from the constructor - /* eslint-disable react/no-direct-mutation-state */ - this.state.siteInfo = siteData; - /* eslint-enable react/no-direct-mutation-state */ - } - // These handle app control by the components /** @@ -349,7 +329,7 @@ class Dashboard extends React.Component { const defaultSource = Object.keys(speciesData)[0]; // This just gives us a partial key, we need to add in the species to get // a complete sourceKey - const sourceKey = `${species}.${defaultSource}` + const sourceKey = `${species}.${defaultSource}`; newSources.add(sourceKey); } @@ -369,35 +349,9 @@ class Dashboard extends React.Component { this.setState({ showSidebar: !this.state.showSidebar }); } - /** Was used to set an overlay of the site and show an image, currently unused - * @deprecated - * - */ - setSiteOverlay(e) { - console.warn("Deprecated function. May be removed."); - const siteCode = String(e.target.dataset.onclickparam).toUpperCase(); - const siteInfo = this.state.siteInfo[siteCode]; - - const siteText = siteInfo["description"]; - const image = siteInfo["image"]; - const alt = `Image of ${siteCode}`; - - const overlay = ( - - ); - - this.toggleOverlay(); - this.setOverlay(overlay); - } - render() { let { error, isLoaded } = this.state; - let overlay = null; - if (this.state.overlayOpen) { - overlay = {this.state.overlay}; - } - let extraSidebarStyle = {}; if (this.state.showSidebar) { extraSidebarStyle = { transform: "translateX(0px)" }; @@ -469,7 +423,6 @@ class Dashboard extends React.Component { } /> - {overlay} ); diff --git a/src/components/LeafletMap/LeafletMap.js b/src/components/LeafletMap/LeafletMap.js index 54c4d4a..b35346f 100644 --- a/src/components/LeafletMap/LeafletMap.js +++ b/src/components/LeafletMap/LeafletMap.js @@ -1,11 +1,7 @@ import PropTypes from "prop-types"; import React from "react"; import { LayerGroup, MapContainer, ImageOverlay, TileLayer, CircleMarker, Popup } from "react-leaflet"; -import { createSourceKey } from "../../util/helpers"; import { get } from "lodash"; -// import TextButton from "../TextButton/TextButton"; -// import "./LeafletMapResponsive.css"; - import TextButton from "../TextButton/TextButton"; import styles from "./LeafletMap.module.css"; diff --git a/src/data/defaults.json b/src/data/defaults.json new file mode 100644 index 0000000..af3f0cf --- /dev/null +++ b/src/data/defaults.json @@ -0,0 +1,7 @@ +{ + "site": "tac", + "network": "decc", + "species": "co2", + "inlet": "185m", + "instrument": "picarro" +} From 1c281161f4a94b7c51dc883183763e7f6fe3ad09 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Thu, 7 Sep 2023 09:25:29 +0100 Subject: [PATCH 18/21] Swap dropdown and clear buttons around --- src/components/ObsBox/ObsBox.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/ObsBox/ObsBox.js b/src/components/ObsBox/ObsBox.js index dabf08b..a191cac 100644 --- a/src/components/ObsBox/ObsBox.js +++ b/src/components/ObsBox/ObsBox.js @@ -57,12 +57,12 @@ class ObsBox extends React.Component {
{this.createEmissionsGraphs()}
-
{clearButton}
+
{clearButton}
); From 6ab626922974cc0241b0e87e35590addc6eb5699 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 20 Sep 2023 09:01:44 +0100 Subject: [PATCH 19/21] Remove unused default settings, added try catch for filename lookup --- src/Dashboard.js | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/Dashboard.js b/src/Dashboard.js index 6027597..13e94db 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -11,8 +11,6 @@ import Explainer from "./components/Explainer/Explainer"; import { createSourceKey } from "./util/helpers"; import styles from "./Dashboard.module.css"; -import siteDefaults from "./data/defaults.json"; - async function retrieveJSON(url) { return await (await fetch(url)).json(); } @@ -28,16 +26,9 @@ class Dashboard extends React.Component { let defaultNetwork = null; let defaultSourceKey = null; - try { - defaultSite = siteDefaults["site"]; - defaultSpecies = siteDefaults["species"]; - defaultInlet = siteDefaults["inlet"]; - defaultInstrument = siteDefaults["instrument"]; - defaultNetwork = siteDefaults["network"]; - defaultSourceKey = createSourceKey(defaultSpecies, defaultNetwork, defaultSite, defaultInlet, defaultInstrument); - } catch (error) { - console.error("Unable to set defaults."); - } + // TODO - read in data from dashboard_config.json + // These settings will control how the interface is built + // so users select data by site or by site inlets this.state = { error: null, @@ -204,7 +195,15 @@ class Dashboard extends React.Component { // Here we add the data directly as this is on first load // We retrieve the data for the default source - const filepath = get(filenameLookup, defaultSourceKey); + // TODO - can we show errors to the user in a clean way instead of just console? + let filepath = null; + try { + filepath = get(filenameLookup, defaultSourceKey); + } catch (error) { + console.error(`Error retrieving filename - ${error}`); + return; + } + const url = new URL(filepath, this.dataRepoURL).href; console.log("Retrieving default source data from ", url); retrieveJSON(url) From eae7cba97d1dedf83c85444e1621305497f683a2 Mon Sep 17 00:00:00 2001 From: Gareth Jones Date: Wed, 20 Sep 2023 09:02:10 +0100 Subject: [PATCH 20/21] Removed commented out regex lines as we won't use those --- src/components/MultiSiteLineChart/MultiSiteLineChart.js | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/components/MultiSiteLineChart/MultiSiteLineChart.js b/src/components/MultiSiteLineChart/MultiSiteLineChart.js index 571a8ff..f337ea4 100644 --- a/src/components/MultiSiteLineChart/MultiSiteLineChart.js +++ b/src/components/MultiSiteLineChart/MultiSiteLineChart.js @@ -137,13 +137,6 @@ class MultiSiteLineChart extends React.Component { plotData.push(trace); } - /*fetching all the site names to pass them as filename - using regex to remove and "-" within the name - */ - // let sites = []; - // sites = plotData.map((item) => item.name); - // sites = sites.map((item) => item.replace(/<\/?b>/g, "").replace(/\s*-\s*/g, "")); - const widthScaleFactor = 0.925; const uniOfBristol = require(`../../images/UniOfBristolLogo.png`); const metOffice = require(`../../images/Metoffice.png`); From d69c4726cd09f9a5e886d4d0dc8178749bb481e1 Mon Sep 17 00:00:00 2001 From: prasad Date: Tue, 10 Sep 2024 18:57:14 +0100 Subject: [PATCH 21/21] auth0 --- package-lock.json | 19 +++++++++++++++++++ package.json | 2 ++ src/Dashboard.js | 4 +++- src/ProtectedRoute/ProtectedRoute.js | 20 ++++++++++++++++++++ src/index.js | 7 +++++++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 src/ProtectedRoute/ProtectedRoute.js diff --git a/package-lock.json b/package-lock.json index 9b8d688..b6f554d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,8 @@ "name": "decc_dashboard", "version": "0.1.0", "dependencies": { + "@auth0/auth0-react": "^2.2.4", + "@auth0/auth0-spa-js": "^2.1.3", "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", "@material-ui/core": "^4.12.3", @@ -89,6 +91,23 @@ "node": ">=6.0.0" } }, + "node_modules/@auth0/auth0-react": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/@auth0/auth0-react/-/auth0-react-2.2.4.tgz", + "integrity": "sha512-l29PQC0WdgkCoOc6WeMAY26gsy/yXJICW0jHfj0nz8rZZphYKrLNqTRWFFCMJY+sagza9tSgB1kG/UvQYgGh9A==", + "dependencies": { + "@auth0/auth0-spa-js": "^2.1.3" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17 || ^18", + "react-dom": "^16.11.0 || ^17 || ^18" + } + }, + "node_modules/@auth0/auth0-spa-js": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@auth0/auth0-spa-js/-/auth0-spa-js-2.1.3.tgz", + "integrity": "sha512-NMTBNuuG4g3rame1aCnNS5qFYIzsTUV5qTFPRfTyYFS1feS6jsCBR+eTq9YkxCp1yuoM2UIcjunPaoPl77U9xQ==" + }, "node_modules/@babel/code-frame": { "version": "7.22.10", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.10.tgz", diff --git a/package.json b/package.json index 7a33da1..93dc623 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,8 @@ "version": "0.1.0", "private": true, "dependencies": { + "@auth0/auth0-react": "^2.2.4", + "@auth0/auth0-spa-js": "^2.1.3", "@emotion/react": "^11.11.1", "@emotion/styled": "^11.11.0", "@material-ui/core": "^4.12.3", diff --git a/src/Dashboard.js b/src/Dashboard.js index 13e94db..8601183 100644 --- a/src/Dashboard.js +++ b/src/Dashboard.js @@ -3,7 +3,7 @@ import { Routes, Route, Link, HashRouter } from "react-router-dom"; import { cloneDeep, has, set, get } from "lodash"; import { Button, MenuItem } from "@mui/material"; import LaunchIcon from "@mui/icons-material/Launch"; - +import ProtectedRoute from "./ProtectedRoute/ProtectedRoute"; import ControlPanel from "./components/ControlPanel/ControlPanel"; import FAQ from "./components/FAQ/FAQ"; import LiveData from "./components/LiveData/LiveData"; @@ -419,7 +419,9 @@ class Dashboard extends React.Component { } /> + + } /> diff --git a/src/ProtectedRoute/ProtectedRoute.js b/src/ProtectedRoute/ProtectedRoute.js new file mode 100644 index 0000000..0a500db --- /dev/null +++ b/src/ProtectedRoute/ProtectedRoute.js @@ -0,0 +1,20 @@ +import React from "react"; +import { useAuth0 } from "@auth0/auth0-react"; +import { Navigate } from "react-router-dom"; + +const ProtectedRoute = ({ children }) => { + const { isAuthenticated, loginWithRedirect, isLoading } = useAuth0(); + + if (isLoading) { + return
Loading...
; + } + + if (!isAuthenticated) { + loginWithRedirect(); // Redirect to Auth0 login + return null; + } + + return children; +}; + +export default ProtectedRoute; diff --git a/src/index.js b/src/index.js index cf7ad43..080df01 100644 --- a/src/index.js +++ b/src/index.js @@ -1,13 +1,20 @@ import React from 'react'; import ReactDOM from 'react-dom'; import './index.css'; +import { Auth0Provider } from "@auth0/auth0-react"; import Dashboard from './Dashboard'; import reportWebVitals from './reportWebVitals'; ReactDOM.render( + , + , document.getElementById('root') );