From 7107f53aa76c514b637b847c4d3b348974c9e3b5 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 1 May 2023 15:13:17 +0100 Subject: [PATCH 001/111] TC update + Box_filter class --- FileReaders.py | 42 ++-- Filters.py | 656 +++++++++++++++++++++++++++++++++++++++---------- MesoModels.py | 5 +- MicroModels.py | 368 +++++++++++++++++++++++++-- 4 files changed, 902 insertions(+), 169 deletions(-) diff --git a/FileReaders.py b/FileReaders.py index 11db94c..a87e5ca 100644 --- a/FileReaders.py +++ b/FileReaders.py @@ -44,23 +44,37 @@ def read_in_data(self, micro_model): micro_model: class MicroModel strs in micromodel have to be the same as hdf5 files output from METHOD. """ + + self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) + for prim_str in micro_model.get_prim_strs(): + if prim_str == "n": + self.translating_prims[prim_str] = "rho" + else: + self.translating_prims[prim_str] = prim_str + for prim_var_str in micro_model.prim_vars: try: + method_str = self.translating_prims[prim_var_str] for counter in range(self.num_files): - micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+prim_var_str][:] ) + micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+method_str][:] ) # The [:] is for returning the arrays not the dataset micro_model.prim_vars[prim_var_str] = np.array(micro_model.prim_vars[prim_var_str]) except KeyError: - print(f'{prim_var_str} is not in the hdf5 dataset: check Primitive/') + print(f'{method_str} is not in the hdf5 dataset: check Primitive/') + self.translating_aux = dict.fromkeys(micro_model.get_aux_strs()) + for aux_str in micro_model.get_aux_strs(): + self.translating_aux[aux_str] = aux_str + for aux_var_str in micro_model.aux_vars: try: + method_str = self.translating_aux[aux_var_str] for counter in range(self.num_files): - micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+aux_var_str][:] ) + micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+method_str][:] ) micro_model.aux_vars[aux_var_str] = np.array(micro_model.aux_vars[aux_var_str]) except KeyError: - print(f'{aux_var_str} is not in the hdf5 dataset: check Auxiliary/') + print(f'{method_str} is not in the hdf5 dataset: check Auxiliary/') # As METHOD saves endTime, the time variables (and points) need to be dealt with separately for dom_var_str in micro_model.domain_int_strs: @@ -91,19 +105,6 @@ def read_in_data(self, micro_model): print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - # for dom_var_str in micro_model.domain_vars: - # try: - # if dom_var_str in ['t','nt','tmin','tmax','points']: - # pass - # if dom_var_str in ['x', 'y']: - # micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] - # if dom_var_str in ['nx', 'ny']: - # micro_model.domain_vars[dom_var_str] = int(self.hdf5_files[0]['Domain/' + dom_var_str][:] - # else: - # micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] - # except KeyError: - # print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - micro_model.domain_vars['nt'] = self.num_files for counter in range(self.num_files): micro_model.domain_vars['t'].append( float(self.hdf5_files[counter]['Domain/endTime'][:])) @@ -113,17 +114,10 @@ def read_in_data(self, micro_model): micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ micro_model.domain_vars['y']] - if __name__ == '__main__': from MicroModels import * FileReader = METHOD_HDF5('./Data/test_res100/') MicroModel = IdealMHD_2D() - - # print(FileReader.get_hdf5_keys()) - # FileReader.micro_model_compatibility(MicroModel) FileReader.read_in_data(MicroModel) - for str in MicroModel.domain_vars: - print(str + ' ',type(MicroModel.domain_vars[str]),' ', MicroModel.domain_vars[str], '\n') - \ No newline at end of file diff --git a/Filters.py b/Filters.py index 4897072..e4175de 100644 --- a/Filters.py +++ b/Filters.py @@ -2,10 +2,11 @@ """ Created on Tue Mar 28 15:36:01 2023 -@author: Marcus +@author: Thomas """ import numpy as np +import time import scipy.integrate as integrate from scipy.optimize import minimize from itertools import product @@ -15,7 +16,10 @@ class Favre_observers(object): - + """ + Class for computing the Favre observer given a micromodel. + Work in any dimension, read on construction from the micro-model. + """ def __init__(self, micro_model, box_len): """ Parameters: @@ -26,9 +30,16 @@ def __init__(self, micro_model, box_len): ----- To-do: think about checking compatibility (dimension + baryon currrent) """ - self.spatial_dims = micro_model.spatial_dims + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() self.L = box_len + self.residuals_methods = { + "gauss" : self.Favre_residual_Gauss , + "linear" : self.Favre_residual , + "inbuilt" : self.Favre_residual_ib + } + def set_box_length(self, bl): self.L = bl @@ -50,7 +61,6 @@ def Mink_dot(self, vec1, vec2): dot += vec1[i] * vec2[i] return dot - def get_U_from_vels(self, spatial_vels): """ Build unit vectors starting from spatial components @@ -72,10 +82,8 @@ def get_U_from_vels(self, spatial_vels): U = [W] for i in range(len(spatial_vels)): U.append(W * spatial_vels[i]) - return U + return np.array(U) - - def get_tetrad_from_vels(self, spatial_vels): """ Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels @@ -104,8 +112,7 @@ def get_tetrad_from_vels(self, spatial_vels): tetrad += [es[i]] return tetrad - - def Favre_residual(self, spatial_vels, point, lin_spacing): + def Favre_residual(self, spatial_vels, point, lin_spacing = 10): """ Compute the drift of baryons through the box built from vx_vy. First get the center of the 2*(d+1) faces of the (d+1)-box, then build coords @@ -127,25 +134,23 @@ def Favre_residual(self, spatial_vels, point, lin_spacing): Notes: ------ - Much faster than method based on inbuilt dblquad: 100 points (per face) gives decent - results, and is 250 times faster. + Much faster than method based on inbuilt dblquad. """ - tetrad = self.get_tetrad_from_vels(spatial_vels) flux = 0 + xs = [] + for i in range(self.spatial_dims): + xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) + coords = [] + for element in product(*xs): + coords.append(np.array(element)) + for vec in tetrad: rem_vecs = [x for x in tetrad if not (x==vec).all()] for i in range(2): center = point + np.multiply( (-1)**i * self.L / 2, vec) - xs = [] - for i in range(self.spatial_dims): - xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) - coords = [] - for element in product(*xs): - coords.append(np.array(element)) - surf_coords = [] for coord in coords: temp = center @@ -154,47 +159,102 @@ def Favre_residual(self, spatial_vels, point, lin_spacing): surf_coords.append(temp) for coord in surf_coords: - U = micro_model.get_interpol_struct('bar_vel', coord) - rho = micro_model.get_interpol_prim(['rho'], coord) - Na = np.multiply(rho, U) + U = self.micro_model.get_interpol_struct('bar_vel', coord) + n = self.micro_model.get_interpol_prim(['n'], coord) + Na = np.multiply(n, U) flux += self.Mink_dot(Na, vec) flux *= (self.L / lin_spacing) ** self.spatial_dims return abs(flux) - - - def point_flux(self, x, y , point, Vx, Vy, normal): + + def Favre_residual_Gauss(self, spatial_vels, point, order = 3): """ - Compute the baryon flux at a point given two coordinates that param the surface - Identified by normal. + Alternative for computing manually the Favre residual, using the Gauss + Legendre method. - Parameters: + Parameters: ----------- - coords: d floats, adapted coordinates of the box face - - point: list of floats (t,x,y) - - Vx, Vy: (2+1)-arrays, tangent vectors to the box face + spatial_vels: list of d (spatial dimension) floats, spatial coord of vel + + point: list of d+1 floats (t,x,y) for the box center - normal: (2+1)-array, normal to the box face + order: integer, order of the Gauss-Legendre sampling method. Returns: -------- - Float: flux at the point - + float: absolute flux + Notes: ------ - As this is used in Favre_residual_ib - which uses dblquad - this method has been - developed for the 2+1 dimensional case only. + Much faster than method based on inbuilt dblquad. Should be more accurate + than the one based on linearly spaced points. """ - coords = point + np.multiply(x, Vx) + np.multiply(y, Vy) - U = micro_model.get_interpol_struct('bar_vel', coords) - rho = micro_model.get_interpol_prim(['rho'], coords) - Na = np.multiply(rho, U) - flux = self.Mink_dot(Na, normal) - return flux + ps1d = [] + ws1d = [] + if order == 3: + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + elif order == 4: + p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) + p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) + w1 = (18. + np.sqrt(30) ) / 36. + w2 = (18. - np.sqrt(30) ) / 36. + ps1d = [p1, -p1, p2, -p2] + ws1d = [w1, w1, w2, w2] + elif order == 5: + p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 + p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 + w1 = (322 + 13 * np.sqrt(70)) /900 + w2 = (322 - 13 * np.sqrt(70)) /900 + ps1d = [0, p1, -p1, p2, -p2] + ws1d = [128/225, w1, w1, w2, w2] + else: + print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") + return [] - def Favre_residual_ib(self, vx_vy, point): + xs = [] + ws = [] + for _ in range(self.spatial_dims): + xs.append(ps1d) + ws.append(ws1d) + + coords = [] + for element in product(*xs): + coords.append(np.multiply(self.L/2, np.array(element) )) + + totws = [] + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + # for i in range(len(coords)): + # print(f'Coord: {coords[i]} , weight: {weights[i]}') + tetrad = self.get_tetrad_from_vels(spatial_vels) + flux = 0. + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + + surf_coords = [] + for coord in coords: + temp = center + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], rem_vecs[i]) + surf_coords.append(temp) + + for i, coord in enumerate(surf_coords): + U = self.micro_model.get_interpol_struct('bar_vel', coord) + n = self.micro_model.get_interpol_prim(['n'], coord) + Na = np.multiply(n, U) + flux += self.Mink_dot(Na, vec) * totws[i] + + flux *= (self.L /2 ) ** self.spatial_dims + return abs(flux) + + def Favre_residual_ib(self, spatial_vels, point, abserr = 1e-7): """ Compute the residual using inbuilt method dblquad Based on function point_flux @@ -214,44 +274,153 @@ def Favre_residual_ib(self, vx_vy, point): Vastly slower than method above. As this is based on dblquad, this method works fine only if it's 2+1 dim + Also, dblquad returns an estimate of the error. This info is presently discarded + in order to have the method return a scalar value so that this can be minimized + by find observer. + """ - if self.spatial_dim != 2: - print('This method uses a 2-dim integrator, so works fine only for 2 spatial dimensions ') - return [] - - xy_range = [- self.L / 2, + self.L / 2] - tetrad = self.get_tetrad_from_vels(vx_vy) + tetrad = self.get_tetrad_from_vels(spatial_vels) flux = 0 partial_flux = 0 error = 0 partial_error = 0 - for vec in tetrad: - rem_vecs = [x for x in tetrad if not (x==vec).all()] - for i in range(2): - partial_flux, partial_error = integrate.dblquad(self.point_flux, xy_range[0], xy_range[1], xy_range[0], xy_range[1], \ - args = (point, rem_vecs[0],rem_vecs[1],vec), epsabs = 1e-6 )[:] - flux += partial_flux - error += partial_error - return abs(flux) , error + if self.spatial_dims == 2: + def point_flux(x, y , center, Vx, Vy, normal): + coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + U = self.micro_model.get_interpol_struct('bar_vel', coords) + n = self.micro_model.get_interpol_prim(['n'], coords) + Na = np.multiply(n, U) + flux = self.Mink_dot(Na, normal) + return flux + + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + partial_flux, partial_error = integrate.dblquad(point_flux, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, \ + args = (center, rem_vecs[0], rem_vecs[1], vec), epsabs = abserr)[:] + flux += partial_flux + error += partial_error + return abs(flux) + + elif self.spatial_dims == 3: + def point_flux(x, y, z , center, Vx, Vy, Vz, normal): + coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + np.multiply(z, Vz) + U = self.micro_model.get_interpol_struct('bar_vel', coords) + n = self.micro_model.get_interpol_prim(['n'], coords) + Na = np.multiply(n, U) + flux = self.Mink_dot(Na, normal) + return flux + + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + partial_flux, partial_error = integrate.tplquad(point_flux, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2,\ + args = (center, rem_vecs[0], rem_vecs[1], vec), epsabs = abserr)[:] + flux += partial_flux + error += partial_error + return abs(flux) + + def find_observer(self, point, residual_str = "gauss"): + """ + Key function: minimize the Favre_residual and find the Favre observer at point. + + Parameters: + ----------- + point: list of spatial_dims+1 floats (t,x,y) + + residual_str: string, default to "gauss" + string for the method used to compute the residual, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + OptimizedResult of the minimization via scipy.optimize.minimize + + Notes: + ------ + To change the number of points to be used with linear or Gauss-Legendre methods, + or the absolute relative of the inbuilt method, change the default values of + the optional arguments in the corresponding methods. + """ + U = self.micro_model.get_interpol_struct('bar_vel', point) + guess = [] + for i in range(1, len(U)): + guess.append(U[i] / U[0]) + guess = np.array(guess) + try: + sol = minimize(self.residuals_methods[residual_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) + return sol + except KeyError: + print(f"The method you want to use for computing the residual, {residual_str}, does not exist!") + return None + + def find_observers_points(self, points, residual_str = "gauss"): + """ + Key function: minimize the Favre_residual and find the Favre observers for points. + Spacing is the param passed to Favre_residual to sample the box faces. - def find_observers(self, num_points, ranges, spacing ): + Parameters: + ----------- + points: list of spatial_dims+1 floats, ordered as (t,x,y) + + residual_str: string, default to "gauss" + string for the method used to compute the residual, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which minimization is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which the minimization failed + + Notes: + ------ + To change the number of points to be used with linear or Gauss-Legendre methods, + change the default values of the optional arguments in the corresponding methods. """ - Main function: minimize the Favre_residual and find the Favre observers for points + observers = [] + funs = [] + success_coords = [] + failed_coords = [] + + for point in points: + sol = self.find_observer(point, residual_str) + if sol.success: + observers.append(self.get_U_from_vels(sol.x)) + funs.append(sol.fun) + success_coords.append(point) + + if (sol.fun > 1e-5): + print(f"Warning, residual is large at {point}: ", sol.fun) + else: + print(f'Failed for coordinates: {point}, due to', sol.message) + failed_coords.append(point) + return [success_coords, observers, funs] , failed_coords + + def find_observers_ranges(self, num_points, ranges, residual_str = "gauss" ): + """ + Key function: minimize the Favre_residual and find the Favre observers for points in linearly spaced (with spacing num_points) in the ranges. Spacing is the param passed to Favre_residual to sample the box faces. Parameters: ----------- - num_points: list of floats (t,x,y,(z)) - number of points to find observers at + num_points: list of integers + number of points to find observers at in each direction ranges: list of lists of two floats [[t_min,t_max], ...] Define the coord ranges to find observers at - spacing: integer - param to be passed to Favre_residual + residual_str: string, default to "gauss" + string for the method used to compute the residual, must be chosen within + ['gauss', 'linear', 'inbuilt'] Returns: -------- @@ -264,88 +433,327 @@ def find_observers(self, num_points, ranges, spacing ): Notes: ------ - This uses the faster residual, not the one based on the inbuilt dblquad + To change the number of points to be used with linear or Gauss-Legendre methods, + change the default values of the optional arguments in the corresponding methods. """ list_of_coords = [] for i in range(len(num_points)): list_of_coords.append( np.linspace( ranges[i][0], ranges[i][-1] , num_points[i]) ) - coords = [] + points = [] for element in product(*list_of_coords): - coords.append(np.array(element)) + points.append(np.array(element)) - observers = [] - funs = [] - success_coords = [] - failed_coords = [] + return self.find_observers_points(points, residual_str) - for coord in coords: - U = micro_model.get_interpol_struct('bar_vel', coord) - guess = [] - for i in range(1, len(U)): - guess.append(U[i] / U[0]) - guess = np.array(guess) - try: - sol = minimize(self.Favre_residual, x0 = guess, args = (coord, spacing), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) - if sol.success: - observers.append(self.get_U_from_vels(sol.x)) - funs.append(sol.fun) - success_coords.append(coord) - - if (sol.fun > 1e-5): - print(f"Warning, residual is large at {coord}: ", sol.fun) - except: - print(f'Failed for coordinates: {coord}, due to', sol.message) - failed_coords.append(coord) - return [success_coords, observers, funs] , failed_coords +class Box_filter(object): + """ + Class for box-filtering the variables of a micro_model. + Work in any dimensions, read on construction from the micro_model. + The observers for covariant filtering are computed separately and must be + passed when the fitlering methods are called. + """ + def __init__(self, micro_model, filter_width): + """ + Parameters: + ---------- + filter_width: float, width of the filter window in each direction + micro_model: instance of micro_model class, so to have access to its structures + """ + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() + self.filter_width = filter_width -if __name__ == '__main__': - CPU_start_time = time.process_time() + def Mink_dot(self, vec1, vec2): + """ + Parameters: + ----------- + vec1, vec2 : list of floats (or np.arrays) - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() - FileReader.read_in_data(micro_model) - micro_model.setup_structures() + Return: + ------- + mink-dot (cartesian) in 1+n dim + """ + if len(vec1) != len(vec2): + print("The two vectors passed to Mink_dot are not of same dimension!") - filter = Favre_observers(micro_model,0.001) - - # tetrad = filter.get_tetrad_from_vxvy([0.5,0.73]) - # print(type(tetrad[0]),' ',type(tetrad[1]),' ',type(tetrad[2]),'\n', tetrad[0],' ',tetrad[1],' ',tetrad[2],'\n') - # print(filter.Mink_dot(tetrad[0],tetrad[1]), filter.Mink_dot(tetrad[0],tetrad[2]), filter.Mink_dot(tetrad[1],tetrad[2])) - # print(type(tetrad)) + dot = -vec1[0]*vec2[0] + for i in range(1,len(vec1)): + dot += vec1[i] * vec2[i] + return dot - # smart_guess = micro_model.get_interpol_prim(['vx','vy'],[0.5,0.5,0.5]) + def set_filter_width(self, filter_width): + """ + Method to change the width of the filter. - # CPU_start_time = time.process_time() - # res = filter.Favre_residual(smart_guess,[0.5,0.5,0.5], 10) - # print('Residual: ',res,f'\nElapsed CPU time is {time.process_time() - CPU_start_time} with {10**filter.spatial_dim} points per face\n') + Parameters: + ----------- + filter_width: float - # CPU_start_time = time.process_time() - # res, error = filter.Favre_residual_ib(smart_guess,[0.5,0.5,0.5])[:] - # print('Residual: ',res,"\nError estimate: ",error,f'\nElapsed CPU time is {time.process_time() - CPU_start_time} with the inbuilt method') + Returns: + -------- + None + """ + self.filter_width = filter_width - CPU_start_time = time.process_time() - coord_range = [[1.502,1.504],[0.5,0.7],[0.5,0.7]] - num_points = [1,1,1] - - min_res, failed_coord = filter.find_observers(num_points, coord_range, 10) - for i in range(len(min_res[0])): - for j in range(len(min_res)): - print(min_res[j][i]) - print('\n') + def set_micro_model(self, micro_model): + """ + Method to change the micro_model to be filtered. - num_minim = 1 - for x in num_points: - num_minim *= x - print(f'Elapsed CPU time for finding {num_minim} observer is {time.process_time() - CPU_start_time}.') - print('Failed coordinates:', failed_coord) + Parameters: + ----------- + micro_model: instance of micro_model class + Returns: + -------- + None + """ + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() + + def complete_U_tetrad(self, U): + """ + Build unit vectors orthogonal to observer U + U has to be normalized to -1. + + Parameters: + ----------- + U: np.array of shape (1+spatial_dims,) + + Returns: + list of spatial_dims numpy arrays of shape (1+spatial_dims,) + which complete U to a orthonormal tetrad + """ + es =[] + for _ in range(self.spatial_dims): + es.append(np.zeros(self.spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1. + triad = [] + for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! + vec = vec + np.multiply(self.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(self.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1 / np.sqrt(self.Mink_dot(vec, vec))) + triad += [es[i]] + return triad + def filter_var_point_ip(self, var_str, point, observer, sample_method = "gauss", num_points = 3): + """ + First complete the observer to a tetrad at the point. Then build coords for + (linearly spaced) sample points in the spatial directions. Then approximate the + filter integral as a Riemann sum. + Parameters: + ----------- + var_str: string corresponding to a variable of the micro_model + + point: list of floats (t,x,y) + + observer: np.array of shape (1+spatial_dims,) . + sample_method: string, either 'gauss' or 'linear'. Decide how to sample the d-volume for + filtering. + + num_points: integer, number of sample points in each spatial direction. + Returns: + -------- + nd.array with shape of the variable, the box_filtered quantity. + + Notes: + ------ + The method uses interpolated values for the structure. So this is expected to be + more costly than using only values on the grid points. Using Gauss-Legendre method + should give accurate results for low values of num-points. + + If Gauss-Legendre is using interpolated values is too expensive, change: + self.micro_model.get_interpol_var() for self.micro_model.get_var_gridpoint(). + This avoids interpolating, and uses the values on the grid closest to input 'point'. + """ + xs = [] + coords = [] + if sample_method == "linear": + for i in range(self.spatial_dims): + xs.append(np.linspace(-self.filter_width /2 , self.filter_width /2, num_points)) + for element in product(*xs): + coords.append(np.array(element)) + + vecs = self.complete_U_tetrad(observer) + sample_points = [] + for coord in coords: + temp = np.array(point) + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], vecs[i]) + sample_points.append(temp) + + filtered_var = np.zeros(self.micro_model.get_interpol_var(var_str, point).shape) + for sample in sample_points: + filtered_var += self.micro_model.get_interpol_var(var_str, sample) + + return np.multiply(filtered_var, 1 / (num_points**self.spatial_dims)) + + elif sample_method == "gauss": + ps1d = [] + ws1d = [] + ws = [] + totws = [] + if num_points == 3: + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + elif num_points == 4: + p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) + p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) + w1 = (18. + np.sqrt(30) ) / 36. + w2 = (18. - np.sqrt(30) ) / 36. + ps1d = [p1, -p1, p2, -p2] + ws1d = [w1, w1, w2, w2] + elif num_points == 5: + p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 + p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 + w1 = (322 + 13 * np.sqrt(70)) /900 + w2 = (322 - 13 * np.sqrt(70)) /900 + ps1d = [0, p1, -p1, p2, -p2] + ws1d = [128/225, w1, w1, w2, w2] + else: + print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") + return None + + for _ in range(self.spatial_dims): + xs.append(ps1d) + ws.append(ws1d) + + for element in product(*xs): + coords.append(np.multiply(self.filter_width/2, np.array(element) )) + + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + vecs = self.complete_U_tetrad(observer) + sample_points = [] + for coord in coords: + temp = np.array(point) + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], vecs[i]) + sample_points.append(temp) + + filtered_var = np.zeros(self.micro_model.get_interpol_var(var_str, point).shape) + for i, sample in enumerate(sample_points): + filtered_var += totws[i] * self.micro_model.get_interpol_var(var_str, sample) + + return np.multiply(filtered_var, 1 / (2**self.spatial_dims)) + + else: + print("Sample methods to filter variable must be either 'gauss' or 'linear'! ") + return None + + def filter_var_manypoints_ip(self, var_str, points, observers, sample_method = "gauss", num_points = 3): + """ + Method to filter a variable in the micro_model given a list of points and observers. + + Parameters: + ----------- + var_str: string corresponding to a variable in the micro_model + + points: list of N lists of floats, ordered as (t,x,y) + + observers: list of N np.array of shape (1 + spatial_dims,) + + sample_method, num_points: string, int passed to filter method at a point to choose how to + sample the d-volume for box filtering. + + Returns: + -------- + List with the filtered var (np.float or nd.array depending on var_str) at all points + """ + if len(points) != len(observers): + print("The number of points and observers do not match!") + return [] + + filtered_var = [] + for i, point in enumerate(points): + filtered_var.append(self.filter_var_point_ip(var_str, point, observers[i], sample_method = sample_method, num_points = num_points )) + + return filtered_var + + def filter_var_point_inbuilt(self, var_str, point, observer): + """ + Computed the filtered variable using the inbuilt scipy quad method and the interpolated + values of the quantity. + + Parameters: + ----------- + var_str: string, must correspond to a string in the micro_model + + point: list of 1+spatial_dims floats, center of the box-filter + + observer: nd.array of shape (1+spatial_dims,) + + Returns: + -------- + The filtered quantity at the point, or none if the dimensionality is more than 3+1. + + Notes: + ------ + Much slower than corresponding filter_var_manypoints_ip method, this has been implemented + to check the accuracy of the alternative methods. + """ + vecs = self.complete_U_tetrad(observer) + integrand = np.zeros(self.micro_model.get_interpol_var(var_str,point).shape) + error = np.zeros(self.micro_model.get_interpol_var(var_str,point).shape) + + if self.spatial_dims == 2: + def interpol_var_adapt_coord(x, y, *ind): + p = np.array(point) + p += np.multiply(x, vecs[0]) + np.multiply(y, vecs[1]) + return self.micro_model.get_interpol_var(var_str, p)[ind] + + for ind in np.ndindex(integrand.shape): + integrand[ind], error[ind] = integrate.dblquad(interpol_var_adapt_coord, -self.filter_width/2 , self.filter_width/2, -self.filter_width/2, self.filter_width/2, \ + args = (ind), epsabs = 1e-7 )[:] + elif self.spatial_dims ==3: + def interpol_var_adapt_coord(self, x, y, z, *ind): + p = np.array(point) + p += np.multiply(x, vecs[0]) + np.multiply(y, vecs[1]) + np.multiply(z, vecs[2]) + return self.micro_model.get_interpol_var(var_str, p)[ind] + + for ind in np.ndindex(integrand.shape): + integrand[ind], error[ind] = integrate.tplquad(interpol_var_adapt_coord, -self.filter_width/2, self.filter_width/2, -self.filter_width/2, self.filter_width/2, \ + -self.filter_width/2, self.filter_width/2, args = (point), epsabs = 1e-7 )[:] + else: + print("The inbuilt method is implemented in 2+1 and 3+1 dims only!") + return None + + return np.multiply( integrand, 1 / (self.filter_width**self.spatial_dims)), error + +if __name__ == '__main__': + CPU_start_time = time.process_time() + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + constraint = Favre_observers(micro_model,0.001) + filter = Box_filter(micro_model, 0.001) + + var = "SET" + point = [1.502,0.3,0.5] + observer = constraint.get_U_from_vels( constraint.find_observer(point).x) + CPU_start_time = time.process_time() + filtvar1 = filter.filter_var_point_inbuilt(var, point, observer) + print(f"CPU time to filter SET with in-built method is {time.process_time()- CPU_start_time}. ") + + CPU_start_time = time.process_time() + filtvar2 = filter.filter_var_point_ip(var, point, observer) + print(f"CPU time to filter SET with Gauss method is {time.process_time()- CPU_start_time}. ") + + print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") + \ No newline at end of file diff --git a/MesoModels.py b/MesoModels.py index 36c8c0b..fa7d6a5 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -155,7 +155,10 @@ def calculate_coefficients(self, point): return -Pi_res/Theta, -q_res/omega, -pi_res/sigma - +class meso_model_example(object): + + def __init__(self, micro_model, constraint, filter): + pass diff --git a/MicroModels.py b/MicroModels.py index 0f7e92d..5d70840 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -9,6 +9,7 @@ from scipy.interpolate import interpn # from scipy import interpolate import numpy as np +import math import time # These are the symbols, so be careful when using these to construct vectors! @@ -51,7 +52,7 @@ def __init__(self, interp_method = "linear"): self.domain_vars[str] = [] #Dictionary for primitive var - self.prim_strs = ("vx","vy","rho","p","Bx","By") + self.prim_strs = ("vx","vy","n","p","Bx","By") self.prim_vars = dict.fromkeys(self.prim_strs) for str in self.prim_strs: self.prim_vars[str] = [] @@ -68,8 +69,11 @@ def __init__(self, interp_method = "linear"): for str in self.structures_strs: self.structures[str] = [] + def get_spatial_dims(self): + return self.spatial_dims + def get_domain_strs(self): - return self.domain_info_strs + self.domain_points_strs + return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs def get_prim_strs(self): return self.prim_strs @@ -78,7 +82,7 @@ def get_aux_strs(self): return self.aux_strs def get_structures_strs(self): - return self.get_structures_strs + return self.structures_strs def setup_structures(self): """ @@ -102,7 +106,7 @@ def setup_structures(self): fibr_b = np.array([self.aux_vars['b0'][h,i,j],self.aux_vars['bx'][h,i,j],self.aux_vars['by'][h,i,j]]) - self.structures["SET"][:,:,h,i,j] = (self.prim_vars["rho"][h,i,j] + self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]) * \ + self.structures["SET"][:,:,h,i,j] = (self.prim_vars["n"][h,i,j] + self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]) * \ np.outer( vel,vel) + (self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]/2) * self.metric \ - np.outer(fibr_b, fibr_b) @@ -114,6 +118,9 @@ def setup_structures(self): self.structures['Faraday'][:,:,h,i,j] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) + self.vars = self.prim_vars + self.vars.update(self.aux_vars) + self.vars.update(self.structures) def get_interpol_prim(self, vars, point): """ @@ -139,7 +146,7 @@ def get_interpol_prim(self, vars, point): try: res.append( interpn(self.domain_vars["points"], self.prim_vars[var_name], point, method = self.interp_method)[0]) #, bounds_error = False) except KeyError: - print(f"{var_name} does not belong to the primitive variables of the micromodel!") + print(f"{var_name} does not belong to the primitive variables of the micro_model!") return res def get_interpol_aux(self, vars, point): @@ -166,9 +173,8 @@ def get_interpol_aux(self, vars, point): try: res.append( interpn(self.domain_vars["points"], self.aux_vars[var_name], point, method = self.interp_method)[0]) except KeyError: - print(f"{var_name} does not belong to the auxiliary variables of the micromodel!") + print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") return res - def get_interpol_struct(self, var, point): """ @@ -184,7 +190,7 @@ def get_interpol_struct(self, var, point): Return ------ Array with the interpolated values of the var structure - Empty list if var is not a structure in the micromodel + Empty list if var is not a structure in the micro_model Notes ----- @@ -206,9 +212,327 @@ def get_interpol_struct(self, var, point): for b in range(len(self.structures[var][0,:,0,0,0])): res[a,b]= interpn(self.domain_vars["points"],self.structures[var][a,b,:,:,:], point, method = self.interp_method)[0] else: - print(f"{var} does not belong to the structures in the Micromodel") + print(f"{var} does not belong to the structures in the micro_model") return res + def get_interpol_var(self, var, point): + """ + Returns the interpolated var at the point. The method calls the correct one + for prim, aux or struct. + + Parameters + ---------- + var : str corresponding to either a primitive, auxiliary or structre variable + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values of the var + Empty list if var is not a primitive, auxiliary o structure variable of the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + if var in self.get_prim_strs(): + return self.get_interpol_prim([var], point)[0] + + elif var in self.get_aux_strs(): + return self.get_interpol_aux([var], point)[0] + + elif var in self.get_structures_strs(): + return self.get_interpol_struct(var, point) + else: + print('The variable to be interpolated does not belong to the micromodel!') + + def find_nearest(self, array, value): + """ + Returns closest value to input 'value' in 'array' + + Parameters: + ----------- + array: np.array of shape (n,) + + value: float + + Returns: + -------- + float + + Note: + ----- + To be used together with find_nearest_cell. + """ + idx = np.searchsorted(array, value, side="left") + if idx > 0 and (idx == len(array) or math.fabs(value - array[idx-1]) < math.fabs(value - array[idx])): + return array[idx-1] + else: + return array[idx] + + def find_nearest_cell(self, point): + """ + Use find nearest to find closest value in the grid to point. + Then returns the grid_point indices for this. + + Parameters: + ----------- + point: list of float, ordered as (t, x ,y) + + Returns: + -------- + List of three indices + + Notes: + ------ + This method is key to avoid using too many interpolations + should this becomes too expensive. + """ + t_pos = self.find_nearest(self.domain_vars["t"], point[0]) + x_pos = self.find_nearest(self.domain_vars["x"], point[1]) + y_pos = self.find_nearest(self.domain_vars["y"], point[2]) + + return [np.asarray(t_pos == self.domain_vars["t"]).nonzero()[0][0], np.asarray(x_pos == self.domain_vars["x"]).nonzero()[0][0], \ + np.asarray(y_pos == self.domain_vars["y"]).nonzero()[0][0]] + + def get_var_gridpoint(self, var, point): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + var: string + + point: list of 2+1 floats + + Returns: + -------- + np.array or np.float + + Notes: + ------ + This method should be used in case using interpolated values + becomes too expensive. + """ + indices = self.find_nearest_cell(point) + + if var in self.get_prim_strs(): + return self.prim_vars[var][tuple(indices)] + + elif var in self.get_aux_strs(): + return self.aux_vars[var][tuple(indices)] + + elif var in self.get_structures_strs(): + if var == "bar_vel": + res = np.zeros(self.structures[var][:,0,0,0].shape) + for a in range(len(self.structures[var][:,0,0,0])): + res[a] = self.structures[var][tuple([a]+indices)] + return res + else: + res = res = np.zeros(self.structures[var][:,:,0,0,0].shape) + for a in range(len(res[:,0])): + for b in range(len(res[0,:])): + res[a,b] = self.structures[var][tuple([a,b]+indices)] + return res + else: + print(f"{var} is not a variable of IdealMHD_2D!") + return None + + +class IdealHydro_2D(object): + + def __init__(self, interp_method = "linear"): + """ + Sets up the variables and dictionaries, strings correspond to + those used in METHOD + Parameters + ---------- + interp_method: str + optional method to be used by interpn + """ + self.spatial_dims = 2 + self.interp_method = interp_method + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + + #Dictionary for grid: info and points + self.domain_int_strs = ('nt','nx','ny') + self.domain_float_strs = ("tmin","tmax","xmin","xmax","ymin","ymax","dt","dx","dy") + self.domain_array_strs = ("t","x","y","points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs + self.domain_array_strs) + for str in self.domain_vars: + self.domain_vars[str] = [] + + #Dictionary for primitive var + self.prim_strs = ("vx","vy","n","p") + self.prim_vars = dict.fromkeys(self.prim_strs) + for str in self.prim_strs: + self.prim_vars[str] = [] + + #Dictionary for auxiliary var + self.aux_strs = ("W","h","T") + self.aux_vars = dict.fromkeys(self.aux_strs) + for str in self.aux_strs: + self.aux_vars[str] = [] + + #Dictionary for structures + self.structures_strs = ("bar_vel","SET") + self.structures = dict.fromkeys(self.structures_strs) + for str in self.structures_strs: + self.structures[str] = [] + + #Dictionary for all vars + self.var_strs = self.prim_strs + self.aux_strs + self.structures_strs + + def get_domain_strs(self): + return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs + + def get_prim_strs(self): + return self.prim_strs + + def get_aux_strs(self): + return self.aux_strs + + def get_structures_strs(self): + return self.structures_strs + + def setup_structures(self): + """ + Set up the structures (i.e baryon vel, SET) + Structures are built as multi-dim np.arrays, with the first indices referring + corrensponding to their components, the last three to the grid coordinates. + """ + self.structures["bar_vel"] = np.zeros((3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) + self.structures["SET"] = np.zeros((3,3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) + + for h in range(self.domain_vars['nt']): + for i in range(self.domain_vars['nx']): + for j in range(self.domain_vars['ny']): + self.structures["bar_vel"][0,h,i,j] = self.aux_vars['W'][h,i,j] + self.structures["bar_vel"][1,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] + self.structures["bar_vel"][2,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j] + vel = np.array(self.structures["bar_vel"][:,h,i,j]) + + + self.structures["SET"][:,:,h,i,j] = (self.prim_vars["n"][h,i,j] + self.prim_vars["p"][h,i,j]) * \ + np.outer(vel,vel) + self.prim_vars["p"][h,i,j] * self.metric + + self.vars = self.prim_vars + self.vars.update(self.aux_vars) + self.vars.update(self.structures) + + def get_interpol_prim(self, var_names, point): + """ + Returns the interpolated variable at the point + Parameters + ---------- + vars : list of strings + strings have to be in to prim_vars keys + point : list of floats + ordered coordinates: t,x,y + Return + ------ + list of floats corresponding to string of vars + Notes + ----- + Interpolation raises a ValueError when out of grid boundaries. + """ + res = [] + + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.prim_vars[var_name], point, method = self.interp_method)[0]) #, bounds_error = False) + except KeyError: + print(f"{var_name} does not belong to the primitive variables of the micro_model!") + return res + + def get_interpol_aux(self, var_names, point): + """ + Returns the interpolated variable at the point + Parameters + ---------- + vars : list of strings + strings have to be in to aux_vars keys + point : list of floats + ordered coordinates: t,x,y + Return + ------ + list of floats corresponding to string of vars + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + res = [] + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.aux_vars[var_name], point, method = self.interp_method)[0]) + except KeyError: + print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") + return res + + def get_interpol_struct(self, var_name, point): + """ + Returns the interpolated structure at the point + Parameters + ---------- + var : str corresponding to one of the structures + + point : list of floats + ordered coordinates: t,x,y + Return + ------ + Array with the interpolated values of the var structure + Empty list if var is not a structure in the micro_model + Notes + ----- + Interpolation gives errors when applied to boundary + """ + res = [] + if var_name == "bar_vel": + res = np.zeros(len(self.structures[var_name][:,0,0,0])) + for a in range(len(self.structures[var_name][:,0,0,0])): + res[a] = interpn(self.domain_vars["points"], self.structures[var_name][a,:,:,:], point, method = self.interp_method)[0] + elif var_name == "SET": + res = np.zeros((len(self.structures[var_name][:,0,0,0,0]),len(self.structures[var][0,:,0,0,0]))) + for a in range(len(self.structures[var_name][:,0,0,0,0])): + for b in range(len(self.structures[var_name][0,:,0,0,0])): + res[a,b] = interpn(self.domain_vars["points"],self.structures[var_name][a,b,:,:,:], point, method = self.interp_method)[0] + else: + print(f"{var} does not belong to the structures in the micro_model") + return res + + def get_interpol_var(self, var_names, point): + """ + Returns the interpolated structure at the point + Parameters + ---------- + var : str corresponding to one of the structures + + point : list of floats + ordered coordinates: t,x,y + Return + ------ + Array with the interpolated values of any var + Empty list if var is not a structure in the micro_model + Notes + ----- + Interpolation gives errors when applied to boundary + """ + res = [] + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) + except KeyError: + print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") + return res + + if __name__ == '__main__': @@ -219,17 +543,21 @@ def get_interpol_struct(self, var, point): FileReader.read_in_data(micro_model) micro_model.setup_structures() - # res = micro_model.get_interpol_prim(['vx'],[0.5, 0.3,0.2]) - # print(type(res),"\n", res) - # res = micro_model.get_interpol_aux(["b0","bx"],[0.5, 0.3,0.2]) - # print(type(res),"\n", res) - - # res = micro_model.get_interpol_struct("SET",[2.5, 0.3, 0.2]) - # print(res.shape, res[0],'\n') + var = "SET" + point = [1.502,0.4,0.2] + + star_time = time.process_time() + v1 = micro_model.get_interpol_var(var, point) + time4interp = time.process_time() - star_time - # print( micro_model.get_domain_strs() ) + star_time = time.process_time() + v2 = micro_model.get_var_gridpoint(var, point) + time4grid = time.process_time() - star_time - CPU_end_time = time.process_time() - CPU_time = CPU_end_time - CPU_start_time - print(f'The CPU time is {CPU_time} seconds') + speedup = time4interp / time4grid + error = np.zeros(v1.shape) + for ind in np.ndindex(error.shape): + error[ind] = (v1[ind] - v2[ind])/v1[ind] + print(f"Reading from the grid corresponds to a speed up of {speedup}, with an associated"+ \ + f" componentwise error of \n {error}") \ No newline at end of file From 041c8bf1a1feff9df4f432aef0cbf8f4dff2df0e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 1 May 2023 15:53:25 +0100 Subject: [PATCH 002/111] TC update + Box_filter class --- FileReaders.py | 3 +++ Filters.py | 41 ++--------------------------------------- MicroModels.py | 11 +++++------ 3 files changed, 10 insertions(+), 45 deletions(-) diff --git a/FileReaders.py b/FileReaders.py index 2f010eb..a87e5ca 100644 --- a/FileReaders.py +++ b/FileReaders.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- """ Created on Fri Mar 31 10:00:00 2023 + @author: Thomas """ @@ -14,6 +15,7 @@ def __init__(self, directory): """ Set up the list of files (from hdf5) and dictionary with dataset names in the hdf5 file. + Parameters ---------- directory: string @@ -36,6 +38,7 @@ def get_hdf5_keys(self): def read_in_data(self, micro_model): """ Store data from files into micro_model + Parameters ---------- micro_model: class MicroModel diff --git a/Filters.py b/Filters.py index 4cb7c34..e4175de 100644 --- a/Filters.py +++ b/Filters.py @@ -733,45 +733,8 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): return np.multiply( integrand, 1 / (self.filter_width**self.spatial_dims)), error - integrand = 0 - counter = 0 - start_cell, end_cell = micro_model.find_nearest_cell([t-(self.L/2),x-(self.L/2),y-(self.L/2)]), \ - micro_model.find_nearest_cell([t+(self.L/2),x+(self.L/2),y+(self.L/2)]) - for i in range(start_cell[0],end_cell[0]+1): - for j in range(start_cell[1],end_cell[1]+1): - for k in range(start_cell[2],end_cell[2]+1): - integrand += self.vars[quant_str][i][j,k] - counter += 1 - return integrand/counter - - - for vec in tetrad: - rem_vecs = [x for x in tetrad if not (x==vec).all()] - for i in range(2): - center = point + np.multiply( (-1)**i * self.L / 2, vec) - - xs = [] - for i in range(self.spatial_dims): - xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) - coords = [] - for element in product(*xs): - coords.append(np.array(element)) - - surf_coords = [] - for coord in coords: - temp = center - for i in range(self.spatial_dims): - temp += np.multiply(coord[i], rem_vecs[i]) - surf_coords.append(temp) - - for coord in surf_coords: - U = self.micro_model.get_interpol_struct('bar_vel', coord) - rho = self.micro_model.get_interpol_prim(['rho'], coord) - Na = np.multiply(rho, U) - flux += self.Mink_dot(Na, vec) - - flux *= (self.L / lin_spacing) ** self.spatial_dims - return abs(flux) +if __name__ == '__main__': + CPU_start_time = time.process_time() FileReader = METHOD_HDF5('./Data/test_res100/') micro_model = IdealMHD_2D() diff --git a/MicroModels.py b/MicroModels.py index 0c0d291..5d70840 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -6,7 +6,7 @@ """ from FileReaders import * -from scipy.interpolate import interpn +from scipy.interpolate import interpn # from scipy import interpolate import numpy as np import math @@ -99,8 +99,8 @@ def setup_structures(self): for i in range(self.domain_vars['nx']): for j in range(self.domain_vars['ny']): self.structures["bar_vel"][0,h,i,j] = self.aux_vars['W'][h,i,j] - self.structures["bar_vel"][1,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['v1'][h,i,j] - self.structures["bar_vel"][2,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['v2'][h,i,j] + self.structures["bar_vel"][1,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] + self.structures["bar_vel"][2,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j] vel = np.array(self.structures["bar_vel"][:,h,i,j]) fibr_b = np.array([self.aux_vars['b0'][h,i,j],self.aux_vars['bx'][h,i,j],self.aux_vars['by'][h,i,j]]) @@ -538,9 +538,8 @@ def get_interpol_var(self, var_names, point): CPU_start_time = time.process_time() - FileReader = METHOD_HDF5('./Data/Testing/') - # micro_model = IdealMHD_2D() - micro_model = IdealHydro_2D() + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() From c355ddcad1a6c3d583fd49127ebd2511bae2122b Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 1 May 2023 15:56:39 +0100 Subject: [PATCH 003/111] TC update + Box_filter class --- MicroModels.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/MicroModels.py b/MicroModels.py index 5d70840..b531dfe 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -118,10 +118,6 @@ def setup_structures(self): self.structures['Faraday'][:,:,h,i,j] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) - self.vars = self.prim_vars - self.vars.update(self.aux_vars) - self.vars.update(self.structures) - def get_interpol_prim(self, vars, point): """ Returns the interpolated variable at the point From 6a515afbe2fb6ccbf29066ae9a376a1d43295896 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 23 May 2023 13:27:34 +0100 Subject: [PATCH 004/111] TC updates to interpolation methods --- Filters.py | 18 ++-- MicroModels.py | 226 +++++++++++++------------------------------------ 2 files changed, 69 insertions(+), 175 deletions(-) diff --git a/Filters.py b/Filters.py index e4175de..d468d4b 100644 --- a/Filters.py +++ b/Filters.py @@ -159,8 +159,8 @@ def Favre_residual(self, spatial_vels, point, lin_spacing = 10): surf_coords.append(temp) for coord in surf_coords: - U = self.micro_model.get_interpol_struct('bar_vel', coord) - n = self.micro_model.get_interpol_prim(['n'], coord) + U = self.micro_model.get_interpol_var('bar_vel', coord) + n = self.micro_model.get_interpol_var('n', coord) Na = np.multiply(n, U) flux += self.Mink_dot(Na, vec) @@ -246,8 +246,8 @@ def Favre_residual_Gauss(self, spatial_vels, point, order = 3): surf_coords.append(temp) for i, coord in enumerate(surf_coords): - U = self.micro_model.get_interpol_struct('bar_vel', coord) - n = self.micro_model.get_interpol_prim(['n'], coord) + U = self.micro_model.get_interpol_var('bar_vel', coord) + n = self.micro_model.get_interpol_var('n', coord) Na = np.multiply(n, U) flux += self.Mink_dot(Na, vec) * totws[i] @@ -288,8 +288,8 @@ def Favre_residual_ib(self, spatial_vels, point, abserr = 1e-7): if self.spatial_dims == 2: def point_flux(x, y , center, Vx, Vy, normal): coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) - U = self.micro_model.get_interpol_struct('bar_vel', coords) - n = self.micro_model.get_interpol_prim(['n'], coords) + U = self.micro_model.get_interpol_var('bar_vel', coords) + n = self.micro_model.get_interpol_vars('n', coords) Na = np.multiply(n, U) flux = self.Mink_dot(Na, normal) return flux @@ -307,8 +307,8 @@ def point_flux(x, y , center, Vx, Vy, normal): elif self.spatial_dims == 3: def point_flux(x, y, z , center, Vx, Vy, Vz, normal): coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + np.multiply(z, Vz) - U = self.micro_model.get_interpol_struct('bar_vel', coords) - n = self.micro_model.get_interpol_prim(['n'], coords) + U = self.micro_model.get_interpol_var('bar_vel', coords) + n = self.micro_model.get_interpol_var('n', coords) Na = np.multiply(n, U) flux = self.Mink_dot(Na, normal) return flux @@ -346,7 +346,7 @@ def find_observer(self, point, residual_str = "gauss"): the optional arguments in the corresponding methods. """ - U = self.micro_model.get_interpol_struct('bar_vel', point) + U = self.micro_model.get_interpol_var('bar_vel', point) guess = [] for i in range(1, len(U)): guess.append(U[i] / U[0]) diff --git a/MicroModels.py b/MicroModels.py index b531dfe..dc60870 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -88,25 +88,25 @@ def setup_structures(self): """ Set up the structures (i.e baryon vel, SET and Faraday) - Structures are built as multi-dim np.arrays, with the first indices referring - corrensponding to their components, the last three to the grid coordinates. + Structures are built as multi-dim np.arrays, with the first three indices referring + to the grid, while the last one or two refer to space-time components. """ - self.structures["bar_vel"] = np.zeros((3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) - self.structures["SET"] = np.zeros((3,3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) - self.structures["Faraday"] = np.zeros((3,3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) + self.structures["bar_vel"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + self.structures["Faraday"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) for h in range(self.domain_vars['nt']): for i in range(self.domain_vars['nx']): for j in range(self.domain_vars['ny']): - self.structures["bar_vel"][0,h,i,j] = self.aux_vars['W'][h,i,j] - self.structures["bar_vel"][1,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] - self.structures["bar_vel"][2,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j] - vel = np.array(self.structures["bar_vel"][:,h,i,j]) + self.structures["bar_vel"][h,i,j,0] = self.aux_vars['W'][h,i,j] + self.structures["bar_vel"][h,i,j,1] = self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] + self.structures["bar_vel"][h,i,j,2] = self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j] + vel = np.array(self.structures["bar_vel"][h,i,j,:]) fibr_b = np.array([self.aux_vars['b0'][h,i,j],self.aux_vars['bx'][h,i,j],self.aux_vars['by'][h,i,j]]) - self.structures["SET"][:,:,h,i,j] = (self.prim_vars["n"][h,i,j] + self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]) * \ + self.structures["SET"][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] + self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]) * \ np.outer( vel,vel) + (self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]/2) * self.metric \ - np.outer(fibr_b, fibr_b) @@ -115,134 +115,9 @@ def setup_structures(self): fol_b_vec = np.array([self.aux_vars["b0"][h,i,j],self.aux_vars["bx"][h,i,j],self.aux_vars["by"][h,i,j]]) fol_e_vec = np.tensordot( self.Levi3D, np.outer(vel,fol_b_vec), axes = ([1,2],[0,1])) - self.structures['Faraday'][:,:,h,i,j] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ + self.structures['Faraday'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) - def get_interpol_prim(self, vars, point): - """ - Returns the interpolated variable at the point - - Parameters - ---------- - vars : list of strings - strings have to be in to prim_vars keys - point : list of floats - ordered coordinates: t,x,y - Return - ------ - list of floats corresponding to string of vars - - Notes - ----- - Interpolation raises a ValueError when out of grid boundaries. - """ - res = [] - - for var_name in vars: - try: - res.append( interpn(self.domain_vars["points"], self.prim_vars[var_name], point, method = self.interp_method)[0]) #, bounds_error = False) - except KeyError: - print(f"{var_name} does not belong to the primitive variables of the micro_model!") - return res - - def get_interpol_aux(self, vars, point): - """ - Returns the interpolated variable at the point - - Parameters - ---------- - vars : list of strings - strings have to be in to aux_vars keys - point : list of floats - ordered coordinates: t,x,y - Return - ------ - list of floats corresponding to string of vars - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - - res = [] - for var_name in vars: - try: - res.append( interpn(self.domain_vars["points"], self.aux_vars[var_name], point, method = self.interp_method)[0]) - except KeyError: - print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") - return res - - def get_interpol_struct(self, var, point): - """ - Returns the interpolated structure at the point - - Parameters - ---------- - var : str corresponding to one of the structures - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Array with the interpolated values of the var structure - Empty list if var is not a structure in the micro_model - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - res = [] - if var == "bar_vel": - res = np.zeros(len(self.structures[var][:,0,0,0])) - for a in range(len(self.structures[var][:,0,0,0])): - res[a] = interpn(self.domain_vars["points"], self.structures[var][a,:,:,:], point, method = self.interp_method)[0] - elif var == "SET": - res = np.zeros((len(self.structures[var][:,0,0,0,0]),len(self.structures[var][0,:,0,0,0]))) - for a in range(len(self.structures[var][:,0,0,0,0])): - for b in range(len(self.structures[var][0,:,0,0,0])): - res[a,b] = interpn(self.domain_vars["points"],self.structures[var][a,b,:,:,:], point, method = self.interp_method)[0] - elif var == "Faraday": - res = np.zeros((len(self.structures[var][:,0,0,0,0]),len(self.structures[var][0,:,0,0,0]))) - for a in range(len(self.structures[var][:,0,0,0,0])): - for b in range(len(self.structures[var][0,:,0,0,0])): - res[a,b]= interpn(self.domain_vars["points"],self.structures[var][a,b,:,:,:], point, method = self.interp_method)[0] - else: - print(f"{var} does not belong to the structures in the micro_model") - return res - - def get_interpol_var(self, var, point): - """ - Returns the interpolated var at the point. The method calls the correct one - for prim, aux or struct. - - Parameters - ---------- - var : str corresponding to either a primitive, auxiliary or structre variable - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Interpolated values of the var - Empty list if var is not a primitive, auxiliary o structure variable of the micro_model - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - if var in self.get_prim_strs(): - return self.get_interpol_prim([var], point)[0] - - elif var in self.get_aux_strs(): - return self.get_interpol_aux([var], point)[0] - - elif var in self.get_structures_strs(): - return self.get_interpol_struct(var, point) - else: - print('The variable to be interpolated does not belong to the micromodel!') - def find_nearest(self, array, value): """ Returns closest value to input 'value' in 'array' @@ -299,13 +174,13 @@ def get_var_gridpoint(self, var, point): Parameters: ----------- - var: string + vars: string corresponding to primitive, auxiliary or structre variable point: list of 2+1 floats Returns: -------- - np.array or np.float + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. Notes: ------ @@ -313,29 +188,58 @@ def get_var_gridpoint(self, var, point): becomes too expensive. """ indices = self.find_nearest_cell(point) - if var in self.get_prim_strs(): return self.prim_vars[var][tuple(indices)] - + elif var in self.get_aux_strs(): return self.aux_vars[var][tuple(indices)] - + elif var in self.get_structures_strs(): if var == "bar_vel": - res = np.zeros(self.structures[var][:,0,0,0].shape) - for a in range(len(self.structures[var][:,0,0,0])): - res[a] = self.structures[var][tuple([a]+indices)] - return res + tmp = np.zeros(self.structures[var][0,0,0,:].shape) + for a in range(len(self.structures[var][0,0,0,:])): + tmp[a] = self.structures[var][tuple(indices+[a])] + return tmp else: - res = res = np.zeros(self.structures[var][:,:,0,0,0].shape) - for a in range(len(res[:,0])): - for b in range(len(res[0,:])): - res[a,b] = self.structures[var][tuple([a,b]+indices)] - return res + tmp = np.zeros(self.structures[var][0,0,0,:,:].shape) + for a in range(len(tmp[:,0])): + for b in range(len(tmp[0,:])): + tmp[a,b] = self.structures[var][tuple(indices+[a,b])] + return tmp else: print(f"{var} is not a variable of IdealMHD_2D!") return None + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + vars : str corresponding to primitive, auxiliary or structre variable + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + if var in self.get_prim_strs(): + return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] + elif var in self.get_aux_strs(): + return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] + elif var in self.get_structures_strs(): + return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] + else: + print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + class IdealHydro_2D(object): @@ -539,21 +443,11 @@ def get_interpol_var(self, var_names, point): FileReader.read_in_data(micro_model) micro_model.setup_structures() - var = "SET" + # var = "SET" point = [1.502,0.4,0.2] - - star_time = time.process_time() - v1 = micro_model.get_interpol_var(var, point) - time4interp = time.process_time() - star_time - - star_time = time.process_time() - v2 = micro_model.get_var_gridpoint(var, point) - time4grid = time.process_time() - star_time - - speedup = time4interp / time4grid - error = np.zeros(v1.shape) - for ind in np.ndindex(error.shape): - error[ind] = (v1[ind] - v2[ind])/v1[ind] - print(f"Reading from the grid corresponds to a speed up of {speedup}, with an associated"+ \ - f" componentwise error of \n {error}") \ No newline at end of file + vars = ['SET', 'vx'] + for var in vars: + res = micro_model.get_interpol_var(var, point) + res2 = micro_model.get_var_gridpoint(var, point) + print(f'{var}: \n {res} \n {res2} \n ********** \n ') \ No newline at end of file From 788becc0e4671d370a2545490ccedd09cf988f75 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 25 May 2023 17:18:42 +0100 Subject: [PATCH 005/111] Update to interpol + structures, stuff moved to base --- Filters.py | 144 +++++++++++------------------------ MicroModels.py | 146 +++++++++++++++++------------------- system/BaseFunctionality.py | 143 ++++++++++++++++++----------------- 3 files changed, 186 insertions(+), 247 deletions(-) diff --git a/Filters.py b/Filters.py index d468d4b..7500b5a 100644 --- a/Filters.py +++ b/Filters.py @@ -10,12 +10,13 @@ import scipy.integrate as integrate from scipy.optimize import minimize from itertools import product +from system.BaseFunctionality import * from MicroModels import * from FileReaders import * -class Favre_observers(object): +class FindObs_drift_min(object): """ Class for computing the Favre observer given a micromodel. Work in any dimension, read on construction from the micro-model. @@ -35,54 +36,13 @@ def __init__(self, micro_model, box_len): self.L = box_len self.residuals_methods = { - "gauss" : self.Favre_residual_Gauss , - "linear" : self.Favre_residual , - "inbuilt" : self.Favre_residual_ib + "gauss" : self.drift_residual_Gauss , + "linear" : self.drift_residual , + "inbuilt" : self.drift_residual_ib } def set_box_length(self, bl): self.L = bl - - def Mink_dot(self, vec1, vec2): - """ - Parameters: - ----------- - vec1, vec2 : list of floats (or np.arrays) - - Return: - ------- - mink-dot (cartesian) in 1+n dim - """ - if len(vec1) != len(vec2): - print("The two vectors passed to Mink_dot are not of same dimension!") - - dot = -vec1[0]*vec2[0] - for i in range(1,len(vec1)): - dot += vec1[i] * vec2[i] - return dot - - def get_U_from_vels(self, spatial_vels): - """ - Build unit vectors starting from spatial components - Needed as this will enter the minimization procedure - - Parameters: - ---------- - spatial_vels: list of floats - - Returns: - -------- - list of floats: the d+1 vector, normalized wrt Mink metric - """ - - temp = 0 - for i in range(len(spatial_vels)): - temp += spatial_vels[i]**2 - W = 1 / np.sqrt(1 - temp) - U = [W] - for i in range(len(spatial_vels)): - U.append(W * spatial_vels[i]) - return np.array(U) def get_tetrad_from_vels(self, spatial_vels): """ @@ -96,8 +56,11 @@ def get_tetrad_from_vels(self, spatial_vels): ------- list of arrays: U + d unit vectors that complete it to a orthonormal basis """ - - U = np.array(self.get_U_from_vels(spatial_vels)) + if len(spatial_vels) != self.spatial_dims: + print('The number of spatial velocities passed is not compatible with \ + micro_model dimensionality!') + return None + U = Base.get_rel_vel(spatial_vels) es =[] for _ in range(self.spatial_dims): es.append(np.zeros(self.spatial_dims+1)) @@ -105,14 +68,14 @@ def get_tetrad_from_vels(self, spatial_vels): es[i][i+1] = 1 tetrad = [U] for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! - vec = vec + np.multiply(self.Mink_dot(vec, U), U) + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) for j in range(i-1,-1,-1): - vec = vec - np.multiply(self.Mink_dot(vec, es[j]), es[j]) - es[i] = np.multiply(vec, 1 / np.sqrt(self.Mink_dot(vec, vec))) + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) tetrad += [es[i]] return tetrad - def Favre_residual(self, spatial_vels, point, lin_spacing = 10): + def drift_residual(self, spatial_vels, point, lin_spacing = 10): """ Compute the drift of baryons through the box built from vx_vy. First get the center of the 2*(d+1) faces of the (d+1)-box, then build coords @@ -159,15 +122,12 @@ def Favre_residual(self, spatial_vels, point, lin_spacing = 10): surf_coords.append(temp) for coord in surf_coords: - U = self.micro_model.get_interpol_var('bar_vel', coord) - n = self.micro_model.get_interpol_var('n', coord) - Na = np.multiply(n, U) - flux += self.Mink_dot(Na, vec) + flux += Base.Mink_dot(self.micro_model.get_interpol_var('BC', coord), vec) flux *= (self.L / lin_spacing) ** self.spatial_dims return abs(flux) - def Favre_residual_Gauss(self, spatial_vels, point, order = 3): + def drift_residual_Gauss(self, spatial_vels, point, order = 3): """ Alternative for computing manually the Favre residual, using the Gauss Legendre method. @@ -246,15 +206,13 @@ def Favre_residual_Gauss(self, spatial_vels, point, order = 3): surf_coords.append(temp) for i, coord in enumerate(surf_coords): - U = self.micro_model.get_interpol_var('bar_vel', coord) - n = self.micro_model.get_interpol_var('n', coord) - Na = np.multiply(n, U) - flux += self.Mink_dot(Na, vec) * totws[i] + Na = self.micro_model.get_interpol_var('BC',coord) + flux += Base.Mink_dot(Na, vec) * totws[i] flux *= (self.L /2 ) ** self.spatial_dims return abs(flux) - def Favre_residual_ib(self, spatial_vels, point, abserr = 1e-7): + def drift_residual_ib(self, spatial_vels, point, abserr = 1e-7): """ Compute the residual using inbuilt method dblquad Based on function point_flux @@ -288,9 +246,7 @@ def Favre_residual_ib(self, spatial_vels, point, abserr = 1e-7): if self.spatial_dims == 2: def point_flux(x, y , center, Vx, Vy, normal): coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) - U = self.micro_model.get_interpol_var('bar_vel', coords) - n = self.micro_model.get_interpol_vars('n', coords) - Na = np.multiply(n, U) + Na = self.micro_model.get_interpol_var('BC', coords) flux = self.Mink_dot(Na, normal) return flux @@ -307,9 +263,7 @@ def point_flux(x, y , center, Vx, Vy, normal): elif self.spatial_dims == 3: def point_flux(x, y, z , center, Vx, Vy, Vz, normal): coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + np.multiply(z, Vz) - U = self.micro_model.get_interpol_var('bar_vel', coords) - n = self.micro_model.get_interpol_var('n', coords) - Na = np.multiply(n, U) + Na = self.micro_model.get_interpol_var('BC', coords) flux = self.Mink_dot(Na, normal) return flux @@ -346,7 +300,7 @@ def find_observer(self, point, residual_str = "gauss"): the optional arguments in the corresponding methods. """ - U = self.micro_model.get_interpol_var('bar_vel', point) + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) guess = [] for i in range(1, len(U)): guess.append(U[i] / U[0]) @@ -467,24 +421,6 @@ def __init__(self, micro_model, filter_width): self.spatial_dims = micro_model.get_spatial_dims() self.filter_width = filter_width - def Mink_dot(self, vec1, vec2): - """ - Parameters: - ----------- - vec1, vec2 : list of floats (or np.arrays) - - Return: - ------- - mink-dot (cartesian) in 1+n dim - """ - if len(vec1) != len(vec2): - print("The two vectors passed to Mink_dot are not of same dimension!") - - dot = -vec1[0]*vec2[0] - for i in range(1,len(vec1)): - dot += vec1[i] * vec2[i] - return dot - def set_filter_width(self, filter_width): """ Method to change the width of the filter. @@ -534,17 +470,17 @@ def complete_U_tetrad(self, U): es[i][i+1] = 1. triad = [] for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! - vec = vec + np.multiply(self.Mink_dot(vec, U), U) + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) for j in range(i-1,-1,-1): - vec = vec - np.multiply(self.Mink_dot(vec, es[j]), es[j]) - es[i] = np.multiply(vec, 1 / np.sqrt(self.Mink_dot(vec, vec))) + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1/np.sqrt(Base.Mink_dot(vec, vec))) triad += [es[i]] return triad - def filter_var_point_ip(self, var_str, point, observer, sample_method = "gauss", num_points = 3): + def filter_var_point(self, var_str, point, observer, sample_method = "gauss", num_points = 3): """ First complete the observer to a tetrad at the point. Then build coords for - (linearly spaced) sample points in the spatial directions. Then approximate the + sample points in the spatial directions adapted to observer. Then approximate the filter integral as a Riemann sum. Parameters: @@ -566,13 +502,12 @@ def filter_var_point_ip(self, var_str, point, observer, sample_method = "gauss", Notes: ------ - The method uses interpolated values for the structure. So this is expected to be - more costly than using only values on the grid points. Using Gauss-Legendre method - should give accurate results for low values of num-points. + Current version uses interpolated values. Should this be too expensive, change + get_interpol_var for get_var_gridpoint! - If Gauss-Legendre is using interpolated values is too expensive, change: - self.micro_model.get_interpol_var() for self.micro_model.get_var_gridpoint(). - This avoids interpolating, and uses the values on the grid closest to input 'point'. + Gauss quadrature gives more accurate results for low values of num-points, hence + reduce number of interpolations. + The alternative is to use the gridpoint method with linearly spaced sampling points. """ xs = [] coords = [] @@ -653,7 +588,7 @@ def filter_var_point_ip(self, var_str, point, observer, sample_method = "gauss", print("Sample methods to filter variable must be either 'gauss' or 'linear'! ") return None - def filter_var_manypoints_ip(self, var_str, points, observers, sample_method = "gauss", num_points = 3): + def filter_var_manypoints(self, var_str, points, observers, sample_method = "gauss", num_points = 3): """ Method to filter a variable in the micro_model given a list of points and observers. @@ -671,6 +606,11 @@ def filter_var_manypoints_ip(self, var_str, points, observers, sample_method = " Returns: -------- List with the filtered var (np.float or nd.array depending on var_str) at all points + + Notes: + ------ + Current version uses interpolated values. Should this be too expensive, change + get_interpol_var for get_var_gridpoint! """ if len(points) != len(observers): print("The number of points and observers do not match!") @@ -678,7 +618,7 @@ def filter_var_manypoints_ip(self, var_str, points, observers, sample_method = " filtered_var = [] for i, point in enumerate(points): - filtered_var.append(self.filter_var_point_ip(var_str, point, observers[i], sample_method = sample_method, num_points = num_points )) + filtered_var.append(self.filter_var_point(var_str, point, observers[i], sample_method = sample_method, num_points = num_points )) return filtered_var @@ -740,18 +680,18 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): micro_model = IdealMHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() - constraint = Favre_observers(micro_model,0.001) + constraint = FindObs_drift_min(micro_model,0.001) filter = Box_filter(micro_model, 0.001) var = "SET" point = [1.502,0.3,0.5] - observer = constraint.get_U_from_vels( constraint.find_observer(point).x) + observer = Base.get_rel_vel(constraint.find_observer(point).x) CPU_start_time = time.process_time() filtvar1 = filter.filter_var_point_inbuilt(var, point, observer) print(f"CPU time to filter SET with in-built method is {time.process_time()- CPU_start_time}. ") CPU_start_time = time.process_time() - filtvar2 = filter.filter_var_point_ip(var, point, observer) + filtvar2 = filter.filter_var_point(var, point, observer) print(f"CPU time to filter SET with Gauss method is {time.process_time()- CPU_start_time}. ") print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") diff --git a/MicroModels.py b/MicroModels.py index dc60870..0060e9c 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -7,7 +7,7 @@ from FileReaders import * from scipy.interpolate import interpn -# from scipy import interpolate +from system.BaseFunctionality import * import numpy as np import math import time @@ -64,7 +64,7 @@ def __init__(self, interp_method = "linear"): self.aux_vars[str] = [] #Dictionary for structures - self.structures_strs = ("bar_vel","SET","Faraday") + self.structures_strs = ("BC","SET","Faraday") self.structures = dict.fromkeys(self.structures_strs) for str in self.structures_strs: self.structures[str] = [] @@ -91,17 +91,17 @@ def setup_structures(self): Structures are built as multi-dim np.arrays, with the first three indices referring to the grid, while the last one or two refer to space-time components. """ - self.structures["bar_vel"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) self.structures["Faraday"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) for h in range(self.domain_vars['nt']): for i in range(self.domain_vars['nx']): for j in range(self.domain_vars['ny']): - self.structures["bar_vel"][h,i,j,0] = self.aux_vars['W'][h,i,j] - self.structures["bar_vel"][h,i,j,1] = self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] - self.structures["bar_vel"][h,i,j,2] = self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j] - vel = np.array(self.structures["bar_vel"][h,i,j,:]) + vel = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ + self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j]]) + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel ) + fibr_b = np.array([self.aux_vars['b0'][h,i,j],self.aux_vars['bx'][h,i,j],self.aux_vars['by'][h,i,j]]) @@ -118,55 +118,6 @@ def setup_structures(self): self.structures['Faraday'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) - def find_nearest(self, array, value): - """ - Returns closest value to input 'value' in 'array' - - Parameters: - ----------- - array: np.array of shape (n,) - - value: float - - Returns: - -------- - float - - Note: - ----- - To be used together with find_nearest_cell. - """ - idx = np.searchsorted(array, value, side="left") - if idx > 0 and (idx == len(array) or math.fabs(value - array[idx-1]) < math.fabs(value - array[idx])): - return array[idx-1] - else: - return array[idx] - - def find_nearest_cell(self, point): - """ - Use find nearest to find closest value in the grid to point. - Then returns the grid_point indices for this. - - Parameters: - ----------- - point: list of float, ordered as (t, x ,y) - - Returns: - -------- - List of three indices - - Notes: - ------ - This method is key to avoid using too many interpolations - should this becomes too expensive. - """ - t_pos = self.find_nearest(self.domain_vars["t"], point[0]) - x_pos = self.find_nearest(self.domain_vars["x"], point[1]) - y_pos = self.find_nearest(self.domain_vars["y"], point[2]) - - return [np.asarray(t_pos == self.domain_vars["t"]).nonzero()[0][0], np.asarray(x_pos == self.domain_vars["x"]).nonzero()[0][0], \ - np.asarray(y_pos == self.domain_vars["y"]).nonzero()[0][0]] - def get_var_gridpoint(self, var, point): """ Returns variable corresponding to input 'var' at gridpoint @@ -187,7 +138,7 @@ def get_var_gridpoint(self, var, point): This method should be used in case using interpolated values becomes too expensive. """ - indices = self.find_nearest_cell(point) + indices = Base.find_nearest_cell(point, self.domain_vars['points']) if var in self.get_prim_strs(): return self.prim_vars[var][tuple(indices)] @@ -195,7 +146,7 @@ def get_var_gridpoint(self, var, point): return self.aux_vars[var][tuple(indices)] elif var in self.get_structures_strs(): - if var == "bar_vel": + if var == "BC": tmp = np.zeros(self.structures[var][0,0,0,:].shape) for a in range(len(self.structures[var][0,0,0,:])): tmp[a] = self.structures[var][tuple(indices+[a])] @@ -241,12 +192,38 @@ def get_interpol_var(self, var, point): print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + + """ + Returns the interpolated structure at the point + Parameters + ---------- + var : str corresponding to one of the structures + + point : list of floats + ordered coordinates: t,x,y + Return + ------ + Array with the interpolated values of any var + Empty list if var is not a structure in the micro_model + Notes + ----- + Interpolation gives errors when applied to boundary + """ + res = [] + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) + except KeyError: + print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") + return res + class IdealHydro_2D(object): def __init__(self, interp_method = "linear"): """ Sets up the variables and dictionaries, strings correspond to those used in METHOD + Parameters ---------- interp_method: str @@ -259,6 +236,9 @@ def __init__(self, interp_method = "linear"): self.metric[0,0] = -1 self.metric[1,1] = self.metric[2,2] = +1 + # This is the Levi-Civita symbol, not tensor, so be careful when using it + self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ + for k in range(3)]for j in range(3) ] for i in range(3) ]) #Dictionary for grid: info and points self.domain_int_strs = ('nt','nx','ny') @@ -269,7 +249,7 @@ def __init__(self, interp_method = "linear"): self.domain_vars[str] = [] #Dictionary for primitive var - self.prim_strs = ("vx","vy","n","p") + self.prim_strs = ("v1","v2","rho","p","n") self.prim_vars = dict.fromkeys(self.prim_strs) for str in self.prim_strs: self.prim_vars[str] = [] @@ -288,9 +268,12 @@ def __init__(self, interp_method = "linear"): #Dictionary for all vars self.var_strs = self.prim_strs + self.aux_strs + self.structures_strs - + # self.vars = self.prim_vars + # self.vars.update(self.aux_vars) + # self.vars.update(self.structures) + def get_domain_strs(self): - return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs + return self.domain_info_strs + self.domain_points_strs def get_prim_strs(self): return self.prim_strs @@ -299,36 +282,38 @@ def get_aux_strs(self): return self.aux_strs def get_structures_strs(self): - return self.structures_strs + return self.get_structures_strs def setup_structures(self): """ - Set up the structures (i.e baryon vel, SET) + Set up the structures (i.e baryon vel, SET and Faraday) + Structures are built as multi-dim np.arrays, with the first indices referring corrensponding to their components, the last three to the grid coordinates. """ - self.structures["bar_vel"] = np.zeros((3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) - self.structures["SET"] = np.zeros((3,3,self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'])) + self.structures["bar_vel"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) for h in range(self.domain_vars['nt']): for i in range(self.domain_vars['nx']): for j in range(self.domain_vars['ny']): - self.structures["bar_vel"][0,h,i,j] = self.aux_vars['W'][h,i,j] - self.structures["bar_vel"][1,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] - self.structures["bar_vel"][2,h,i,j] = self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j] - vel = np.array(self.structures["bar_vel"][:,h,i,j]) + self.structures["bar_vel"][h,i,j,0] = self.aux_vars['W'][h,i,j] + self.structures["bar_vel"][h,i,j,1] = self.aux_vars['W'][h,i,j] * self.prim_vars['v1'][h,i,j] + self.structures["bar_vel"][h,i,j,2] = self.aux_vars['W'][h,i,j] * self.prim_vars['v2'][h,i,j] + vel = np.array(self.structures["bar_vel"][h,i,j,:]) - self.structures["SET"][:,:,h,i,j] = (self.prim_vars["n"][h,i,j] + self.prim_vars["p"][h,i,j]) * \ + self.structures["SET"][h,i,j,:,:] = (self.prim_vars["rho"][h,i,j] + self.prim_vars["p"][h,i,j]) * \ np.outer(vel,vel) + self.prim_vars["p"][h,i,j] * self.metric self.vars = self.prim_vars self.vars.update(self.aux_vars) - self.vars.update(self.structures) + self.vars.update(self.structures) def get_interpol_prim(self, var_names, point): """ Returns the interpolated variable at the point + Parameters ---------- vars : list of strings @@ -338,6 +323,7 @@ def get_interpol_prim(self, var_names, point): Return ------ list of floats corresponding to string of vars + Notes ----- Interpolation raises a ValueError when out of grid boundaries. @@ -354,6 +340,7 @@ def get_interpol_prim(self, var_names, point): def get_interpol_aux(self, var_names, point): """ Returns the interpolated variable at the point + Parameters ---------- vars : list of strings @@ -363,6 +350,7 @@ def get_interpol_aux(self, var_names, point): Return ------ list of floats corresponding to string of vars + Notes ----- Interpolation gives errors when applied to boundary @@ -379,16 +367,19 @@ def get_interpol_aux(self, var_names, point): def get_interpol_struct(self, var_name, point): """ Returns the interpolated structure at the point + Parameters ---------- var : str corresponding to one of the structures point : list of floats ordered coordinates: t,x,y + Return ------ Array with the interpolated values of the var structure Empty list if var is not a structure in the micro_model + Notes ----- Interpolation gives errors when applied to boundary @@ -407,19 +398,22 @@ def get_interpol_struct(self, var_name, point): print(f"{var} does not belong to the structures in the micro_model") return res - def get_interpol_var(self, var_names, point): + def get_interpol_var(self, var_names, point): """ Returns the interpolated structure at the point + Parameters ---------- var : str corresponding to one of the structures point : list of floats ordered coordinates: t,x,y + Return ------ Array with the interpolated values of any var Empty list if var is not a structure in the micro_model + Notes ----- Interpolation gives errors when applied to boundary @@ -429,11 +423,9 @@ def get_interpol_var(self, var_names, point): try: res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) except KeyError: - print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") + print(f"{var_name} does not belong to the variables of the micro_model!") return res - - if __name__ == '__main__': CPU_start_time = time.process_time() @@ -443,10 +435,8 @@ def get_interpol_var(self, var_names, point): FileReader.read_in_data(micro_model) micro_model.setup_structures() - # var = "SET" point = [1.502,0.4,0.2] - - vars = ['SET', 'vx'] + vars = ['SET', 'BC'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) diff --git a/system/BaseFunctionality.py b/system/BaseFunctionality.py index d51e32f..73f2212 100644 --- a/system/BaseFunctionality.py +++ b/system/BaseFunctionality.py @@ -21,79 +21,53 @@ class Base(object): - def Mink_dot(self,vec1,vec2): + @staticmethod + def Mink_dot(vec1, vec2): """ - Inner-product in (n+1)-dimensions + Parameters: + ----------- + vec1, vec2 : list of floats (or np.arrays) + + Return: + ------- + mink-dot (cartesian) in 1+n dim """ - dot = -vec1[0]*vec2[0] # time component + if len(vec1) != len(vec2): + print("The two vectors passed to Mink_dot are not of same dimension!") + + dot = -vec1[0]*vec2[0] for i in range(1,len(vec1)): - dot += vec1[i]*vec2[i] # spatial components + dot += vec1[i] * vec2[i] return dot - def get_rel_vel(self, spatial_vels): - """ - Construct (n+1)-velocity (meso) from spatial Cartesian (x,y,...) components + @staticmethod + def get_rel_vel(spatial_vels): """ - W = 1 / np.sqrt(1-np.sum(spatial_vels**2)) - return spatial_vels.insert(spatial_vels,0,W) + Build unit vectors starting from spatial Cartesian components - def get_U_mu_MagTheta(self, Vmag_Vtheta): - """ - Construct (2+1)-velocity (meso) from spatial Polar (r, theta) components - """ - Vmag, Vtheta = Vmag_Vtheta[0], Vmag_Vtheta[1] - return self.get_U_mu([Vmag*np.cos(Vtheta),Vmag*np.sin(Vtheta)]) - - def construct_tetrad(self, U): - """ - Construct 2 tetrad vectors that are perpendicular to (2+1)-velocity U, - and each other. These are used to define the box for filtering. - """ - e_x = np.array([0.0,1.0,0.0]) # 1 + 2D - E_x = e_x + np.multiply(self.Mink_dot(U,e_x),U) - E_x = E_x / np.sqrt(self.Mink_dot(E_x,E_x)) # normalization - e_y = np.array([0.0,0.0,1.0]) - E_y = e_y + np.multiply(self.Mink_dot(U,e_y),U) - np.multiply(self.Mink_dot(E_x,e_y),E_x) - E_y = E_y / np.sqrt(self.Mink_dot(E_y,E_y)) - return E_x, E_y - - def find_boundary_pts(self, E_x,E_y,P,L): - """ - Find the (four) corners of the box that is the filtering region. - - Parameters + Parameters: ---------- - E_x : list of floats - One tetrad vector. - E_y : list of floats - Second tetrad vector. - P : list of floats - Coordinate of the centre of the box (t,x,y). - L : float - Filtering lengthscale (length of one side of the box). - - Returns - ------- - corners : list of list of floats - list of the coordinates of the box's corners. + spatial_vels: list of floats + Returns: + -------- + list of floats: the d+1 vector, normalized wrt Mink metric """ - c1 = P + (L/2)*(E_x + E_y) - c2 = P + (L/2)*(E_x - E_y) - c3 = P + (L/2)*(-E_x - E_y) - c4 = P + (L/2)*(-E_x + E_y) - corners = [c1,c2,c3,c4] - return corners - - def surface_flux(self, x,E_x,E_y,P,direc_vec): - point = P + x*(E_x + E_y) - u, n = self.interpolate_u_n_point(point) - n_mu = np.multiply(u,n) - return self.Mink_dot(n_mu,direc_vec) - + + temp = 0 + for i in range(len(spatial_vels)): + temp += spatial_vels[i]**2 + W = 1 / np.sqrt(1 - temp) + U = [W] + for i in range(len(spatial_vels)): + U.append(W * spatial_vels[i]) + return np.array(U) + + @staticmethod def project_tensor(self, vector1_wrt, vector2_wrt, to_project): return np.inner(vector1_wrt,np.inner(vector2_wrt,to_project)) + @staticmethod def orthogonal_projector(self, u): return self.metric + np.outer(u,u) @@ -103,19 +77,54 @@ def orthogonal_projector(self, u): find_nearest returns the closest value to in put 'value' in 'array', find_nearest_cell then takes this closest value and returns its indices. """ - def find_nearest(self, array, value): + @staticmethod + def find_nearest(array, value): + """ + Returns closest value to input 'value' in 'array' + + Parameters: + ----------- + array: np.array of shape (n,) + + value: float + + Returns: + -------- + float + + Note: + ----- + To be used together with find_nearest_cell. + """ idx = np.searchsorted(array, value, side="left") if idx > 0 and (idx == len(array) or math.fabs(value - array[idx-1]) < math.fabs(value - array[idx])): return array[idx-1] else: return array[idx] - def find_nearest_cell(self, point): - t_pos = self.find_nearest(self.ts,point[0]) - x_pos = self.find_nearest(self.xs,point[1]) - y_pos = self.find_nearest(self.ys,point[2]) - return [np.where(self.ts==t_pos)[0][0], np.where(self.xs==x_pos)[0][0], np.where(self.ys==y_pos)[0][0]] - + @staticmethod + def find_nearest_cell(point, points): + """ + Use find nearest to find closest value in a list of input 'points' to + input 'point'. + + Parameters: + ----------- + point: list of d+1 float + + Returns: + -------- + List of d+1 indices corresponding to closest value to point in points + """ + if len(points) != len(point): + print("find_nearest_cell: The length of the coordinate vector\ + does not match the length of the coordinates.") + positions = [] + for dim in range(len(point)): + positions.append(Base.find_nearest(points[dim], point[dim])) + return [np.where(points[i] == positions[i])[0][0] for i in range(len(positions))] + + def profile(self, fnc): """A decorator that uses cProfile to profile a function""" def inner(*args, **kwargs): From 51311cf46f16395ac8537b1f6c570804f64ec8d0 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 25 May 2023 17:49:05 +0100 Subject: [PATCH 006/111] Update to interpol + structures, stuff moved to base --- Filters.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/Filters.py b/Filters.py index 7500b5a..eca4c77 100644 --- a/Filters.py +++ b/Filters.py @@ -683,17 +683,19 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): constraint = FindObs_drift_min(micro_model,0.001) filter = Box_filter(micro_model, 0.001) - var = "SET" + vars = ['BC','SET'] point = [1.502,0.3,0.5] observer = Base.get_rel_vel(constraint.find_observer(point).x) - CPU_start_time = time.process_time() - filtvar1 = filter.filter_var_point_inbuilt(var, point, observer) - print(f"CPU time to filter SET with in-built method is {time.process_time()- CPU_start_time}. ") + for var in vars: + CPU_start_time = time.process_time() + filtvar1 = filter.filter_var_point_inbuilt(var, point, observer) + print(f"CPU time to filter {var} with in-built method is {time.process_time()- CPU_start_time}. ") - CPU_start_time = time.process_time() - filtvar2 = filter.filter_var_point(var, point, observer) - print(f"CPU time to filter SET with Gauss method is {time.process_time()- CPU_start_time}. ") + CPU_start_time = time.process_time() + filtvar2 = filter.filter_var_point(var, point, observer) + print(f"CPU time to filter {var} with Gauss method is {time.process_time()- CPU_start_time}. ") - print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") + print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") + print('\n************************\n') \ No newline at end of file From ae8d616eaebb8e70ea2660a665dd7ad93d2515e7 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 26 May 2023 12:41:32 +0100 Subject: [PATCH 007/111] Added class for finding obs via root-finding --- Filters.py | 454 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 395 insertions(+), 59 deletions(-) diff --git a/Filters.py b/Filters.py index eca4c77..c5da55d 100644 --- a/Filters.py +++ b/Filters.py @@ -8,7 +8,7 @@ import numpy as np import time import scipy.integrate as integrate -from scipy.optimize import minimize +from scipy.optimize import minimize, root from itertools import product from system.BaseFunctionality import * @@ -16,9 +16,10 @@ from FileReaders import * -class FindObs_drift_min(object): +class FindObs_flux_min(object): """ - Class for computing the Favre observer given a micromodel. + Class for computing the observer by minimizing the (micro) baryon current + surface flux over a box. Work in any dimension, read on construction from the micro-model. """ def __init__(self, micro_model, box_len): @@ -27,18 +28,16 @@ def __init__(self, micro_model, box_len): ----------- micro_model: instance of class containing the microdata - Note: - ----- - To-do: think about checking compatibility (dimension + baryon currrent) + box_len: float, side of the box """ self.micro_model = micro_model self.spatial_dims = micro_model.get_spatial_dims() self.L = box_len - self.residuals_methods = { - "gauss" : self.drift_residual_Gauss , - "linear" : self.drift_residual , - "inbuilt" : self.drift_residual_ib + self.flux_methods = { + "gauss" : self.flux_residual_Gauss , + "linear" : self.flux_residual , + "inbuilt" : self.flux_residual_ib } def set_box_length(self, bl): @@ -75,9 +74,9 @@ def get_tetrad_from_vels(self, spatial_vels): tetrad += [es[i]] return tetrad - def drift_residual(self, spatial_vels, point, lin_spacing = 10): + def flux_residual(self, spatial_vels, point, lin_spacing = 10): """ - Compute the drift of baryons through the box built from vx_vy. + Compute the drift of baryons through the box built from spatial_vels. First get the center of the 2*(d+1) faces of the (d+1)-box, then build coords for points to sample the flux through each face. Next, approximate the flux integral as a sum @@ -127,9 +126,9 @@ def drift_residual(self, spatial_vels, point, lin_spacing = 10): flux *= (self.L / lin_spacing) ** self.spatial_dims return abs(flux) - def drift_residual_Gauss(self, spatial_vels, point, order = 3): + def flux_residual_Gauss(self, spatial_vels, point, order = 3): """ - Alternative for computing manually the Favre residual, using the Gauss + Alternative for computing manually the flux residual, using the Gauss Legendre method. Parameters: @@ -189,8 +188,6 @@ def drift_residual_Gauss(self, spatial_vels, point, order = 3): temp *= w totws.append(temp) - # for i in range(len(coords)): - # print(f'Coord: {coords[i]} , weight: {weights[i]}') tetrad = self.get_tetrad_from_vels(spatial_vels) flux = 0. for vec in tetrad: @@ -212,10 +209,10 @@ def drift_residual_Gauss(self, spatial_vels, point, order = 3): flux *= (self.L /2 ) ** self.spatial_dims return abs(flux) - def drift_residual_ib(self, spatial_vels, point, abserr = 1e-7): + def flux_residual_ib(self, spatial_vels, point, abserr = 1e-7): """ - Compute the residual using inbuilt method dblquad - Based on function point_flux + Compute the flux residual using inbuilt method dblquad or tplquad + Based on function point_flux (nested definition) Parameters: ----------- @@ -247,7 +244,7 @@ def drift_residual_ib(self, spatial_vels, point, abserr = 1e-7): def point_flux(x, y , center, Vx, Vy, normal): coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) Na = self.micro_model.get_interpol_var('BC', coords) - flux = self.Mink_dot(Na, normal) + flux = Base.Mink_dot(Na, normal) return flux for vec in tetrad: @@ -264,7 +261,7 @@ def point_flux(x, y , center, Vx, Vy, normal): def point_flux(x, y, z , center, Vx, Vy, Vz, normal): coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + np.multiply(z, Vz) Na = self.micro_model.get_interpol_var('BC', coords) - flux = self.Mink_dot(Na, normal) + flux = Base.Mink_dot(Na, normal) return flux for vec in tetrad: @@ -277,16 +274,16 @@ def point_flux(x, y, z , center, Vx, Vy, Vz, normal): error += partial_error return abs(flux) - def find_observer(self, point, residual_str = "gauss"): + def find_observer(self, point, flux_str = "gauss"): """ - Key function: minimize the Favre_residual and find the Favre observer at point. + Key function: minimize the flux residual and find the observer at point. Parameters: ----------- point: list of spatial_dims+1 floats (t,x,y) - residual_str: string, default to "gauss" - string for the method used to compute the residual, must be chosen within + flux_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within ['gauss', 'linear', 'inbuilt'] Returns: @@ -306,23 +303,23 @@ def find_observer(self, point, residual_str = "gauss"): guess.append(U[i] / U[0]) guess = np.array(guess) try: - sol = minimize(self.residuals_methods[residual_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) + sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) return sol except KeyError: - print(f"The method you want to use for computing the residual, {residual_str}, does not exist!") + print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") return None - def find_observers_points(self, points, residual_str = "gauss"): + def find_observers_points(self, points, flux_str = "gauss"): """ - Key function: minimize the Favre_residual and find the Favre observers for points. - Spacing is the param passed to Favre_residual to sample the box faces. + Key function: minimize the flux residual and find the observers for points. + flux_str determines the method to compute the flux residual. Parameters: ----------- points: list of spatial_dims+1 floats, ordered as (t,x,y) - residual_str: string, default to "gauss" - string for the method used to compute the residual, must be chosen within + flux_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within ['gauss', 'linear', 'inbuilt'] Returns: @@ -345,9 +342,9 @@ def find_observers_points(self, points, residual_str = "gauss"): failed_coords = [] for point in points: - sol = self.find_observer(point, residual_str) + sol = self.find_observer(point, flux_str) if sol.success: - observers.append(self.get_U_from_vels(sol.x)) + observers.append(Base.get_rel_vel(sol.x)) funs.append(sol.fun) success_coords.append(point) @@ -358,11 +355,10 @@ def find_observers_points(self, points, residual_str = "gauss"): failed_coords.append(point) return [success_coords, observers, funs] , failed_coords - def find_observers_ranges(self, num_points, ranges, residual_str = "gauss" ): + def find_observers_ranges(self, num_points, ranges, flux_str = "gauss" ): """ - Key function: minimize the Favre_residual and find the Favre observers for points - in linearly spaced (with spacing num_points) in the ranges. - Spacing is the param passed to Favre_residual to sample the box faces. + Key function: minimize the flux_residual and find the observers for points + in the ranges. flux_str determines the method to compute the flux residual. Parameters: ----------- @@ -372,8 +368,8 @@ def find_observers_ranges(self, num_points, ranges, residual_str = "gauss" ): ranges: list of lists of two floats [[t_min,t_max], ...] Define the coord ranges to find observers at - residual_str: string, default to "gauss" - string for the method used to compute the residual, must be chosen within + flux_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within ['gauss', 'linear', 'inbuilt'] Returns: @@ -399,10 +395,348 @@ def find_observers_ranges(self, num_points, ranges, residual_str = "gauss" ): for element in product(*list_of_coords): points.append(np.array(element)) - return self.find_observers_points(points, residual_str) + return self.find_observers_points(points, flux_str) -class Box_filter(object): +class FindObs_drift_root(object): + """ + Class for computing the observer by minimizing the (micro) baryon current + drift over a box. + Work in any dimension, read on construction from the micro-model. + """ + def __init__(self, micro_model, box_len): + """ + Parameters: + ----------- + micro_model: instance of micro_model, the micro data to be filtered + + box_len: float, side of the box for computing drift + """ + self.micro_model = micro_model + self.L = box_len + self.spatial_dims = micro_model.get_spatial_dims() + + self.drift_methods = { + "gauss" : self.drift_residual_gauss , + "linear" : self.drift_residual , + "inbuilt" : self.drift_residual_ib + } + + def set_box_len(self, box_len): + self.L = box_len + + def get_tetrad_from_vels(self, spatial_vels): + """ + Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels + + Parameters: + ----------- + spatial_vels: list of d floats, with d the number of spatial dimensions + + Return: + ------- + list of arrays: U + d unit vectors that complete it to a orthonormal basis + """ + if len(spatial_vels) != self.spatial_dims: + print('The number of spatial velocities passed is not compatible with \ + micro_model dimensionality!') + return None + U = Base.get_rel_vel(spatial_vels) + es =[] + for _ in range(self.spatial_dims): + es.append(np.zeros(self.spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1 + tetrad = [U] + for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) + tetrad += [es[i]] + return tetrad + + def drift_residual(self, spatial_vels, point, lin_spacing = 10): + """ + Compute the averaged baryon current over box, using linearly spaced sample points. + Then compute the drift with respect to each element in the triad completing U (built from + spatial vels) to a tetrad. + + Parameters: + ----------- + spatial_vels: list of d floats, the spatial vels of the observer + + point: list of d+1 floats, the coord of the center of the box + + lin_spacing: integer, number of points (in each direction) used for sampling the box + + Returns: + -------- + list of d floats, the drifts in each direction of the triad. + """ + tetrad = self.get_tetrad_from_vels(spatial_vels) + + xs = [] + for i in range(self.spatial_dims+1): + xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) + adapt_coords = [] + for element in product(*xs): + adapt_coords.append(np.array(element)) + + coords = [] + for coord in adapt_coords: + temp = np.array(point) + for i in range(self.spatial_dims+1): + temp += np.multiply(coord[i], tetrad[i]) + coords.append(temp) + + integral = np.zeros(1+self.spatial_dims) + for coord in coords: + DeltaV = (self.L / lin_spacing)**(1+self.spatial_dims) + integral += np.multiply(DeltaV, micro_model.get_interpol_var('BC', coord)) + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + + return drifts + + def drift_residual_gauss(self, spatial_vels, point, order = 3): + """ + Method to compute net drift residual through the box, using Gauss-Legendre method + + Parameters: + ----------- + spatial_vels: list of d floats, the spatial velocities of the observer + + point: list of d+1 floats ordered as (t,x,y,...), the centre of the box + + order: integer, the order of the Gauss-Legendre approximation + + Returns: + -------- + list of d floats, the average drift in each direction of the triad. + + """ + ps1d = [] + ws1d = [] + if order == 3: + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + elif order == 4: + p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) + p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) + w1 = (18. + np.sqrt(30) ) / 36. + w2 = (18. - np.sqrt(30) ) / 36. + ps1d = [p1, -p1, p2, -p2] + ws1d = [w1, w1, w2, w2] + elif order == 5: + p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 + p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 + w1 = (322 + 13 * np.sqrt(70)) /900 + w2 = (322 - 13 * np.sqrt(70)) /900 + ps1d = [0, p1, -p1, p2, -p2] + ws1d = [128/225, w1, w1, w2, w2] + else: + print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") + return [] + + xs = [] + ws = [] + for _ in range(self.spatial_dims+1): + xs.append(ps1d) + ws.append(ws1d) + + adapt_coords = [] + for element in product(*xs): + adapt_coords.append(np.multiply(self.L/2, np.array(element) )) + + totws = [] + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + tetrad = self.get_tetrad_from_vels(spatial_vels) + coords = [] + for coord in adapt_coords: + temp = np.array(point) + for i in range(self.spatial_dims+1): + temp += np.multiply(coord[i], tetrad[i]) + coords.append(temp) + + integral = np.zeros(1+self.spatial_dims) + for i, coord in enumerate(coords): + integral += np.multiply(totws[i], micro_model.get_interpol_var('BC', coord)) + integral *= (self.L /2 ) ** (self.spatial_dims+1) + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + + return drifts + + def drift_residual_ib(self, spatial_vels, point, abserr = 1e-7): + """ + Method to compute net drift residual through the box, using Gauss-Legendre method + + Parameters: + ----------- + spatial_vels: list of d floats, the spatial velocities of the observer + + point: list of d+1 floats ordered as (t,x,y,...), the centre of the box + + abserr = optional float, absolute error to be passed to tplquad + + Returns: + -------- + list of d floats, the average drift in each direction of the triad. + """ + if self.spatial_dims != 2: + print('In built methods for integrals can be used only in 2+1 dimensions (i.e. tplquad). The 1+1 \ + dimensional case is not really interesting so not implemented.') + return None + else: + tetrad = self.get_tetrad_from_vels(spatial_vels) + integral = np.zeros(3) + error = np.zeros(3) + + for i in range(3): + def BC_point_value(t, x, y): + coord = point + np.multiply(t, tetrad[0]) + np.multiply(x, tetrad[1]) + np.multiply(y, tetrad[2]) + return self.micro_model.get_interpol_var('BC', coord)[i] + + integral[i], error[i] = integrate.tplquad(BC_point_value, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, \ + -self.L/2, self.L/2, args = (), epsabs = abserr)[:] + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + + return drifts + + def find_observer(self, point, drift_str): + """ + Key method: use optimize.root to find the roots of the drift function. + + Parameters: + ----------- + point: list of d+1 floats, ordered (t,x,y,...), the coordinates of the box centre + + drift_str: string, the method used to compute the net drift + must be either 'gauss', 'linear', 'inbuilt' + + Returns: + -------- + OptimizedResult obtained via optimize.root + + Notes: + ------ + Change the optional values of the corresponding drift methods to change + the number of points used to sample the d+1 box. + """ + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + guess = [] + for i in range(1, len(U)): + guess.append(U[i] / U[0]) + guess = np.array(guess) + + try: + sol = root(self.drift_methods[drift_str], x0 = guess, args = (point)) + return sol + except KeyError: + print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") + return None + + def find_observers_points(self, points, drift_str = "gauss"): + """ + Key method: use optimize.root to find the roots of the drift function. + + Parameters: + ----------- + points: list of spatial_dims+1 floats, ordered as (t,x,y) + + drift_str: string, default to "gauss" + string for the method used to compute the drifts, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which root finding is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which root finding failed + + Notes: + ------ + Change the optional values of the corresponding drift methods to change + the number of points used to sample the d+1 box. + """ + observers = [] + funs = [] + success_coords = [] + failed_coords = [] + + for point in points: + sol = self.find_observer(point, drift_str) + if sol.success: + observers.append(Base.get_rel_vel(sol.x)) + funs.append(sol.fun) + success_coords.append(point) + + if (sol.fun > 1e-5).any(): + print(f"Warning, residual is large in at least one direction at {point}: ", sol.fun) + else: + print(f'Failed for coordinates: {point}, due to', sol.message) + failed_coords.append(point) + return [success_coords, observers, funs] , failed_coords + + def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): + """ + Key function: minimize the flux_residual and find the observers for points + in the ranges. drift_str determines the method to compute the flux residual. + + Parameters: + ----------- + num_points: list of integers + number of points to find observers at in each direction + + ranges: list of lists of two floats [[t_min,t_max], ...] + Define the coord ranges to find observers at + + drift_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which root finding is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which the root finding failed + + Notes: + ------ + Change the optional values of the corresponding drift methods to change + the number of points used to sample the d+1 box. + """ + + list_of_coords = [] + for i in range(len(num_points)): + list_of_coords.append( np.linspace( ranges[i][0], ranges[i][-1] , num_points[i]) ) + + points = [] + for element in product(*list_of_coords): + points.append(np.array(element)) + + return self.find_observers_points(points, drift_str) + +class spatial_box_filter(object): """ Class for box-filtering the variables of a micro_model. Work in any dimensions, read on construction from the micro_model. @@ -680,22 +1014,24 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): micro_model = IdealMHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() - constraint = FindObs_drift_min(micro_model,0.001) - filter = Box_filter(micro_model, 0.001) - vars = ['BC','SET'] - point = [1.502,0.3,0.5] - observer = Base.get_rel_vel(constraint.find_observer(point).x) - for var in vars: - CPU_start_time = time.process_time() - filtvar1 = filter.filter_var_point_inbuilt(var, point, observer) - print(f"CPU time to filter {var} with in-built method is {time.process_time()- CPU_start_time}. ") + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) - CPU_start_time = time.process_time() - filtvar2 = filter.filter_var_point(var, point, observer) - print(f"CPU time to filter {var} with Gauss method is {time.process_time()- CPU_start_time}. ") - - print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") - print('\n************************\n') - - \ No newline at end of file + vars = ['BC','SET'] + points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] + observers = find_obs.find_observers_points(points)[0][1] + + for i in range(len(points)): + for var in vars: + CPU_start_time = time.process_time() + filtvar1 = filter.filter_var_point_inbuilt(var, points[i], observers[i]) + inbuilt_time = time.process_time() - CPU_start_time + + CPU_start_time = time.process_time() + filtvar2 = filter.filter_var_point(var, points[i], observers[i]) + gauss_time = time.process_time() - CPU_start_time + print(f"CPU time speed-up to filter {var} with Gauss method at {points[i]} is {inbuilt_time/gauss_time}. ") + print(f'Filtered quantity with Gauss is \n {filtvar2}') + print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") + print('\n************************\n') \ No newline at end of file From ad212eb15b30b2ebd526b09846bafdaeff7e23dc Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 13 Jun 2023 10:37:18 +0200 Subject: [PATCH 008/111] TC latest update --- Filters.py | 84 ++++-- MesoModels.py | 550 +++++++++++++++++++++++++++--------- MicroModels.py | 6 +- system/BaseFunctionality.py | 2 + 4 files changed, 479 insertions(+), 163 deletions(-) diff --git a/Filters.py b/Filters.py index c5da55d..6874772 100644 --- a/Filters.py +++ b/Filters.py @@ -288,7 +288,8 @@ def find_observer(self, point, flux_str = "gauss"): Returns: -------- - OptimizedResult of the minimization via scipy.optimize.minimize + Successful minimization: Boolean True, observer, error + Failed minimization: Boolean False, coordinates Notes: ------ @@ -302,11 +303,24 @@ def find_observer(self, point, flux_str = "gauss"): for i in range(1, len(U)): guess.append(U[i] / U[0]) guess = np.array(guess) + # try: + # sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) + # return sol + # except KeyError: + # print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") + # return None + try: - sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) - return sol + sol = sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) + if sol.success: + observer = Base.get_rel_vel(sol.x) + if sol.fun > 1e-5: + print(f'Warning: residual is large at {point}', avg_error) + return sol.success, observer, sol.fun + if not sol.success: + return sol.success, point except KeyError: - print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") + print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") return None def find_observers_points(self, points, flux_str = "gauss"): @@ -337,23 +351,21 @@ def find_observers_points(self, points, flux_str = "gauss"): change the default values of the optional arguments in the corresponding methods. """ observers = [] - funs = [] + errors = [] success_coords = [] failed_coords = [] for point in points: sol = self.find_observer(point, flux_str) - if sol.success: - observers.append(Base.get_rel_vel(sol.x)) - funs.append(sol.fun) + if sol[0]: + observers.append(sol[1]) + errors.append(sol[2]) success_coords.append(point) + + if not sol[0]: + failed_coords.append[point] - if (sol.fun > 1e-5): - print(f"Warning, residual is large at {point}: ", sol.fun) - else: - print(f'Failed for coordinates: {point}, due to', sol.message) - failed_coords.append(point) - return [success_coords, observers, funs] , failed_coords + return [success_coords, observers, errors] , failed_coords def find_observers_ranges(self, num_points, ranges, flux_str = "gauss" ): """ @@ -493,7 +505,7 @@ def drift_residual(self, spatial_vels, point, lin_spacing = 10): integral = np.zeros(1+self.spatial_dims) for coord in coords: DeltaV = (self.L / lin_spacing)**(1+self.spatial_dims) - integral += np.multiply(DeltaV, micro_model.get_interpol_var('BC', coord)) + integral += np.multiply(DeltaV, self.micro_model.get_interpol_var('BC', coord)) drifts = [] for i in range(len(tetrad)-1): @@ -568,7 +580,7 @@ def drift_residual_gauss(self, spatial_vels, point, order = 3): integral = np.zeros(1+self.spatial_dims) for i, coord in enumerate(coords): - integral += np.multiply(totws[i], micro_model.get_interpol_var('BC', coord)) + integral += np.multiply(totws[i], self.micro_model.get_interpol_var('BC', coord)) integral *= (self.L /2 ) ** (self.spatial_dims+1) drifts = [] @@ -616,7 +628,7 @@ def BC_point_value(t, x, y): return drifts - def find_observer(self, point, drift_str): + def find_observer(self, point, drift_str = "gauss"): """ Key method: use optimize.root to find the roots of the drift function. @@ -629,7 +641,8 @@ def find_observer(self, point, drift_str): Returns: -------- - OptimizedResult obtained via optimize.root + Successful root-finding: Boolean True, observer, avg error + Failed minimization: Boolean False, coordinates Notes: ------ @@ -644,7 +657,17 @@ def find_observer(self, point, drift_str): try: sol = root(self.drift_methods[drift_str], x0 = guess, args = (point)) - return sol + if sol.success: + observer = Base.get_rel_vel(sol.x) + error = 0 + for i in range(len(sol.fun)): + error += sol.fun[i] + avg_error = error/ len(sol.fun) + if avg_error > 1e-5: + print(f'Warning: residual is large at {point}', avg_error) + return sol.success, observer, avg_error + if not sol.success: + return sol.success, point except KeyError: print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") return None @@ -676,23 +699,21 @@ def find_observers_points(self, points, drift_str = "gauss"): the number of points used to sample the d+1 box. """ observers = [] - funs = [] + avg_errors = [] success_coords = [] failed_coords = [] for point in points: sol = self.find_observer(point, drift_str) - if sol.success: - observers.append(Base.get_rel_vel(sol.x)) - funs.append(sol.fun) + if sol[0]: + observers.append(sol[1]) + avg_errors.append(sol[2]) success_coords.append(point) + + if not sol[0]: + failed_coords.append[point] - if (sol.fun > 1e-5).any(): - print(f"Warning, residual is large in at least one direction at {point}: ", sol.fun) - else: - print(f'Failed for coordinates: {point}, due to', sol.message) - failed_coords.append(point) - return [success_coords, observers, funs] , failed_coords + return [success_coords, observers, avg_errors] , failed_coords def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): """ @@ -1015,7 +1036,8 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): FileReader.read_in_data(micro_model) micro_model.setup_structures() - find_obs = FindObs_drift_root(micro_model, 0.001) + # find_obs = FindObs_drift_root(micro_model, 0.001) + find_obs = FindObs_flux_min(micro_model, 0.001) filter = spatial_box_filter(micro_model, 0.003) vars = ['BC','SET'] @@ -1033,5 +1055,5 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): gauss_time = time.process_time() - CPU_start_time print(f"CPU time speed-up to filter {var} with Gauss method at {points[i]} is {inbuilt_time/gauss_time}. ") print(f'Filtered quantity with Gauss is \n {filtvar2}') - print(f"Difference between the two is: \n {filtvar1[0] - filtvar2}") + print(f"Difference with method based on inbuilt integration is: \n {filtvar1[0] - filtvar2}") print('\n************************\n') \ No newline at end of file diff --git a/MesoModels.py b/MesoModels.py index 0be7422..b8c60a5 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -8,6 +8,14 @@ import numpy as np from scipy.integrate import quad from multiprocessing import Process, Pool +from itertools import product +from system.BaseFunctionality import * +from FileReaders import * +from MicroModels import * +from Filters import * + + + class NonIdealHydro2D(object): @@ -16,115 +24,242 @@ def __init__(self, MicroModel, Filter): self.MicroModel = MicroModel #self.Observers = Observers self.spatial_dims = 2 + + self.domain_var_strs = ('Nt','Nx','Ny','dT','dX','dY') + self.domain_vars = dict.fromkeys(self.domain_var_strs) + + #Dictionary for 'local' variables, + #obtained from filtering the appropriate MicroModel variables + self.filter_var_strs = ("n","SET") + self.filter_vars = dict.fromkeys(self.filter_var_strs) + + #Dictionary for MesoModel variables + self.meso_var_strs = ("U","U_coords","U_errors","T~") + self.meso_vars = dict.fromkeys(self.meso_var_strs) + + #Strings for 'non-local' variables - ones we need to take derivatives of + self.nonlocal_var_strs = ("U","T~") + + #Dictionary for derivative variables - calculated by finite differencing + self.deriv_var_strs = ("dtU","dxU","dyU","dtT~","dxT~","dyT~") + self.deriv_vars = dict.fromkeys(self.deriv_var_strs) + + #Dictionary for dissipative residuals - from the NonId-SET that we take + #projections of + self.diss_residual_strs = ("Pi","q","pi") + self.diss_residuals = dict.fromkeys(self.diss_residual_strs) + + self.diss_var_strs = ("Theta","Omega","Sigma") + self.diss_vars = dict.fromkeys(self.diss_residual_strs) + + self.diss_coeff_strs = ("Zeta", "Kappa", "Eta") + self.diss_coeffs = dict.fromkeys(self.diss_coeff_strs) + + self.coefficient_strs = ("Gamma") + self.coefficients = dict.fromkeys(self.diss_coeff_strs) + self.coefficients['Gamma'] = 4.0/3.0 + + # Stencils for finite-differencing + self.cen_SO_stencil = [1/12, -2/3, 0, 2/3, -1/12] + self.cen_FO_stencil = [-1/2, 0, 1/2] + self.fw_FO_stencil = [-1, 1] + self.bw_FO_stencil = [-1, 1] + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 # Run some compatability test... compatible = True - if(compatible): + if self.spatial_dims != self.MicroModel.spatial_dims: + compatible = False + for filter_var_str in self.filter_var_strs: + if not (filter_var_str in MicroModel.vars.keys()): + compatible = False + + if compatible: print("Meso and Micro models are compatible!") else: - print("Meso and Micro models are incompatible!") - - #Dictionary for 'local' variables - ones we won't need derivatives of - self.local_var_strs = ("n","SET") - self.local_vars = dict.fromkeys(self.local_var_strs) - for str in self.local_vars: - self.local_vars[str] = [] - - #Dictionary for 'non-local' variables - ones we need to take derivatives of - self.nonlocal_var_strs = ("T_tilde","U") - self.nonlocal_vars = dict.fromkeys(self.nonlocal_var_strs) - for str in self.nonlocal_var_strs: - self.nonlocal_vars[str] = [] + print("Meso and Micro models are incompatible!") def find_observers(self, num_points, ranges, spacing): - self.U_coords, self.Us, self.U_errors = self.Filter.find_observers(num_points, ranges, spacing)[0] - + self.meso_vars['U_coords'], self.meso_vars['U'], self.meso_vars['U_errors'] = \ + self.Filter.find_observers(num_points, ranges, spacing)[0] + self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] = num_points[:] + self.domain_vars['dT'] = (ranges[0][-1] - ranges[0][0]) / self.domain_vars['Nt'] + self.domain_vars['dX'] = (ranges[1][-1] - ranges[1][0]) / self.domain_vars['Nx'] + self.domain_vars['dY'] = (ranges[2][-1] - ranges[2][0]) / self.domain_vars['Ny'] + + def setup_variables(self): + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + n_dims = self.spatial_dims+1 + self.meso_vars['U_coords'] = np.array(self.meso_vars['U_coords']).reshape([Nt, Nx, Ny, n_dims]) + self.meso_vars['U'] = np.array(self.meso_vars['U']).reshape([Nt, Nx, Ny, n_dims]) + self.meso_vars['U_errors'] = np.array(self.meso_vars['U_errors']).reshape([Nt, Nx, Ny]) + self.meso_vars['T~'] = np.zeros((Nt, Nx, Ny)) + + self.filter_vars['n'] = np.zeros((Nt, Nx, Ny)) + self.filter_vars['SET'] = np.zeros((Nt, Nx, Ny,n_dims,n_dims)) + + for nonlocal_var_str in self.nonlocal_var_strs: + self.deriv_vars['dt'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) + self.deriv_vars['dx'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) + self.deriv_vars['dy'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) + + self.diss_residuals['Pi'] = np.zeros((Nt, Nx, Ny)) + self.diss_vars['Theta'] = np.zeros((Nt, Nx, Ny)) + self.diss_residuals['q'] = np.zeros((Nt, Nx, Ny,n_dims)) + self.diss_vars['Omega'] = np.zeros((Nt, Nx, Ny,n_dims)) + self.diss_residuals['pi'] = np.zeros((Nt, Nx, Ny,n_dims,n_dims)) + self.diss_vars['Sigma'] = np.zeros((Nt, Nx, Ny,n_dims,n_dims)) + + # Single value for each coefficient (per data point) for now... + self.diss_coeffs['Zeta'] = np.zeros((Nt, Nx, Ny)) + self.diss_coeffs['Kappa'] = np.zeros((Nt, Nx, Ny)) + self.diss_coeffs['Eta'] = np.zeros((Nt, Nx, Ny)) + def p_from_EoS(self, rho, n): """ - Calculate pressure from EoS + Calculate pressure from EoS using rho (energy density) and n (number density) """ - p = (self.coefficients['gamma']-1)*(rho-n) + p = (self.coefficients['Gamma']-1)*(rho-n) return p + + def filter_micro_variables(self): + """ + 'Spatially' average required variables from the micromodel w.r.t. + the observers that have been found. + """ + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + self.filter_vars['n'][h,i,j] =\ + self.Filter.filter_prim_var(self.meso_vars['U_coords'][h,i,j], self.meso_vars['U'][h,i,j], 'n') + self.filter_vars['SET'][h,i,j] =\ + self.Filter.filter_struc(self.meso_vars['U_coords'][h,i,j], self.meso_vars['U'][h,i,j], 'SET') + + # Should be able to convert here to only 1 filter function... (not prim/struct) + # for filter_var_str in self.filter_var_strs: + # filter_args = [(coord, U, filter_var_str) for coord, U in zip(self.U_coords, self.Us)] + # with Pool(2) as p: + # self.micro_vars[micro_var_str] = p.starmap(self.Filter.filter_var, filter_args) + # self.micro_vars[micro_var_str] = p.starmap(self.Filter.filter_prim_var, filter_args) - - def filter_variables(self): - # Filter necessary variables from the micromodel - filter_args = [(coord, U) for coord, U in zip(self.U_coords, self.Us)] - print(filter_args) - for local_var_str in self.local_var_strs: - with Pool(2) as p: - self.local_vars[local_var_str] = p.starmap(self.Filter.filter_var, filter_args) - - for coord, U in zip(self.U_coords, self.Us): - self.Ns = self.Filter.filter_var(coord, U, 'n') - self.SETs = self.Filter.filter_var(coord, U, 'SET') + def calculate_derivatives(self): + """ + Calculate the required derivatives of MesoModel variables for + constructing the non-ideal terms. + """ + for nonlocal_var_str in self.nonlocal_var_strs: + self.calculate_time_derivatives(nonlocal_var_str) + self.calculate_x_derivatives(nonlocal_var_str) + self.calculate_y_derivatives(nonlocal_var_str) - def calculate_nonlocal_variables(self, coords): - # Size of box for spatial filtering - # the numerical coefficient ~ determines #cells along side of filtering box - self.L = 4*np.sqrt(self.dx*self.dy) - self.dT = 0.01 # steps to take for differential calculations - self.dX = 0.01 - self.dY = 0.01 - # Stencils for finite-differencing - self.cen_SO_stencil = [1/12, -2/3, 0, 2/3, -1/12] - self.cen_FO_stencil = [-1/2, 0, 1/2] - self.fw_FO_stencil = [-1, 1] - self.bw_FO_stencil = [-1, 1] - - # Calculate time derivatives for central slices - for t_slice in range(self.n_obs_t): - # Central-difference - for i in range(1,self.n_obs_x-1): - for j in range(1,self.n_obs_y-1): # fix these to use self.n_x_pts etc. - self.dtUts[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uts[t_slice][i][j] / self.dt_obs - self.dtUxs[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uxs[t_slice][i][j] / self.dt_obs - self.dtUys[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uys[t_slice][i][j] / self.dt_obs - # pick out central slice with first [1] - self.dxUts[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uts[1][i-1+t_slice][j] / self.dx_obs - self.dxUxs[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uxs[1][i-1+t_slice][j] / self.dx_obs - self.dxUys[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uys[1][i-1+t_slice][j] / self.dx_obs - - self.dyUts[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uts[1][i][j-1+t_slice] / self.dy_obs - self.dyUxs[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uxs[1][i][j-1+t_slice] / self.dy_obs - self.dyUys[i,j] += self.cen_FO_stencil[t_slice]*\ - self.Uys[1][i][j-1+t_slice] / self.dy_obs - - - def calculate_dissipative_residuals(self, coord, U): - # Filter the scalar fields - N = self.Filter.filter_scalar(coord, U, 'n') + def calculate_time_derivatives(self, nonlocal_var_str): + deriv_var_str = 'dt'+nonlocal_var_str + for h in range(self.domain_vars['Nt']): + if h == 0: + stencil = self.fw_FO_stencil + samples = [0,1] + elif h == (self.domain_vars['Nt']-1): + stencil = self.fw_FO_stencil + samples = [-1,0] + else: + stencil = self.cen_FO_stencil + samples = [-1,0,1] + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + for s in range(len(samples)): + self.deriv_vars[deriv_var_str][h,i,j] \ + += (stencil[s]*self.meso_var_strs[nonlocal_var_str][h+samples[s],i,j]) / self.domain_vars['dT'] + + def calculate_x_derivatives(self, nonlocal_var_str): + deriv_var_str = 'dx'+nonlocal_var_str + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + if i == 0: + stencil = self.fw_FO_stencil + samples = [0,1] + elif i == (self.domain_vars['Nx']-1): + stencil = self.fw_FO_stencil + samples = [-1,0] + else: + stencil = self.cen_FO_stencil + samples = [-1,0,1] + for j in range(self.domain_vars['Ny']): + for s in range(len(samples)): + self.deriv_vars[deriv_var_str][h,i,j] \ + += (stencil[s]*self.meso_var_strs[nonlocal_var_str][h,i+samples[s],j]) / self.domain_vars['dX'] + + def calculate_y_derivatives(self, nonlocal_var_str): + deriv_var_str = 'dy'+nonlocal_var_str + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + if j == 0: + stencil = self.fw_FO_stencil + samples = [0,1] + elif j == (self.domain_vars['Ny']-1): + stencil = self.fw_FO_stencil + samples = [-1,0] + else: + stencil = self.cen_FO_stencil + samples = [-1,0,1] + for s in range(len(samples)): + self.deriv_vars[deriv_var_str][h,i,j] \ + += (stencil[s]*self.meso_var_strs[nonlocal_var_str][h,i,j+samples[s]]) / self.domain_vars['dY'] + + def calculate_dissipative_residuals(self, h, i, j): + """ + Returns the inferred (residual) values of the + dissipative terms from the filtered MicroModel SET. + + Parameters + ---------- + indices : TYPE + DESCRIPTION. + + Returns + ------- + Pi_res : scalar float + Bulk viscosity + q_res : (d+1) vector of floats + Heat flux. + pi_res : (d+1)x(d+1) tensor of floats + Shear viscosity. - # Construct filtered Id SET - filtered_Id_SET = self.Filter.filter_scalar(coord, U, 'Id_SET') + """ + # Move this + # h, i, j = indices + # Filter the scalar fields + N = self.filter_vars['n'][h,i,j] + Id_SET = self.filter_vars['SET'][h,i,j] + U = self.meso_vars['U'][h,i,j] # Do required projections of SET - h_mu_nu = self.orthogonal_projector(U) - rho_res = self.project_tensor(U,U,filtered_Id_SET) - q_res = np.einsum('ij,i,jk',filtered_Id_SET,U,h_mu_nu) - tau_res = np.einsum('ij,ik,jl',filtered_Id_SET,h_mu_nu,h_mu_nu) # tau = p + Pi+ pi - - # Calculate Pi and pi residuals - tau_trace = np.trace(tau_res)# + h_mu_nu = Base.orthogonal_projector(U, self.metric) + rho_res = Base.project_tensor(U,U,Id_SET) + + # Set Meso temperature from filtered quantities using EoS p_tilde = self.p_from_EoS(rho_res, N) T_tilde = p_tilde/N - self.T_tildes[h, i, j] = T_tilde + self.meso_vars['T~'][h,i,j] = T_tilde + + # Calculate dissipative residual with tensor manipulation + q_res = np.einsum('ij,i,jk',Id_SET,U,h_mu_nu) + tau_res = np.einsum('ij,ik,jl',Id_SET,h_mu_nu,h_mu_nu) + tau_trace = np.trace(tau_res) Pi_res = tau_trace - p_tilde pi_res = tau_res - np.dot((p_tilde + Pi_res),h_mu_nu) - - return Pi_res, q_res, pi_res - def calculate_nonlocal_variables(self, coord, indices): + self.diss_residuals['Pi'][h,i,j] = Pi_res + self.diss_residuals['q'][h,i,j] = q_res + self.diss_residuals['pi'][h,i,j] = pi_res + + def calculate_dissipative_variables(self, h, i, j): """ - Calculate the non-ideal, dissipation terms (without coefficeints). + Calculates the non-ideal, dissipation terms (without coefficeints) + for the non-ideal MesoModel SET. Parameters ---------- @@ -135,64 +270,221 @@ def calculate_nonlocal_variables(self, coord, indices): Returns ------- - Theta : float - Divergence of the observer velocity. + Decomposition of the observer velocity: + + Theta : float (scalar) + Divergence (isotropic). omega : vector of floats Transverse momentum. - sigma : tensor of floats - Symmetric, trace-free bla bla. + sigma : (d+1)x(d+1) tensor of floats + Symmetric, trace-free. """ - h, i, j = indices - # T = self.values_from_hdf5(coord, 'T') # Fix this - should be from EoS(N,p_tilde) - T = self.T_tildes[h, i, j] - dtT = self.calc_t_deriv('T',coord)[0] - dxT = self.calc_x_deriv('T',coord)[0] - dyT = self.calc_y_deriv('T',coord)[0] - # print(T.shape,dxT.shape) - Ut = self.Uts[h,i,j] - Ux = self.Uxs[h,i,j] - Uy = self.Uys[h,i,j] - dtUt = self.dtUts[h,i,j] - dtUx = self.dtUxs[h,i,j] - dtUy = self.dtUys[h,i,j] - dxUt = self.dxUts[h,i,j] - dxUx = self.dxUxs[h,i,j] - dxUy = self.dxUys[h,i,j] - dyUt = self.dyUts[h,i,j] - dyUx = self.dyUxs[h,i,j] - dyUy = self.dyUys[h,i,j] - - Theta = dtUt + dxUx + dyUy - a = np.array([Ut*dtUt + Ux*dxUt + Uy*dyUt, Ut*dtUx + Ux*dxUx + Uy*dyUx, Ut*dtUy + Ux*dxUy + Uy*dyUy])#,ux*dxuz+uy*dyuz+uz*dzuz]) + # h, i, j = indices - omega = np.array([dtT, dxT, dyT]) + np.multiply(T,a) # FIX - sigma = np.array([[2*dtUt - (2/3)*Theta, dtUx + dxUt, dtUy + dyUt],\ - [dxUt + dtUx, 2*dxUx - (2/3)*Theta, dxUy + dyUx], - [dyUt + dtUy, dyUx + dxUy, 2*dyUy - (2/3)*Theta]]) + T = self.meso_vars['T~'][h, i, j] + dtT = self.deriv_vars['dtT~'][h, i, j] + dxT = self.deriv_vars['dxT~'][h, i, j] + dyT = self.deriv_vars['dyT~'][h, i, j] - return Theta, omega, sigma + # U = self.meso_vars['U'][h,i,j][:] + Ut, Ux, Uy = self.meso_vars['U'][h,i,j][:] + # dtU = self.deriv_vars['dtU'][h,i,j] + dtUt, dtUx, dtUy = self.deriv_vars['dtU'][h,i,j][:] + dxUt, dxUx, dxUy = self.deriv_vars['dxU'][h,i,j][:] + dyUt, dyUx, dyUy = self.deriv_vars['dyU'][h,i,j][:] - def calculate_coefficients(self, point): - Pi_res, q_res, pi_res = self.calculate_dissipative_residuals(point) - Theta, omega, sigma = self.calculate_nonlocal_variables(point) - return -Pi_res/Theta, -q_res/omega, -pi_res/sigma - - -class meso_model_example(object): + # Need to do this with Einsum... + Theta = dtUt + dxUx + dyUy + a = np.array([Ut*dtUt + Ux*dxUt + Uy*dyUt, Ut*dtUx + Ux*dxUx + Uy*dyUx, Ut*dtUy + Ux*dxUy + Uy*dyUy]) - def __init__(self, micro_model, constraint, filter): - pass - + Omega = np.array([dtT, dxT, dyT]) + np.multiply(T,a) + Sigma = np.array([[2*dtUt - (2/3)*Theta, dtUx + dxUt, dtUy + dyUt],\ + [dxUt + dtUx, 2*dxUx - (2/3)*Theta, dxUy + dyUx], + [dyUt + dtUy, dyUx + dxUy, 2*dyUy - (2/3)*Theta]]) + self.diss_vars['Theta'][h,i,j] = Theta + self.diss_vars['Omega'][h,i,j] = Omega + self.diss_vars['Sigma'][h,i,j] = Sigma + def calculate_dissipative_coefficients(self): + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + # parallel_args = [(h, i, j) for h in range(Nt) for i in range(Nx) for j in range(Ny)] + # # print(parallel_args) + # with Pool(2) as p: + # p.starmap(self.calculate_dissipative_residuals, parallel_args) + # p.starmap(self.calculate_dissipative_variables, parallel_args) + + # self.diss_coeffs['Zeta'] = -self.diss_residuals['Pi'] / self.diss_vars['Theta'] + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + self.calculate_dissipative_residuals(h,i,j) + self.calculate_dissipative_variables(h,i,j) + self.diss_coeffs['Zeta'][h,i,j] = -self.diss_residuals['Pi'][h,i,j] / self.diss_vars['Theta'][h,i,j] + + for diss_coeff_str in self.diss_coeff_strs: + coeffs_handle = open(diss_coeff_str+'.pickle', 'wb') + pickle.dump(self.diss_coeffs[diss_coeff_str], coeffs_handle, protocol=pickle.HIGHEST_PROTOCOL) +class resMHD2D(object): + """ + Work in Progress + """ + def __init__(self, micro_model, find_obs, filter): + self.micro_model = micro_model + self.find_obs = find_obs + self.filter = filter + + self.spatial_dims = 2 + + self.meso_structures_strs = ['SET', 'BC', 'Fab'] + self.meso_structures = dict.fromkeys(self.meso_structures_strs) + for var in self.meso_structures: + self.meso_structures[var] = [] + + self.domain_int_strs = ('Nt','Nx','Ny') + self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") + self.domain_array_strs = ("T","X","Y","Points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) + for var in self.domain_vars: + self.domain_vars[var] = [] + + self.filter_vars_strs = ['U', 'U_errors', 'U_success'] + self.filter_vars = dict.fromkeys(self.filter_vars_strs) + for var in self.filter_vars: + var = [] + + # Run some compatability test... + compatible = True + error = '' + if self.spatial_dims != micro_model.get_spatial_dims(): + compatible = False + error += '\nError: different dimensions.' + for struct in self.meso_structures_strs: + if struct not in self.micro_model.get_structures_strs(): + compatible = False + error += f'\nError: {struct} not in micro_model!' + + if not compatible: + print("Meso and Micro models are incompatible"+error) + + + def setup_meso_grid(self, patch_bdrs, coarse_factor): + """ + Builds the meso_model grid using the micro_model grid points contained in + the input patch (defined via 'patch_bdrs'). The method allows for coarse graining + the grid in the spatial directions. + + Parameters: + ----------- + patch_bdrs: list of lists of two floats, + [[tmin, tmax],[xmin,xmax],[ymin,ymax]] + + coarse_factor: integer + """ + # Is the patch within the micro_model domain? + conditions = patch_bdrs[0][0] < self.micro_model.domain_vars['tmin'] and \ + patch_bdrs[0][1] > self.micro_model.domain_vars['tmax'] and \ + patch_bdrs[1][0] < self.micro_model.domain_vars['xmin'] and \ + patch_bdrs[1][1] > self.micro_model.domain_vars['xmax'] and \ + patch_bdrs[2][0] < self.micro_model.domain_vars['ymin'] and \ + patch_bdrs[2][1] > self.micro_model.domain_vars['ymax'] + if conditions: + print('The input region for filtering is larger than micro_model domain!') + return None + + patch_min = [patch_bdrs[0][0], patch_bdrs[1][0], patch_bdrs[2][0]] + patch_max = [patch_bdrs[0][1], patch_bdrs[1][1], patch_bdrs[2][1]] + idx_mins = Base.find_nearest_cell(patch_min, micro_model.domain_vars['points']) + idx_maxs = Base.find_nearest_cell(patch_max, micro_model.domain_vars['points']) + + self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] + self.domain_vars['DX'] = self.micro_model.domain_vars['dx'] * coarse_factor + self.domain_vars['DY'] = self.micro_model.domain_vars['dy'] * coarse_factor + + h, i, j = idx_mins[0], idx_mins[1], idx_mins[2] + while h <= idx_maxs[0]: + self.domain_vars['T'].append(self.micro_model.domain_vars['t'][h]) + h += 1 + while i <= idx_maxs[1]: + self.domain_vars['X'].append(self.micro_model.domain_vars['x'][i]) + i += coarse_factor + while j <= idx_maxs[2]: + self.domain_vars['Y'].append(self.micro_model.domain_vars['x'][j]) + j += coarse_factor + + self.domain_vars['Points'] = [self.domain_vars['T'], self.domain_vars['X'], self.domain_vars['Y']] + self.domain_vars['Tmin'] = np.amin(self.domain_vars['T']) + self.domain_vars['Xmin'] = np.amin(self.domain_vars['X']) + self.domain_vars['Ymin'] = np.amin(self.domain_vars['Y']) + self.domain_vars['Tmax'] = np.amax(self.domain_vars['T']) + self.domain_vars['Xmax'] = np.amax(self.domain_vars['X']) + self.domain_vars['Ymax'] = np.amax(self.domain_vars['Y']) + self.domain_vars['Nt'] = len(self.domain_vars['T']) + self.domain_vars['Nx'] = len(self.domain_vars['X']) + self.domain_vars['Ny'] = len(self.domain_vars['Y']) + + def build_observers(self): + """ + Method to compute filtering observers at grid points built with setup_meso_grid. + The observers found (and relative errors) are saved in the dictionary self.filter_vars. + Set up a dictionary self.filter_vars['U_successes] with info on where observers have been found. + Notes: + ------ + The dictionary self.filter_vars['U_successes] has as key:value pairs the following + + keys: tuples corresponding to grid points + values: [bool, index] + The bool is true/false according to whether the observer has been found at the point, + the index is the tuple needed to access the observers found at the corresponding point + as ' self.filter_vars['U'][index]' + """ + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + self.filter_vars['U'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) + self.filter_vars['U_success'] = dict() + + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,x,y] + sol = self.find_obs.find_observer(point) + if sol[0]: + self.filter_vars['U'][h,i,j] = sol[1] + self.filter_vars['U_errors'][h,i,j] = sol[2] + self.filter_vars['U_success'].update({tuple(point) : [True, (h,i,j)]}) + + if not sol[0]: + self.filter_vars['U_success'].update({tuple(point) : [False, (h,i,j)]}) + print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) + + + def compute_local(self): - \ No newline at end of file + +if __name__ == '__main__': + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + + meso_model = resMHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([[1.503, 1.503],[0.37, 0.42],[0.43, 0.54]],2) + meso_model.build_observers() + for elem in meso_model.filter_vars['U_success']: + idx = meso_model.filter_vars['U_success'][elem][1] + if meso_model.filter_vars['U_success'][elem][0]: + print('Success: ', elem, meso_model.filter_vars['U'][idx], meso_model.filter_vars['U_errors'][idx], '\n') + else: + print('Failed: ', elem,'\n') diff --git a/MicroModels.py b/MicroModels.py index 0060e9c..b54a9ad 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -64,7 +64,7 @@ def __init__(self, interp_method = "linear"): self.aux_vars[str] = [] #Dictionary for structures - self.structures_strs = ("BC","SET","Faraday") + self.structures_strs = ("BC","SET","Fab") self.structures = dict.fromkeys(self.structures_strs) for str in self.structures_strs: self.structures[str] = [] @@ -93,7 +93,7 @@ def setup_structures(self): """ self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) - self.structures["Faraday"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + self.structures["Fab"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) for h in range(self.domain_vars['nt']): for i in range(self.domain_vars['nx']): @@ -115,7 +115,7 @@ def setup_structures(self): fol_b_vec = np.array([self.aux_vars["b0"][h,i,j],self.aux_vars["bx"][h,i,j],self.aux_vars["by"][h,i,j]]) fol_e_vec = np.tensordot( self.Levi3D, np.outer(vel,fol_b_vec), axes = ([1,2],[0,1])) - self.structures['Faraday'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ + self.structures['Fab'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) def get_var_gridpoint(self, var, point): diff --git a/system/BaseFunctionality.py b/system/BaseFunctionality.py index 73f2212..5a69769 100644 --- a/system/BaseFunctionality.py +++ b/system/BaseFunctionality.py @@ -112,6 +112,8 @@ def find_nearest_cell(point, points): ----------- point: list of d+1 float + points: list of lists of d+1 floats + Returns: -------- List of d+1 indices corresponding to closest value to point in points From be09fbc08a164361bc2c2529a97b10602f8785d2 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 13 Jun 2023 11:47:02 +0200 Subject: [PATCH 009/111] Merging with MH --- MicroModels.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/MicroModels.py b/MicroModels.py index b77bc1d..a896e02 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -64,27 +64,11 @@ def __init__(self, interp_method = "linear"): self.aux_vars[str] = [] #Dictionary for structures -<<<<<<< HEAD self.structures_strs = ("BC","SET","Fab") -======= - self.structures_strs = ("BC","SET","Faraday") ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 self.structures = dict.fromkeys(self.structures_strs) for str in self.structures_strs: self.structures[str] = [] -<<<<<<< HEAD -======= - #Dictionary for all vars - self.all_var_strs = self.prim_strs + self.aux_strs + self.structures_strs - self.all_vars = self.prim_vars.copy() - self.all_vars.update(self.aux_vars) - self.all_vars.update(self.structures) - - def get_model_name(self): - return 'IdealMHD_2D' - ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 def get_spatial_dims(self): return self.spatial_dims @@ -99,12 +83,6 @@ def get_aux_strs(self): def get_structures_strs(self): return self.structures_strs -<<<<<<< HEAD -======= - - def get_all_var_strs(self): - return self.all_var_strs ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 def setup_structures(self): """ @@ -115,11 +93,7 @@ def setup_structures(self): """ self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) -<<<<<<< HEAD self.structures["Fab"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) -======= - self.structures["Faraday"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 for h in range(self.domain_vars['nt']): for i in range(self.domain_vars['nx']): @@ -141,11 +115,7 @@ def setup_structures(self): fol_b_vec = np.array([self.aux_vars["b0"][h,i,j],self.aux_vars["bx"][h,i,j],self.aux_vars["by"][h,i,j]]) fol_e_vec = np.tensordot( self.Levi3D, np.outer(vel,fol_b_vec), axes = ([1,2],[0,1])) -<<<<<<< HEAD self.structures['Fab'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ -======= - self.structures['Faraday'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) def get_var_gridpoint(self, var, point): @@ -155,11 +125,7 @@ def get_var_gridpoint(self, var, point): Parameters: ----------- -<<<<<<< HEAD vars: string corresponding to primitive, auxiliary or structre variable -======= - vars: string corresponding to primitive, auxiliary or structure variable ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 point: list of 2+1 floats @@ -301,13 +267,11 @@ def __init__(self, interp_method = "linear"): self.structures[str] = [] #Dictionary for all vars -<<<<<<< HEAD self.var_strs = self.prim_strs + self.aux_strs + self.structures_strs # self.vars = self.prim_vars # self.vars.update(self.aux_vars) # self.vars.update(self.structures) -======= self.all_var_strs = self.prim_strs + self.aux_strs + self.structures_strs def get_model_name(self): @@ -316,7 +280,6 @@ def get_model_name(self): def get_spatial_dims(self): return self.spatial_dims ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 def get_domain_strs(self): return self.domain_info_strs + self.domain_points_strs @@ -327,14 +290,10 @@ def get_aux_strs(self): return self.aux_strs def get_structures_strs(self): -<<<<<<< HEAD - return self.get_structures_strs -======= return self.get_structures_strs def get_all_var_strs(self): return self.all_var_strs ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 def setup_structures(self): """ @@ -478,8 +437,6 @@ def get_interpol_var(self, var_names, point): print(f"{var_name} does not belong to the variables of the micro_model!") return res -<<<<<<< HEAD -======= # TC if __name__ == '__main__': @@ -499,7 +456,6 @@ def get_interpol_var(self, var_names, point): print(f'{var}: \n {res} \n {res2} \n ********** \n ') # MH ->>>>>>> 7775d53b835ac3366603f777af6eadd768abd049 if __name__ == '__main__': CPU_start_time = time.process_time() From 81b689dd0fe0bd52a65becdd877111ccef1d94f7 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 13 Jun 2023 11:50:27 +0200 Subject: [PATCH 010/111] Merging with MH --- MicroModels.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/MicroModels.py b/MicroModels.py index a896e02..35ff25e 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -456,18 +456,32 @@ def get_interpol_var(self, var_names, point): print(f'{var}: \n {res} \n {res2} \n ********** \n ') # MH -if __name__ == '__main__': +# if __name__ == '__main__': - CPU_start_time = time.process_time() +# CPU_start_time = time.process_time() - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() - FileReader.read_in_data(micro_model) - micro_model.setup_structures() +# FileReader = METHOD_HDF5('./Data/Testing/') +# # micro_model = IdealMHD_2D() +# micro_model = IdealHydro_2D() +# FileReader.read_in_data(micro_model) +# micro_model.setup_structures() - point = [1.502,0.4,0.2] - vars = ['SET', 'BC'] - for var in vars: - res = micro_model.get_interpol_var(var, point) - res2 = micro_model.get_var_gridpoint(var, point) - print(f'{var}: \n {res} \n {res2} \n ********** \n ') \ No newline at end of file +# res = micro_model.get_interpol_var(['v1','rho'],[10.0,0.3,0.2]) +# print(type(res),"\n", res) + +# res = micro_model.get_interpol_var(['T','W'],[10.0,0.3,0.2]) +# print(type(res),"\n", res) + +# # res = micro_model.get_interpol_var(['bar_vel','SET'],[10.0,0.3,0.2]) +# # print(type(res),"\n", res) +# # res = micro_model.get_interpol_aux(["b0","bx"],[0.5, 0.3,0.2]) +# # print(type(res),"\n", res) + +# # res = micro_model.get_interpol_struct("SET",[2.5, 0.3, 0.2]) +# # print(res.shape, res[0],'\n') + +# # print( micro_model.get_domain_strs() ) + +# CPU_end_time = time.process_time() +# CPU_time = CPU_end_time - CPU_start_time +# print(f'The CPU time is {CPU_time} seconds') \ No newline at end of file From a5bdefdd3505eae2d85bb18152f6def21513bf51 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 13 Jun 2023 11:57:53 +0200 Subject: [PATCH 011/111] Merging with MH --- MesoModels.py | 53 --------------------------------------------------- 1 file changed, 53 deletions(-) diff --git a/MesoModels.py b/MesoModels.py index 47a9118..a07c1f7 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -342,56 +342,3 @@ def calculate_dissipative_coefficients(self): coeffs_handle = open(diss_coeff_str+'.pickle', 'wb') pickle.dump(self.diss_coeffs[diss_coeff_str], coeffs_handle, protocol=pickle.HIGHEST_PROTOCOL) -# Notes: -# ------ -# The dictionary self.filter_vars['U_successes] has as key:value pairs the following - -# keys: tuples corresponding to grid points - -# values: [bool, index] - -# The bool is true/false according to whether the observer has been found at the point, -# the index is the tuple needed to access the observers found at the corresponding point -# as ' self.filter_vars['U'][index]' -# """ -# Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] -# self.filter_vars['U'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) -# self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) -# self.filter_vars['U_success'] = dict() - -# for h, t in enumerate(self.domain_vars['T']): -# for i, x in enumerate(self.domain_vars['X']): -# for j, y in enumerate(self.domain_vars['Y']): -# point = [t,x,y] -# sol = self.find_obs.find_observer(point) -# if sol[0]: -# self.filter_vars['U'][h,i,j] = sol[1] -# self.filter_vars['U_errors'][h,i,j] = sol[2] -# self.filter_vars['U_success'].update({tuple(point) : [True, (h,i,j)]}) - -# if not sol[0]: -# self.filter_vars['U_success'].update({tuple(point) : [False, (h,i,j)]}) -# print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) - - -# def compute_local(self): - - -# if __name__ == '__main__': - -# FileReader = METHOD_HDF5('./Data/test_res100/') -# micro_model = IdealMHD_2D() -# FileReader.read_in_data(micro_model) -# micro_model.setup_structures() -# find_obs = FindObs_drift_root(micro_model, 0.001) -# filter = spatial_box_filter(micro_model, 0.003) - -# meso_model = resMHD2D(micro_model, find_obs, filter) -# meso_model.setup_meso_grid([[1.503, 1.503],[0.37, 0.42],[0.43, 0.54]],2) -# meso_model.build_observers() -# for elem in meso_model.filter_vars['U_success']: -# idx = meso_model.filter_vars['U_success'][elem][1] -# if meso_model.filter_vars['U_success'][elem][0]: -# print('Success: ', elem, meso_model.filter_vars['U'][idx], meso_model.filter_vars['U_errors'][idx], '\n') -# else: -# print('Failed: ', elem,'\n') From 7ff01c4bdcb94c87be8d8ced6e090d9e9bd7d369 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 19 Jul 2023 09:57:16 +0200 Subject: [PATCH 012/111] TC work on MHD meso class --- Filters.py | 49 ++-- MesoModels.py | 491 +++++++++++++++++++++++++++++++++++- MicroModels.py | 93 +++---- system/BaseFunctionality.py | 12 + 4 files changed, 578 insertions(+), 67 deletions(-) diff --git a/Filters.py b/Filters.py index 61ea8b9..26e0f9a 100644 --- a/Filters.py +++ b/Filters.py @@ -294,7 +294,7 @@ def point_flux(x, y, z , center, Vx, Vy, Vz, normal): error += partial_error return abs(flux) - def find_observer(self, point, flux_str = "gauss"): + def find_observer(self, point, flux_str = "gauss", initial_guess = None): """ Key function: minimize the flux residual and find the observer at point. @@ -306,6 +306,10 @@ def find_observer(self, point, flux_str = "gauss"): string for the method used to compute the flux, must be chosen within ['gauss', 'linear', 'inbuilt'] + initial_guess: DEPRECATED, list of d floats: the spatial velocities of the initial guess + if the number of spatial vels passed does not match the micro_model dimensionality, the + pointwise velocity is used instead. + Returns: -------- Successful minimization: Boolean True, observer, error @@ -318,29 +322,26 @@ def find_observer(self, point, flux_str = "gauss"): the optional arguments in the corresponding methods. """ - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) guess = [] - for i in range(1, len(U)): - guess.append(U[i] / U[0]) - guess = np.array(guess) - # try: - # sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) - # return sol - # except KeyError: - # print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") - # return None + if initial_guess is not None and len(initial_guess) == self.spatial_dims: + guess = initial_guess + else: + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + for i in range(1, len(U)): + guess.append(U[i] / U[0]) + guess = np.array(guess) try: sol = sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) if sol.success: observer = Base.get_rel_vel(sol.x) if sol.fun > 1e-5: - print(f'Warning: residual is large at {point}', avg_error) + print(f'Warning: residual is large at {point}', sol.fun) return sol.success, observer, sol.fun if not sol.success: return sol.success, point except KeyError: - print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") + print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") return None def find_observers_points(self, points, flux_str = "gauss"): @@ -648,7 +649,7 @@ def BC_point_value(t, x, y): return drifts - def find_observer(self, point, drift_str = "gauss"): + def find_observer(self, point, drift_str = "gauss", initial_guess= None): """ Key method: use optimize.root to find the roots of the drift function. @@ -659,6 +660,10 @@ def find_observer(self, point, drift_str = "gauss"): drift_str: string, the method used to compute the net drift must be either 'gauss', 'linear', 'inbuilt' + initial_guess: DEPRECATED, list of d floats: the spatial velocities of the initial guess + if the number of spatial vels passed does not match the micro_model dimensionality, the + pointwise velocity is used instead. + Returns: -------- Successful root-finding: Boolean True, observer, avg error @@ -669,10 +674,13 @@ def find_observer(self, point, drift_str = "gauss"): Change the optional values of the corresponding drift methods to change the number of points used to sample the d+1 box. """ - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) guess = [] - for i in range(1, len(U)): - guess.append(U[i] / U[0]) + if initial_guess is not None and len(initial_guess) == self.spatial_dims: + guess = initial_guess + else: + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + for i in range(1, len(U)): + guess.append(U[i] / U[0]) guess = np.array(guess) try: @@ -777,6 +785,7 @@ def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): return self.find_observers_points(points, drift_str) + class spatial_box_filter(object): """ Class for box-filtering the variables of a micro_model. @@ -1056,11 +1065,11 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): FileReader.read_in_data(micro_model) micro_model.setup_structures() - # find_obs = FindObs_drift_root(micro_model, 0.001) - find_obs = FindObs_flux_min(micro_model, 0.001) + find_obs = FindObs_drift_root(micro_model, 0.001) + # find_obs = FindObs_flux_min(micro_model, 0.001) filter = spatial_box_filter(micro_model, 0.003) - vars = ['BC','SET'] + vars = ['BC','SETfl', 'Fab', 'SETem'] points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] observers = find_obs.find_observers_points(points)[0][1] diff --git a/MesoModels.py b/MesoModels.py index a07c1f7..e291e05 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -6,11 +6,14 @@ """ import numpy as np -from scipy.integrate import quad from multiprocessing import Process, Pool -from system.BaseFunctionality import * import pickle +from system.BaseFunctionality import * +from MicroModels import * +from FileReaders import * +from Filters import * + class NonIdealHydro2D(object): def __init__(self, MicroModel, Filter, interp_method = "linear"): @@ -342,3 +345,487 @@ def calculate_dissipative_coefficients(self): coeffs_handle = open(diss_coeff_str+'.pickle', 'wb') pickle.dump(self.diss_coeffs[diss_coeff_str], coeffs_handle, protocol=pickle.HIGHEST_PROTOCOL) +class resMHD2D(object): + """ + Idea: alternative meso_models (e.g. with different closures scheme) will + only have to change the method model_residuals and list of non_local_vars + (possibly but less likely decompose_structure_gridpoint) + """ + def __init__(self, micro_model, find_obs, filter): + """ + + """ + self.micro_model = micro_model + self.find_obs = find_obs + self.filter = filter + + self.spatial_dims = 2 + + self.domain_int_strs = ('Nt','Nx','Ny') + self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") + self.domain_array_strs = ("T","X","Y","Points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) + for var in self.domain_vars: + self.domain_vars[var] = [] + + # This is the Levi-Civita symbol, not tensor, so be careful when using it + self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ + for k in range(3)]for j in range(3) ] for i in range(3) ]) + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + self.filter_vars_strs = ['U', 'U_errors', 'U_success'] + self.filter_vars = dict.fromkeys(self.filter_vars_strs) + for var in self.filter_vars: + var = [] + + self.meso_structures_strs = ['SETfl', 'SETem', 'BC', 'Fab'] + self.meso_structures = dict.fromkeys(self.meso_structures_strs) + for var in self.meso_structures: + self.meso_structures[var] = [] + + + # self.meso_fluid_vars_strs = ['eps_tilde', 'u_tilde', 'n_tilde', 'p_tilde', 'p', 'eos_res', 'Pi_res', 'q_res', 'pi_res'] + # self.meso_em_vars_strs = ['e', 'b', 'j', 'sigma', 'J'] + # self.meso_vars_strs = self.meso_fluid_vars_strs + self.meso_em_vars_strs + # self.meso_vars = dict.fromkeys(self.meso_vars_strs) + + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde'] + self.meso_vectors_strs = ['u_tilde', 'q_res', 'e_tilde', 'j', 'J_tilde'] + self.meso_tensors_strs = ['pi_res'] + self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_tensors_strs + self.meso_vars = dict.fromkeys(self.meso_vars_strs) + + # Dictionary with stencils and coefficients for finite differencing + self.differencing = dict.fromkeys((1, 2)) + self.differencing[1] = dict.fromkeys(['fw', 'bw', 'cen']) + self.differencing[1]['fw'] = {'coefficients' : [-1, 1] , 'stencil' : [0, 1]} + self.differencing[1]['bw'] = {'coefficients' : [1, -1] , 'stencil' : [0, -1]} + self.differencing[1]['cen'] = {'coefficients' : [-1/2., 0, 1/2.] , 'stencil' : [-1, 0, 1]} + self.differencing[2] = dict.fromkeys(['fw', 'bw', 'cen']) + self.differencing[2]['fw'] = {'coefficients' : [-3/2., 2., -1/2.] , 'stencil' : [0, 1, 2]} + self.differencing[2]['bw'] = {'coefficients' : [3/2., -2., 1/2.] , 'stencil' : [0, -1, -2]} + self.differencing[2]['cen'] = {'coefficients' : [1/12., -2/3., 0., 2/3., -1/12.] , 'stencil' : [-2, -1, 0, 1, 2]} + + self.nonlocal_vars_strs = ['u_tilde', 'Fab'] # These have to belong to either a structure or meso_vars + Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] + self.deriv_vars = dict.fromkeys(Dstrs) + + # Run some compatibility test... + compatible = True + error = '' + if self.spatial_dims != micro_model.get_spatial_dims(): + compatible = False + error += '\nError: different dimensions.' + for struct in self.meso_structures_strs: + if struct not in self.micro_model.get_structures_strs(): + compatible = False + error += f'\nError: {struct} not in micro_model!' + + if not compatible: + print("Meso and Micro models are incompatible:"+error) + + def get_model_name(self): + return 'resMHD2D' + + def set_find_obs_method(self, find_obs): + self.find_obs = find_obs + + def setup_meso_grid(self, patch_bdrs, coarse_factor): + """ + Builds the meso_model grid using the micro_model grid points contained in + the input patch (defined via 'patch_bdrs'). The method allows for coarse graining + the grid in the spatial directions ONLY. + + Parameters: + ----------- + patch_bdrs: list of lists of two floats, + [[tmin, tmax],[xmin,xmax],[ymin,ymax]] + + coarse_factor: integer + + Notes: + ------ + If the patch_bdrs are larger than micro_grid, the method will not set-up the meso_grid, + and an error message is printed. This is extra safety measure! + """ + + # Is the patch within the micro_model domain? + conditions = patch_bdrs[0][0] < self.micro_model.domain_vars['tmin'] or \ + patch_bdrs[0][1] > self.micro_model.domain_vars['tmax'] or \ + patch_bdrs[1][0] < self.micro_model.domain_vars['xmin'] or \ + patch_bdrs[1][1] > self.micro_model.domain_vars['xmax'] or \ + patch_bdrs[2][0] < self.micro_model.domain_vars['ymin'] or \ + patch_bdrs[2][1] > self.micro_model.domain_vars['ymax'] + + if conditions: + print('Error: the input region for filtering is larger than micro_model domain!') + return None + + #Find the nearest cell to input patch bdrs + patch_min = [patch_bdrs[0][0], patch_bdrs[1][0], patch_bdrs[2][0]] + patch_max = [patch_bdrs[0][1], patch_bdrs[1][1], patch_bdrs[2][1]] + idx_mins = Base.find_nearest_cell(patch_min, self.micro_model.domain_vars['points']) + idx_maxs = Base.find_nearest_cell(patch_max, self.micro_model.domain_vars['points']) + + # Set meso_grid spacings + self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] + self.domain_vars['Dx'] = self.micro_model.domain_vars['dx'] * coarse_factor + self.domain_vars['Dy'] = self.micro_model.domain_vars['dy'] * coarse_factor + + # Building the meso_grid + h, i, j = idx_mins[0], idx_mins[1], idx_mins[2] + while h <= idx_maxs[0]: + t = self.micro_model.domain_vars['t'][h] + self.domain_vars['T'].append(t) + h += 1 + while i <= idx_maxs[1]: + x = self.micro_model.domain_vars['x'][i] + self.domain_vars['X'].append(x) + i += coarse_factor + while j <= idx_maxs[2]: + y = self.micro_model.domain_vars['y'][j] + self.domain_vars['Y'].append(y) + j += coarse_factor + + # Saving the info about the meso_grid + self.domain_vars['Points'] = [self.domain_vars['T'], self.domain_vars['X'], self.domain_vars['Y']] + self.domain_vars['Tmin'] = np.amin(self.domain_vars['T']) + self.domain_vars['Xmin'] = np.amin(self.domain_vars['X']) + self.domain_vars['Ymin'] = np.amin(self.domain_vars['Y']) + self.domain_vars['Tmax'] = np.amax(self.domain_vars['T']) + self.domain_vars['Xmax'] = np.amax(self.domain_vars['X']) + self.domain_vars['Ymax'] = np.amax(self.domain_vars['Y']) + self.domain_vars['Nt'] = len(self.domain_vars['T']) + self.domain_vars['Nx'] = len(self.domain_vars['X']) + self.domain_vars['Ny'] = len(self.domain_vars['Y']) + + def setup_variables(self): + """ + Set up the arrays for meso_structures, meso_vars and filter_vars + + Notes: + ------ + Use after setup meso_grid + """ + # Setup arrays for structures + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + self.meso_structures['BC'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.meso_structures['SETfl'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.meso_structures['SETem'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.meso_structures['Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + + # Setup arrays for meso_vars + for str in self.meso_scalars_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny)) + for str in self.meso_vectors_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + for str in self.meso_tensors_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + + # Setup arrays for filter_vars + self.filter_vars['U'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) + self.filter_vars['U_success'] = dict() + + + # MARCUS'S WAY + # for str in self.non_local_vars_strs: + # if str in self.meso_vars: + # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_vars[str])}) + # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_vars[str])}) + # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_vars[str])}) + # if str in self.meso_structures: + # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_structures[str])}) + # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) + # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) + + # THOMAS'S WAY (SAVE THEM AS TENSORS) + self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.deriv_vars['D_Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1, self.spatial_dims+1)) + + def find_observers(self): + """ + Method to compute filtering observers at grid points built with setup_meso_grid. + The observers found (and relative errors) are saved in the dictionary self.filter_vars. + Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices + on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) + + Notes: + ------ + Requires setup_meso_grid() and setup_variables() to be called first. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,x,y] + sol = self.find_obs.find_observer(point) + if sol[0]: + self.filter_vars['U'][h,i,j] = sol[1] + self.filter_vars['U_errors'][h,i,j] = sol[2] + self.filter_vars['U_success'].update({(h,i,j) : True}) + + if not sol[0]: + self.filter_vars['U_success'].update({(h,i,j) : False}) + print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) + + def filter_micro_variables(self): + """ + Filter all meso_model structures AND micro pressure at the grid_points built with setup_meso_grid(). + This method relies on filter_var_point implemented separately for the filter, e.g. spatial_box_filter + + Parameters: + ----------- + + Notes: + ------ + Requires build_observers(), setup_meso_grid() and setup_variables() to be called first. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t, x, y] + if self.filter_vars['U_success'][h,i,j]: + obs = self.filter_vars['U'][h,i,j] + for struct in self.meso_structures: + self.meso_structures[struct][h,i,j] = self.filter.filter_var_point(struct, point, obs) + self.meso_vars['p'][h,i,j] = self.filter.filter_var_point('p', point, obs) + else: + print('Could not filter at {}: observer not found.'.format(point)) + + def decompose_structures_gridpoint(self, h, i ,j): + """ + Decompose the fluid part of SET as well as Fab at grid point (h,i,j) + + Parameters: + ----------- + h, i, j: integers + the indices on the grid where the decomposition is performed. + + Returns: + -------- + None + + Notes: + ------ + Decomposition of fluid SET and Faraday tensor. The EM part of SET is decomposed later + via non local operations (same story for the charge current). + + """ + # Computing the Favre density and velocity + n_tilde = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) + u_tilde = np.multiply(1 / n_tilde, self.meso_structures['BC'][h,i,j]) + self.meso_vars['n_tilde'][h,i,j] = n_tilde + self.meso_vars['u_tilde'][h,i,j,:] = u_tilde + + # Decomposing the fluid part of the stress energy tensor + # To change after update to Base.project_tensor() ? + h_ab = Base.orthogonal_projector(u_tilde, self.metric) + T_ab = self.meso_structures['SETfl'][h,i,j,:,:] + for l, m in np.ndindex(T_ab.shape): + if l==0: + T_ab[l,m] *= -1 + if m==0: + T_ab[l,m] *= -1 + p_filt = self.meso_vars['p'][h,i,j] + + self.meso_vars['eps_tilde'][h,i,j] = np.tensordot(np.tensordot(T_ab, u_tilde, axes = ([0],[0])), u_tilde, axes = ([0],[0])) + self.meso_vars['Pi_res'][h,i,j] = np.tensordot(T_ab, h_ab, axes = ([0,1],[0,1])) - p_filt + self.meso_vars['q_res'][h,i,j:] = np.tensordot( h_ab, np.tensordot(T_ab, u_tilde, axes = ([1],[0])), axes = ([1],[0])) + self.meso_vars['pi_res'][h,i,j,:,:] = np.tensordot(h_ab, np.tensordot(h_ab, T_ab, axes = ([1],[0])), axes = ([1],[1])) + # CHANGE THIS + self.meso_vars['eos_res'][h,i,j] = 0 + # TO + # p_tilde = p_from_EOS method(eps_tilde, n_tilde) + # self.meso_vars['eos_res'][h,i,j] = p_filt - p_tilde + + + # Decompose the Fab + Fab = self.meso_structures['Fab'][h,i,j] + self.meso_vars['e_tilde'][h,i,j,:] = np.tensordot(Fab, u_tilde, axes = ([1],[0])) + self.meso_vars['b_tilde'][h,i,j] = 1/2. * np.tensordot(np.tensordot(self.Levi3D, Fab, axes = ([1,2],[0,1])), u_tilde, axes = ([0],[0])) + + def decompose_structures(self): + """ + Decompose structures at all points on the meso_grid where observers could be found. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,y,x] + if self.filter_vars['U_success'][h,i,j]: + self.decompose_structures_gridpoint(h,i,j) + else: + print('Structures not decomposed at {}: observer could not be found.'.format(point)) + + def calculate_derivative_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): + """ + Calculate partial derivative in the input 'direction' of the variable corresponding to 'nonlocal_var_str' + at the position on the grid identified by indices h,i,j. The order of the differencing scheme + can also be specified, default to 1. + + Parameters: + ----------- + nonlocal_var_str: string + quantity to be taken derivate of, must be in self.nonlocal_vars_strs() + + h, i ,j: integers + indices for point on the meso_grid + + direction: integer < self.spatial_dim + 1 + + order: integer, defatult to 1 + order of the differencing scheme + + Returns: + -------- + Finite-differenced quantity at (h,i,j) + + Notes: + ------ + The method returns the value instead of storing it, so that these can be + rearranged as preferred later, that is in calculate_derivatives() + """ + + if direction > self.spatial_dims: + print('Directions are numbered from 0 up to {}'.format(self.spatial_dims)) + return None + + if order > len(self.differencing): + print('Maximum order implemented is {}: continuing with it.'.format(len(self.differencing))) + order = len(self.differencing) + + # Forward, backward or centered differencing? + k = [h,i,j][direction] + N = len(self.domain_vars['Points'][direction]) + + if k in [l for l in range(order)]: + coefficients = self.differencing[order]['fw']['coefficients'] + stencil = self.differencing[order]['fw']['stencil'] + elif k in [N-1-l for l in range(order)]: + coefficients = self.differencing[order]['bw']['coefficients'] + stencil = self.differencing[order]['bw']['stencil'] + else: + coefficients = self.differencing[order]['cen']['coefficients'] + stencil = self.differencing[order]['cen']['stencil'] + + # Is it a structure or a meso_var? + if nonlocal_var_str in self.meso_vars_strs: + temp = 0 + for s, sample in enumerate(stencil): + idxs = [h,i,j] + idxs[direction] += sample + temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_vars[nonlocal_var_str][tuple(idxs)] ) + # self.deriv_vars[deriv_var_str][h,i,j] += \ + # np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_vars[nonlocal_var_str][tuple(idxs)] ) + return temp + else: + temp = 0 + for s, sample in enumerate(stencil): + idxs = [h,i,j] + idxs[direction] += sample + temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_structures[nonlocal_var_str][tuple(idxs)] ) + # self.deriv_vars[deriv_var_str][h,i,j] += \ + # np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_structures[nonlocal_var_str][tuple(idxs)] ) + return temp + + def calculate_derivatives(self): + """ + Compute all the derivatives of the quantities corresponding to nonlocal_vars_strs, for all + gridpoints on the meso-grid. + + Notes: + ------ + The derived quantities are stored as 'tensors' as follows: + 1st three indices refer to the position on the grid + + 4th index refers to the directionality of the derivative + + last indices (1 or 2) correspond to the components of the quantity to be derived + + The index corresponding to the derivative is covariant, i.e. down. + + Example: + + Fab [h,i,j,a,b] : h,i,j grid; a,b, spacetime components + + D_Fab[h,i,j,c,a,b]: h,i,j grid; c direction of derivative; a,b as for Fab + + """ + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + for str in self.nonlocal_vars_strs: + for dir in range(self.spatial_dims+1): + dstr = 'D_' + str + self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivative_gridpoint(str, h, i, j, dir) + + def model_residuals(self, h, i, j): + """ + + """ + u_tilde = self.meso_vars['u_tilde'][h,i,j] + u_tilde_down = np.einsum('ij,i', self.metric, u_tilde) + + # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS + #Computing four-current: raise the last two indices of D_Fab and then sum over 1,3 component + temp = self.deriv_vars['D_Fab'][h,i,j] + temp = np.einsum('ij,klj->ikl', self.metric, temp) + temp = np.einsum('ij,klj->ikl', self.metric, temp) + current = np.einsum('iji->j', temp) + + # Decompose wrt u_tilde: again careful with indices up or down! + sigma_tilde = np.einsum('i,i', current, u_tilde_down) + J_tilde = current - np.multiply(sigma_tilde, u_tilde) + + self.meso_vars['sigma_tilde'][h,i,j] = sigma_tilde + self.meso_vars['j'][h,i,j] = current + self.meso_vars['J_tilde'][h,i,j] = J_tilde + + print(u_tilde, '\n',u_tilde_down, '\n', current, '\n', sigma_tilde, '\n', J_tilde,'\n***********') + + # MODELLING THE DISSIPATIVE TERMS IN THE FLUID SET + # DECOMPOSING NABLA_U + nabla_u = np.einsum('ij,jl->il', self.metric, self.deriv_vars['D_u_tilde'][h,i,j]) + acc = np.einsum('i,ij', u_tilde_down, nabla_u) + should_be_zero = np.einsum('i,ji', u_tilde_down, nabla_u) + Daub = nabla_u + np.outer(u_tilde, acc) + np.outer(should_be_zero, u_tilde) +\ + np.multiply(np.inner(should_be_zero, u_tilde_down), np.outer(u_tilde, u_tilde)) + exp_rate = np.einsum('ii', np.einsum('ij,jk->ik', self.metric, Daub)) + shear_rate = np.multiply(1/2., Daub + np.einsum('ji', Daub)) - \ + np.multiply( 1 / self.spatial_dims * exp_rate, Base.orthogonal_projector(u_tilde, self.metric)) + + print(Daub, '\n', exp_rate, '\n', shear_rate, '\n', should_be_zero) + + # EXTRACTING THE DISSIPATIVE COEFFICIENTS + # LATER: SAVE THIS TO A FILE (PICKLE) AND PLOT IT --> MARCUS'S ROUTINES + + # EXTRACT COEFFICIENTS COMPONENTWISE AND AVERAGE (FOR NOW) + # THINK WHAT TO DO WITH THE HEAT FLUX --> GRADIENTS IN T OR N AND EPS + # THINK ABOUT THE EM PART, WHAT COEFFICIENTS DO YOU WANT TO EXTRACT? SAVE THEM! + + + +if __name__ == '__main__': + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + + + CPU_start_time = time.process_time() + meso_model = resMHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([[1.501, 1.505],[0.37, 0.40],[0.43, 0.46]],1) + meso_model.setup_variables() + meso_model.find_observers() + meso_model.filter_micro_variables() + meso_model.decompose_structures() + + # print(meso_model.calculate_derivative('u_tilde', 1,1,0,1, order=2)) + meso_model.calculate_derivatives() + meso_model.model_residuals(1,1,0) diff --git a/MicroModels.py b/MicroModels.py index 35ff25e..8a579f5 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -52,19 +52,19 @@ def __init__(self, interp_method = "linear"): self.domain_vars[str] = [] #Dictionary for primitive var - self.prim_strs = ("vx","vy","n","p","Bx","By") + self.prim_strs = ("vx","vy","n","p", "Bz") self.prim_vars = dict.fromkeys(self.prim_strs) for str in self.prim_strs: self.prim_vars[str] = [] #Dictionary for auxiliary var - self.aux_strs = ("W","h","b0","bx","by","bsq") + self.aux_strs = ("W","h","bz","bsq", "e") self.aux_vars = dict.fromkeys(self.aux_strs) for str in self.aux_strs: self.aux_vars[str] = [] #Dictionary for structures - self.structures_strs = ("BC","SET","Fab") + self.structures_strs = ("BC", "SETfl", "SETem", "Fab") self.structures = dict.fromkeys(self.structures_strs) for str in self.structures_strs: self.structures[str] = [] @@ -72,6 +72,9 @@ def __init__(self, interp_method = "linear"): def get_spatial_dims(self): return self.spatial_dims + def get_model_name(self): + return 'IdealMHD_2D' + def get_domain_strs(self): return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs @@ -83,40 +86,65 @@ def get_aux_strs(self): def get_structures_strs(self): return self.structures_strs - + + def get_all_var_strs(self): + return self.get_prim_strs() + self.get_aux_strs() + self.get_structures_strs() + def setup_structures(self): """ - Set up the structures (i.e baryon vel, SET and Faraday) + Set up the structures (i.e baryon (mass) current BC, Stress-Energy and Faraday tensors + Notes: + ------ Structures are built as multi-dim np.arrays, with the first three indices referring to the grid, while the last one or two refer to space-time components. + + The Faraday tensor is stored as a fully co-variant tensor (both indices down) + The stress-energy tensor is stored as a fully contra-variant tensor (both indices up) """ self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) - self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + self.structures["SETfl"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + self.structures["SETem"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) self.structures["Fab"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) for h in range(self.domain_vars['nt']): for i in range(self.domain_vars['nx']): for j in range(self.domain_vars['ny']): - vel = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ + vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j]]) - self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel ) - + + fibr_b = self.aux_vars['bz'][h,i,j] - fibr_b = np.array([self.aux_vars['b0'][h,i,j],self.aux_vars['bx'][h,i,j],self.aux_vars['by'][h,i,j]]) + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) + + self.structures['SETfl'][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + + self.prim_vars['p'][h,i,j] * self.metric + + self.structures['SETem'][h,i,j:,:] = (fibr_b**2) * np.outer(vel_vec, vel_vec) + (fibr_b/2) * self.metric + self.structures['Fab'][h,i,j,:,:] = np.tensordot(self.Levi3D, vel_vec, axes = ([2,0])) * fibr_b - self.structures["SET"][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] + self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]) * \ - np.outer( vel,vel) + (self.prim_vars["p"][h,i,j] + self.aux_vars["bsq"][h,i,j]/2) * self.metric \ - - np.outer(fibr_b, fibr_b) - fol_vel_vec = np.zeros(3) - fol_vel_vec[0]=+1 - fol_b_vec = np.array([self.aux_vars["b0"][h,i,j],self.aux_vars["bx"][h,i,j],self.aux_vars["by"][h,i,j]]) - fol_e_vec = np.tensordot( self.Levi3D, np.outer(vel,fol_b_vec), axes = ([1,2],[0,1])) + ################################ + # CORRESPONDING 3+1-d FORMULAE + ################################ + + # vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ + # self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j], self.aux_vars['W'][h,i,j] * self.prim_vars['vz'][h,i,j]]) + + # fibr_b = [0, self.aux_vars['bx'][h,i,j], self.aux_vars['by'][h,i,j], self.aux_vars['bz'][h,i,j]] #3+1d version - self.structures['Fab'][h,i,j,:,:] = np.outer( fol_vel_vec,fol_e_vec) - np.outer(fol_e_vec,fol_vel_vec) -\ - np.tensordot(self.Levi3D,fol_b_vec,axes=([2],[0])) + # self.structures['Fab'][h,i,j,:,:] = np.tensordot(np.tensordot(self.Levi4D, vel_vec, axes = ([2,0])), fibr_b, axes= ([2,0])) + + # self.structures['SETem'][h,i,j,:,:] = Base.Mink_dot(fibr_b, fibr_b) * np.outer(vel_vec, vel_vec) +\ + # (Base.Mink_dot(fibr_b, fibr_b)/2) * self.metric -\ + # np.multiply(1/2., np.outer(fibr_b, fibr_b) ) + + + # This might be useful/needed + # self.all_vars = self.prim_vars + # self.all_vars.update(self.aux_vars) + # self.all_vars.update(self.structures) def get_var_gridpoint(self, var, point): """ @@ -192,31 +220,6 @@ def get_interpol_var(self, var, point): print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') - - """ - Returns the interpolated structure at the point - Parameters - ---------- - var : str corresponding to one of the structures - - point : list of floats - ordered coordinates: t,x,y - Return - ------ - Array with the interpolated values of any var - Empty list if var is not a structure in the micro_model - Notes - ----- - Interpolation gives errors when applied to boundary - """ - res = [] - for var_name in var_names: - try: - res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) - except KeyError: - print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") - return res - class IdealHydro_2D(object): def __init__(self, interp_method = "linear"): @@ -449,7 +452,7 @@ def get_interpol_var(self, var_names, point): micro_model.setup_structures() point = [1.502,0.4,0.2] - vars = ['SET', 'BC'] + vars = ['SETfl', 'BC', 'Fab', 'SETem'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) diff --git a/system/BaseFunctionality.py b/system/BaseFunctionality.py index 9e7fe1d..c410d74 100644 --- a/system/BaseFunctionality.py +++ b/system/BaseFunctionality.py @@ -59,10 +59,22 @@ def get_rel_vel(spatial_vels): @staticmethod def project_tensor(vector1_wrt, vector2_wrt, to_project): + """ + """ return np.inner(vector1_wrt,np.inner(vector2_wrt,to_project)) + @staticmethod def orthogonal_projector(u, metric): + """ + Returns: + -------- + Orthogonal projector wrt vector u + + Notes: + ------ + The vector u must be time-like + """ return metric + np.outer(u,u) From 389410f739080ba7ee559520a056deafce0d6357 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 18 Sep 2023 16:49:02 +0200 Subject: [PATCH 013/111] Work on MHD meso class --- Filters.py | 2 +- MesoModels.py | 380 +++++++++++++++++++++++++++++++++++-------------- MicroModels.py | 28 ++-- 3 files changed, 284 insertions(+), 126 deletions(-) diff --git a/Filters.py b/Filters.py index 26e0f9a..501bb58 100644 --- a/Filters.py +++ b/Filters.py @@ -332,7 +332,7 @@ def find_observer(self, point, flux_str = "gauss", initial_guess = None): guess = np.array(guess) try: - sol = sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) + sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) if sol.success: observer = Base.get_rel_vel(sol.x) if sol.fun > 1e-5: diff --git a/MesoModels.py b/MesoModels.py index e291e05..2520391 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -6,13 +6,19 @@ """ import numpy as np -from multiprocessing import Process, Pool +from scipy.interpolate import interpn import pickle +from multiprocessing import Process, Pool + +import matplotlib.pyplot as plt +import seaborn as sns +import scipy.stats as scst from system.BaseFunctionality import * from MicroModels import * from FileReaders import * from Filters import * +from Visualization import * class NonIdealHydro2D(object): @@ -351,15 +357,15 @@ class resMHD2D(object): only have to change the method model_residuals and list of non_local_vars (possibly but less likely decompose_structure_gridpoint) """ - def __init__(self, micro_model, find_obs, filter): + def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): """ """ self.micro_model = micro_model self.find_obs = find_obs self.filter = filter - self.spatial_dims = 2 + self.interp_method = interp_method self.domain_int_strs = ('Nt','Nx','Ny') self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") @@ -386,16 +392,10 @@ def __init__(self, micro_model, find_obs, filter): for var in self.meso_structures: self.meso_structures[var] = [] - - # self.meso_fluid_vars_strs = ['eps_tilde', 'u_tilde', 'n_tilde', 'p_tilde', 'p', 'eos_res', 'Pi_res', 'q_res', 'pi_res'] - # self.meso_em_vars_strs = ['e', 'b', 'j', 'sigma', 'J'] - # self.meso_vars_strs = self.meso_fluid_vars_strs + self.meso_em_vars_strs - # self.meso_vars = dict.fromkeys(self.meso_vars_strs) - - self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde'] + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde'] self.meso_vectors_strs = ['u_tilde', 'q_res', 'e_tilde', 'j', 'J_tilde'] - self.meso_tensors_strs = ['pi_res'] - self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_tensors_strs + self.meso_r2tensors_strs = ['pi_res'] + self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs self.meso_vars = dict.fromkeys(self.meso_vars_strs) # Dictionary with stencils and coefficients for finite differencing @@ -409,10 +409,14 @@ def __init__(self, micro_model, find_obs, filter): self.differencing[2]['bw'] = {'coefficients' : [3/2., -2., 1/2.] , 'stencil' : [0, -1, -2]} self.differencing[2]['cen'] = {'coefficients' : [1/12., -2/3., 0., 2/3., -1/12.] , 'stencil' : [-2, -1, 0, 1, 2]} - self.nonlocal_vars_strs = ['u_tilde', 'Fab'] # These have to belong to either a structure or meso_vars + + # dictionary with non-local quantities (keys must match one of meso_vars or structure) + self.nonlocal_vars_strs = ['u_tilde', 'Fab'] Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] self.deriv_vars = dict.fromkeys(Dstrs) + # Additional, closure-scheme specific vars must be added appropriately using model_residuals() + # Run some compatibility test... compatible = True error = '' @@ -427,6 +431,70 @@ def __init__(self, micro_model, find_obs, filter): if not compatible: print("Meso and Micro models are incompatible:"+error) + def get_all_var_strs(self): + return self.all_var_strs + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + var: str corresponding to meso_structure, meso_vars or deriv_vars + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is not meso_structures/meso_vars or deriv_vars of the model. + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + if var in self.meso_structures: + return interpn(self.domain_vars['Points'], self.meso_structures[var], point, method = self.interp_method)[0] + elif var in self.meso_vars: + return interpn(self.domain_vars['Points'], self.meso_vars[var], point, method = self.interp_method)[0] + elif var in self.deriv_vars: + return interpn(self.domain_vars['Points'], self.deriv_vars[var], point, method = self.interp_method)[0] + else: + print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) + + def get_var_gridpoint(self, var, point): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + vars: string corresponding to structure, meso_vars or deriv_vars + + point: list of 2+1 floats + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest + grid-point to input 'point'. + + Notes: + ------ + This method should be used in case using interpolated values becomes + too expensive. + """ + indices = Base.find_nearest_cell(point, self.domain_vars['Points']) + if var in self.meso_structures: + return self.meso_structures[var][tuple(indices)] + elif var in self.meso_vars: + return self.meso_vars[var][tuple(indices)] + elif var in self.deriv_vars: + return self.deriv_vars[var][tuple(indices)] + else: + print("{} is not a variable of resMHD_2D!".format(var)) + return None + def get_model_name(self): return 'resMHD2D' @@ -438,6 +506,8 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor): Builds the meso_model grid using the micro_model grid points contained in the input patch (defined via 'patch_bdrs'). The method allows for coarse graining the grid in the spatial directions ONLY. + Then store the info about the meso grid and set up arrays of definite rank and size + for the quantities needed later. Parameters: ----------- @@ -502,14 +572,6 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor): self.domain_vars['Nx'] = len(self.domain_vars['X']) self.domain_vars['Ny'] = len(self.domain_vars['Y']) - def setup_variables(self): - """ - Set up the arrays for meso_structures, meso_vars and filter_vars - - Notes: - ------ - Use after setup meso_grid - """ # Setup arrays for structures Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] self.meso_structures['BC'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) @@ -522,7 +584,7 @@ def setup_variables(self): self.meso_vars[str] = np.zeros((Nt, Nx, Ny)) for str in self.meso_vectors_strs: self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) - for str in self.meso_tensors_strs: + for str in self.meso_r2tensors_strs: self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) # Setup arrays for filter_vars @@ -531,6 +593,11 @@ def setup_variables(self): self.filter_vars['U_success'] = dict() + # Setup arrays for derivatives of the model. + # THOMAS'S WAY (SAVE THEM AS TENSORS) + self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.deriv_vars['D_Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1, self.spatial_dims+1)) + # MARCUS'S WAY # for str in self.non_local_vars_strs: # if str in self.meso_vars: @@ -541,11 +608,7 @@ def setup_variables(self): # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_structures[str])}) # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) - - # THOMAS'S WAY (SAVE THEM AS TENSORS) - self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - self.deriv_vars['D_Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1, self.spatial_dims+1)) - + def find_observers(self): """ Method to compute filtering observers at grid points built with setup_meso_grid. @@ -581,7 +644,7 @@ def filter_micro_variables(self): Notes: ------ - Requires build_observers(), setup_meso_grid() and setup_variables() to be called first. + Requires find_observers() and setup_meso_grid() to be called first. """ for h, t in enumerate(self.domain_vars['T']): for i, x in enumerate(self.domain_vars['X']): @@ -610,42 +673,42 @@ def decompose_structures_gridpoint(self, h, i ,j): Notes: ------ - Decomposition of fluid SET and Faraday tensor. The EM part of SET is decomposed later + Decomposition of BC, fluid SET and Faraday tensor. The EM part of SET is decomposed later via non local operations (same story for the charge current). """ # Computing the Favre density and velocity - n_tilde = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) - u_tilde = np.multiply(1 / n_tilde, self.meso_structures['BC'][h,i,j]) - self.meso_vars['n_tilde'][h,i,j] = n_tilde - self.meso_vars['u_tilde'][h,i,j,:] = u_tilde - - # Decomposing the fluid part of the stress energy tensor - # To change after update to Base.project_tensor() ? - h_ab = Base.orthogonal_projector(u_tilde, self.metric) - T_ab = self.meso_structures['SETfl'][h,i,j,:,:] - for l, m in np.ndindex(T_ab.shape): - if l==0: - T_ab[l,m] *= -1 - if m==0: - T_ab[l,m] *= -1 + n_t = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) + u_t = np.multiply(1 / n_t, self.meso_structures['BC'][h,i,j]) + T_ab = self.meso_structures['SETfl'][h,i,j,:,:] # Remember this is rank (2,0) + + # Computing the decomposition at each point + eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, self.metric, self.metric, T_ab) + h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) # This is a rank (1,1) tensor, i.e. a real projector. + q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) + s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, T_ab) + s = np.einsum('ii',s_ab) + + # Storing the decomposition with appropriate names. + self.meso_vars['n_tilde'][h,i,j] = n_t + self.meso_vars['u_tilde'][h,i,j,:] = u_t + self.meso_vars['eps_tilde'][h,i,j] = eps_t + self.meso_vars['q_res'][h,i,j,:] = q_a + self.meso_vars['pi_res'][h,i,j,:,:] = s_ab - np.multiply(s / self.spatial_dims , self.metric +np.outer(u_t,u_t)) p_filt = self.meso_vars['p'][h,i,j] - - self.meso_vars['eps_tilde'][h,i,j] = np.tensordot(np.tensordot(T_ab, u_tilde, axes = ([0],[0])), u_tilde, axes = ([0],[0])) - self.meso_vars['Pi_res'][h,i,j] = np.tensordot(T_ab, h_ab, axes = ([0,1],[0,1])) - p_filt - self.meso_vars['q_res'][h,i,j:] = np.tensordot( h_ab, np.tensordot(T_ab, u_tilde, axes = ([1],[0])), axes = ([1],[0])) - self.meso_vars['pi_res'][h,i,j,:,:] = np.tensordot(h_ab, np.tensordot(h_ab, T_ab, axes = ([1],[0])), axes = ([1],[1])) - # CHANGE THIS self.meso_vars['eos_res'][h,i,j] = 0 - # TO - # p_tilde = p_from_EOS method(eps_tilde, n_tilde) - # self.meso_vars['eos_res'][h,i,j] = p_filt - p_tilde - + # p_t = p_from_EOS method(eps_t, n_t) + # self.meso_vars['eos_res'][h,i,j] = p_filt - p_t + self.meso_vars['Pi_res'][h,i,j] = s - p_filt -self.meso_vars['eos_res'][h,i,j] + - # Decompose the Fab + # Repeat for the Faraday tensor (which is rank (0,2)) Fab = self.meso_structures['Fab'][h,i,j] - self.meso_vars['e_tilde'][h,i,j,:] = np.tensordot(Fab, u_tilde, axes = ([1],[0])) - self.meso_vars['b_tilde'][h,i,j] = 1/2. * np.tensordot(np.tensordot(self.Levi3D, Fab, axes = ([1,2],[0,1])), u_tilde, axes = ([0],[0])) + self.meso_vars['e_tilde'][h,i,j,:] = np.einsum('ij,j->i',Fab, u_t) + self.meso_vars['b_tilde'][h,i,j] = np.einsum('ijk,i,jl,km,lm', self.Levi3D, u_t, self.metric, self.metric, Fab) + + # Corresponding 3+1 d formulae: only change in B + # np.einsum('ijkl,i,km,ln,mn->i', self.Levi4D, u_t, self.metric, self.metric, Fab) def decompose_structures(self): """ @@ -718,17 +781,14 @@ def calculate_derivative_gridpoint(self, nonlocal_var_str, h, i, j, direction, o idxs = [h,i,j] idxs[direction] += sample temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_vars[nonlocal_var_str][tuple(idxs)] ) - # self.deriv_vars[deriv_var_str][h,i,j] += \ - # np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_vars[nonlocal_var_str][tuple(idxs)] ) return temp + else: temp = 0 for s, sample in enumerate(stencil): idxs = [h,i,j] idxs[direction] += sample temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_structures[nonlocal_var_str][tuple(idxs)] ) - # self.deriv_vars[deriv_var_str][h,i,j] += \ - # np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_structures[nonlocal_var_str][tuple(idxs)] ) return temp def calculate_derivatives(self): @@ -754,59 +814,161 @@ def calculate_derivatives(self): D_Fab[h,i,j,c,a,b]: h,i,j grid; c direction of derivative; a,b as for Fab """ - for h in range(self.domain_vars['Nt']): - for i in range(self.domain_vars['Nx']): - for j in range(self.domain_vars['Ny']): - for str in self.nonlocal_vars_strs: + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,y,x] + if self.filter_vars['U_success'][h,i,j]: for dir in range(self.spatial_dims+1): - dstr = 'D_' + str - self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivative_gridpoint(str, h, i, j, dir) + for str in self.nonlocal_vars_strs: + dstr = 'D_' + str + self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivative_gridpoint(str, h, i, j, dir) + else: + print('Derivatives not calculated at {}: observer could not be found.'.format(point)) - def model_residuals(self, h, i, j): + def model_residuals_gridpoint(self, h, i, j): """ + Decompose quantities obatined via non_local operations (i.e. derivatives) as needed + for the closure scheme. This is done at gridpoint (h,i,j) and then relevant values are + returned - """ - u_tilde = self.meso_vars['u_tilde'][h,i,j] - u_tilde_down = np.einsum('ij,i', self.metric, u_tilde) + Parameters: + ----------- + h, i, j: integers + the indices of the gridpoint - # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS - #Computing four-current: raise the last two indices of D_Fab and then sum over 1,3 component - temp = self.deriv_vars['D_Fab'][h,i,j] - temp = np.einsum('ij,klj->ikl', self.metric, temp) - temp = np.einsum('ij,klj->ikl', self.metric, temp) - current = np.einsum('iji->j', temp) + Returns: + -------- + list of strings: names of the quantities computed - # Decompose wrt u_tilde: again careful with indices up or down! - sigma_tilde = np.einsum('i,i', current, u_tilde_down) - J_tilde = current - np.multiply(sigma_tilde, u_tilde) + list of nd.arrays with quantities computed - self.meso_vars['sigma_tilde'][h,i,j] = sigma_tilde - self.meso_vars['j'][h,i,j] = current - self.meso_vars['J_tilde'][h,i,j] = J_tilde - - print(u_tilde, '\n',u_tilde_down, '\n', current, '\n', sigma_tilde, '\n', J_tilde,'\n***********') + Notes: + ------ + The ingredients for the closure scheme are model specific: it makes sense that the relevant + dictionary is set up not on construction. + + UNDER CONSTRUCTION: THE EM PART NEEDS MORE THINKING + """ + # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS + u_t = self.meso_vars['u_tilde'][h,i,j] + DFab = self.deriv_vars['D_Fab'][h,i,j] + + current = np.einsum('ij, kl, kjl->i', self.metric, self.metric, DFab) + sigma_t = np.einsum('i,ij,j->',current, self.metric, u_t) + J_t = current - np.multiply(sigma_t, u_t) # MODELLING THE DISSIPATIVE TERMS IN THE FLUID SET - # DECOMPOSING NABLA_U - nabla_u = np.einsum('ij,jl->il', self.metric, self.deriv_vars['D_u_tilde'][h,i,j]) - acc = np.einsum('i,ij', u_tilde_down, nabla_u) - should_be_zero = np.einsum('i,ji', u_tilde_down, nabla_u) - Daub = nabla_u + np.outer(u_tilde, acc) + np.outer(should_be_zero, u_tilde) +\ - np.multiply(np.inner(should_be_zero, u_tilde_down), np.outer(u_tilde, u_tilde)) - exp_rate = np.einsum('ii', np.einsum('ij,jk->ik', self.metric, Daub)) - shear_rate = np.multiply(1/2., Daub + np.einsum('ji', Daub)) - \ - np.multiply( 1 / self.spatial_dims * exp_rate, Base.orthogonal_projector(u_tilde, self.metric)) - - print(Daub, '\n', exp_rate, '\n', shear_rate, '\n', should_be_zero) - - # EXTRACTING THE DISSIPATIVE COEFFICIENTS - # LATER: SAVE THIS TO A FILE (PICKLE) AND PLOT IT --> MARCUS'S ROUTINES - - # EXTRACT COEFFICIENTS COMPONENTWISE AND AVERAGE (FOR NOW) + nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor + acc_t = np.einsum('i,ij->j',u_t, nabla_u) + should_be_zero = np.einsum('ij,i,kj',self.metric, u_t, nabla_u) + + h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) + Daub = nabla_u + np.einsum('ij,j,l', self.metric, u_t, acc_t) + np.einsum('ij,j,l', self.metric, should_be_zero, u_t) + exp_t = np.einsum('ii->', Daub) + shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/self.spatial_dims * exp_t, h_ab) + # THINK WHAT TO DO WITH THE HEAT FLUX --> GRADIENTS IN T OR N AND EPS - # THINK ABOUT THE EM PART, WHAT COEFFICIENTS DO YOU WANT TO EXTRACT? SAVE THEM! + # THINK ABOUT THE EM PART, DO YOU NEED MORE? + # COME BACK TO THIS LATER? + # self.meso_vars['j'][h,i,j,:] = current ?? + + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'sigma_tilde', 'J_tilde'] + closure_vars = [shear_t, exp_t, acc_t, sigma_t, J_t] + + return closure_vars_strs, closure_vars + + def model_residuals(self): + """ + Wrapper of the corresponding gridpoint method. + Set up the dictionary for the closure variables not yet defined. + Loop over the grid and store the decomposed variables. + """ + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][h,i,j]: + keys, values = self.model_residuals_gridpoint(h,i,j) + # try-except block to extend the meso_vars dictionary + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars, creating it!'.format(key)) + shape = values[idx].shape + self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + finally: + self.meso_vars[key][h,i,j] = values[idx] + + def shear_regression_test(self): + """ + WORK IN PROGRESS: A MINIMAL REGRESSION ANALYSIS ON THE COMPONENTS OF THE SHEAR TENSOR + NEED TO FIRST WORK ON WRAPPER OF MODEL_RESIDUALS THAT STORES ALL THE INFO AT ALL POINTS + Notes: + ------ + Automatic looping over all components: how to make use of the fact that the shear is symmetric? + Better to use a wrapper? + """ + # LINEAR REGRESSION: LOOP OVER COMPONENTS AND AVERAGE + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + shear = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + intercepts = [] + slopes = [] + r_values = [] + for components in np.ndindex(shear[0,0,0,:,:].shape): + # components = (1,1) + xs, ys = np.zeros((Nx,Ny)), np.zeros((Nx,Ny)) + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][0,i,j]: + shear = self.model_residuals_gridpoint(0,i,j)[1][0] + xs[i,j] = shear[components] + ys[i,j] = self.meso_vars['pi_res'][tuple([0,i,j]+list(components))] + + xs = xs.flatten() + ys = ys.flatten() + sns_plot = sns.regplot(x = xs, y = ys) + # print(scst.pearsonr(xs,ys)) + intercept, slope, rvalue = scst.linregress(xs,ys)[:3] + intercepts.append(intercept) + slopes.append(slope) + r_values.append(rvalue) + # print(intercept, slope, rvalue) + + fig = sns_plot.get_figure() + fig.tight_layout() + plt.show() + + + m_intercept = np.mean(intercepts) + m_slope = np.mean(slopes) + m_rvalue = np.mean(r_values) + print(m_intercept, m_slope, m_rvalue) + + + # BEFORE WORKING ON THESE: WRAPPER OF MODEL_RESIDUAL, THEN CONTINUE ON THIS + # PREPARING THE DATA FOR CONTRASTING RESIDUAL AND MODEL + eta = m_slope + pi_model = np.zeros(self.meso_vars['pi_res'][0,:,:,:,:].shape) + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][0,i,j]: + shear = self.model_residuals_gridpoint(0,i,j)[1][0] + pi_model[i,j,:,:] = np.multiply(eta, shear) # Calling the same function twice! Save data! + pi_res = self.meso_vars['pi_res'][0,:,:,:,:] + + + # SETTING UP AND PLOTTING VIA MARCUS'S ROUTINES + visualizer = Plotter_2D() + t_range, x_range, y_range = [self.domain_vars['Tmin'], self.domain_vars['Tmax']] ,\ + [self.domain_vars['Xmin'], self.domain_vars['Xmax']] ,\ + [self.domain_vars['Ymin'], self.domain_vars['Ymax']] + print(t_range, x_range, y_range) + # visualizer.plot_vars(self, ['pi_res', ,], t_range, x_range, y_range, interp_dims=(), method = 'interpolate', component_indices) + + - if __name__ == '__main__': @@ -820,12 +982,18 @@ def model_residuals(self, h, i, j): CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([[1.501, 1.505],[0.37, 0.40],[0.43, 0.46]],1) - meso_model.setup_variables() + meso_model.setup_meso_grid([[1.501, 1.503],[0.37, 0.39],[0.43, 0.45]],1) + # print(meso_model.domain_vars['Points']) + meso_model.find_observers() meso_model.filter_micro_variables() + # meso_model.decompose_structures_gridpoint(1,1,1) meso_model.decompose_structures() - # print(meso_model.calculate_derivative('u_tilde', 1,1,0,1, order=2)) + # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,0,1, order=2),) meso_model.calculate_derivatives() - meso_model.model_residuals(1,1,0) + # meso_model.model_residuals_gridpoint(1,1,0) + meso_model.model_residuals() + meso_model.shear_regression_test() + + diff --git a/MicroModels.py b/MicroModels.py index 8a579f5..d4839ed 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -132,7 +132,7 @@ def setup_structures(self): # vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ # self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j], self.aux_vars['W'][h,i,j] * self.prim_vars['vz'][h,i,j]]) - # fibr_b = [0, self.aux_vars['bx'][h,i,j], self.aux_vars['by'][h,i,j], self.aux_vars['bz'][h,i,j]] #3+1d version + # fibr_b = [0, self.aux_vars['bx'][h,i,j], self.aux_vars['by'][h,i,j], self.aux_vars['bz'][h,i,j]] # self.structures['Fab'][h,i,j,:,:] = np.tensordot(np.tensordot(self.Levi4D, vel_vec, axes = ([2,0])), fibr_b, axes= ([2,0])) @@ -153,7 +153,7 @@ def get_var_gridpoint(self, var, point): Parameters: ----------- - vars: string corresponding to primitive, auxiliary or structre variable + vars: string corresponding to primitive, auxiliary or structure variable point: list of 2+1 floats @@ -168,34 +168,23 @@ def get_var_gridpoint(self, var, point): """ indices = Base.find_nearest_cell(point, self.domain_vars['points']) if var in self.get_prim_strs(): - return self.prim_vars[var][tuple(indices)] - + return self.prim_vars[var][tuple(indices)] elif var in self.get_aux_strs(): - return self.aux_vars[var][tuple(indices)] - + return self.aux_vars[var][tuple(indices)] elif var in self.get_structures_strs(): - if var == "BC": - tmp = np.zeros(self.structures[var][0,0,0,:].shape) - for a in range(len(self.structures[var][0,0,0,:])): - tmp[a] = self.structures[var][tuple(indices+[a])] - return tmp - else: - tmp = np.zeros(self.structures[var][0,0,0,:,:].shape) - for a in range(len(tmp[:,0])): - for b in range(len(tmp[0,:])): - tmp[a,b] = self.structures[var][tuple(indices+[a,b])] - return tmp + return self.structures[var][tuple(indices)] else: print(f"{var} is not a variable of IdealMHD_2D!") return None + def get_interpol_var(self, var, point): """ Returns the interpolated variables at the point. Parameters ---------- - vars : str corresponding to primitive, auxiliary or structre variable + var: str corresponding to primitive, auxiliary or structre variable point : list of floats ordered coordinates: t,x,y @@ -453,10 +442,11 @@ def get_interpol_var(self, var_names, point): point = [1.502,0.4,0.2] vars = ['SETfl', 'BC', 'Fab', 'SETem'] + # vars = ['BC'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) - print(f'{var}: \n {res} \n {res2} \n ********** \n ') + print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') # MH # if __name__ == '__main__': From 7314516246aee0a1a3e7effb10f79f087d04fe7e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 18 Sep 2023 17:14:57 +0200 Subject: [PATCH 014/111] Syncing with MH, work on resMHD --- MesoModels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MesoModels.py b/MesoModels.py index fa40dda..54c8162 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1018,6 +1018,6 @@ def shear_regression_test(self): meso_model.calculate_derivatives() # meso_model.model_residuals_gridpoint(1,1,0) meso_model.model_residuals() - meso_model.shear_regression_test() + # meso_model.shear_regression_test() From 79e7f09e6760c700329978e8968d665f3e672df1 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 20 Sep 2023 11:53:43 +0200 Subject: [PATCH 015/111] TC tweaks on Visualization tools. --- MesoModels.py | 55 ++++++++++++-- MicroModels.py | 175 +++++++++++++++++++++++++++----------------- Visualization.py | 186 +++++++++++++++++++++++++++++++++-------------- 3 files changed, 290 insertions(+), 126 deletions(-) diff --git a/MesoModels.py b/MesoModels.py index 54c8162..a4864f3 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -9,6 +9,7 @@ from scipy.interpolate import interpn import pickle from multiprocessing import Process, Pool +from multimethod import multimethod import matplotlib.pyplot as plt import seaborn as sns @@ -383,7 +384,7 @@ class resMHD2D(object): """ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): """ - + ADD DESCRIPTION OF THIS """ self.micro_model = micro_model self.find_obs = find_obs @@ -441,6 +442,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): # Additional, closure-scheme specific vars must be added appropriately using model_residuals() + self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs # Run some compatibility test... compatible = True error = '' @@ -458,6 +460,12 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): def get_all_var_strs(self): return self.all_var_strs + def get_gridpoints(self): + """ + Pretty self-explanatory + """ + return self.domain_vars['Points'] + def get_interpol_var(self, var, point): """ Returns the interpolated variables at the point. @@ -487,7 +495,40 @@ def get_interpol_var(self, var, point): else: print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) - def get_var_gridpoint(self, var, point): + @multimethod + def get_var_gridpoint(self, var: str, h: object, i: object, j: object): + """ + Returns variable corresponding to input 'var' at gridpoint + identified by h,i,j + + Parameters: + ----------- + var: string corresponding to structure, meso_vars or deriv_vars + + h,i,j: int + integers corresponding to position on the grid. + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest + grid-point to input 'point'. + + Notes: + ------ + This method is useful e.g. for plotting the raw data. + """ + if var in self.meso_structures: + return self.meso_structures[var][h,i,j] + elif var in self.meso_vars: + return self.meso_vars[var][h,i,j] + elif var in self.deriv_vars: + return self.deriv_vars[var][h,i,j] + else: + print("{} is not a variable of resMHD_2D!".format(var)) + return None + + @multimethod + def get_var_gridpoint(self, var: str, point: object): """ Returns variable corresponding to input 'var' at gridpoint closest to input 'point'. @@ -525,7 +566,7 @@ def get_model_name(self): def set_find_obs_method(self, find_obs): self.find_obs = find_obs - def setup_meso_grid(self, patch_bdrs, coarse_factor): + def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): """ Builds the meso_model grid using the micro_model grid points contained in the input patch (defined via 'patch_bdrs'). The method allows for coarse graining @@ -682,7 +723,7 @@ def filter_micro_variables(self): else: print('Could not filter at {}: observer not found.'.format(point)) - def decompose_structures_gridpoint(self, h, i ,j): + def decompose_structures_gridpoint(self, h, i, j): """ Decompose the fluid part of SET as well as Fab at grid point (h,i,j) @@ -919,9 +960,10 @@ def model_residuals(self): try: self.meso_vars[key] except KeyError: - print('The key {} does not belong to meso_vars, creating it!'.format(key)) + print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) shape = values[idx].shape self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + self.all_var_strs += key finally: self.meso_vars[key][h,i,j] = values[idx] @@ -992,8 +1034,6 @@ def shear_regression_test(self): # visualizer.plot_vars(self, ['pi_res', ,], t_range, x_range, y_range, interp_dims=(), method = 'interpolate', component_indices) - - if __name__ == '__main__': FileReader = METHOD_HDF5('./Data/test_res100/') @@ -1020,4 +1060,5 @@ def shear_regression_test(self): meso_model.model_residuals() # meso_model.shear_regression_test() + diff --git a/MicroModels.py b/MicroModels.py index 7af8d9b..7e55142 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -5,13 +5,17 @@ @author: Thomas """ -from FileReaders import * -from scipy.interpolate import interpn -from system.BaseFunctionality import * import numpy as np import math import time +from scipy.interpolate import interpn +from multimethod import multimethod + +from FileReaders import * +from system.BaseFunctionality import * + + # These are the symbols, so be careful when using these to construct vectors! # levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ # for k in range(3)]for j in range(3) ] for i in range(3) ]) @@ -21,7 +25,9 @@ class IdealMHD_2D(object): - + """ + ADD SHORT DESCRIPTION OF THE CLASS AND ITS METHODS + """ def __init__(self, interp_method = "linear"): """ Sets up the variables and dictionaries, strings correspond to @@ -90,6 +96,105 @@ def get_structures_strs(self): def get_all_var_strs(self): return self.get_prim_strs() + self.get_aux_strs() + self.get_structures_strs() + def get_gridpoints(self): + """ + Pretty self-explanatory + """ + return self.domain_vars['points'] + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + var: str corresponding to primitive, auxiliary or structre variable + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + if var in self.get_prim_strs(): + return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] + elif var in self.get_aux_strs(): + return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] + elif var in self.get_structures_strs(): + return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] + else: + print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + + @multimethod + def get_var_gridpoint(self, var: str, h: object, i: object, j: object): + """ + Returns variable corresponding to input 'var' at gridpoint identified by h,i,j + + Parameters: + ----------- + var: str + String corresponding to primitive, auxiliary or structure variable + + h,i,j: int + integers corresponding to position on the grid. + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. + + Notes: + ------ + This method is useful e.g. for plotting the raw data. + """ + if var in self.get_prim_strs(): + return self.prim_vars[var][h,i,j] + elif var in self.get_aux_strs(): + return self.aux_vars[var][h,i,j] + elif var in self.get_structures_strs(): + return self.structures[var][h,i,j] + else: + print('{} is not a variable of model {}'.format(var, self.get_model_name())) + return None + + @multimethod + def get_var_gridpoint(self, var: str, point: object): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + vars: string corresponding to primitive, auxiliary or structure variable + + point: list of 2+1 floats + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. + + Notes: + ------ + This method should be used in case using interpolated values + becomes too expensive. + """ + indices = Base.find_nearest_cell(point, self.domain_vars['points']) + if var in self.get_prim_strs(): + return self.prim_vars[var][tuple(indices)] + elif var in self.get_aux_strs(): + return self.aux_vars[var][tuple(indices)] + elif var in self.get_structures_strs(): + return self.structures[var][tuple(indices)] + else: + print(f"{var} is not a variable of IdealMHD_2D!") + return None + def setup_structures(self): """ Set up the structures (i.e baryon (mass) current BC, Stress-Energy and Faraday tensors @@ -149,67 +254,7 @@ def setup_structures(self): self.vars = self.prim_vars self.vars.update(self.aux_vars) self.vars.update(self.structures) - - def get_var_gridpoint(self, var, point): - """ - Returns variable corresponding to input 'var' at gridpoint - closest to input 'point'. - - Parameters: - ----------- - vars: string corresponding to primitive, auxiliary or structure variable - - point: list of 2+1 floats - - Returns: - -------- - Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. - - Notes: - ------ - This method should be used in case using interpolated values - becomes too expensive. - """ - indices = Base.find_nearest_cell(point, self.domain_vars['points']) - if var in self.get_prim_strs(): - return self.prim_vars[var][tuple(indices)] - elif var in self.get_aux_strs(): - return self.aux_vars[var][tuple(indices)] - elif var in self.get_structures_strs(): - return self.structures[var][tuple(indices)] - else: - print(f"{var} is not a variable of IdealMHD_2D!") - return None - - def get_interpol_var(self, var, point): - """ - Returns the interpolated variables at the point. - - Parameters - ---------- - var: str corresponding to primitive, auxiliary or structre variable - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Interpolated values/arrays corresponding to variable. - Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - - if var in self.get_prim_strs(): - return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] - elif var in self.get_aux_strs(): - return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] - elif var in self.get_structures_strs(): - return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] - else: - print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + class IdealHydro_2D(object): diff --git a/Visualization.py b/Visualization.py index e0f8583..67e2390 100644 --- a/Visualization.py +++ b/Visualization.py @@ -13,11 +13,18 @@ from scipy.interpolate import interpn from system.BaseFunctionality import * +from MicroModels import * +from MesoModels import * +from Filters import * + class Plotter_2D(object): - def __init__(self): + def __init__(self, screen_size): """ - Blank as yet. + Parameters: + ----------- + screen_size = list of 2 floats + width and height of the screen: used to rescale plots' size. """ self.subplots_dims = {1 : (1,1), 2 : (1,2), @@ -25,9 +32,10 @@ def __init__(self): 4 : (2,2), 5 : (2,3), 6 : (2,3)} - - def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims, method, component_indices): + self.screen_size = np.array(screen_size) + + def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, method= 'raw_data', component_indices=()): """ Retrieves the required data from model to plot a variable defined by var_str over coordinates t, x_range, y_range, either from the model's @@ -55,57 +63,73 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims, method, ------- data_to_plot : numpy array of floats the (2D) data to be plotted by plt.imshow(). - points : numpy array of floats - the coordinates of the data-points. + extent: list of floats + + Notes: + ------ + Logic: if method is raw_data, then no interp_dims are needed. + Better to have 'raw_data' and no interp_dims = None as default """ if var_str in model.get_all_var_strs(): - + + # Block to check component_indices passed are compatible with shape of var to be plotted. + st1 = len(component_indices) + st2 = len(model.get_var_gridpoint(var_str, 0, 0, 0).shape) + compatible = st1==st2 + if not compatible: + if st2 == 0 : + print('WARNING: {} is a scalar but you passed some indices, ignoring this and moving on.'.format(var_str)) + component_indices = () + elif st2 != 0: + print('WARNING: {} is a tensor but you passed more/fewer indices than required. Retrieving the "first component"!'.format(var_str)) + component_indices = tuple([0 for _ in range(st2)]) + + extent = [*x_range, *y_range] + if method == 'interpolate': - + compatible = interp_dims != None and len(interp_dims) ==2 + if not compatible: + print('Error: when using (linearly spaced) interpolated data, you must \ + specify # points in each direction! Exiting.') + return None + nx, ny = interp_dims[:] - xs, ys = np.linspace(x_range[0],x_range[1],nx), np.linspace(y_range[0],y_range[1],ny) - points = [t,xs,ys] - data_to_plot = np.zeros((nx,ny)) + xs, ys = np.linspace(x_range[0], x_range[1], nx), np.linspace(y_range[0], y_range[1], ny) + data_to_plot = np.zeros((nx, ny)) for i in range(nx): for j in range(ny): - point = [t,xs[i],ys[j]] - data_to_plot[i,j] = interpn(model.domain_vars['points'],\ - model.vars[var_str], point,\ - method = model.interp_method)[0][component_indices] + point = [t, xs[i], ys[j]] + data_to_plot[i, j] = model.get_interpol_var(var_str, point)[component_indices] + elif method == 'raw_data': - - start_indices = Base.find_nearest_cell([t, x_range[0], y_range[0]], model.domain_vars['points']) - end_indices = Base.find_nearest_cell([t, x_range[1], y_range[1]], model.domain_vars['points']) - + start_indices = Base.find_nearest_cell([t, x_range[0], y_range[0]], model.get_gridpoints()) + end_indices = Base.find_nearest_cell([t, x_range[1], y_range[1]], model.get_gridpoints()) + h = start_indices[0] i_s, i_f = start_indices[1], end_indices[1] - j_s, j_f = start_indices[2], end_indices[2] - - points = [model.domain_vars['points'][0][h],\ - model.domain_vars['points'][1][i_s:i_f+1],\ - model.domain_vars['points'][2][j_s:j_f+1]] - - data_to_plot = model.vars[var_str][h:h+1, i_s:i_f+1, j_s:j_f+1][0]#[:, :, component_indices] - # print(data_to_plot) - if component_indices: - for component_index in component_indices: - data_to_plot = data_to_plot[:, :, component_index] + j_s, j_f = start_indices[2], end_indices[2] + + data_shape = (i_f+ 1- i_s, j_f+ 1- j_s) + data_to_plot = np.zeros(data_shape) + for i in range(i_f+ 1 - i_s): + for j in range(j_f+ 1 - j_s): + data_to_plot[i,j] = model.get_var_gridpoint(var_str, h, i, j)[component_indices] - else: print('Data method is not a valid choice! Must be interpolate or raw_data.') + return None + + # return data_to_plot, points + return data_to_plot, extent else: print(f'{var_str} is not a plottable variable of the model!') + return None - return data_to_plot, points - - - def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=(), \ - method='interpolate', components_indices=None): + def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=[()]): """ Plot variable(s) from model, defined by var_strs, over coordinates t, x_range, y_range. Either from the model's raw data or by interpolating @@ -128,7 +152,7 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=(), \ currently either raw_data or interpolate. components_indices : list of tuple(s) the indices of the components to pick out if the variables are vectors/tensors. - Can be omitted is all variables are scalars, otherwise must be a list + Can be omitted if all variables are scalars, otherwise must be a list of tuples matching the length of var_strs that corresponds with each variable in the list. @@ -142,31 +166,42 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=(), \ """ n_plots = len(var_strs) n_rows, n_cols = self.subplots_dims[n_plots] - fig, axes = plt.subplots(n_rows,n_cols, figsize=(16,16)) # figsize should be adaptive.. + + # Block to determine adaptively the figsize. + figsizes = {1 : (1/3.,1/3.), + 2 : (2/3.,1/3.), + 3 : (1,1/3.), + 4 : (2/3.,2/3.), + 5 : (1,2/3.), + 6 : (1,2/3.)} + for item in figsizes: + figsizes[item] = tuple(figsizes[item] * self.screen_size) + figsize = figsizes[n_plots] + + + fig, axes = plt.subplots(n_rows,n_cols, figsize=figsize) if n_plots == 1: axes = [axes] else: axes = axes.flatten() for var_str, component_indices, ax in zip(var_strs, components_indices, axes): - data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) im = ax.imshow(data_to_plot, extent=extent) divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title(model.get_model_name()) - # fig.suptitle(model.get_model_name(), fontsize=16) ax.set_title(var_str) - ax.set_xlabel(r'$y$') + ax.set_xlabel(r'$y$') # WHY? ax.set_ylabel(r'$x$') + fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), t), fontsize = 12) fig.tight_layout() plt.show() - - def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, \ - interp_dims=(), method='interpolate', component_indices=(), diff_plot=True): + def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp_dims=None, method='raw_data', component_indices=(), diff_plot=True): """ Plot a variable from a number of models. If 2 models are given, a third plot of the difference will be automatically plotted, too. If 'raw_data' @@ -205,11 +240,16 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, \ n_rows = 1 if diff_plot: n_cols+=1 - fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=(16,16)) + + # Block to determine adaptively the figsize. + figsize = np.array([1,2/3.]) * self.screen_size + + fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=figsize) for model, ax in zip(models, axes.flatten()): - data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) im = ax.imshow(data_to_plot, extent=extent) divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) @@ -220,8 +260,10 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, \ if diff_plot: ax = axes.flatten()[-1] - data_to_plot1, points1 = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) - data_to_plot2, points2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices) + # data_to_plot1, points1 = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) + # data_to_plot2, points2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices) + data_to_plot1, extent = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) + data_to_plot2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices)[0] # if len(points1) != len(points2): # diff_plot = False # pass @@ -234,27 +276,63 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, \ # diff_plot = False # if diff_plot: try: - extent = [points1[2][0],points1[2][-1],points1[1][0],points1[1][-1]] + # extent = [points1[2][0],points1[2][-1],points1[1][0],points1[1][-1]] im = ax.imshow(data_to_plot1 - data_to_plot2, extent=extent) divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title(model.get_model_name()) - ax.set_title('Model Difference') + ax.set_title('Models difference') ax.set_xlabel(r'$y$') ax.set_ylabel(r'$x$') except(ValueError): print(f"Cannot plot the difference between {var_str} in the two " "models. Likely due to the data coordinates not coinciding.") + fig.suptitle('Contrasting {} against different models'.format(var_str)) fig.tight_layout() plt.show() +if __name__ == '__main__': + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + visualizer = Plotter_2D([11.97, 8.36]) + # TESTING GET_VAR_DATA + ###################### + # var = 'BC' + # components = (0,2) + # data1= visualizer.get_var_data(micro_model, var, 1.502, [0.3, 0.4], [0.3,0.4], component_indices=components)[0] + # data, extent= visualizer.get_var_data(micro_model, var, 1.502, [0.3, 0.4], [0.3,0.4], component_indices=components, method='interpolate', interp_dims=(20,20)) + # print(extent) + # TESTING PLOT_VARS + ################### + vars = ['BC', 'vx', 'vy', 'Bz', 'p', 'W'] + components = [(0,), (), (), (), (), ()] + model = micro_model + # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], components_indices=components) + visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], method = 'interpolate', interp_dims=(100,100), components_indices=components) + # TESTING PLOT_VAR_MODEL_COMPARISON + ################################### + # find_obs = FindObs_drift_root(micro_model, 0.001) + # filter = spatial_box_filter(micro_model, 0.003) + # meso_model = resMHD2D(micro_model, find_obs, filter) + # ranges = [0.3, 0.4] + # meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) + # meso_model.find_observers() + # meso_model.filter_micro_variables() + # var = 'BC' + # component = (0,) + # models = [micro_model, meso_model] + # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries + # # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ + # # method='interpolate', interp_dims=(30,30), component_indices=component) + # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, component_indices=component) \ No newline at end of file From b6b0cd6a181b8373db70df90e548e0fb3857bbda Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 20 Sep 2023 16:35:15 +0200 Subject: [PATCH 016/111] TC tweaks to visualization tools --- FileReaders.py | 3 +++ MesoModels.py | 10 ++++++++-- MicroModels.py | 15 ++++++++++++++- Visualization.py | 1 + 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/FileReaders.py b/FileReaders.py index a87e5ca..6cf447f 100644 --- a/FileReaders.py +++ b/FileReaders.py @@ -114,6 +114,9 @@ def read_in_data(self, micro_model): micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ micro_model.domain_vars['y']] + + + if __name__ == '__main__': from MicroModels import * diff --git a/MesoModels.py b/MesoModels.py index a4864f3..d4418ea 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1046,19 +1046,25 @@ def shear_regression_test(self): CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([[1.501, 1.503],[0.37, 0.39],[0.43, 0.45]],1) # print(meso_model.domain_vars['Points']) - meso_model.find_observers() meso_model.filter_micro_variables() + # meso_model.decompose_structures_gridpoint(1,1,1) meso_model.decompose_structures() + # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,0,1, order=2),) meso_model.calculate_derivatives() + # meso_model.model_residuals_gridpoint(1,1,0) meso_model.model_residuals() # meso_model.shear_regression_test() - + with open('prova_class.pickle', 'wb') as filehandle: + pickle.dump(meso_model, filehandle) + filehandle.close() + diff --git a/MicroModels.py b/MicroModels.py index 7e55142..7bd245a 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -256,7 +256,6 @@ def setup_structures(self): self.vars.update(self.structures) - class IdealHydro_2D(object): def __init__(self, interp_method = "linear"): @@ -534,6 +533,20 @@ def get_interpol_var(self, var, point): print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") return res + +class A(object): + def __init__(self, s, i, t, d): + self.s = s + self.i = i + self.t = t + self.d = d + + def check_equality(self, B): + compatible = True + if self.s != B.s or self.i != B.i or self.t != B.t or self.d != B.d: + compatible = False + return compatible + # TC if __name__ == '__main__': diff --git a/Visualization.py b/Visualization.py index 67e2390..e27b606 100644 --- a/Visualization.py +++ b/Visualization.py @@ -300,6 +300,7 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp FileReader.read_in_data(micro_model) micro_model.setup_structures() + visualizer = Plotter_2D([11.97, 8.36]) # TESTING GET_VAR_DATA From 7ea76c343fd98de600e37e7d14c947dc5aeaa966 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 20 Sep 2023 16:37:51 +0200 Subject: [PATCH 017/111] TC updates to get data methods --- MesoModels.py | 3 --- MicroModels.py | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/MesoModels.py b/MesoModels.py index d4418ea..007e822 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1063,8 +1063,5 @@ def shear_regression_test(self): meso_model.model_residuals() # meso_model.shear_regression_test() - with open('prova_class.pickle', 'wb') as filehandle: - pickle.dump(meso_model, filehandle) - filehandle.close() diff --git a/MicroModels.py b/MicroModels.py index 7bd245a..79ad68a 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -558,8 +558,8 @@ def check_equality(self, B): micro_model.setup_structures() point = [1.502,0.4,0.2] - vars = ['SETfl', 'BC', 'Fab', 'SETem'] - # vars = ['BC'] + # vars = ['SETfl', 'BC', 'Fab', 'SETem'] + vars = ['BC'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) From 4b83d38e6b48fc41b4a802f37bb4c63603b4868d Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 21 Sep 2023 11:41:57 +0200 Subject: [PATCH 018/111] TC tweaks to visualization tools --- MesoModels.py | 89 +++++++++++++++++++++++------------------------- MicroModels.py | 8 ++--- Visualization.py | 39 ++++++++++----------- 3 files changed, 66 insertions(+), 70 deletions(-) diff --git a/MesoModels.py b/MesoModels.py index 007e822..ba192e7 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -442,7 +442,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): # Additional, closure-scheme specific vars must be added appropriately using model_residuals() - self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs + # self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs # Run some compatibility test... compatible = True error = '' @@ -458,7 +458,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): print("Meso and Micro models are incompatible:"+error) def get_all_var_strs(self): - return self.all_var_strs + return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) def get_gridpoints(self): """ @@ -963,7 +963,6 @@ def model_residuals(self): print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) shape = values[idx].shape self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) - self.all_var_strs += key finally: self.meso_vars[key][h,i,j] = values[idx] @@ -977,65 +976,61 @@ def shear_regression_test(self): Better to use a wrapper? """ # LINEAR REGRESSION: LOOP OVER COMPONENTS AND AVERAGE - Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] - shear = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - intercepts = [] - slopes = [] - r_values = [] - for components in np.ndindex(shear[0,0,0,:,:].shape): - # components = (1,1) - xs, ys = np.zeros((Nx,Ny)), np.zeros((Nx,Ny)) - for i in range(Nx): - for j in range(Ny): - if self.filter_vars['U_success'][0,i,j]: - shear = self.model_residuals_gridpoint(0,i,j)[1][0] - xs[i,j] = shear[components] - ys[i,j] = self.meso_vars['pi_res'][tuple([0,i,j]+list(components))] + shape = self.meso_vars['shear_tilde'][0,0,0].shape + statistics = np.zeros((tuple(list(shape)+ [3]))) + for component_idx in np.ndindex(shape): + xs = self.meso_vars['shear_tilde'][0,:,:,component_idx] + ys = self.meso_vars['pi_res'][0,:,:,component_idx] xs = xs.flatten() ys = ys.flatten() sns_plot = sns.regplot(x = xs, y = ys) # print(scst.pearsonr(xs,ys)) - intercept, slope, rvalue = scst.linregress(xs,ys)[:3] - intercepts.append(intercept) - slopes.append(slope) - r_values.append(rvalue) - # print(intercept, slope, rvalue) + statistics[component_idx] = np.array(scst.linregress(xs,ys)[:3]) fig = sns_plot.get_figure() fig.tight_layout() - plt.show() + # plt.show() + m_statistics = np.zeros((3,)) + for i in range(len(m_statistics)): + # for idx in np.ndindex(shape): + idx = np.ndindex(shape) + m_statistics[i] = np.mean(statistics[:,:,i]) - m_intercept = np.mean(intercepts) - m_slope = np.mean(slopes) - m_rvalue = np.mean(r_values) - print(m_intercept, m_slope, m_rvalue) - - # BEFORE WORKING ON THESE: WRAPPER OF MODEL_RESIDUAL, THEN CONTINUE ON THIS # PREPARING THE DATA FOR CONTRASTING RESIDUAL AND MODEL - eta = m_slope - pi_model = np.zeros(self.meso_vars['pi_res'][0,:,:,:,:].shape) - for i in range(Nx): - for j in range(Ny): - if self.filter_vars['U_success'][0,i,j]: - shear = self.model_residuals_gridpoint(0,i,j)[1][0] - pi_model[i,j,:,:] = np.multiply(eta, shear) # Calling the same function twice! Save data! - pi_res = self.meso_vars['pi_res'][0,:,:,:,:] - + eta = m_statistics[0] + # pi_model = np.zeros(self.meso_vars['pi_res'][0,:,:,:,:].shape) + + # Nx, Ny = self.domain_vars['Nx'], self.domain_vars['Ny'] + # for i in range(Nx): + # for j in range(Ny): + # if self.filter_vars['U_success'][0,i,j]: + # shear = self.model_residuals_gridpoint(0,i,j)[1][0] + # pi_model[i,j,:,:] = np.multiply(eta, shear) # Calling the same function twice! Save data! + # pi_res = self.meso_vars['pi_res'][0,:,:,:,:] + + # THIS IS A TRICK: THINK ABOUT HOW TO DEAL WITH THIS. + # self.meso_vars['shear_tilde'][:,:,:] = np.multiply(eta, self.meso_vars['shear_tilde'][:,:,:] ) + for idx in np.ndindex((3,3)): + self.meso_vars['shear_tilde'][:,:,:,idx] = np.multiply(eta, self.meso_vars['shear_tilde'][:,:,:,idx]) + - # SETTING UP AND PLOTTING VIA MARCUS'S ROUTINES + # # SETTING UP AND PLOTTING VIA MARCUS'S ROUTINES visualizer = Plotter_2D() - t_range, x_range, y_range = [self.domain_vars['Tmin'], self.domain_vars['Tmax']] ,\ - [self.domain_vars['Xmin'], self.domain_vars['Xmax']] ,\ + t = self.domain_vars['T'][0] + x_range, y_range = [self.domain_vars['Xmin'], self.domain_vars['Xmax']] ,\ [self.domain_vars['Ymin'], self.domain_vars['Ymax']] - print(t_range, x_range, y_range) - # visualizer.plot_vars(self, ['pi_res', ,], t_range, x_range, y_range, interp_dims=(), method = 'interpolate', component_indices) + # print(t, x_range, y_range) + + # print(visualizer.get_var_data(self, 'shear_tilde', t, x_range, y_range, component_indices=(0,0))) + visualizer.plot_vars(self, ['pi_res', 'shear_tilde'], t, x_range, y_range, components_indices = [(0,0),(0,0)]) if __name__ == '__main__': + FileReader = METHOD_HDF5('./Data/test_res100/') micro_model = IdealMHD_2D() FileReader.read_in_data(micro_model) @@ -1047,8 +1042,8 @@ def shear_regression_test(self): CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([[1.501, 1.503],[0.37, 0.39],[0.43, 0.45]],1) - # print(meso_model.domain_vars['Points']) + meso_model.setup_meso_grid([[1.501, 1.504],[0.1, 0.5],[0.1, 0.5]],1) + print(meso_model.domain_vars['Points']) meso_model.find_observers() meso_model.filter_micro_variables() @@ -1061,7 +1056,7 @@ def shear_regression_test(self): # meso_model.model_residuals_gridpoint(1,1,0) meso_model.model_residuals() - # meso_model.shear_regression_test() + meso_model.shear_regression_test() - + print('Total time is {}'.format(time.process_time() - CPU_start_time)) diff --git a/MicroModels.py b/MicroModels.py index 79ad68a..5fd2a44 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -323,16 +323,16 @@ def get_domain_strs(self): return self.domain_info_strs + self.domain_points_strs def get_prim_strs(self): - return self.prim_strs + return list(self.prim_vars.keys()) def get_aux_strs(self): - return self.aux_strs + return list(self.aux_vars.keys()) def get_structures_strs(self): - return self.structures_strs + return list(self.structures.keys()) def get_all_var_strs(self): - return self.all_var_strs + return list(self.prim_vars.keys()) + list(self.aux_vars_vars.keys()) + list(self.structures.keys()) def setup_structures(self): """ diff --git a/Visualization.py b/Visualization.py index e27b606..b5d719e 100644 --- a/Visualization.py +++ b/Visualization.py @@ -19,7 +19,7 @@ class Plotter_2D(object): - def __init__(self, screen_size): + def __init__(self, screen_size = [11.97, 8.36]): """ Parameters: ----------- @@ -313,27 +313,28 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp # TESTING PLOT_VARS ################### - vars = ['BC', 'vx', 'vy', 'Bz', 'p', 'W'] - components = [(0,), (), (), (), (), ()] - model = micro_model + # vars = ['BC', 'vx', 'vy', 'Bz', 'p', 'W'] + # components = [(0,), (), (), (), (), ()] + # model = micro_model # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], components_indices=components) - visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], method = 'interpolate', interp_dims=(100,100), components_indices=components) + # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], method = 'interpolate', interp_dims=(100,100), components_indices=components) # TESTING PLOT_VAR_MODEL_COMPARISON ################################### - # find_obs = FindObs_drift_root(micro_model, 0.001) - # filter = spatial_box_filter(micro_model, 0.003) - # meso_model = resMHD2D(micro_model, find_obs, filter) - # ranges = [0.3, 0.4] - # meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) - # meso_model.find_observers() - # meso_model.filter_micro_variables() + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + meso_model = resMHD2D(micro_model, find_obs, filter) + ranges = [0.3, 0.4] + meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) + meso_model.find_observers() + meso_model.filter_micro_variables() + + var = 'BC' + component = (0,) + models = [micro_model, meso_model] + smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries + # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ + # method='interpolate', interp_dims=(30,30), component_indices=component) + visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, component_indices=component) - # var = 'BC' - # component = (0,) - # models = [micro_model, meso_model] - # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries - # # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ - # # method='interpolate', interp_dims=(30,30), component_indices=component) - # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, component_indices=component) \ No newline at end of file From aed37b2f3aa0d195175666e52a9432ff01834a8e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 21 Sep 2023 18:08:18 +0200 Subject: [PATCH 019/111] Tweaks to MicroModels --- MesoModels.py | 2 +- MicroModels.py | 14 -------------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/MesoModels.py b/MesoModels.py index ba192e7..a6910be 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1011,7 +1011,7 @@ def shear_regression_test(self): # pi_model[i,j,:,:] = np.multiply(eta, shear) # Calling the same function twice! Save data! # pi_res = self.meso_vars['pi_res'][0,:,:,:,:] - # THIS IS A TRICK: THINK ABOUT HOW TO DEAL WITH THIS. + # THIS IS A TRICK: YOU WANT TO COMPARE THE RESIDUAL WITH MODEL = SHEAR x ETA # self.meso_vars['shear_tilde'][:,:,:] = np.multiply(eta, self.meso_vars['shear_tilde'][:,:,:] ) for idx in np.ndindex((3,3)): self.meso_vars['shear_tilde'][:,:,:,idx] = np.multiply(eta, self.meso_vars['shear_tilde'][:,:,:,idx]) diff --git a/MicroModels.py b/MicroModels.py index 5fd2a44..9663177 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -359,7 +359,6 @@ def setup_structures(self): self.vars.update(self.aux_vars) self.vars.update(self.structures) - def get_interpol_prim(self, var_names, point): """ Returns the interpolated variable at the point @@ -534,19 +533,6 @@ def get_interpol_var(self, var, point): return res -class A(object): - def __init__(self, s, i, t, d): - self.s = s - self.i = i - self.t = t - self.d = d - - def check_equality(self, B): - compatible = True - if self.s != B.s or self.i != B.i or self.t != B.t or self.d != B.d: - compatible = False - return compatible - # TC if __name__ == '__main__': From f653a79883ff4375ece8f6ae82caceeb1a701327 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 22 Sep 2023 11:57:01 +0200 Subject: [PATCH 020/111] TC update to derivatives method --- MesoModels.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/MesoModels.py b/MesoModels.py index a6910be..9d3bea5 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -839,23 +839,13 @@ def calculate_derivative_gridpoint(self, nonlocal_var_str, h, i, j, direction, o coefficients = self.differencing[order]['cen']['coefficients'] stencil = self.differencing[order]['cen']['stencil'] - # Is it a structure or a meso_var? - if nonlocal_var_str in self.meso_vars_strs: - temp = 0 - for s, sample in enumerate(stencil): - idxs = [h,i,j] - idxs[direction] += sample - temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_vars[nonlocal_var_str][tuple(idxs)] ) - return temp - - else: - temp = 0 - for s, sample in enumerate(stencil): - idxs = [h,i,j] - idxs[direction] += sample - temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.meso_structures[nonlocal_var_str][tuple(idxs)] ) - return temp - + temp = 0 + for s, sample in enumerate(stencil): + idxs = [h,i,j] + idxs[direction] += sample + temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.get_var_gridpoint(nonlocal_var_str, *idxs)) + return temp + def calculate_derivatives(self): """ Compute all the derivatives of the quantities corresponding to nonlocal_vars_strs, for all @@ -1042,7 +1032,7 @@ def shear_regression_test(self): CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([[1.501, 1.504],[0.1, 0.5],[0.1, 0.5]],1) + meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.4],[0.4, 0.5]],1) print(meso_model.domain_vars['Points']) meso_model.find_observers() meso_model.filter_micro_variables() @@ -1051,12 +1041,12 @@ def shear_regression_test(self): meso_model.decompose_structures() - # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,0,1, order=2),) + # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,5,5,1, order=2)) meso_model.calculate_derivatives() # meso_model.model_residuals_gridpoint(1,1,0) meso_model.model_residuals() meso_model.shear_regression_test() - print('Total time is {}'.format(time.process_time() - CPU_start_time)) + # print('Total time is {}'.format(time.process_time() - CPU_start_time)) From 35c63165dfa80df19f40bde468ef69eb9f832cbe Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 17 Oct 2023 10:31:38 +0200 Subject: [PATCH 021/111] TC restructuring analysis --- Analysis.py | 102 +++++++++++++++++++++++++++++++++++++++++++++----- MesoModels.py | 29 +++++++++++++- 2 files changed, 121 insertions(+), 10 deletions(-) diff --git a/Analysis.py b/Analysis.py index f03d660..18f8b20 100644 --- a/Analysis.py +++ b/Analysis.py @@ -10,10 +10,15 @@ import h5py import pickle import seaborn as sns - -class CoefficientAnalysis(object): +import statsmodels.api as sm - def __init__(self, visualizer): + + +class CoefficientsAnalysis(object): # 2D function? + """ + """ + + def __init__(self, visualizer, spatial_dims): """ Nothing as yet... @@ -23,7 +28,8 @@ def __init__(self, visualizer): """ self.visualizer = visualizer # need to re-structure this... or do I - + self.spatial_dims = spatial_dims + def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ interp_dims, method, y_component_indices, x_component_indices): y_data_to_plot, points = \ @@ -49,16 +55,94 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met fig.tight_layout() plt.show() + def scalar_regression(self, y, X): + """ + Routine to perform ordinary/weighted/multivariate regression on some gridded data. + Parameters: + ----------- + y: ndarray of shape (n,m) + the "measurements" of the dependent quantity + + X: list of ndarrays of shape (n,m) + the data for the regressors + weights: array of shape (n,m) + the weights of the data (variance of the measurement) + """ + dep_shape = np.shape(y) + n_reg = len(X) + + for i in range(n_reg): + if np.shape(X[i]) != dep_shape: + print('Dependent data is not aligned with regressor, removing regressor data.') + X.remove(X[i]) + else: + X[i] = X[i].flatten() + + y = y.flatten() + model = sm.OLS(y, X) + result = model.fit() + return result.params + + def scalar_weighted_regression(): + """ + Based on sm.WLS + Weights are computed externally by the MesoModel class + """ + pass + def tensor_components_regression(self, y, X, weights = None, components = None): + """ + JUST JOTTING DOWN IDEAS FOR NOW + SHOULD BE A WRAPPER OF scalar_weighted_regression() + """ + if not components: + components = [] + skipping_idx = tuple([0 for _ in range(self.spatial_dims)]) + for i in np.ndindex(y[skipping_idx].shape): + components.append(i) + + # component_regress = np.zeros((len(components), len(X))) + components_params = [] + for component in components: + dep = y[component] + reg = X[component] + if weights: + ws = weights[component] + # component_regress.append(self.simple_regression(dep, reg, ws)) + components_params.append( self.simple_regression(dep, reg, ws)) + components_params.reshape(len(X), len(y)) + + avg_params = np.mean(components_params) + return avg_params + + def tensor_components_weighted_regression(): + """ + Should be a wrapper of scalar_weighted_regression() + Weights are computed externally by the MesoModel class + """ + pass + def PCA_check_regressor_correlation(): + """ + Idea: pass a large list of quantities that are correlated with a residual/closure coefficient. + Check if there is a smaller subset: pass percentage of variance as parameter, like 70%, + and the # of principal components to be retained will be computed from the eigenvalues of the + correlation matrix, that is the loadings. + Then return the PCA weights: this will be later used as regressors for the residuals. + """ + pass - - - - - + def PCA_find_regressors(): + """ + Idea pass both the residuals and a large list of quantities, identify which among these are + correlated with the residual. Do this by checking the scores of the residual (the first var + of the dataset, say) on the various principal components. + Then return the weights. This will then be used in regression (possibyly after checking they're + not correlated). + """ + pass diff --git a/MesoModels.py b/MesoModels.py index 9d3bea5..6238ea5 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -881,6 +881,7 @@ def calculate_derivatives(self): else: print('Derivatives not calculated at {}: observer could not be found.'.format(point)) + # closure_ingredients_gridpoint() ?? def model_residuals_gridpoint(self, h, i, j): """ Decompose quantities obatined via non_local operations (i.e. derivatives) as needed @@ -931,8 +932,10 @@ def model_residuals_gridpoint(self, h, i, j): closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'sigma_tilde', 'J_tilde'] closure_vars = [shear_t, exp_t, acc_t, sigma_t, J_t] + # I THINK THERE'S A MISTAKE WITH THE SHEAR CALCULATION HERE, CHECK!!! + # PRINT IT TO CHECK IT IS REALLY SYMMETRIC AS IT SHOULD! return closure_vars_strs, closure_vars - + #closure_ingredients ?? def model_residuals(self): """ Wrapper of the corresponding gridpoint method. @@ -1017,6 +1020,30 @@ def shear_regression_test(self): # print(visualizer.get_var_data(self, 'shear_tilde', t, x_range, y_range, component_indices=(0,0))) visualizer.plot_vars(self, ['pi_res', 'shear_tilde'], t, x_range, y_range, components_indices = [(0,0),(0,0)]) + def EL_style_closure_gridpoint(): + """ + Compute bulk, shear via scalarization + PA trick, thermal conductivity later. + At a point. + Returns: one coefficient at a point. + """ + pass + + def EL_style_closure(): + """ + Wrapper of EL_style_closure_gridpoint() + """ + pass + + def EL_style_closure_regression(): + """ + Takes in a list of correlation quantities (strings? The regressors needed are computed + previously via closure ingredients) + Pass the dataset to a CoefficientAnalysis instance. + Plot the correlations with the regressors and perform the regression using methods from a Coefficient + Analysis instance. + """ + pass + if __name__ == '__main__': From 553e2c577060599ec95508776f5cdd972dcfb00e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 19 Oct 2023 18:21:04 +0200 Subject: [PATCH 022/111] TC work on regression + visualizing correlations --- Analysis.py | 276 ++++++++++++++++++++++++++++++++++++++++++++++---- MesoModels.py | 128 ++++++++++++++++------- 2 files changed, 344 insertions(+), 60 deletions(-) diff --git a/Analysis.py b/Analysis.py index 18f8b20..92a5639 100644 --- a/Analysis.py +++ b/Analysis.py @@ -7,10 +7,20 @@ import matplotlib.pyplot as plt import numpy as np +import pandas as pd import h5py import pickle import seaborn as sns +import warnings import statsmodels.api as sm + +from system.BaseFunctionality import * +from MicroModels import * +from FileReaders import * +from Filters import * +from Visualization import * +from MesoModels import * + @@ -27,8 +37,46 @@ def __init__(self, visualizer, spatial_dims): Also nothing... """ - self.visualizer = visualizer # need to re-structure this... or do I - self.spatial_dims = spatial_dims + self.visualizer=visualizer # need to re-structure this... or do I + self.spatial_dims=spatial_dims + + def trim_data(self, data, ranges, model_points): + """ + Takes input gridded data 'data' (grid points given by input 'model_points') + and trim this to lie within ranges. + + Parameters: + ----------- + data: ndarray + gridded data, shape must be compatible with model_points + + ranges: list of lists of 2 floats + the min and max in each direction + + model_points: list of list of floats + + Returns: + -------- + data trimmed to within ranges + """ + if not (len(ranges)==len(model_points) and len(ranges) == len(data.shape)): + print('Check: i) data incompatible with ranges or ii) ranges incompatible with model_points. Skipping!') + return data + else: + mins = [i[0] for i in ranges] + maxs = [i[1] for i in ranges] + start_indices = Base.find_nearest_cell(mins, model_points) + end_indices = Base.find_nearest_cell(maxs, model_points) + + IdxsToRemove = [] + num_points = [len(model_points[i]) for i in range(len(model_points))] + for i in range(len(ranges)): + IdxsToRemove.append([ j for j in range(num_points[i]) if j < start_indices[i] or j > end_indices[i]]) + + for i in range(len(ranges)): + data = np.delete(data, IdxsToRemove[i], axis=i) + + return data def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ interp_dims, method, y_component_indices, x_component_indices): @@ -55,42 +103,229 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met fig.tight_layout() plt.show() - def scalar_regression(self, y, X): + def scalar_regression(self, y, X, ranges = None, model_points = None): """ - Routine to perform ordinary/weighted/multivariate regression on some gridded data. + Routine to perform ordinary (multivariate) regression on some gridded data. Parameters: ----------- - y: ndarray of shape (n,m) + y: ndarray of gridded scalar data the "measurements" of the dependent quantity - X: list of ndarrays of shape (n,m) - the data for the regressors + X: list of ndarrays of gridded data (treated as indep scalars) + the "data" for the regressors + + ranges: list of lists of 2 floats + mins and max in each direction - weights: array of shape (n,m) - the weights of the data (variance of the measurement) + model_points: list of lists containing the gridpoints in each direction + + Returns: + -------- + list of fitted parameters + list of std errors associated with the fitted parameters, + (i.e. parameter * std error of the corresponding regressor.) + + Notes: + ------ + Gridded data and ranges are chosen at the model level, and passed here + as parameters, so that this external method will not change/access any + model quantity. + + Works in any dimensions! """ + # CHECKING ALIGNMENT OF PASSED DATA dep_shape = np.shape(y) n_reg = len(X) - for i in range(n_reg): if np.shape(X[i]) != dep_shape: - print('Dependent data is not aligned with regressor, removing regressor data.') + print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') X.remove(X[i]) - else: - X[i] = X[i].flatten() + # TRIMMING THE DATA TO WITHIN RANGES + if ranges != None and model_points != None: + print('Trimming dataset for regression') + y = self.trim_data(y, ranges, model_points) + for x in X: + x = self.trim_data(x, ranges, model_points) + + # FLATTENING + FITTING y = y.flatten() + for x in X: + x = x.flatten() + n_reg = len(X) + n_data = len(y) + X = np.reshape(X, (n_data, n_reg)) model = sm.OLS(y, X) result = model.fit() - return result.params + return result.params, result.bse - def scalar_weighted_regression(): + def scalar_weighted_regression(self, y, X, W, ranges = None, model_points = None): """ - Based on sm.WLS - Weights are computed externally by the MesoModel class + Routine to perform ordinary (multivariate) regression on some gridded data. + + Parameters: + ----------- + y: ndarray of gridded scalar data + the "measurements" of the dependent quantity + + X: list of ndarrays of gridded data (treated as indep scalars) + the "data" for the regressors + + W: ndarray of gridded weights (one per gridpoint) + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + Returns: + -------- + list of fitted parameters + list of std errors associated with the fitted parameters, + (i.e. parameter * std error of the corresponding regressor.) + + Notes: + ------ + For the future, is it worth coding this as a decorator? """ - pass + # CHECKING ALIGNMENT OF PASSED DATA + dep_shape = np.shape(y) + n_reg = len(X) + + for i in range(n_reg): + if np.shape(X[i]) != dep_shape: + print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') + X.remove(X[i]) + elif np.shape(y) != np.shape(W): + print(f'The weights passed are not not aligned with data, setting these to 1') + W = [1 for i in range(len(W))] + + + # TRIMMING THE DATA TO WITHIN RANGES + if ranges != None and model_points != None: + print('Trimming dataset for regression') + y = self.trim_data(y, ranges, model_points) + for x in X: + x = self.trim_data(x, ranges, model_points) + + # FLATTENING + FITTING + y = y.flatten() + for x in X: + x = x.flatten() + n_reg = len(X) + n_data = len(y) + X = np.reshape(X, (n_data, n_reg)) + model = sm.WLS(y, X, W) + result = model.fit() + return result.params, result.bse + + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None): + """ + Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. + Possibility to cut data to lie within ranges. + + Parameters: + ----------- + x,y: nd.arrays + must be of the same shape + + xlabel, ylabel: strs + labels for the quantites plotted + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + + Returns: + -------- + Instance of JointGrid. Figure can then be accessed at the model level + for further adjustments. + """ + if x.shape != y.shape: + print('Cannot check correlation: data is misaligned!') + return None + + if ranges != None and model_points != None: + print('Trimming dataset for correlation plot') + x = self.trim_data(x, ranges, model_points) + y = self.trim_data(y, ranges, model_points) + + x=x.flatten() + y=y.flatten() + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + g=sns.JointGrid() + sns.scatterplot(x=x, y=y, ax=g.ax_joint) + sns.kdeplot(x=x, ax=g.ax_marg_x) + sns.histplot(y=y, ax=g.ax_marg_y, kde=True) + + g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) + g.fig.tight_layout() + return g + + def visualize_correlations(self, data, labels, ranges=None, model_points=None): + """ + Method that returns an instance of PairGrid of correlation plots for a list of vars. + Plotted is: + daigonal: univariate histogram (with kde superimposed) + upper triangle: scatteplots + lower triangle: bivariate kde(s) + + Possibility to cut data to lie within ranges. + + Parameters: + ----------- + data: list of nd.arrays + must be of the same shape + + labels: list of strs + labels for the quantites plotted + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + + Returns: + -------- + Instance of PairGrid. Figure can then be accessed at the model level + for further adjustments. + """ + ref_shape=data[0].shape + Idx_to_delete = [] + for i in range(1,len(data)): + if data[i].shape != ref_shape: + print(f'Cannot use {labels[i]} data: misaligned with first array. Removing it.') + Idx_to_delete.append(i) + + if len(Idx_to_delete) >= 1: + data=list(np.delete(data, Idx_to_delete, axis=0)) + + if ranges != None and model_points != None: + print('Trimming dataset for correlation plot') + for i in range(len(data)): + data[i]=self.trim_data(data[i], ranges, model_points) + + for i in range(len(data)): + data[i]=data[i].flatten() + + data=np.column_stack(data) + data_df = pd.DataFrame(data, columns=labels) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + g=sns.PairGrid(data_df) + g.map_upper(sns.scatterplot) + g.map_lower(sns.kdeplot) + g.map_diag(sns.histplot, kde=True) + g.fig.tight_layout() + return g def tensor_components_regression(self, y, X, weights = None, components = None): """ @@ -143,10 +378,7 @@ def PCA_find_regressors(): not correlated). """ pass - - - - + diff --git a/MesoModels.py b/MesoModels.py index 6238ea5..f259b41 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -20,6 +20,7 @@ from FileReaders import * from Filters import * from Visualization import * +from Analysis import * class NonIdealHydro2D(object): @@ -881,8 +882,7 @@ def calculate_derivatives(self): else: print('Derivatives not calculated at {}: observer could not be found.'.format(point)) - # closure_ingredients_gridpoint() ?? - def model_residuals_gridpoint(self, h, i, j): + def closure_ingredients_gridpoint(self, h, i, j): """ Decompose quantities obatined via non_local operations (i.e. derivatives) as needed for the closure scheme. This is done at gridpoint (h,i,j) and then relevant values are @@ -905,38 +905,50 @@ def model_residuals_gridpoint(self, h, i, j): dictionary is set up not on construction. UNDER CONSTRUCTION: THE EM PART NEEDS MORE THINKING + + Rename: closure_ingredients_gridpoint() ?? + Coefficients can be computed here directly, but more general models will require more work. + Do not split this, at least for now """ # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS u_t = self.meso_vars['u_tilde'][h,i,j] + u_t_cov = np.einsum('ij,j->i', self.metric, u_t) DFab = self.deriv_vars['D_Fab'][h,i,j] - current = np.einsum('ij, kl, kjl->i', self.metric, self.metric, DFab) - sigma_t = np.einsum('i,ij,j->',current, self.metric, u_t) - J_t = current - np.multiply(sigma_t, u_t) - - # MODELLING THE DISSIPATIVE TERMS IN THE FLUID SET + # MODELLING THE DISSIPATIVE TERMS IN THE FLUID SET - WORKING WITH (2,0) nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor - acc_t = np.einsum('i,ij->j',u_t, nabla_u) - should_be_zero = np.einsum('ij,i,kj',self.metric, u_t, nabla_u) - - h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) - Daub = nabla_u + np.einsum('ij,j,l', self.metric, u_t, acc_t) + np.einsum('ij,j,l', self.metric, should_be_zero, u_t) - exp_t = np.einsum('ii->', Daub) + nabla_u = np.einsum('ij,jk->ik', self.metric, nabla_u) #this is a rank (2,0) tensor + acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector + + # The following two quantities should be identically zero, but numerical errors. + # Computing them and removing to project velocity gradients + unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector + acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) + Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ + np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor + # Checking orthogonality + # print('Checking orthogonality of u_t with Daub. \n First component: {}\n *********\n Second component: {}'.format(np.einsum('i,ij->j', u_t_cov, Daub), \ + # np.einsum('ij,j->i', Daub, u_t_cov))) + h_ab = self.metric + np.einsum('i,j->ij', u_t, u_t) + exp_t = np.einsum('ii->',Daub) shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/self.spatial_dims * exp_t, h_ab) + vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + # print('Expansion: \n {} \n ***** \n Shear: \n {} \n ***** \n Vorticity: \n{}'.format(exp_t, shear_t, vort_t)) + + # # THINK WHAT TO DO WITH THE HEAT FLUX --> GRADIENTS IN T OR N AND EPS + # # THINK ABOUT THE EM PART, DO YOU NEED MORE? + # # COME BACK TO THIS LATER? - # THINK WHAT TO DO WITH THE HEAT FLUX --> GRADIENTS IN T OR N AND EPS - # THINK ABOUT THE EM PART, DO YOU NEED MORE? - # COME BACK TO THIS LATER? - # self.meso_vars['j'][h,i,j,:] = current ?? - - closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'sigma_tilde', 'J_tilde'] - closure_vars = [shear_t, exp_t, acc_t, sigma_t, J_t] + # current = np.einsum('ij, kl, kjl->i', self.metric, self.metric, DFab) + # sigma_t = np.einsum('i,ij,j->',current, self.metric, u_t) + # J_t = current - np.multiply(sigma_t, u_t) + # # self.meso_vars['j'][h,i,j,:] = current ?? - # I THINK THERE'S A MISTAKE WITH THE SHEAR CALCULATION HERE, CHECK!!! - # PRINT IT TO CHECK IT IS REALLY SYMMETRIC AS IT SHOULD! + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde'] + closure_vars = [shear_t, exp_t, acc_t] return closure_vars_strs, closure_vars - #closure_ingredients ?? - def model_residuals(self): + + def closure_ingredients(self): """ Wrapper of the corresponding gridpoint method. Set up the dictionary for the closure variables not yet defined. @@ -947,7 +959,7 @@ def model_residuals(self): for i in range(Nx): for j in range(Ny): if self.filter_vars['U_success'][h,i,j]: - keys, values = self.model_residuals_gridpoint(h,i,j) + keys, values = self.closure_ingredients_gridpoint(h,i,j) # try-except block to extend the meso_vars dictionary for idx, key in enumerate(keys): try: @@ -1020,21 +1032,39 @@ def shear_regression_test(self): # print(visualizer.get_var_data(self, 'shear_tilde', t, x_range, y_range, component_indices=(0,0))) visualizer.plot_vars(self, ['pi_res', 'shear_tilde'], t, x_range, y_range, components_indices = [(0,0),(0,0)]) - def EL_style_closure_gridpoint(): + def EL_style_closure_gridpoint(self, h, i, j): """ Compute bulk, shear via scalarization + PA trick, thermal conductivity later. At a point. - Returns: one coefficient at a point. + Returns: coefficient(s) at a point. """ - pass + coefficients_names = ['zeta'] + zeta = self.meso_vars['Pi_res'][h,i,j] / self.meso_vars['exp_tilde'][h,i,j] + coefficients = [zeta] + return coefficients_names, coefficients - def EL_style_closure(): + def EL_style_closure(self): """ Wrapper of EL_style_closure_gridpoint() """ - pass - - def EL_style_closure_regression(): + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][h,i,j]: + keys, values = self.EL_style_closure_gridpoint(h,i,j) + # try-except block to extend the meso_vars dictionary with the dissipative coefficients + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) + shape = values[idx].shape + self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + finally: + self.meso_vars[key][h,i,j] = values[idx] + + def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Points = None): """ Takes in a list of correlation quantities (strings? The regressors needed are computed previously via closure ingredients) @@ -1042,7 +1072,24 @@ def EL_style_closure_regression(): Plot the correlations with the regressors and perform the regression using methods from a Coefficient Analysis instance. """ - pass + y = self.meso_vars['zeta'] + # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + X = self.meso_vars['eps_tilde'] + + ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ + [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ + [self.domain_vars['Ymin'], self.domain_vars['Ymax']]] + points = self.domain_vars['Points'] + # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) + # print(*stats_result) + + # g1 = CoefficientAnalysis.visualize_correlation(X,y,'eps_tilde','zeta',ranges=ranges, model_points=points) + + data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + labels=['b_tilde', 'eps_tilde', 'n_tilde'] + g2=CoefficientAnalysis.visualize_correlations(data, labels) + plt.show() + if __name__ == '__main__': @@ -1059,7 +1106,7 @@ def EL_style_closure_regression(): CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.4],[0.4, 0.5]],1) + meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.35],[0.4, 0.45]],1) print(meso_model.domain_vars['Points']) meso_model.find_observers() meso_model.filter_micro_variables() @@ -1067,13 +1114,18 @@ def EL_style_closure_regression(): # meso_model.decompose_structures_gridpoint(1,1,1) meso_model.decompose_structures() - - # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,5,5,1, order=2)) + # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) meso_model.calculate_derivatives() # meso_model.model_residuals_gridpoint(1,1,0) - meso_model.model_residuals() - meso_model.shear_regression_test() + meso_model.closure_ingredients() + # meso_model.shear_regression_test() + meso_model.EL_style_closure() + + visualizer = Plotter_2D() + regressor = CoefficientsAnalysis(visualizer, meso_model.spatial_dims) + result = meso_model.EL_style_closure_regression(regressor) + # print('\n {} \n ******** \n {}'.format(result[0], result[1])) # print('Total time is {}'.format(time.process_time() - CPU_start_time)) From 0dd74daa45b05fdd04bec88b8a3bb2785ae01863 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 20 Oct 2023 11:49:23 +0200 Subject: [PATCH 023/111] TC work on tensor regression --- Analysis.py | 220 ++++++++++++++++++++++++++++++++++++-------------- MesoModels.py | 29 +++++-- 2 files changed, 179 insertions(+), 70 deletions(-) diff --git a/Analysis.py b/Analysis.py index 92a5639..7094842 100644 --- a/Analysis.py +++ b/Analysis.py @@ -24,10 +24,11 @@ -class CoefficientsAnalysis(object): # 2D function? +class CoefficientsAnalysis(object): """ + Class containing a number of methods for performing statistical analysis on gridded data. + Methods include: regression, visualizing correlations, PCAs """ - def __init__(self, visualizer, spatial_dims): """ Nothing as yet... @@ -37,8 +38,9 @@ def __init__(self, visualizer, spatial_dims): Also nothing... """ - self.visualizer=visualizer # need to re-structure this... or do I - self.spatial_dims=spatial_dims + # self.visualizer=visualizer # need to re-structure this... or do I + # Do you really need the visualizer here? I don't think so! + # self.spatial_dims=spatial_dims def trim_data(self, data, ranges, model_points): """ @@ -78,31 +80,6 @@ def trim_data(self, data, ranges, model_points): return data - def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ - interp_dims, method, y_component_indices, x_component_indices): - y_data_to_plot, points = \ - self.visualizer.get_var_data(model, y_var_str, t, x_range, y_range, interp_dims, method, y_component_indices) - x_data_to_plot, points = \ - self.visualizer.get_var_data(model, x_var_str, t, x_range, y_range, interp_dims, method, x_component_indices) - fig = plt.figure(figsize=(16,16)) - sns.jointplot(x=x_data_to_plot.flatten(), y=y_data_to_plot.flatten(), kind="hex", color="#4CB391") - plt.title(y_var_str+'('+x_var_str+')') - fig.tight_layout() - plt.show() - - def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, method, component_indices): - - data_to_plot, points = \ - self.visualizer.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - - # print(data_to_plot) - - fig = plt.figure(figsize=(16,16)) - sns.displot(data_to_plot) - plt.title(var_str) - fig.tight_layout() - plt.show() - def scalar_regression(self, y, X, ranges = None, model_points = None): """ Routine to perform ordinary (multivariate) regression on some gridded data. @@ -206,6 +183,7 @@ def scalar_weighted_regression(self, y, X, W, ranges = None, model_points = None if ranges != None and model_points != None: print('Trimming dataset for regression') y = self.trim_data(y, ranges, model_points) + W = self.trim_data(W, ranges, model_points) for x in X: x = self.trim_data(x, ranges, model_points) @@ -220,6 +198,133 @@ def scalar_weighted_regression(self, y, X, W, ranges = None, model_points = None result = model.fit() return result.params, result.bse + def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_points=None, components=None): + """ + Wrapper of scalar_regression: if no components is passed, perform regression on each tensor component + independently, otherwise only on a subset of these. All the component-wise results are returned. + + Parameters: + ----------- + y: ndarray of gridded tensorial data + the "measurements" of the dependent quantity + + X: list of ndarrays of gridded data + the "data" for the regressors + + spatial_dims: int + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + components: list of tuples + the tensor components to be considered for regression + Returns: + -------- + list of fitted parameters + list of std errors associated with the fitted parameters, + (i.e. parameter * std error of the corresponding regressor.) + + Notes: + ------ + Gridded data and ranges are chosen at the model level, and passed here + as parameters, so that this external method will not change/access any + model quantity. + + Works in any dimensions! + """ + # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE + l=[0 for i in range(spatial_dims+1)] + dep_shape = y[tuple(l)].shape + for i, x in enumerate(X): + if x[tuple(l)].shape != dep_shape: + print('The {}-th regressor shape {} is not compatible with dependent shape {}. Removing it!'.format(i, x[0,0,0].shape, dep_shape)) + X.remove(x) + if len(X)==0: + print('None of the passed regressors is compatible with dep data. Exiting.') + return None + + # PREPARING COMPONENTS TO LOOP OVER: user-provided or all? + if components: + for c in components: + if len(c) != len(dep_shape): + print(f'The components indices passed {c} are not compatible with data. Ignoring this and moving on!') + components.remove(c) + else: + components = [] + for i in np.ndindex(dep_shape): + components.append(i) + + # RESHAPING: now grid indices come last, needed for flattening! + tot_shape=y.shape + reshaping=[tot_shape[i] for i in range(spatial_dims+1)] + reshaping=tuple(list(dep_shape)+reshaping) + y = y.reshape(reshaping) + for i in range(len(X)): + X[i]= X[i].reshape(reshaping) + + # SCALAR REGRESSION ON EACH COMPONENT + betas = [] + bses = [] + for c in components: + yc = y[c] + Xc = [ x[c] for x in X] + beta, bse = self.scalar_regression(yc, Xc, ranges, model_points) + betas.append(beta) + bses.append(bse) + return betas, bses + + def tensor_components_weighted_regression(self, y, X, spatial_dims, W, ranges=None, model_points=None, components=None): + """ + Should be a wrapper of scalar_weighted_regression() + Weights are computed externally by the MesoModel class + + For now: each component has the same weight at a point, this may change in the future + as data along different components may have different errors (say, if the shear components are + of different odm for example.) + """ + # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE + l=[0 for i in range(spatial_dims+1)] + dep_shape = y[tuple(l)].shape + for i, x in enumerate(X): + if x[tuple(l)].shape != dep_shape: + print('The {}-th regressor shape {} is not compatible with dependent shape {}. Removing it!'.format(i, x[0,0,0].shape, dep_shape)) + X.remove(x) + if len(X)==0: + print('None of the passed regressors is compatible with dep data. Exiting.') + return None + + # PREPARING COMPONENTS TO LOOP OVER: user-provided or all? + if components: + for c in components: + if len(c) != len(dep_shape): + print(f'The components indices passed {c} are not compatible with data. Ignoring this and moving on!') + components.remove(c) + else: + components = [] + for i in np.ndindex(dep_shape): + components.append(i) + + # RESHAPING: now grid indices come last, needed for flattening! + tot_shape=y.shape + reshaping=[tot_shape[i] for i in range(spatial_dims+1)] + reshaping=tuple(list(dep_shape)+reshaping) + y = y.reshape(reshaping) + for i in range(len(X)): + X[i]= X[i].reshape(reshaping) + + # SCALAR REGRESSION ON EACH COMPONENT + betas = [] + bses = [] + for c in components: + yc = y[c] + Xc = [ x[c] for x in X] + beta, bse = self.scalar_weighted_regression(yc, Xc, W, ranges, model_points) + betas.append(*beta) + bses.append(*bse) + return betas, bses + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None): """ Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. @@ -268,7 +373,7 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod g.fig.tight_layout() return g - def visualize_correlations(self, data, labels, ranges=None, model_points=None): + def visualize_many_correlations(self, data, labels, ranges=None, model_points=None): """ Method that returns an instance of PairGrid of correlation plots for a list of vars. Plotted is: @@ -327,38 +432,6 @@ def visualize_correlations(self, data, labels, ranges=None, model_points=None): g.fig.tight_layout() return g - def tensor_components_regression(self, y, X, weights = None, components = None): - """ - JUST JOTTING DOWN IDEAS FOR NOW - SHOULD BE A WRAPPER OF scalar_weighted_regression() - """ - if not components: - components = [] - skipping_idx = tuple([0 for _ in range(self.spatial_dims)]) - for i in np.ndindex(y[skipping_idx].shape): - components.append(i) - - # component_regress = np.zeros((len(components), len(X))) - components_params = [] - for component in components: - dep = y[component] - reg = X[component] - if weights: - ws = weights[component] - # component_regress.append(self.simple_regression(dep, reg, ws)) - components_params.append( self.simple_regression(dep, reg, ws)) - components_params.reshape(len(X), len(y)) - - avg_params = np.mean(components_params) - return avg_params - - def tensor_components_weighted_regression(): - """ - Should be a wrapper of scalar_weighted_regression() - Weights are computed externally by the MesoModel class - """ - pass - def PCA_check_regressor_correlation(): """ Idea: pass a large list of quantities that are correlated with a residual/closure coefficient. @@ -379,7 +452,30 @@ def PCA_find_regressors(): """ pass + # Not sure about these two methods. + def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ + interp_dims, method, y_component_indices, x_component_indices): + y_data_to_plot, points = \ + self.visualizer.get_var_data(model, y_var_str, t, x_range, y_range, interp_dims, method, y_component_indices) + x_data_to_plot, points = \ + self.visualizer.get_var_data(model, x_var_str, t, x_range, y_range, interp_dims, method, x_component_indices) + fig = plt.figure(figsize=(16,16)) + sns.jointplot(x=x_data_to_plot.flatten(), y=y_data_to_plot.flatten(), kind="hex", color="#4CB391") + plt.title(y_var_str+'('+x_var_str+')') + fig.tight_layout() + plt.show() + + def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, method, component_indices): + data_to_plot, points = \ + self.visualizer.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + + # print(data_to_plot) + fig = plt.figure(figsize=(16,16)) + sns.displot(data_to_plot) + plt.title(var_str) + fig.tight_layout() + plt.show() diff --git a/MesoModels.py b/MesoModels.py index f259b41..9dbb5ee 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1074,23 +1074,36 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po """ y = self.meso_vars['zeta'] # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - X = self.meso_vars['eps_tilde'] + X = [self.meso_vars['eps_tilde']] ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ [self.domain_vars['Ymin'], self.domain_vars['Ymax']]] points = self.domain_vars['Points'] + + # TESTING SCALAR REGRESSION ROUTINE # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) # print(*stats_result) - # g1 = CoefficientAnalysis.visualize_correlation(X,y,'eps_tilde','zeta',ranges=ranges, model_points=points) - - data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - labels=['b_tilde', 'eps_tilde', 'n_tilde'] - g2=CoefficientAnalysis.visualize_correlations(data, labels) - plt.show() + # TESTING TENSOR REGRESSION ROUTINE + # y=self.meso_vars['q_res'] + # X=[self.meso_vars['e_tilde'], self.meso_vars['u_tilde']] + # # print(y.shape, X.shape) + # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, components=[(0,),(2,)]) + # print(stats_result) + + # TESTING CORRELATION VISUALIZATION ROUTINES + # x = self.meso_vars['zeta'] + # y = self.meso_vars['eps_tilde'] + # labels = ['zeta','eps_tilde'] + # g1=CoefficientAnalysis.visualize_correlation(x, y, labels) + + # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['b_tilde', 'eps_tilde', 'n_tilde'] + # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) + # plt.show() + - if __name__ == '__main__': From fe746262b18123ce4a02dd6ac2c9abd9761c8ef9 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 20 Oct 2023 11:56:37 +0200 Subject: [PATCH 024/111] TC work on tensor regression --- Analysis.py | 46 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 41 insertions(+), 5 deletions(-) diff --git a/Analysis.py b/Analysis.py index 7094842..933d9ec 100644 --- a/Analysis.py +++ b/Analysis.py @@ -277,12 +277,48 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po def tensor_components_weighted_regression(self, y, X, spatial_dims, W, ranges=None, model_points=None, components=None): """ - Should be a wrapper of scalar_weighted_regression() - Weights are computed externally by the MesoModel class + Wrapper of scalar_weighted_regression. If no components is passed, perform regression on each tensor component + independently, otherwise only on a subset of these. All the component-wise results are returned. + + Weights computed and passed externally. + + Parameters: + ----------- + y: ndarray of gridded tensorial data + the "measurements" of the dependent quantity + + X: list of ndarrays of gridded data + the "data" for the regressors + + spatial_dims: int + + W: ndarray of gridded weights (one per gridpoint for now) + + ranges: list of lists of 2 floats + mins and max in each direction - For now: each component has the same weight at a point, this may change in the future - as data along different components may have different errors (say, if the shear components are - of different odm for example.) + model_points: list of lists containing the gridpoints in each direction + + components: list of tuples + the tensor components to be considered for regression + + Returns: + -------- + list of lists. Each of these lists is built as follows: + [ [b1, b2, ...], [bse1, bse2, ... ]] + namely coefficients + errors of multivariate regressions + each list correspond to a component + + Notes: + ------ + Gridded data and ranges are chosen at the model level, and passed here + as parameters, so that this external method will not change/access any + model quantity. + + For now each component has the same weight at a point, although this + may change in the future as data along different components may have different errors. + + Works in any dimensions! """ # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE l=[0 for i in range(spatial_dims+1)] From 6a7879bc7aa4c673937d4a26b3202ccb6ac5f1d2 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 20 Oct 2023 17:24:45 +0200 Subject: [PATCH 025/111] TC update to regression routines: weighted + sklearn --- Analysis.py | 220 +++++++++++++------------------------------------- MesoModels.py | 24 +++--- 2 files changed, 69 insertions(+), 175 deletions(-) diff --git a/Analysis.py b/Analysis.py index 933d9ec..51318fc 100644 --- a/Analysis.py +++ b/Analysis.py @@ -5,6 +5,8 @@ @author: marcu """ +# USE !SCIKIT LEARN INSTEAD? IT'S THE PACKAGE FOR MACHINE LEARNING SO! + import matplotlib.pyplot as plt import numpy as np import pandas as pd @@ -12,6 +14,7 @@ import pickle import seaborn as sns import warnings +from sklearn import linear_model import statsmodels.api as sm from system.BaseFunctionality import * @@ -80,9 +83,9 @@ def trim_data(self, data, ranges, model_points): return data - def scalar_regression(self, y, X, ranges = None, model_points = None): + def scalar_regression(self, y, X, ranges = None, model_points = None, weights=None, add_intercept=False): """ - Routine to perform ordinary (multivariate) regression on some gridded data. + Routine to perform ordinary or weighted (multivariate) regression on some gridded data. Parameters: ----------- @@ -97,6 +100,8 @@ def scalar_regression(self, y, X, ranges = None, model_points = None): model_points: list of lists containing the gridpoints in each direction + weights: ndarray of gridded weights + Returns: -------- list of fitted parameters @@ -110,6 +115,8 @@ def scalar_regression(self, y, X, ranges = None, model_points = None): model quantity. Works in any dimensions! + + Intercept must be added manually: for the future, add this via true/false block. """ # CHECKING ALIGNMENT OF PASSED DATA dep_shape = np.shape(y) @@ -118,87 +125,64 @@ def scalar_regression(self, y, X, ranges = None, model_points = None): if np.shape(X[i]) != dep_shape: print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') X.remove(X[i]) + if weights: + if np.shape(y) != np.shape(weights): + print(f'The weights passed are not not aligned with data, setting these to 1') + weights=np.ones(y.shape) + # TRIMMING THE DATA TO WITHIN RANGES if ranges != None and model_points != None: print('Trimming dataset for regression') y = self.trim_data(y, ranges, model_points) + if weights: + weights = self.trim_data(weights, ranges, model_points) for x in X: x = self.trim_data(x, ranges, model_points) # FLATTENING + FITTING y = y.flatten() - for x in X: - x = x.flatten() - n_reg = len(X) - n_data = len(y) - X = np.reshape(X, (n_data, n_reg)) - model = sm.OLS(y, X) - result = model.fit() - return result.params, result.bse - - def scalar_weighted_regression(self, y, X, W, ranges = None, model_points = None): - """ - Routine to perform ordinary (multivariate) regression on some gridded data. - - Parameters: - ----------- - y: ndarray of gridded scalar data - the "measurements" of the dependent quantity - - X: list of ndarrays of gridded data (treated as indep scalars) - the "data" for the regressors - - W: ndarray of gridded weights (one per gridpoint) - - ranges: list of lists of 2 floats - mins and max in each direction + if weights: + weights = weights.flatten() - model_points: list of lists containing the gridpoints in each direction + for i in range(len(X)): + X[i] = X[i].flatten() - Returns: - -------- - list of fitted parameters - list of std errors associated with the fitted parameters, - (i.e. parameter * std error of the corresponding regressor.) + if add_intercept: + const = np.ones(y.shape) + X.insert(0,const) - Notes: - ------ - For the future, is it worth coding this as a decorator? - """ - # CHECKING ALIGNMENT OF PASSED DATA - dep_shape = np.shape(y) - n_reg = len(X) - - for i in range(n_reg): - if np.shape(X[i]) != dep_shape: - print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') - X.remove(X[i]) - elif np.shape(y) != np.shape(W): - print(f'The weights passed are not not aligned with data, setting these to 1') - W = [1 for i in range(len(W))] - - - # TRIMMING THE DATA TO WITHIN RANGES - if ranges != None and model_points != None: - print('Trimming dataset for regression') - y = self.trim_data(y, ranges, model_points) - W = self.trim_data(W, ranges, model_points) - for x in X: - x = self.trim_data(x, ranges, model_points) - - # FLATTENING + FITTING - y = y.flatten() - for x in X: - x = x.flatten() n_reg = len(X) n_data = len(y) X = np.reshape(X, (n_data, n_reg)) - model = sm.WLS(y, X, W) - result = model.fit() - return result.params, result.bse - - def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_points=None, components=None): + + # VERSION USING STATSMODELS + # if weights: + # model=sm.WLS(y, X, W) + # result= model.fit() + # return result.params, result.bse + # else: + # model=sm.OLS(y, X) + # result=model.fit() + # return result.params, result.bse + + # VERSION USING SKLEARN: + model=linear_model.LinearRegression(fit_intercept=False) + if weights: + model.fit(X,y,sample_weight=weights) + else: + model.fit(X,y) + y_hat = model.predict(X) + res = y - y_hat + res_sum_of_sq = np.dot(res,res) + sigma_res_sq = res_sum_of_sq / (n_data-n_reg) + var_beta = np.linalg.inv(np.einsum('ji,jl->il',X,X)) * sigma_res_sq + bse = [var_beta[i,i]**0.5 for i in range(n_reg)] + + result = [(model.coef_[i], bse[i]) for i in range(n_reg)] + return result + + def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_points=None, components=None, weights=None, add_intercept=False): """ Wrapper of scalar_regression: if no components is passed, perform regression on each tensor component independently, otherwise only on a subset of these. All the component-wise results are returned. @@ -228,98 +212,9 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po Notes: ------ - Gridded data and ranges are chosen at the model level, and passed here - as parameters, so that this external method will not change/access any - model quantity. - - Works in any dimensions! - """ - # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE - l=[0 for i in range(spatial_dims+1)] - dep_shape = y[tuple(l)].shape - for i, x in enumerate(X): - if x[tuple(l)].shape != dep_shape: - print('The {}-th regressor shape {} is not compatible with dependent shape {}. Removing it!'.format(i, x[0,0,0].shape, dep_shape)) - X.remove(x) - if len(X)==0: - print('None of the passed regressors is compatible with dep data. Exiting.') - return None - - # PREPARING COMPONENTS TO LOOP OVER: user-provided or all? - if components: - for c in components: - if len(c) != len(dep_shape): - print(f'The components indices passed {c} are not compatible with data. Ignoring this and moving on!') - components.remove(c) - else: - components = [] - for i in np.ndindex(dep_shape): - components.append(i) - - # RESHAPING: now grid indices come last, needed for flattening! - tot_shape=y.shape - reshaping=[tot_shape[i] for i in range(spatial_dims+1)] - reshaping=tuple(list(dep_shape)+reshaping) - y = y.reshape(reshaping) - for i in range(len(X)): - X[i]= X[i].reshape(reshaping) - - # SCALAR REGRESSION ON EACH COMPONENT - betas = [] - bses = [] - for c in components: - yc = y[c] - Xc = [ x[c] for x in X] - beta, bse = self.scalar_regression(yc, Xc, ranges, model_points) - betas.append(beta) - bses.append(bse) - return betas, bses - - def tensor_components_weighted_regression(self, y, X, spatial_dims, W, ranges=None, model_points=None, components=None): + Weights are taken as the same for each component here. + For future: do you need to change this? """ - Wrapper of scalar_weighted_regression. If no components is passed, perform regression on each tensor component - independently, otherwise only on a subset of these. All the component-wise results are returned. - - Weights computed and passed externally. - - Parameters: - ----------- - y: ndarray of gridded tensorial data - the "measurements" of the dependent quantity - - X: list of ndarrays of gridded data - the "data" for the regressors - - spatial_dims: int - - W: ndarray of gridded weights (one per gridpoint for now) - - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - - components: list of tuples - the tensor components to be considered for regression - - Returns: - -------- - list of lists. Each of these lists is built as follows: - [ [b1, b2, ...], [bse1, bse2, ... ]] - namely coefficients + errors of multivariate regressions - each list correspond to a component - - Notes: - ------ - Gridded data and ranges are chosen at the model level, and passed here - as parameters, so that this external method will not change/access any - model quantity. - - For now each component has the same weight at a point, although this - may change in the future as data along different components may have different errors. - - Works in any dimensions! - """ # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE l=[0 for i in range(spatial_dims+1)] dep_shape = y[tuple(l)].shape @@ -351,15 +246,12 @@ def tensor_components_weighted_regression(self, y, X, spatial_dims, W, ranges=No X[i]= X[i].reshape(reshaping) # SCALAR REGRESSION ON EACH COMPONENT - betas = [] - bses = [] + results = [] for c in components: yc = y[c] Xc = [ x[c] for x in X] - beta, bse = self.scalar_weighted_regression(yc, Xc, W, ranges, model_points) - betas.append(*beta) - bses.append(*bse) - return betas, bses + results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) + return results def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None): """ diff --git a/MesoModels.py b/MesoModels.py index 9dbb5ee..c82485a 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1072,36 +1072,39 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po Plot the correlations with the regressors and perform the regression using methods from a Coefficient Analysis instance. """ - y = self.meso_vars['zeta'] - # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - X = [self.meso_vars['eps_tilde']] + # FICTITIOUS RANGES AND POINTS TO TEST TRIM_DATA ROUTINE ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ [self.domain_vars['Ymin'], self.domain_vars['Ymax']]] points = self.domain_vars['Points'] # TESTING SCALAR REGRESSION ROUTINE + # y = self.meso_vars['zeta'] + # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) # print(*stats_result) # TESTING TENSOR REGRESSION ROUTINE # y=self.meso_vars['q_res'] # X=[self.meso_vars['e_tilde'], self.meso_vars['u_tilde']] - # # print(y.shape, X.shape) - # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, components=[(0,),(2,)]) - # print(stats_result) + # # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, components=[(0,),(2,)]) + # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, add_intercept=False) + # for i in range(len(stats_result)): + # print('{}-th component: \n {} \n\n'.format(i, stats_result[i])) # TESTING CORRELATION VISUALIZATION ROUTINES - # x = self.meso_vars['zeta'] + # x = self.meso_vars['n_tilde'] # y = self.meso_vars['eps_tilde'] - # labels = ['zeta','eps_tilde'] - # g1=CoefficientAnalysis.visualize_correlation(x, y, labels) + # labels = ['n_tilde','eps_tilde'] + # g1=CoefficientAnalysis.visualize_correlation(x, y, xlabel=labels[0], ylabel=labels[1]) + # fig=g1.figure + # fig.savefig('Single_correlation_plot.pdf', format='pdf') # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] # labels=['b_tilde', 'eps_tilde', 'n_tilde'] # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) - # plt.show() + # plt.savefig('Many_correlations_plot.pdf', format='pdf') @@ -1138,7 +1141,6 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po visualizer = Plotter_2D() regressor = CoefficientsAnalysis(visualizer, meso_model.spatial_dims) result = meso_model.EL_style_closure_regression(regressor) - # print('\n {} \n ******** \n {}'.format(result[0], result[1])) # print('Total time is {}'.format(time.process_time() - CPU_start_time)) From 508fb0bfcd469aa9cec313fc5c069de30c95cf43 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 25 Oct 2023 16:23:27 +0200 Subject: [PATCH 026/111] TC work PCA tools --- Analysis.py | 188 +++++++++++++++++++++++++++++++++++++------------- MesoModels.py | 27 +++++--- 2 files changed, 156 insertions(+), 59 deletions(-) diff --git a/Analysis.py b/Analysis.py index 51318fc..61edf49 100644 --- a/Analysis.py +++ b/Analysis.py @@ -14,7 +14,8 @@ import pickle import seaborn as sns import warnings -from sklearn import linear_model +from sklearn.linear_model import LinearRegression +from sklearn.decomposition import PCA import statsmodels.api as sm from system.BaseFunctionality import * @@ -26,13 +27,12 @@ - class CoefficientsAnalysis(object): """ Class containing a number of methods for performing statistical analysis on gridded data. Methods include: regression, visualizing correlations, PCAs """ - def __init__(self, visualizer, spatial_dims): + def __init__(self): #, visualizer, spatial_dims): """ Nothing as yet... @@ -102,6 +102,10 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No weights: ndarray of gridded weights + add_intercept: bool + whether to extemd the dataset to account for a constant offset in the + regression model. + Returns: -------- list of fitted parameters @@ -125,58 +129,62 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No if np.shape(X[i]) != dep_shape: print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') X.remove(X[i]) - if weights: - if np.shape(y) != np.shape(weights): - print(f'The weights passed are not not aligned with data, setting these to 1') - weights=np.ones(y.shape) - - + + if len(X)==0: + print('None of the passed regressors is compatible with dep data. Exiting.') + return None + + if weights: + if np.shape(y) != np.shape(weights): + print(f'The weights passed are not not aligned with data, setting these to 1') + weights=np.ones(y.shape) + # TRIMMING THE DATA TO WITHIN RANGES if ranges != None and model_points != None: print('Trimming dataset for regression') - y = self.trim_data(y, ranges, model_points) + Y = self.trim_data(y, ranges, model_points) if weights: - weights = self.trim_data(weights, ranges, model_points) + Weights = self.trim_data(weights, ranges, model_points) + XX = [] for x in X: - x = self.trim_data(x, ranges, model_points) + XX.append(self.trim_data(x, ranges, model_points)) # FLATTENING + FITTING - y = y.flatten() + Y = Y.flatten() if weights: - weights = weights.flatten() + Weights = Weights.flatten() for i in range(len(X)): - X[i] = X[i].flatten() + XX[i]=XX[i].flatten() if add_intercept: - const = np.ones(y.shape) - X.insert(0,const) - - n_reg = len(X) - n_data = len(y) - X = np.reshape(X, (n_data, n_reg)) + const = np.ones(Y.shape) + XX.insert(0,const) + n_reg = len(XX) + n_data = len(Y) + XX= np.reshape(XX, (n_data, n_reg)) # VERSION USING STATSMODELS # if weights: - # model=sm.WLS(y, X, W) + # model=sm.WLS(y, Xfl, W) # result= model.fit() # return result.params, result.bse # else: - # model=sm.OLS(y, X) + # model=sm.OLS(y, Xfl) # result=model.fit() # return result.params, result.bse # VERSION USING SKLEARN: - model=linear_model.LinearRegression(fit_intercept=False) + model=LinearRegression(fit_intercept=False) if weights: - model.fit(X,y,sample_weight=weights) + model.fit(XX,Y,sample_weight=weights) else: - model.fit(X,y) - y_hat = model.predict(X) - res = y - y_hat + model.fit(XX,Y) + Y_hat = model.predict(XX) + res = Y - Y_hat res_sum_of_sq = np.dot(res,res) sigma_res_sq = res_sum_of_sq / (n_data-n_reg) - var_beta = np.linalg.inv(np.einsum('ji,jl->il',X,X)) * sigma_res_sq + var_beta = np.linalg.inv(np.einsum('ji,jl->il',XX,XX)) * sigma_res_sq bse = [var_beta[i,i]**0.5 for i in range(n_reg)] result = [(model.coef_[i], bse[i]) for i in range(n_reg)] @@ -204,6 +212,11 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po components: list of tuples the tensor components to be considered for regression + + add_intercept: bool + whether to extemd the dataset to account for a constant offset in the + regression model. + Returns: -------- list of fitted parameters @@ -283,20 +296,19 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod if ranges != None and model_points != None: print('Trimming dataset for correlation plot') - x = self.trim_data(x, ranges, model_points) - y = self.trim_data(y, ranges, model_points) + X = self.trim_data(x, ranges, model_points) + Y = self.trim_data(y, ranges, model_points) - x=x.flatten() - y=y.flatten() + X=X.flatten() + Y=Y.flatten() with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') g=sns.JointGrid() - sns.scatterplot(x=x, y=y, ax=g.ax_joint) - sns.kdeplot(x=x, ax=g.ax_marg_x) - sns.histplot(y=y, ax=g.ax_marg_y, kde=True) - + sns.scatterplot(x=X, y=Y, ax=g.ax_joint) + sns.kdeplot(x=X, ax=g.ax_marg_x) + sns.histplot(y=Y, ax=g.ax_marg_y, kde=True) g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) g.fig.tight_layout() return g @@ -345,30 +357,108 @@ def visualize_many_correlations(self, data, labels, ranges=None, model_points=No for i in range(len(data)): data[i]=self.trim_data(data[i], ranges, model_points) + Data = [] for i in range(len(data)): - data[i]=data[i].flatten() + Data.append(data[i].flatten()) - data=np.column_stack(data) - data_df = pd.DataFrame(data, columns=labels) + Data=np.column_stack(Data) + Data_df = pd.DataFrame(Data, columns=labels) with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') - g=sns.PairGrid(data_df) + g=sns.PairGrid(Data_df) g.map_upper(sns.scatterplot) g.map_lower(sns.kdeplot) g.map_diag(sns.histplot, kde=True) g.fig.tight_layout() return g - def PCA_check_regressor_correlation(): + def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_wanted = 1.): """ - Idea: pass a large list of quantities that are correlated with a residual/closure coefficient. - Check if there is a smaller subset: pass percentage of variance as parameter, like 70%, - and the # of principal components to be retained will be computed from the eigenvalues of the - correlation matrix, that is the loadings. - Then return the PCA weights: this will be later used as regressors for the residuals. + Idea: pass a large list of quantities that are correlated with a residual/closure coeff. + Check if there is a smaller subset of principal components to be retained that are sufficient + to explain the enough of the observed variance in the dataset. + + AIM: linear dimensionality reductions for regressors + + Parameters: + ----------- + data: list of gridded data + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + var_wanted: float + should be a number between 0. and 1. : the percetage of variance to be retained. + + Returns: + -------- + comp_decomp: array of shape (n_var, n_comp) + Matrix whose columns contains the decomposition of the principal components + written in the basis of the untransformed (original) variables. + + g: result of self.visualize_many_correlations --> show the PCs are indeed uncorrelated. + + Notes: + ------ + """ - pass + # CHECKING AND PREPROCESSING THE DATA + ref_shape = data[0].shape + n_vars = len(data) + for i in range(1, n_vars): + if data[i].shape != ref_shape: + print(f'The {i}-th feature passed is not aligned with the first, removing {i}-th feature.') + data.remove(data[i]) + + if len(data)==0: + print('No two vars are compatible. Exiting.') + return None + + if ranges != None and model_points != None: + print('Trimming dataset for PCA analysis.') + for i in range(len(data)): + data[i] = self.trim_data(data[i], ranges, model_points) + + for i in range(n_vars): + data[i] = data[i].flatten() + + #STANDARDIZING DATA TO ZERO MEAN AND UNIT VARIANCE + Data = [] + for i in range(len(data)): + x = data[i] + mean = np.mean(x) + var = np.var(x) + y = np.array([x[j]-mean for j in range(len(x))]) + Data.append(y/var) + Data = np.column_stack(tuple([Data[i] for i in range(n_vars)])) + + # HOW MANY PRINCIPAL COMPONENTS HAVE TO BE RETAINED? + pca_model = PCA().fit(Data) + n_comp=0 + exp_var_sum = np.cumsum(pca_model.explained_variance_ratio_) + var_captured = exp_var_sum[n_comp] + while var_captured < var_wanted: + n_comp +=1 + var_captured= exp_var_sum[n_comp] + n_comp = n_comp+1 + + # WORKING OUT THE COMPONENTS DECOMPOSITION + var2comp = pca_model.components_ # shape: n_comp * n_var + comp_decomp = [] + for i in range(n_comp): + # should be the inverse but var2comp is unitary, so transpose + comp_decomp.append(np.einsum('ij->ji', var2comp)[:,i]) + + # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEIR INDEP + pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) + labels = [f'{i} comp' for i in range(n_comp)] + pc_for_plotting = [pc_profiles[:,i] for i in range(n_comp)] + g = self.visualize_many_correlations(pc_for_plotting, labels) + + return comp_decomp, g def PCA_find_regressors(): """ @@ -407,3 +497,5 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met plt.show() +if __name__ == '__main__': + pass diff --git a/MesoModels.py b/MesoModels.py index c82485a..9226848 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -1064,7 +1064,7 @@ def EL_style_closure(self): finally: self.meso_vars[key][h,i,j] = values[idx] - def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Points = None): + def EL_style_closure_regression(self, CoefficientAnalysis): """ Takes in a list of correlation quantities (strings? The regressors needed are computed previously via closure ingredients) @@ -1072,7 +1072,6 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po Plot the correlations with the regressors and perform the regression using methods from a Coefficient Analysis instance. """ - # FICTITIOUS RANGES AND POINTS TO TEST TRIM_DATA ROUTINE ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ @@ -1080,10 +1079,10 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po points = self.domain_vars['Points'] # TESTING SCALAR REGRESSION ROUTINE - # y = self.meso_vars['zeta'] - # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) - # print(*stats_result) + y = self.meso_vars['zeta'] + X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) + print(*stats_result) # TESTING TENSOR REGRESSION ROUTINE # y=self.meso_vars['q_res'] @@ -1106,6 +1105,13 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) # plt.savefig('Many_correlations_plot.pdf', format='pdf') + # TESTING THE CHECK REGRESSORS ROUTINE: + data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + labels=['b_tilde', 'eps_tilde', 'n_tilde'] + g1=CoefficientAnalysis.visualize_many_correlations(data, labels) + comps, g2 = CoefficientAnalysis.PCA_find_regressors_subset(data, ranges = ranges, model_points = points) + plt.show() + if __name__ == '__main__': @@ -1122,7 +1128,8 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.35],[0.4, 0.45]],1) + # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.4],[0.4, 0.5]],1) + meso_model.setup_meso_grid([[1.501, 1.502],[0.3, 0.35],[0.4, 0.45]],1) print(meso_model.domain_vars['Points']) meso_model.find_observers() meso_model.filter_micro_variables() @@ -1137,10 +1144,8 @@ def EL_style_closure_regression(self, CoefficientAnalysis, ranges = None, num_Po meso_model.closure_ingredients() # meso_model.shear_regression_test() meso_model.EL_style_closure() - - visualizer = Plotter_2D() - regressor = CoefficientsAnalysis(visualizer, meso_model.spatial_dims) - result = meso_model.EL_style_closure_regression(regressor) + regressor = CoefficientsAnalysis() + meso_model.EL_style_closure_regression(regressor) # print('Total time is {}'.format(time.process_time() - CPU_start_time)) From 6c336ab30628281bdc960761fc1e1d87e63f991b Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 1 Nov 2023 15:34:53 +0100 Subject: [PATCH 027/111] TC work on eta+kappa via squaring --- Analysis.py | 4 +- MesoModels.py | 206 ++++++++++++++++++++++++------------------------- MicroModels.py | 29 ------- 3 files changed, 103 insertions(+), 136 deletions(-) diff --git a/Analysis.py b/Analysis.py index 61edf49..4ceb68c 100644 --- a/Analysis.py +++ b/Analysis.py @@ -433,6 +433,7 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w var = np.var(x) y = np.array([x[j]-mean for j in range(len(x))]) Data.append(y/var) + # Data.append(y) Data = np.column_stack(tuple([Data[i] for i in range(n_vars)])) # HOW MANY PRINCIPAL COMPONENTS HAVE TO BE RETAINED? @@ -452,7 +453,8 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w # should be the inverse but var2comp is unitary, so transpose comp_decomp.append(np.einsum('ij->ji', var2comp)[:,i]) - # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEIR INDEP + # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEY'RE UNCORRELATED + # this block is temporary and will be removed later. pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) labels = [f'{i} comp' for i in range(n_comp)] pc_for_plotting = [pc_profiles[:,i] for i in range(n_comp)] diff --git a/MesoModels.py b/MesoModels.py index 9226848..7992df3 100644 --- a/MesoModels.py +++ b/MesoModels.py @@ -418,12 +418,16 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): for var in self.meso_structures: self.meso_structures[var] = [] - self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde'] + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde', 'T_tilde'] self.meso_vectors_strs = ['u_tilde', 'q_res', 'e_tilde', 'j', 'J_tilde'] self.meso_r2tensors_strs = ['pi_res'] self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs self.meso_vars = dict.fromkeys(self.meso_vars_strs) + self.coefficient_strs = ["Gamma"] + self.coefficients = dict.fromkeys(self.coefficient_strs) + self.coefficients['Gamma'] = 4.0/3.0 + # Dictionary with stencils and coefficients for finite differencing self.differencing = dict.fromkeys((1, 2)) self.differencing[1] = dict.fromkeys(['fw', 'bw', 'cen']) @@ -437,7 +441,8 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): # dictionary with non-local quantities (keys must match one of meso_vars or structure) - self.nonlocal_vars_strs = ['u_tilde', 'Fab'] + self.nonlocal_vars_strs = ['u_tilde', 'Fab', 'T_tilde'] + # self.nonlocal_vars_strs = ['u_tilde', 'Fab'] Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] self.deriv_vars = dict.fromkeys(Dstrs) @@ -663,6 +668,7 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): # THOMAS'S WAY (SAVE THEM AS TENSORS) self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) self.deriv_vars['D_Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1, self.spatial_dims+1)) + self.deriv_vars['D_T_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) # MARCUS'S WAY # for str in self.non_local_vars_strs: @@ -720,10 +726,16 @@ def filter_micro_variables(self): obs = self.filter_vars['U'][h,i,j] for struct in self.meso_structures: self.meso_structures[struct][h,i,j] = self.filter.filter_var_point(struct, point, obs) - self.meso_vars['p'][h,i,j] = self.filter.filter_var_point('p', point, obs) + self.meso_vars['p_filt'][h,i,j] = self.filter.filter_var_point('p', point, obs) else: print('Could not filter at {}: observer not found.'.format(point)) - + + def p_from_EOS(self, eps, n): + """ + Compute pressure from Gamma-law EoS + """ + return (self.coefficients['Gamma']-1)*eps*n + def decompose_structures_gridpoint(self, h, i, j): """ Decompose the fluid part of SET as well as Fab at grid point (h,i,j) @@ -754,6 +766,8 @@ def decompose_structures_gridpoint(self, h, i, j): q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, T_ab) s = np.einsum('ii',s_ab) + p_t = self.p_from_EOS(eps_t, n_t) + # Storing the decomposition with appropriate names. self.meso_vars['n_tilde'][h,i,j] = n_t @@ -761,12 +775,10 @@ def decompose_structures_gridpoint(self, h, i, j): self.meso_vars['eps_tilde'][h,i,j] = eps_t self.meso_vars['q_res'][h,i,j,:] = q_a self.meso_vars['pi_res'][h,i,j,:,:] = s_ab - np.multiply(s / self.spatial_dims , self.metric +np.outer(u_t,u_t)) - p_filt = self.meso_vars['p'][h,i,j] - self.meso_vars['eos_res'][h,i,j] = 0 - # p_t = p_from_EOS method(eps_t, n_t) - # self.meso_vars['eos_res'][h,i,j] = p_filt - p_t - self.meso_vars['Pi_res'][h,i,j] = s - p_filt -self.meso_vars['eos_res'][h,i,j] - + self.meso_vars['p_tilde'][h,i,j] = p_t + self.meso_vars['Pi_res'][h,i,j] = s - p_t + self.meso_vars['eos_res'][h,i,j] = self.meso_vars['p_filt'][h,i,j] - p_t + self.meso_vars['T_tilde'][h,i,j] = p_t / n_t # Repeat for the Faraday tensor (which is rank (0,2)) Fab = self.meso_structures['Fab'][h,i,j] @@ -789,7 +801,7 @@ def decompose_structures(self): else: print('Structures not decomposed at {}: observer could not be found.'.format(point)) - def calculate_derivative_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): + def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): """ Calculate partial derivative in the input 'direction' of the variable corresponding to 'nonlocal_var_str' at the position on the grid identified by indices h,i,j. The order of the differencing scheme @@ -878,7 +890,7 @@ def calculate_derivatives(self): for dir in range(self.spatial_dims+1): for str in self.nonlocal_vars_strs: dstr = 'D_' + str - self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivative_gridpoint(str, h, i, j, dir) + self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivatives_gridpoint(str, h, i, j, dir) else: print('Derivatives not calculated at {}: observer could not be found.'.format(point)) @@ -913,39 +925,41 @@ def closure_ingredients_gridpoint(self, h, i, j): # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS u_t = self.meso_vars['u_tilde'][h,i,j] u_t_cov = np.einsum('ij,j->i', self.metric, u_t) - DFab = self.deriv_vars['D_Fab'][h,i,j] + Nabla_Fab = self.deriv_vars['D_Fab'][h,i,j] - # MODELLING THE DISSIPATIVE TERMS IN THE FLUID SET - WORKING WITH (2,0) + # CLOSURE INGREDIENTS: FLUID BIT - WORKING WITH (2,0) nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor nabla_u = np.einsum('ij,jk->ik', self.metric, nabla_u) #this is a rank (2,0) tensor acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector - # The following two quantities should be identically zero, but numerical errors. + # The following two quantities should be identically zero, but they won't be due to numerical errors. # Computing them and removing to project velocity gradients unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor - # Checking orthogonality - # print('Checking orthogonality of u_t with Daub. \n First component: {}\n *********\n Second component: {}'.format(np.einsum('i,ij->j', u_t_cov, Daub), \ - # np.einsum('ij,j->i', Daub, u_t_cov))) h_ab = self.metric + np.einsum('i,j->ij', u_t, u_t) exp_t = np.einsum('ii->',Daub) shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/self.spatial_dims * exp_t, h_ab) vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) - # print('Expansion: \n {} \n ***** \n Shear: \n {} \n ***** \n Vorticity: \n{}'.format(exp_t, shear_t, vort_t)) - - # # THINK WHAT TO DO WITH THE HEAT FLUX --> GRADIENTS IN T OR N AND EPS - # # THINK ABOUT THE EM PART, DO YOU NEED MORE? - # # COME BACK TO THIS LATER? + + # CLOSURE INGREDIENTS: HEAT FLUX + nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] + projector = np.zeros((3,3)) + np.fill_diagonal(projector, 1) + projector += np.einsum('i,j->ij',u_t_cov, u_t) + DaT = np.einsum('ij,j->i', projector, nabla_T) + Theta_tilde = DaT + np.multiply(self.meso_vars['T_tilde'][h,i,j], np.einsum('ij,j->i', self.metric, acc_t)) + + # CLOSURE INGREDIENTS: EM PART - DO YOU NEED MORE? # current = np.einsum('ij, kl, kjl->i', self.metric, self.metric, DFab) # sigma_t = np.einsum('i,ij,j->',current, self.metric, u_t) # J_t = current - np.multiply(sigma_t, u_t) # # self.meso_vars['j'][h,i,j,:] = current ?? - closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde'] - closure_vars = [shear_t, exp_t, acc_t] + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'Theta_tilde'] + closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] return closure_vars_strs, closure_vars def closure_ingredients(self): @@ -971,76 +985,45 @@ def closure_ingredients(self): finally: self.meso_vars[key][h,i,j] = values[idx] - def shear_regression_test(self): - """ - WORK IN PROGRESS: A MINIMAL REGRESSION ANALYSIS ON THE COMPONENTS OF THE SHEAR TENSOR - NEED TO FIRST WORK ON WRAPPER OF MODEL_RESIDUALS THAT STORES ALL THE INFO AT ALL POINTS - Notes: - ------ - Automatic looping over all components: how to make use of the fact that the shear is symmetric? - Better to use a wrapper? - """ - # LINEAR REGRESSION: LOOP OVER COMPONENTS AND AVERAGE - - shape = self.meso_vars['shear_tilde'][0,0,0].shape - statistics = np.zeros((tuple(list(shape)+ [3]))) - for component_idx in np.ndindex(shape): - xs = self.meso_vars['shear_tilde'][0,:,:,component_idx] - ys = self.meso_vars['pi_res'][0,:,:,component_idx] - xs = xs.flatten() - ys = ys.flatten() - sns_plot = sns.regplot(x = xs, y = ys) - # print(scst.pearsonr(xs,ys)) - statistics[component_idx] = np.array(scst.linregress(xs,ys)[:3]) - - fig = sns_plot.get_figure() - fig.tight_layout() - # plt.show() - - m_statistics = np.zeros((3,)) - for i in range(len(m_statistics)): - # for idx in np.ndindex(shape): - idx = np.ndindex(shape) - m_statistics[i] = np.mean(statistics[:,:,i]) - - - # PREPARING THE DATA FOR CONTRASTING RESIDUAL AND MODEL - eta = m_statistics[0] - # pi_model = np.zeros(self.meso_vars['pi_res'][0,:,:,:,:].shape) - - # Nx, Ny = self.domain_vars['Nx'], self.domain_vars['Ny'] - # for i in range(Nx): - # for j in range(Ny): - # if self.filter_vars['U_success'][0,i,j]: - # shear = self.model_residuals_gridpoint(0,i,j)[1][0] - # pi_model[i,j,:,:] = np.multiply(eta, shear) # Calling the same function twice! Save data! - # pi_res = self.meso_vars['pi_res'][0,:,:,:,:] - - # THIS IS A TRICK: YOU WANT TO COMPARE THE RESIDUAL WITH MODEL = SHEAR x ETA - # self.meso_vars['shear_tilde'][:,:,:] = np.multiply(eta, self.meso_vars['shear_tilde'][:,:,:] ) - for idx in np.ndindex((3,3)): - self.meso_vars['shear_tilde'][:,:,:,idx] = np.multiply(eta, self.meso_vars['shear_tilde'][:,:,:,idx]) - - - # # SETTING UP AND PLOTTING VIA MARCUS'S ROUTINES - visualizer = Plotter_2D() - t = self.domain_vars['T'][0] - x_range, y_range = [self.domain_vars['Xmin'], self.domain_vars['Xmax']] ,\ - [self.domain_vars['Ymin'], self.domain_vars['Ymax']] - # print(t, x_range, y_range) - - # print(visualizer.get_var_data(self, 'shear_tilde', t, x_range, y_range, component_indices=(0,0))) - visualizer.plot_vars(self, ['pi_res', 'shear_tilde'], t, x_range, y_range, components_indices = [(0,0),(0,0)]) - def EL_style_closure_gridpoint(self, h, i, j): """ Compute bulk, shear via scalarization + PA trick, thermal conductivity later. At a point. Returns: coefficient(s) at a point. """ - coefficients_names = ['zeta'] + coefficients_names=[] + coefficients=[] + + # CALCULATING BULK VISCOUS COEFF zeta = self.meso_vars['Pi_res'][h,i,j] / self.meso_vars['exp_tilde'][h,i,j] - coefficients = [zeta] + coefficients_names.append('zeta') + coefficients.append(zeta) + + # CALCULATING SHEAR VISCOUS COEFF + pi_res_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['pi_res'][h,i,j], self.meso_vars['pi_res'][h,i,j], self.metric, self.metric) + shear_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['shear_tilde'][h,i,j], self.meso_vars['shear_tilde'][h,i,j], self.metric, self.metric) + eta = pi_res_sq/shear_sq + # Compute sign of eta by looking at the PA of pi_res with higher eigenvalue. + # Change shear to the eigenbasis of pi_res (using that the matrix is unitary as pi_res is sym) + pi_eig, pi_eigv = np.linalg.eigh(self.meso_vars['pi_res'][h,i,j]) + transformed_shear = np.einsum('ji,jk,kl', pi_eigv, self.meso_vars['shear_tilde'][h,i,j], pi_eigv) + abs_pi_eig = np.abs(pi_eig) + pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) + eta = eta * np.sign(pi_eig[-1] / transformed_shear[pos,pos]) + coefficients_names.append('eta') + coefficients.append(eta) + + # CALCULATING THE HEAT CONDUCTIVITIY + q_res = self.meso_vars['q_res'][h,i,j] + Theta_t = self.meso_vars['Theta_tilde'][h,i,j] + q_res_sq = np.einsum('i,ij,j', q_res, self.metric, q_res) + Theta_t_sq = np.einsum('i,ij,j', Theta_t, self.metric, Theta_t) + cos_angle = np.einsum('i,ij,j', q_res, self.metric, Theta_t) / (q_res_sq * Theta_t_sq) + sign = np.sign(cos_angle) + kappa = sign * np.sqrt(q_res_sq / Theta_t_sq) + coefficients_names.append('kappa') + coefficients.append(kappa) + return coefficients_names, coefficients def EL_style_closure(self): @@ -1079,10 +1062,10 @@ def EL_style_closure_regression(self, CoefficientAnalysis): points = self.domain_vars['Points'] # TESTING SCALAR REGRESSION ROUTINE - y = self.meso_vars['zeta'] - X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) - print(*stats_result) + # y = self.meso_vars['zeta'] + # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) + # print(*stats_result) # TESTING TENSOR REGRESSION ROUTINE # y=self.meso_vars['q_res'] @@ -1100,17 +1083,25 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # fig=g1.figure # fig.savefig('Single_correlation_plot.pdf', format='pdf') - # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - # labels=['b_tilde', 'eps_tilde', 'n_tilde'] + # data=[self.meso_vars['eta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['eta', 'b_tilde', 'eps_tilde', 'n_tilde'] # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) - # plt.savefig('Many_correlations_plot.pdf', format='pdf') + + # data=[self.meso_vars['zeta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['zeta', 'b_tilde', 'eps_tilde', 'n_tilde'] + # gbis= CoefficientAnalysis.visualize_many_correlations(data, labels) + # # plt.savefig('Many_correlations_plot.pdf', format='pdf') + # plt.show() # TESTING THE CHECK REGRESSORS ROUTINE: - data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - labels=['b_tilde', 'eps_tilde', 'n_tilde'] - g1=CoefficientAnalysis.visualize_many_correlations(data, labels) - comps, g2 = CoefficientAnalysis.PCA_find_regressors_subset(data, ranges = ranges, model_points = points) - plt.show() + # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['b_tilde', 'eps_tilde', 'n_tilde'] + # g1=CoefficientAnalysis.visualize_many_correlations(data, labels) + # comps, g2 = CoefficientAnalysis.PCA_find_regressors_subset(data, ranges = ranges, model_points = points) + # g2.fig.suptitle("Correlation between PC (standardized data)") + # g2.fig.subplots_adjust(top=0.94) + # print(comps) + # plt.show() @@ -1128,24 +1119,27 @@ def EL_style_closure_regression(self, CoefficientAnalysis): CPU_start_time = time.process_time() meso_model = resMHD2D(micro_model, find_obs, filter) - # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.4],[0.4, 0.5]],1) + # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.5],[0.4, 0.6]],1) meso_model.setup_meso_grid([[1.501, 1.502],[0.3, 0.35],[0.4, 0.45]],1) - print(meso_model.domain_vars['Points']) + # print(meso_model.domain_vars['Points']) meso_model.find_observers() meso_model.filter_micro_variables() + print('Filter stage ended') # meso_model.decompose_structures_gridpoint(1,1,1) meso_model.decompose_structures() # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) meso_model.calculate_derivatives() - # meso_model.model_residuals_gridpoint(1,1,0) + # meso_model.closure_ingredients_gridpoint(1,1,0) meso_model.closure_ingredients() - # meso_model.shear_regression_test() + # print('Derivatives and Decomposition stage ended') + + # meso_model.EL_style_closure_gridpoint(1,2,3) meso_model.EL_style_closure() - regressor = CoefficientsAnalysis() - meso_model.EL_style_closure_regression(regressor) + # regressor = CoefficientsAnalysis() + # meso_model.EL_style_closure_regression(regressor) # print('Total time is {}'.format(time.process_time() - CPU_start_time)) diff --git a/MicroModels.py b/MicroModels.py index 9663177..3656499 100644 --- a/MicroModels.py +++ b/MicroModels.py @@ -447,35 +447,6 @@ def get_interpol_struct(self, var_name, point): print(f"{var} does not belong to the structures in the micro_model") return res - # def get_interpol_var(self, var_names, point): - # """ - # Returns the interpolated structure at the point - - # Parameters - # ---------- - # var : str corresponding to one of the structures - - # point : list of floats - # ordered coordinates: t,x,y - - # Return - # ------ - # Array with the interpolated values of any var - # Empty list if var is not a structure in the micro_model - - # Notes - # ----- - # Interpolation gives errors when applied to boundary - # """ - # res = [] - # for var_name in var_names: - # try: - # res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) - # except KeyError: - # print(f"{var_name} does not belong to the variables of the micro_model!") - # return res - - def get_interpol_var(self, var, point): """ Returns the interpolated variables at the point. From 4de15116151cbeb411c8e2424e08442b344017fc Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 6 Nov 2023 19:29:39 +0100 Subject: [PATCH 028/111] FileReader: add rouitne to get domain vars not written by METHOD --- FileReaders.py | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/FileReaders.py b/FileReaders.py index 6cf447f..1779599 100644 --- a/FileReaders.py +++ b/FileReaders.py @@ -114,7 +114,96 @@ def read_in_data(self, micro_model): micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ micro_model.domain_vars['y']] + def read_in_data_HDF5_missing_xy(self, micro_model): + """ + Store data from files into micro_model + + Parameters + ---------- + micro_model: class MicroModel + strs in micromodel have to be the same as hdf5 files output from METHOD. + """ + + self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) + for prim_str in micro_model.get_prim_strs(): + if prim_str == "n": + self.translating_prims[prim_str] = "rho" + else: + self.translating_prims[prim_str] = prim_str + for prim_var_str in micro_model.prim_vars: + try: + method_str = self.translating_prims[prim_var_str] + for counter in range(self.num_files): + micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+method_str][:] ) + # The [:] is for returning the arrays not the dataset + micro_model.prim_vars[prim_var_str] = np.array(micro_model.prim_vars[prim_var_str]) + except KeyError: + print(f'{method_str} is not in the hdf5 dataset: check Primitive/') + + + self.translating_aux = dict.fromkeys(micro_model.get_aux_strs()) + for aux_str in micro_model.get_aux_strs(): + self.translating_aux[aux_str] = aux_str + + for aux_var_str in micro_model.aux_vars: + try: + method_str = self.translating_aux[aux_var_str] + for counter in range(self.num_files): + micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+method_str][:] ) + micro_model.aux_vars[aux_var_str] = np.array(micro_model.aux_vars[aux_var_str]) + except KeyError: + print(f'{method_str} is not in the hdf5 dataset: check Auxiliary/') + + # As METHOD saves endTime, the time variables (and points) need to be dealt with separately + # Similar is for x,y which are not stored properly in by METHOD parallelSaveDataHDF5 + for dom_var_str in micro_model.domain_int_strs: + try: + if dom_var_str == 'nt': + pass + else: + micro_model.domain_vars[dom_var_str] = int( self.hdf5_files[0]['Domain/' + dom_var_str][:]) + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + for dom_var_str in micro_model.domain_float_strs: + try: + if dom_var_str in ['tmin', 'tmax']: + pass + else: + micro_model.domain_vars[dom_var_str] = float( self.hdf5_files[0]['Domain/' + dom_var_str][:]) + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + for dom_var_str in micro_model.domain_array_strs: + try: + if dom_var_str in ['t','points', 'x', 'y']: + pass + else: + micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + + micro_model.domain_vars['nt'] = self.num_files + for counter in range(self.num_files): + micro_model.domain_vars['t'].append( float(self.hdf5_files[counter]['Domain/endTime'][:])) + micro_model.domain_vars['t'] = np.array(micro_model.domain_vars['t']) + micro_model.domain_vars['tmin'] = np.amin(micro_model.domain_vars['t']) + micro_model.domain_vars['tmax'] = np.amax(micro_model.domain_vars['t']) + + micro_model.domain_vars['x'] = np.zeros(micro_model.domain_vars['nx']) + for i in range(len(micro_model.domain_vars['x'])): + micro_model.domain_vars['x'][i] = (micro_model.domain_vars['xmax']- micro_model.domain_vars['xmin']) / (2 * micro_model.domain_vars['nx']) + \ + i * micro_model.domain_vars['dx'] + + micro_model.domain_vars['y'] = np.zeros(micro_model.domain_vars['ny']) + for i in range(len(micro_model.domain_vars['y'])): + micro_model.domain_vars['y'][i] = (micro_model.domain_vars['ymax']- micro_model.domain_vars['ymin']) / (2 * micro_model.domain_vars['ny']) + \ + i * micro_model.domain_vars['dy'] + + micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ + micro_model.domain_vars['y']] if __name__ == '__main__': @@ -124,3 +213,4 @@ def read_in_data(self, micro_model): FileReader = METHOD_HDF5('./Data/test_res100/') MicroModel = IdealMHD_2D() FileReader.read_in_data(MicroModel) + # FileReader.read_in_data_HDF5_missing_xy(MicroModel) From a53c335bc96eaa4ddafae8182895311e9e50aafa Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 7 Nov 2023 11:01:25 +0100 Subject: [PATCH 029/111] Restructuring folder --- master_files/Analysis.py | 503 ++++++++++ master_files/FileReaders.py | 216 ++++ master_files/Filters.py | 1088 ++++++++++++++++++++ master_files/MesoModels.py | 1145 ++++++++++++++++++++++ master_files/MicroModels.py | 756 ++++++++++++++ master_files/Visualization.py | 342 +++++++ master_files/system/BaseFunctionality.py | 149 +++ 7 files changed, 4199 insertions(+) create mode 100644 master_files/Analysis.py create mode 100644 master_files/FileReaders.py create mode 100644 master_files/Filters.py create mode 100644 master_files/MesoModels.py create mode 100644 master_files/MicroModels.py create mode 100644 master_files/Visualization.py create mode 100644 master_files/system/BaseFunctionality.py diff --git a/master_files/Analysis.py b/master_files/Analysis.py new file mode 100644 index 0000000..4ceb68c --- /dev/null +++ b/master_files/Analysis.py @@ -0,0 +1,503 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Jan 24 18:02:05 2023 + +@author: marcu +""" + +# USE !SCIKIT LEARN INSTEAD? IT'S THE PACKAGE FOR MACHINE LEARNING SO! + +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +import h5py +import pickle +import seaborn as sns +import warnings +from sklearn.linear_model import LinearRegression +from sklearn.decomposition import PCA +import statsmodels.api as sm + +from system.BaseFunctionality import * +from MicroModels import * +from FileReaders import * +from Filters import * +from Visualization import * +from MesoModels import * + + + +class CoefficientsAnalysis(object): + """ + Class containing a number of methods for performing statistical analysis on gridded data. + Methods include: regression, visualizing correlations, PCAs + """ + def __init__(self): #, visualizer, spatial_dims): + """ + Nothing as yet... + + Returns + ------- + Also nothing... + + """ + # self.visualizer=visualizer # need to re-structure this... or do I + # Do you really need the visualizer here? I don't think so! + # self.spatial_dims=spatial_dims + + def trim_data(self, data, ranges, model_points): + """ + Takes input gridded data 'data' (grid points given by input 'model_points') + and trim this to lie within ranges. + + Parameters: + ----------- + data: ndarray + gridded data, shape must be compatible with model_points + + ranges: list of lists of 2 floats + the min and max in each direction + + model_points: list of list of floats + + Returns: + -------- + data trimmed to within ranges + """ + if not (len(ranges)==len(model_points) and len(ranges) == len(data.shape)): + print('Check: i) data incompatible with ranges or ii) ranges incompatible with model_points. Skipping!') + return data + else: + mins = [i[0] for i in ranges] + maxs = [i[1] for i in ranges] + start_indices = Base.find_nearest_cell(mins, model_points) + end_indices = Base.find_nearest_cell(maxs, model_points) + + IdxsToRemove = [] + num_points = [len(model_points[i]) for i in range(len(model_points))] + for i in range(len(ranges)): + IdxsToRemove.append([ j for j in range(num_points[i]) if j < start_indices[i] or j > end_indices[i]]) + + for i in range(len(ranges)): + data = np.delete(data, IdxsToRemove[i], axis=i) + + return data + + def scalar_regression(self, y, X, ranges = None, model_points = None, weights=None, add_intercept=False): + """ + Routine to perform ordinary or weighted (multivariate) regression on some gridded data. + + Parameters: + ----------- + y: ndarray of gridded scalar data + the "measurements" of the dependent quantity + + X: list of ndarrays of gridded data (treated as indep scalars) + the "data" for the regressors + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + weights: ndarray of gridded weights + + add_intercept: bool + whether to extemd the dataset to account for a constant offset in the + regression model. + + Returns: + -------- + list of fitted parameters + list of std errors associated with the fitted parameters, + (i.e. parameter * std error of the corresponding regressor.) + + Notes: + ------ + Gridded data and ranges are chosen at the model level, and passed here + as parameters, so that this external method will not change/access any + model quantity. + + Works in any dimensions! + + Intercept must be added manually: for the future, add this via true/false block. + """ + # CHECKING ALIGNMENT OF PASSED DATA + dep_shape = np.shape(y) + n_reg = len(X) + for i in range(n_reg): + if np.shape(X[i]) != dep_shape: + print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') + X.remove(X[i]) + + if len(X)==0: + print('None of the passed regressors is compatible with dep data. Exiting.') + return None + + if weights: + if np.shape(y) != np.shape(weights): + print(f'The weights passed are not not aligned with data, setting these to 1') + weights=np.ones(y.shape) + + # TRIMMING THE DATA TO WITHIN RANGES + if ranges != None and model_points != None: + print('Trimming dataset for regression') + Y = self.trim_data(y, ranges, model_points) + if weights: + Weights = self.trim_data(weights, ranges, model_points) + XX = [] + for x in X: + XX.append(self.trim_data(x, ranges, model_points)) + + # FLATTENING + FITTING + Y = Y.flatten() + if weights: + Weights = Weights.flatten() + + for i in range(len(X)): + XX[i]=XX[i].flatten() + + if add_intercept: + const = np.ones(Y.shape) + XX.insert(0,const) + n_reg = len(XX) + n_data = len(Y) + XX= np.reshape(XX, (n_data, n_reg)) + + # VERSION USING STATSMODELS + # if weights: + # model=sm.WLS(y, Xfl, W) + # result= model.fit() + # return result.params, result.bse + # else: + # model=sm.OLS(y, Xfl) + # result=model.fit() + # return result.params, result.bse + + # VERSION USING SKLEARN: + model=LinearRegression(fit_intercept=False) + if weights: + model.fit(XX,Y,sample_weight=weights) + else: + model.fit(XX,Y) + Y_hat = model.predict(XX) + res = Y - Y_hat + res_sum_of_sq = np.dot(res,res) + sigma_res_sq = res_sum_of_sq / (n_data-n_reg) + var_beta = np.linalg.inv(np.einsum('ji,jl->il',XX,XX)) * sigma_res_sq + bse = [var_beta[i,i]**0.5 for i in range(n_reg)] + + result = [(model.coef_[i], bse[i]) for i in range(n_reg)] + return result + + def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_points=None, components=None, weights=None, add_intercept=False): + """ + Wrapper of scalar_regression: if no components is passed, perform regression on each tensor component + independently, otherwise only on a subset of these. All the component-wise results are returned. + + Parameters: + ----------- + y: ndarray of gridded tensorial data + the "measurements" of the dependent quantity + + X: list of ndarrays of gridded data + the "data" for the regressors + + spatial_dims: int + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + components: list of tuples + the tensor components to be considered for regression + + add_intercept: bool + whether to extemd the dataset to account for a constant offset in the + regression model. + + Returns: + -------- + list of fitted parameters + list of std errors associated with the fitted parameters, + (i.e. parameter * std error of the corresponding regressor.) + + Notes: + ------ + Weights are taken as the same for each component here. + For future: do you need to change this? + """ + # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE + l=[0 for i in range(spatial_dims+1)] + dep_shape = y[tuple(l)].shape + for i, x in enumerate(X): + if x[tuple(l)].shape != dep_shape: + print('The {}-th regressor shape {} is not compatible with dependent shape {}. Removing it!'.format(i, x[0,0,0].shape, dep_shape)) + X.remove(x) + if len(X)==0: + print('None of the passed regressors is compatible with dep data. Exiting.') + return None + + # PREPARING COMPONENTS TO LOOP OVER: user-provided or all? + if components: + for c in components: + if len(c) != len(dep_shape): + print(f'The components indices passed {c} are not compatible with data. Ignoring this and moving on!') + components.remove(c) + else: + components = [] + for i in np.ndindex(dep_shape): + components.append(i) + + # RESHAPING: now grid indices come last, needed for flattening! + tot_shape=y.shape + reshaping=[tot_shape[i] for i in range(spatial_dims+1)] + reshaping=tuple(list(dep_shape)+reshaping) + y = y.reshape(reshaping) + for i in range(len(X)): + X[i]= X[i].reshape(reshaping) + + # SCALAR REGRESSION ON EACH COMPONENT + results = [] + for c in components: + yc = y[c] + Xc = [ x[c] for x in X] + results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) + return results + + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None): + """ + Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. + Possibility to cut data to lie within ranges. + + Parameters: + ----------- + x,y: nd.arrays + must be of the same shape + + xlabel, ylabel: strs + labels for the quantites plotted + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + + Returns: + -------- + Instance of JointGrid. Figure can then be accessed at the model level + for further adjustments. + """ + if x.shape != y.shape: + print('Cannot check correlation: data is misaligned!') + return None + + if ranges != None and model_points != None: + print('Trimming dataset for correlation plot') + X = self.trim_data(x, ranges, model_points) + Y = self.trim_data(y, ranges, model_points) + + X=X.flatten() + Y=Y.flatten() + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + g=sns.JointGrid() + sns.scatterplot(x=X, y=Y, ax=g.ax_joint) + sns.kdeplot(x=X, ax=g.ax_marg_x) + sns.histplot(y=Y, ax=g.ax_marg_y, kde=True) + g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) + g.fig.tight_layout() + return g + + def visualize_many_correlations(self, data, labels, ranges=None, model_points=None): + """ + Method that returns an instance of PairGrid of correlation plots for a list of vars. + Plotted is: + daigonal: univariate histogram (with kde superimposed) + upper triangle: scatteplots + lower triangle: bivariate kde(s) + + Possibility to cut data to lie within ranges. + + Parameters: + ----------- + data: list of nd.arrays + must be of the same shape + + labels: list of strs + labels for the quantites plotted + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + + Returns: + -------- + Instance of PairGrid. Figure can then be accessed at the model level + for further adjustments. + """ + ref_shape=data[0].shape + Idx_to_delete = [] + for i in range(1,len(data)): + if data[i].shape != ref_shape: + print(f'Cannot use {labels[i]} data: misaligned with first array. Removing it.') + Idx_to_delete.append(i) + + if len(Idx_to_delete) >= 1: + data=list(np.delete(data, Idx_to_delete, axis=0)) + + if ranges != None and model_points != None: + print('Trimming dataset for correlation plot') + for i in range(len(data)): + data[i]=self.trim_data(data[i], ranges, model_points) + + Data = [] + for i in range(len(data)): + Data.append(data[i].flatten()) + + Data=np.column_stack(Data) + Data_df = pd.DataFrame(Data, columns=labels) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + g=sns.PairGrid(Data_df) + g.map_upper(sns.scatterplot) + g.map_lower(sns.kdeplot) + g.map_diag(sns.histplot, kde=True) + g.fig.tight_layout() + return g + + def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_wanted = 1.): + """ + Idea: pass a large list of quantities that are correlated with a residual/closure coeff. + Check if there is a smaller subset of principal components to be retained that are sufficient + to explain the enough of the observed variance in the dataset. + + AIM: linear dimensionality reductions for regressors + + Parameters: + ----------- + data: list of gridded data + + ranges: list of lists of 2 floats + mins and max in each direction + + model_points: list of lists containing the gridpoints in each direction + + var_wanted: float + should be a number between 0. and 1. : the percetage of variance to be retained. + + Returns: + -------- + comp_decomp: array of shape (n_var, n_comp) + Matrix whose columns contains the decomposition of the principal components + written in the basis of the untransformed (original) variables. + + g: result of self.visualize_many_correlations --> show the PCs are indeed uncorrelated. + + Notes: + ------ + + """ + # CHECKING AND PREPROCESSING THE DATA + ref_shape = data[0].shape + n_vars = len(data) + for i in range(1, n_vars): + if data[i].shape != ref_shape: + print(f'The {i}-th feature passed is not aligned with the first, removing {i}-th feature.') + data.remove(data[i]) + + if len(data)==0: + print('No two vars are compatible. Exiting.') + return None + + if ranges != None and model_points != None: + print('Trimming dataset for PCA analysis.') + for i in range(len(data)): + data[i] = self.trim_data(data[i], ranges, model_points) + + for i in range(n_vars): + data[i] = data[i].flatten() + + #STANDARDIZING DATA TO ZERO MEAN AND UNIT VARIANCE + Data = [] + for i in range(len(data)): + x = data[i] + mean = np.mean(x) + var = np.var(x) + y = np.array([x[j]-mean for j in range(len(x))]) + Data.append(y/var) + # Data.append(y) + Data = np.column_stack(tuple([Data[i] for i in range(n_vars)])) + + # HOW MANY PRINCIPAL COMPONENTS HAVE TO BE RETAINED? + pca_model = PCA().fit(Data) + n_comp=0 + exp_var_sum = np.cumsum(pca_model.explained_variance_ratio_) + var_captured = exp_var_sum[n_comp] + while var_captured < var_wanted: + n_comp +=1 + var_captured= exp_var_sum[n_comp] + n_comp = n_comp+1 + + # WORKING OUT THE COMPONENTS DECOMPOSITION + var2comp = pca_model.components_ # shape: n_comp * n_var + comp_decomp = [] + for i in range(n_comp): + # should be the inverse but var2comp is unitary, so transpose + comp_decomp.append(np.einsum('ij->ji', var2comp)[:,i]) + + # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEY'RE UNCORRELATED + # this block is temporary and will be removed later. + pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) + labels = [f'{i} comp' for i in range(n_comp)] + pc_for_plotting = [pc_profiles[:,i] for i in range(n_comp)] + g = self.visualize_many_correlations(pc_for_plotting, labels) + + return comp_decomp, g + + def PCA_find_regressors(): + """ + Idea pass both the residuals and a large list of quantities, identify which among these are + correlated with the residual. Do this by checking the scores of the residual (the first var + of the dataset, say) on the various principal components. + Then return the weights. This will then be used in regression (possibyly after checking they're + not correlated). + """ + pass + + # Not sure about these two methods. + def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ + interp_dims, method, y_component_indices, x_component_indices): + y_data_to_plot, points = \ + self.visualizer.get_var_data(model, y_var_str, t, x_range, y_range, interp_dims, method, y_component_indices) + x_data_to_plot, points = \ + self.visualizer.get_var_data(model, x_var_str, t, x_range, y_range, interp_dims, method, x_component_indices) + fig = plt.figure(figsize=(16,16)) + sns.jointplot(x=x_data_to_plot.flatten(), y=y_data_to_plot.flatten(), kind="hex", color="#4CB391") + plt.title(y_var_str+'('+x_var_str+')') + fig.tight_layout() + plt.show() + + def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, method, component_indices): + + data_to_plot, points = \ + self.visualizer.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + + # print(data_to_plot) + + fig = plt.figure(figsize=(16,16)) + sns.displot(data_to_plot) + plt.title(var_str) + fig.tight_layout() + plt.show() + + +if __name__ == '__main__': + pass diff --git a/master_files/FileReaders.py b/master_files/FileReaders.py new file mode 100644 index 0000000..1779599 --- /dev/null +++ b/master_files/FileReaders.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Mar 31 10:00:00 2023 + +@author: Thomas +""" + +import h5py +import glob +import numpy as np + +class METHOD_HDF5(object): + + def __init__(self, directory): + """ + Set up the list of files (from hdf5) and dictionary with dataset names + in the hdf5 file. + + Parameters + ---------- + directory: string + the filenames in the directory have to be incremental (sorted is used) + """ + + hdf5_filenames = sorted( glob.glob(directory+str('*.hdf5'))) + self.hdf5_files = [] + for filename in hdf5_filenames: + self.hdf5_files.append(h5py.File(filename,'r')) + self.num_files = len(self.hdf5_files) + + self.hdf5_keys = dict.fromkeys(list(self.hdf5_files[0].keys())) + for key in self.hdf5_keys: + self.hdf5_keys[key] = list(self.hdf5_files[0][key].keys()) + + def get_hdf5_keys(self): + return self.hdf5_keys + + def read_in_data(self, micro_model): + """ + Store data from files into micro_model + + Parameters + ---------- + micro_model: class MicroModel + strs in micromodel have to be the same as hdf5 files output from METHOD. + """ + + self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) + for prim_str in micro_model.get_prim_strs(): + if prim_str == "n": + self.translating_prims[prim_str] = "rho" + else: + self.translating_prims[prim_str] = prim_str + + for prim_var_str in micro_model.prim_vars: + try: + method_str = self.translating_prims[prim_var_str] + for counter in range(self.num_files): + micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+method_str][:] ) + # The [:] is for returning the arrays not the dataset + micro_model.prim_vars[prim_var_str] = np.array(micro_model.prim_vars[prim_var_str]) + except KeyError: + print(f'{method_str} is not in the hdf5 dataset: check Primitive/') + + + self.translating_aux = dict.fromkeys(micro_model.get_aux_strs()) + for aux_str in micro_model.get_aux_strs(): + self.translating_aux[aux_str] = aux_str + + for aux_var_str in micro_model.aux_vars: + try: + method_str = self.translating_aux[aux_var_str] + for counter in range(self.num_files): + micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+method_str][:] ) + micro_model.aux_vars[aux_var_str] = np.array(micro_model.aux_vars[aux_var_str]) + except KeyError: + print(f'{method_str} is not in the hdf5 dataset: check Auxiliary/') + + # As METHOD saves endTime, the time variables (and points) need to be dealt with separately + for dom_var_str in micro_model.domain_int_strs: + try: + if dom_var_str == 'nt': + pass + else: + micro_model.domain_vars[dom_var_str] = int( self.hdf5_files[0]['Domain/' + dom_var_str][:]) + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + for dom_var_str in micro_model.domain_float_strs: + try: + if dom_var_str in ['tmin', 'tmax']: + pass + else: + micro_model.domain_vars[dom_var_str] = float( self.hdf5_files[0]['Domain/' + dom_var_str][:]) + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + for dom_var_str in micro_model.domain_array_strs: + try: + if dom_var_str in ['t','points']: + pass + else: + micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + + micro_model.domain_vars['nt'] = self.num_files + for counter in range(self.num_files): + micro_model.domain_vars['t'].append( float(self.hdf5_files[counter]['Domain/endTime'][:])) + micro_model.domain_vars['t'] = np.array(micro_model.domain_vars['t']) + micro_model.domain_vars['tmin'] = np.amin(micro_model.domain_vars['t']) + micro_model.domain_vars['tmax'] = np.amax(micro_model.domain_vars['t']) + micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ + micro_model.domain_vars['y']] + + def read_in_data_HDF5_missing_xy(self, micro_model): + """ + Store data from files into micro_model + + Parameters + ---------- + micro_model: class MicroModel + strs in micromodel have to be the same as hdf5 files output from METHOD. + """ + + self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) + for prim_str in micro_model.get_prim_strs(): + if prim_str == "n": + self.translating_prims[prim_str] = "rho" + else: + self.translating_prims[prim_str] = prim_str + + for prim_var_str in micro_model.prim_vars: + try: + method_str = self.translating_prims[prim_var_str] + for counter in range(self.num_files): + micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+method_str][:] ) + # The [:] is for returning the arrays not the dataset + micro_model.prim_vars[prim_var_str] = np.array(micro_model.prim_vars[prim_var_str]) + except KeyError: + print(f'{method_str} is not in the hdf5 dataset: check Primitive/') + + + self.translating_aux = dict.fromkeys(micro_model.get_aux_strs()) + for aux_str in micro_model.get_aux_strs(): + self.translating_aux[aux_str] = aux_str + + for aux_var_str in micro_model.aux_vars: + try: + method_str = self.translating_aux[aux_var_str] + for counter in range(self.num_files): + micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+method_str][:] ) + micro_model.aux_vars[aux_var_str] = np.array(micro_model.aux_vars[aux_var_str]) + except KeyError: + print(f'{method_str} is not in the hdf5 dataset: check Auxiliary/') + + # As METHOD saves endTime, the time variables (and points) need to be dealt with separately + # Similar is for x,y which are not stored properly in by METHOD parallelSaveDataHDF5 + for dom_var_str in micro_model.domain_int_strs: + try: + if dom_var_str == 'nt': + pass + else: + micro_model.domain_vars[dom_var_str] = int( self.hdf5_files[0]['Domain/' + dom_var_str][:]) + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + for dom_var_str in micro_model.domain_float_strs: + try: + if dom_var_str in ['tmin', 'tmax']: + pass + else: + micro_model.domain_vars[dom_var_str] = float( self.hdf5_files[0]['Domain/' + dom_var_str][:]) + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + for dom_var_str in micro_model.domain_array_strs: + try: + if dom_var_str in ['t','points', 'x', 'y']: + pass + else: + micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] + except KeyError: + print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') + + + micro_model.domain_vars['nt'] = self.num_files + for counter in range(self.num_files): + micro_model.domain_vars['t'].append( float(self.hdf5_files[counter]['Domain/endTime'][:])) + micro_model.domain_vars['t'] = np.array(micro_model.domain_vars['t']) + micro_model.domain_vars['tmin'] = np.amin(micro_model.domain_vars['t']) + micro_model.domain_vars['tmax'] = np.amax(micro_model.domain_vars['t']) + + micro_model.domain_vars['x'] = np.zeros(micro_model.domain_vars['nx']) + for i in range(len(micro_model.domain_vars['x'])): + micro_model.domain_vars['x'][i] = (micro_model.domain_vars['xmax']- micro_model.domain_vars['xmin']) / (2 * micro_model.domain_vars['nx']) + \ + i * micro_model.domain_vars['dx'] + + micro_model.domain_vars['y'] = np.zeros(micro_model.domain_vars['ny']) + for i in range(len(micro_model.domain_vars['y'])): + micro_model.domain_vars['y'][i] = (micro_model.domain_vars['ymax']- micro_model.domain_vars['ymin']) / (2 * micro_model.domain_vars['ny']) + \ + i * micro_model.domain_vars['dy'] + + micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ + micro_model.domain_vars['y']] + + +if __name__ == '__main__': + + from MicroModels import * + + FileReader = METHOD_HDF5('./Data/test_res100/') + MicroModel = IdealMHD_2D() + FileReader.read_in_data(MicroModel) + # FileReader.read_in_data_HDF5_missing_xy(MicroModel) diff --git a/master_files/Filters.py b/master_files/Filters.py new file mode 100644 index 0000000..501bb58 --- /dev/null +++ b/master_files/Filters.py @@ -0,0 +1,1088 @@ +# -*- coding: utf-8 -*- +""" +Created on Tue Mar 28 15:36:01 2023 + +@author: Thomas +""" + +import numpy as np +import time +import scipy.integrate as integrate +from scipy.optimize import minimize, root +from itertools import product +from system.BaseFunctionality import * + +from MicroModels import * +from FileReaders import * +from system.BaseFunctionality import * + +class FindObs_flux_min(object): + """ + Class for computing the observer by minimizing the (micro) baryon current + surface flux over a box. + Work in any dimension, read on construction from the micro-model. + """ + def __init__(self, micro_model, box_len): + """ + Parameters: + ----------- + micro_model: instance of class containing the microdata + + box_len: float, side of the box + """ + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() + self.L = box_len + + self.flux_methods = { + "gauss" : self.flux_residual_Gauss , + "linear" : self.flux_residual , + "inbuilt" : self.flux_residual_ib + } + + def set_box_length(self, bl): + self.L = bl + + def get_tetrad_from_U(self, U): + """ + Build tetrad orthogonal to unit velocity with from complete velocity vector + + Parameters: + ----------- + U: list of d+1 floats, with d the number of spatial dimensions + + Return: + ------- + list of arrays: U + d unit vectors that complete it to a orthonormal basis + """ + if len(U) != 1+self.spatial_dims: + print('The dimension of U passed is not compatible with \ + micro_model dimensionality!') + return None + + es =[] + for _ in range(self.spatial_dims): + es.append(np.zeros(self.spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1 + tetrad = [U] + for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) + tetrad += [es[i]] + return tetrad + + def get_tetrad_from_vels(self, spatial_vels): + """ + Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels + + Parameters: + ----------- + spatial_vels: list of d floats, with d the number of spatial dimensions + + Return: + ------- + list of arrays: U + d unit vectors that complete it to a orthonormal basis + """ + if len(spatial_vels) != self.spatial_dims: + print('The number of spatial velocities passed is not compatible with \ + micro_model dimensionality!') + return None + + U = np.array(Base.get_rel_vel(spatial_vels)) + return self.get_tetrad_from_U(U) + + def flux_residual(self, spatial_vels, point, lin_spacing = 10): + """ + Compute the drift of baryons through the box built from spatial_vels. + First get the center of the 2*(d+1) faces of the (d+1)-box, then build coords + for points to sample the flux through each face. + Next, approximate the flux integral as a sum + + Parameters: + ----------- + spatial_vels: list of d (spatial dimension) floats, spatial coord of vel + + point: list of d+1 floats (t,x,y) for the box center + + lin_spacing: integer, lin_spacing**spatial_dim is the # of points used to + sample the flux through each face. + + Returns: + -------- + float: absolute flux + + Notes: + ------ + Much faster than method based on inbuilt dblquad. + """ + tetrad = self.get_tetrad_from_vels(spatial_vels) + flux = 0 + + xs = [] + for i in range(self.spatial_dims): + xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) + coords = [] + for element in product(*xs): + coords.append(np.array(element)) + + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + + surf_coords = [] + for coord in coords: + temp = center + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], rem_vecs[i]) + surf_coords.append(temp) + + for coord in surf_coords: + flux += Base.Mink_dot(self.micro_model.get_interpol_var('BC', coord), vec) + + flux *= (self.L / lin_spacing) ** self.spatial_dims + return abs(flux) + + def flux_residual_Gauss(self, spatial_vels, point, order = 3): + """ + Alternative for computing manually the flux residual, using the Gauss + Legendre method. + + Parameters: + ----------- + spatial_vels: list of d (spatial dimension) floats, spatial coord of vel + + point: list of d+1 floats (t,x,y) for the box center + + order: integer, order of the Gauss-Legendre sampling method. + + Returns: + -------- + float: absolute flux + + Notes: + ------ + Much faster than method based on inbuilt dblquad. Should be more accurate + than the one based on linearly spaced points. + """ + ps1d = [] + ws1d = [] + if order == 3: + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + elif order == 4: + p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) + p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) + w1 = (18. + np.sqrt(30) ) / 36. + w2 = (18. - np.sqrt(30) ) / 36. + ps1d = [p1, -p1, p2, -p2] + ws1d = [w1, w1, w2, w2] + elif order == 5: + p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 + p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 + w1 = (322 + 13 * np.sqrt(70)) /900 + w2 = (322 - 13 * np.sqrt(70)) /900 + ps1d = [0, p1, -p1, p2, -p2] + ws1d = [128/225, w1, w1, w2, w2] + else: + print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") + return [] + + xs = [] + ws = [] + for _ in range(self.spatial_dims): + xs.append(ps1d) + ws.append(ws1d) + + coords = [] + for element in product(*xs): + coords.append(np.multiply(self.L/2, np.array(element) )) + + totws = [] + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + tetrad = self.get_tetrad_from_vels(spatial_vels) + flux = 0. + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + + surf_coords = [] + for coord in coords: + temp = center + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], rem_vecs[i]) + surf_coords.append(temp) + + for i, coord in enumerate(surf_coords): + Na = self.micro_model.get_interpol_var('BC',coord) + flux += Base.Mink_dot(Na, vec) * totws[i] + + flux *= (self.L /2 ) ** self.spatial_dims + return abs(flux) + + def flux_residual_ib(self, spatial_vels, point, abserr = 1e-7): + """ + Compute the flux residual using inbuilt method dblquad or tplquad + Based on function point_flux (nested definition) + + Parameters: + ----------- + Vs: list of d floats, spatial components of the velocity vec + + point: float, center of the box + + Returns: + -------- + tuple: absolute flux and error estimate + + Notes: + ------ + Vastly slower than method above. + As this is based on dblquad, this method works fine only if it's 2+1 dim + + Also, dblquad returns an estimate of the error. This info is presently discarded + in order to have the method return a scalar value so that this can be minimized + by find observer. + + """ + tetrad = self.get_tetrad_from_vels(spatial_vels) + flux = 0 + partial_flux = 0 + error = 0 + partial_error = 0 + + if self.spatial_dims == 2: + def point_flux(x, y , center, Vx, Vy, normal): + coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + Na = self.micro_model.get_interpol_var('BC', coords) + flux = Base.Mink_dot(Na, normal) + return flux + + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + partial_flux, partial_error = integrate.dblquad(point_flux, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, \ + args = (center, rem_vecs[0], rem_vecs[1], vec), epsabs = abserr)[:] + flux += partial_flux + error += partial_error + return abs(flux) + + elif self.spatial_dims == 3: + def point_flux(x, y, z , center, Vx, Vy, Vz, normal): + coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + np.multiply(z, Vz) + Na = self.micro_model.get_interpol_var('BC', coords) + flux = Base.Mink_dot(Na, normal) + return flux + + for vec in tetrad: + rem_vecs = [x for x in tetrad if not (x==vec).all()] + for i in range(2): + center = point + np.multiply( (-1)**i * self.L / 2, vec) + partial_flux, partial_error = integrate.tplquad(point_flux, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2,\ + args = (center, rem_vecs[0], rem_vecs[1], vec), epsabs = abserr)[:] + flux += partial_flux + error += partial_error + return abs(flux) + + def find_observer(self, point, flux_str = "gauss", initial_guess = None): + """ + Key function: minimize the flux residual and find the observer at point. + + Parameters: + ----------- + point: list of spatial_dims+1 floats (t,x,y) + + flux_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + initial_guess: DEPRECATED, list of d floats: the spatial velocities of the initial guess + if the number of spatial vels passed does not match the micro_model dimensionality, the + pointwise velocity is used instead. + + Returns: + -------- + Successful minimization: Boolean True, observer, error + Failed minimization: Boolean False, coordinates + + Notes: + ------ + To change the number of points to be used with linear or Gauss-Legendre methods, + or the absolute relative of the inbuilt method, change the default values of + the optional arguments in the corresponding methods. + """ + + guess = [] + if initial_guess is not None and len(initial_guess) == self.spatial_dims: + guess = initial_guess + else: + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + for i in range(1, len(U)): + guess.append(U[i] / U[0]) + guess = np.array(guess) + + try: + sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) + if sol.success: + observer = Base.get_rel_vel(sol.x) + if sol.fun > 1e-5: + print(f'Warning: residual is large at {point}', sol.fun) + return sol.success, observer, sol.fun + if not sol.success: + return sol.success, point + except KeyError: + print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") + return None + + def find_observers_points(self, points, flux_str = "gauss"): + """ + Key function: minimize the flux residual and find the observers for points. + flux_str determines the method to compute the flux residual. + + Parameters: + ----------- + points: list of spatial_dims+1 floats, ordered as (t,x,y) + + flux_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which minimization is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which the minimization failed + + Notes: + ------ + To change the number of points to be used with linear or Gauss-Legendre methods, + change the default values of the optional arguments in the corresponding methods. + """ + observers = [] + errors = [] + success_coords = [] + failed_coords = [] + + for point in points: + sol = self.find_observer(point, flux_str) + if sol[0]: + observers.append(sol[1]) + errors.append(sol[2]) + success_coords.append(point) + + if not sol[0]: + failed_coords.append[point] + + return [success_coords, observers, errors] , failed_coords + + def find_observers_ranges(self, num_points, ranges, flux_str = "gauss" ): + """ + Key function: minimize the flux_residual and find the observers for points + in the ranges. flux_str determines the method to compute the flux residual. + + Parameters: + ----------- + num_points: list of integers + number of points to find observers at in each direction + + ranges: list of lists of two floats [[t_min,t_max], ...] + Define the coord ranges to find observers at + + flux_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which minimization is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which the minimization failed + + Notes: + ------ + To change the number of points to be used with linear or Gauss-Legendre methods, + change the default values of the optional arguments in the corresponding methods. + """ + + list_of_coords = [] + for i in range(len(num_points)): + list_of_coords.append( np.linspace( ranges[i][0], ranges[i][-1] , num_points[i]) ) + + points = [] + for element in product(*list_of_coords): + points.append(np.array(element)) + + return self.find_observers_points(points, flux_str) + + +class FindObs_drift_root(object): + """ + Class for computing the observer by minimizing the (micro) baryon current + drift over a box. + Work in any dimension, read on construction from the micro-model. + """ + def __init__(self, micro_model, box_len): + """ + Parameters: + ----------- + micro_model: instance of micro_model, the micro data to be filtered + + box_len: float, side of the box for computing drift + """ + self.micro_model = micro_model + self.L = box_len + self.spatial_dims = micro_model.get_spatial_dims() + + self.drift_methods = { + "gauss" : self.drift_residual_gauss , + "linear" : self.drift_residual , + "inbuilt" : self.drift_residual_ib + } + + def set_box_len(self, box_len): + self.L = box_len + + def get_tetrad_from_vels(self, spatial_vels): + """ + Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels + + Parameters: + ----------- + spatial_vels: list of d floats, with d the number of spatial dimensions + + Return: + ------- + list of arrays: U + d unit vectors that complete it to a orthonormal basis + """ + if len(spatial_vels) != self.spatial_dims: + print('The number of spatial velocities passed is not compatible with \ + micro_model dimensionality!') + return None + U = Base.get_rel_vel(spatial_vels) + es =[] + for _ in range(self.spatial_dims): + es.append(np.zeros(self.spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1 + tetrad = [U] + for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) + tetrad += [es[i]] + return tetrad + + def drift_residual(self, spatial_vels, point, lin_spacing = 10): + """ + Compute the averaged baryon current over box, using linearly spaced sample points. + Then compute the drift with respect to each element in the triad completing U (built from + spatial vels) to a tetrad. + + Parameters: + ----------- + spatial_vels: list of d floats, the spatial vels of the observer + + point: list of d+1 floats, the coord of the center of the box + + lin_spacing: integer, number of points (in each direction) used for sampling the box + + Returns: + -------- + list of d floats, the drifts in each direction of the triad. + """ + tetrad = self.get_tetrad_from_vels(spatial_vels) + + xs = [] + for i in range(self.spatial_dims+1): + xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) + adapt_coords = [] + for element in product(*xs): + adapt_coords.append(np.array(element)) + + coords = [] + for coord in adapt_coords: + temp = np.array(point) + for i in range(self.spatial_dims+1): + temp += np.multiply(coord[i], tetrad[i]) + coords.append(temp) + + integral = np.zeros(1+self.spatial_dims) + for coord in coords: + DeltaV = (self.L / lin_spacing)**(1+self.spatial_dims) + integral += np.multiply(DeltaV, self.micro_model.get_interpol_var('BC', coord)) + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + + return drifts + + def drift_residual_gauss(self, spatial_vels, point, order = 3): + """ + Method to compute net drift residual through the box, using Gauss-Legendre method + + Parameters: + ----------- + spatial_vels: list of d floats, the spatial velocities of the observer + + point: list of d+1 floats ordered as (t,x,y,...), the centre of the box + + order: integer, the order of the Gauss-Legendre approximation + + Returns: + -------- + list of d floats, the average drift in each direction of the triad. + + """ + ps1d = [] + ws1d = [] + if order == 3: + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + elif order == 4: + p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) + p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) + w1 = (18. + np.sqrt(30) ) / 36. + w2 = (18. - np.sqrt(30) ) / 36. + ps1d = [p1, -p1, p2, -p2] + ws1d = [w1, w1, w2, w2] + elif order == 5: + p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 + p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 + w1 = (322 + 13 * np.sqrt(70)) /900 + w2 = (322 - 13 * np.sqrt(70)) /900 + ps1d = [0, p1, -p1, p2, -p2] + ws1d = [128/225, w1, w1, w2, w2] + else: + print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") + return [] + + xs = [] + ws = [] + for _ in range(self.spatial_dims+1): + xs.append(ps1d) + ws.append(ws1d) + + adapt_coords = [] + for element in product(*xs): + adapt_coords.append(np.multiply(self.L/2, np.array(element) )) + + totws = [] + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + tetrad = self.get_tetrad_from_vels(spatial_vels) + coords = [] + for coord in adapt_coords: + temp = np.array(point) + for i in range(self.spatial_dims+1): + temp += np.multiply(coord[i], tetrad[i]) + coords.append(temp) + + integral = np.zeros(1+self.spatial_dims) + for i, coord in enumerate(coords): + integral += np.multiply(totws[i], self.micro_model.get_interpol_var('BC', coord)) + integral *= (self.L /2 ) ** (self.spatial_dims+1) + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + + return drifts + + def drift_residual_ib(self, spatial_vels, point, abserr = 1e-7): + """ + Method to compute net drift residual through the box, using Gauss-Legendre method + + Parameters: + ----------- + spatial_vels: list of d floats, the spatial velocities of the observer + + point: list of d+1 floats ordered as (t,x,y,...), the centre of the box + + abserr = optional float, absolute error to be passed to tplquad + + Returns: + -------- + list of d floats, the average drift in each direction of the triad. + """ + if self.spatial_dims != 2: + print('In built methods for integrals can be used only in 2+1 dimensions (i.e. tplquad). The 1+1 \ + dimensional case is not really interesting so not implemented.') + return None + else: + tetrad = self.get_tetrad_from_vels(spatial_vels) + integral = np.zeros(3) + error = np.zeros(3) + + for i in range(3): + def BC_point_value(t, x, y): + coord = point + np.multiply(t, tetrad[0]) + np.multiply(x, tetrad[1]) + np.multiply(y, tetrad[2]) + return self.micro_model.get_interpol_var('BC', coord)[i] + + integral[i], error[i] = integrate.tplquad(BC_point_value, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, \ + -self.L/2, self.L/2, args = (), epsabs = abserr)[:] + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + + return drifts + + def find_observer(self, point, drift_str = "gauss", initial_guess= None): + """ + Key method: use optimize.root to find the roots of the drift function. + + Parameters: + ----------- + point: list of d+1 floats, ordered (t,x,y,...), the coordinates of the box centre + + drift_str: string, the method used to compute the net drift + must be either 'gauss', 'linear', 'inbuilt' + + initial_guess: DEPRECATED, list of d floats: the spatial velocities of the initial guess + if the number of spatial vels passed does not match the micro_model dimensionality, the + pointwise velocity is used instead. + + Returns: + -------- + Successful root-finding: Boolean True, observer, avg error + Failed minimization: Boolean False, coordinates + + Notes: + ------ + Change the optional values of the corresponding drift methods to change + the number of points used to sample the d+1 box. + """ + guess = [] + if initial_guess is not None and len(initial_guess) == self.spatial_dims: + guess = initial_guess + else: + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + for i in range(1, len(U)): + guess.append(U[i] / U[0]) + guess = np.array(guess) + + try: + sol = root(self.drift_methods[drift_str], x0 = guess, args = (point)) + if sol.success: + observer = Base.get_rel_vel(sol.x) + error = 0 + for i in range(len(sol.fun)): + error += sol.fun[i] + avg_error = error/ len(sol.fun) + if avg_error > 1e-5: + print(f'Warning: residual is large at {point}', avg_error) + return sol.success, observer, avg_error + if not sol.success: + return sol.success, point + except KeyError: + print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") + return None + + def find_observers_points(self, points, drift_str = "gauss"): + """ + Key method: use optimize.root to find the roots of the drift function. + + Parameters: + ----------- + points: list of spatial_dims+1 floats, ordered as (t,x,y) + + drift_str: string, default to "gauss" + string for the method used to compute the drifts, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which root finding is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which root finding failed + + Notes: + ------ + Change the optional values of the corresponding drift methods to change + the number of points used to sample the d+1 box. + """ + observers = [] + avg_errors = [] + success_coords = [] + failed_coords = [] + + for point in points: + sol = self.find_observer(point, drift_str) + if sol[0]: + observers.append(sol[1]) + avg_errors.append(sol[2]) + success_coords.append(point) + + if not sol[0]: + failed_coords.append[point] + + return [success_coords, observers, avg_errors] , failed_coords + + def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): + """ + Key function: minimize the flux_residual and find the observers for points + in the ranges. drift_str determines the method to compute the flux residual. + + Parameters: + ----------- + num_points: list of integers + number of points to find observers at in each direction + + ranges: list of lists of two floats [[t_min,t_max], ...] + Define the coord ranges to find observers at + + drift_str: string, default to "gauss" + string for the method used to compute the flux, must be chosen within + ['gauss', 'linear', 'inbuilt'] + + Returns: + -------- + list of: + 1) coordinates at which root finding is successful + 2) corresponding observers + 3) corresponding residual + + list of coordinates at which the root finding failed + + Notes: + ------ + Change the optional values of the corresponding drift methods to change + the number of points used to sample the d+1 box. + """ + + list_of_coords = [] + for i in range(len(num_points)): + list_of_coords.append( np.linspace( ranges[i][0], ranges[i][-1] , num_points[i]) ) + + points = [] + for element in product(*list_of_coords): + points.append(np.array(element)) + + return self.find_observers_points(points, drift_str) + + +class spatial_box_filter(object): + """ + Class for box-filtering the variables of a micro_model. + Work in any dimensions, read on construction from the micro_model. + The observers for covariant filtering are computed separately and must be + passed when the fitlering methods are called. + """ + def __init__(self, micro_model, filter_width): + """ + Parameters: + ---------- + filter_width: float, width of the filter window in each direction + + micro_model: instance of micro_model class, so to have access to its structures + """ + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() + self.filter_width = filter_width + + def set_filter_width(self, filter_width): + """ + Method to change the width of the filter. + + Parameters: + ----------- + filter_width: float + + Returns: + -------- + None + """ + self.filter_width = filter_width + + def set_micro_model(self, micro_model): + """ + Method to change the micro_model to be filtered. + + Parameters: + ----------- + micro_model: instance of micro_model class + + Returns: + -------- + None + """ + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() + + def complete_U_tetrad(self, U): + """ + Build unit vectors orthogonal to observer U + U has to be normalized to -1. + + Parameters: + ----------- + U: np.array of shape (1+spatial_dims,) + + Returns: + list of spatial_dims numpy arrays of shape (1+spatial_dims,) + which complete U to a orthonormal tetrad + """ + es =[] + for _ in range(self.spatial_dims): + es.append(np.zeros(self.spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1. + triad = [] + for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1/np.sqrt(Base.Mink_dot(vec, vec))) + triad += [es[i]] + return triad + + def filter_var_point(self, var_str, point, observer, sample_method = "gauss", num_points = 3): + """ + First complete the observer to a tetrad at the point. Then build coords for + sample points in the spatial directions adapted to observer. Then approximate the + filter integral as a Riemann sum. + + Parameters: + ----------- + var_str: string corresponding to a variable of the micro_model + + point: list of floats (t,x,y) + + observer: np.array of shape (1+spatial_dims,) . + + sample_method: string, either 'gauss' or 'linear'. Decide how to sample the d-volume for + filtering. + + num_points: integer, number of sample points in each spatial direction. + + Returns: + -------- + nd.array with shape of the variable, the box_filtered quantity. + + Notes: + ------ + Current version uses interpolated values. Should this be too expensive, change + get_interpol_var for get_var_gridpoint! + + Gauss quadrature gives more accurate results for low values of num-points, hence + reduce number of interpolations. + The alternative is to use the gridpoint method with linearly spaced sampling points. + """ + xs = [] + coords = [] + if sample_method == "linear": + for i in range(self.spatial_dims): + xs.append(np.linspace(-self.filter_width /2 , self.filter_width /2, num_points)) + for element in product(*xs): + coords.append(np.array(element)) + + vecs = self.complete_U_tetrad(observer) + sample_points = [] + for coord in coords: + temp = np.array(point) + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], vecs[i]) + sample_points.append(temp) + + filtered_var = np.zeros(self.micro_model.get_interpol_var(var_str, point).shape) + for sample in sample_points: + filtered_var += self.micro_model.get_interpol_var(var_str, sample) + + return np.multiply(filtered_var, 1 / (num_points**self.spatial_dims)) + + elif sample_method == "gauss": + ps1d = [] + ws1d = [] + ws = [] + totws = [] + if num_points == 3: + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + elif num_points == 4: + p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) + p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) + w1 = (18. + np.sqrt(30) ) / 36. + w2 = (18. - np.sqrt(30) ) / 36. + ps1d = [p1, -p1, p2, -p2] + ws1d = [w1, w1, w2, w2] + elif num_points == 5: + p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 + p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 + w1 = (322 + 13 * np.sqrt(70)) /900 + w2 = (322 - 13 * np.sqrt(70)) /900 + ps1d = [0, p1, -p1, p2, -p2] + ws1d = [128/225, w1, w1, w2, w2] + else: + print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") + return None + + for _ in range(self.spatial_dims): + xs.append(ps1d) + ws.append(ws1d) + + for element in product(*xs): + coords.append(np.multiply(self.filter_width/2, np.array(element) )) + + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + vecs = self.complete_U_tetrad(observer) + sample_points = [] + for coord in coords: + temp = np.array(point) + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], vecs[i]) + sample_points.append(temp) + + filtered_var = np.zeros(self.micro_model.get_interpol_var(var_str, point).shape) + for i, sample in enumerate(sample_points): + filtered_var += totws[i] * self.micro_model.get_interpol_var(var_str, sample) + + return np.multiply(filtered_var, 1 / (2**self.spatial_dims)) + + else: + print("Sample methods to filter variable must be either 'gauss' or 'linear'! ") + return None + + def filter_var_manypoints(self, var_str, points, observers, sample_method = "gauss", num_points = 3): + """ + Method to filter a variable in the micro_model given a list of points and observers. + + Parameters: + ----------- + var_str: string corresponding to a variable in the micro_model + + points: list of N lists of floats, ordered as (t,x,y) + + observers: list of N np.array of shape (1 + spatial_dims,) + + sample_method, num_points: string, int passed to filter method at a point to choose how to + sample the d-volume for box filtering. + + Returns: + -------- + List with the filtered var (np.float or nd.array depending on var_str) at all points + + Notes: + ------ + Current version uses interpolated values. Should this be too expensive, change + get_interpol_var for get_var_gridpoint! + """ + if len(points) != len(observers): + print("The number of points and observers do not match!") + return [] + + filtered_var = [] + for i, point in enumerate(points): + filtered_var.append(self.filter_var_point(var_str, point, observers[i], sample_method = sample_method, num_points = num_points )) + + return filtered_var + + def filter_var_point_inbuilt(self, var_str, point, observer): + """ + Computed the filtered variable using the inbuilt scipy quad method and the interpolated + values of the quantity. + + Parameters: + ----------- + var_str: string, must correspond to a string in the micro_model + + point: list of 1+spatial_dims floats, center of the box-filter + + observer: nd.array of shape (1+spatial_dims,) + + Returns: + -------- + The filtered quantity at the point, or none if the dimensionality is more than 3+1. + + Notes: + ------ + Much slower than corresponding filter_var_manypoints_ip method, this has been implemented + to check the accuracy of the alternative methods. + """ + vecs = self.complete_U_tetrad(observer) + integrand = np.zeros(self.micro_model.get_interpol_var(var_str,point).shape) + error = np.zeros(self.micro_model.get_interpol_var(var_str,point).shape) + + if self.spatial_dims == 2: + def interpol_var_adapt_coord(x, y, *ind): + p = np.array(point) + p += np.multiply(x, vecs[0]) + np.multiply(y, vecs[1]) + return self.micro_model.get_interpol_var(var_str, p)[ind] + + for ind in np.ndindex(integrand.shape): + integrand[ind], error[ind] = integrate.dblquad(interpol_var_adapt_coord, -self.filter_width/2 , self.filter_width/2, -self.filter_width/2, self.filter_width/2, \ + args = (ind), epsabs = 1e-7 )[:] + + elif self.spatial_dims ==3: + def interpol_var_adapt_coord(self, x, y, z, *ind): + p = np.array(point) + p += np.multiply(x, vecs[0]) + np.multiply(y, vecs[1]) + np.multiply(z, vecs[2]) + return self.micro_model.get_interpol_var(var_str, p)[ind] + + for ind in np.ndindex(integrand.shape): + integrand[ind], error[ind] = integrate.tplquad(interpol_var_adapt_coord, -self.filter_width/2, self.filter_width/2, -self.filter_width/2, self.filter_width/2, \ + -self.filter_width/2, self.filter_width/2, args = (point), epsabs = 1e-7 )[:] + else: + print("The inbuilt method is implemented in 2+1 and 3+1 dims only!") + return None + + return np.multiply( integrand, 1 / (self.filter_width**self.spatial_dims)), error + +if __name__ == '__main__': + CPU_start_time = time.process_time() + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + + find_obs = FindObs_drift_root(micro_model, 0.001) + # find_obs = FindObs_flux_min(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + + vars = ['BC','SETfl', 'Fab', 'SETem'] + points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] + observers = find_obs.find_observers_points(points)[0][1] + + for i in range(len(points)): + for var in vars: + CPU_start_time = time.process_time() + filtvar1 = filter.filter_var_point_inbuilt(var, points[i], observers[i]) + inbuilt_time = time.process_time() - CPU_start_time + + CPU_start_time = time.process_time() + filtvar2 = filter.filter_var_point(var, points[i], observers[i]) + gauss_time = time.process_time() - CPU_start_time + print(f"CPU time speed-up to filter {var} with Gauss method at {points[i]} is {inbuilt_time/gauss_time}. ") + print(f'Filtered quantity with Gauss is \n {filtvar2}') + print(f"Difference with method based on inbuilt integration is: \n {filtvar1[0] - filtvar2}") + print('\n************************\n') \ No newline at end of file diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py new file mode 100644 index 0000000..7992df3 --- /dev/null +++ b/master_files/MesoModels.py @@ -0,0 +1,1145 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Mar 27 18:53:53 2023 + +@author: Marcus +""" + +import numpy as np +from scipy.interpolate import interpn +import pickle +from multiprocessing import Process, Pool +from multimethod import multimethod + +import matplotlib.pyplot as plt +import seaborn as sns +import scipy.stats as scst + +from system.BaseFunctionality import * +from MicroModels import * +from FileReaders import * +from Filters import * +from Visualization import * +from Analysis import * + +class NonIdealHydro2D(object): + + def __init__(self, MicroModel, ObsFinder, Filter, interp_method = "linear"): + self.MicroModel = MicroModel + self.ObsFinder = ObsFinder + self.Filter = Filter + self.spatial_dims = 2 + self.n_dims = self.spatial_dims + 1 + self.interp_method = interp_method + + self.domain_var_strs = ('Nt','Nx','Ny','dT','dX','dY','points') + self.domain_vars = dict.fromkeys(self.domain_var_strs) + + #Dictionary for 'local' variables, + #obtained from filtering the appropriate MicroModel variables + self.filter_var_strs = ("BC","SET") + self.filter_vars = dict.fromkeys(self.filter_var_strs) + + #Dictionary for MesoModel variables + self.meso_var_strs = ("U","U_coords","U_errors","T~","N") + self.meso_vars = dict.fromkeys(self.meso_var_strs) + + #Strings for 'non-local' variables - ones we need to take derivatives of + self.nonlocal_var_strs = ("U","T~") + + #Dictionary for derivative variables - calculated by finite differencing + self.deriv_var_strs = ("dtU","dxU","dyU","dtT~","dxT~","dyT~") + self.deriv_vars = dict.fromkeys(self.deriv_var_strs) + + #Dictionary for dissipative residuals - from the NonId-SET that we take + #projections of + self.diss_residual_strs = ("Pi","q","pi") + self.diss_residuals = dict.fromkeys(self.diss_residual_strs) + + self.diss_var_strs = ("Theta","Omega","Sigma") + self.diss_vars = dict.fromkeys(self.diss_residual_strs) + + self.diss_coeff_strs = ("Zeta", "Kappa", "Eta") + self.diss_coeffs = dict.fromkeys(self.diss_coeff_strs) + + self.coefficient_strs = ("Gamma") + self.coefficients = dict.fromkeys(self.diss_coeff_strs) + self.coefficients['Gamma'] = 4.0/3.0 + + #Dictionary for all vars - useful for e.g. plotting + self.all_var_strs = self.filter_var_strs + self.meso_var_strs + self.deriv_var_strs\ + + self.diss_residual_strs + self.diss_var_strs + self.diss_coeff_strs + + # Stencils for finite-differencing + self.cen_SO_stencil = [1/12, -2/3, 0, 2/3, -1/12] + self.cen_FO_stencil = [-1/2, 0, 1/2] + self.fw_FO_stencil = [-1, 1] + self.bw_FO_stencil = [-1, 1] + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + # Run some compatability test... + compatible = True + if self.spatial_dims != self.MicroModel.spatial_dims: + compatible = False + for filter_var_str in self.filter_var_strs: + if not (filter_var_str in MicroModel.vars.keys()): + compatible = False + + if compatible: + print("Meso and Micro models are compatible!") + else: + print("Meso and Micro models are incompatible!") + + def get_all_var_strs(self): + return self.all_var_strs + + def get_model_name(self): + return 'NonIdealHydro2D' + + def find_observers(self, num_points, ranges):#, spacing): + # self.meso_vars['U_coords'], self.meso_vars['U'], self.meso_vars['U_errors'] = \ + # self.ObsFinder.find_observers(num_points, ranges, spacing)[0] + self.meso_vars['U_coords'], self.meso_vars['U'], self.meso_vars['U_errors'] = \ + self.ObsFinder.find_observers_ranges(num_points, ranges)[0] + self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] = num_points[:] + self.domain_vars['dT'] = (ranges[0][-1] - ranges[0][0]) / self.domain_vars['Nt'] + self.domain_vars['dX'] = (ranges[1][-1] - ranges[1][0]) / self.domain_vars['Nx'] + self.domain_vars['dY'] = (ranges[2][-1] - ranges[2][0]) / self.domain_vars['Ny'] + self.domain_vars['points'] = [np.linspace(ranges[0][0], ranges[0][-1], num_points[0]),\ + np.linspace(ranges[1][0], ranges[1][-1], num_points[1]),\ + np.linspace(ranges[2][0], ranges[2][-1], num_points[2])] + + def setup_variables(self): + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + n_dims = self.spatial_dims+1 + self.meso_vars['U_coords'] = np.array(self.meso_vars['U_coords']).reshape([Nt, Nx, Ny, n_dims]) + self.meso_vars['U'] = np.array(self.meso_vars['U']).reshape([Nt, Nx, Ny, n_dims]) + self.meso_vars['U_errors'] = np.array(self.meso_vars['U_errors']).reshape([Nt, Nx, Ny, 2]) + self.meso_vars['T~'] = np.zeros((Nt, Nx, Ny)) + self.meso_vars['N'] = np.zeros((Nt, Nx, Ny)) + + self.filter_vars['BC'] = np.zeros((Nt, Nx, Ny, n_dims)) + self.filter_vars['SET'] = np.zeros((Nt, Nx, Ny, n_dims,n_dims)) + + for nonlocal_var_str in self.nonlocal_var_strs: + self.deriv_vars['dt'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) + self.deriv_vars['dx'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) + self.deriv_vars['dy'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) + + self.diss_residuals['Pi'] = np.zeros((Nt, Nx, Ny)) + self.diss_vars['Theta'] = np.zeros((Nt, Nx, Ny)) + self.diss_residuals['q'] = np.zeros((Nt, Nx, Ny, n_dims)) + self.diss_vars['Omega'] = np.zeros((Nt, Nx, Ny, n_dims)) + self.diss_residuals['pi'] = np.zeros((Nt, Nx, Ny, n_dims, n_dims)) + self.diss_vars['Sigma'] = np.zeros((Nt, Nx, Ny, n_dims, n_dims)) + + # Single value for each coefficient (per data point) for now... + self.diss_coeffs['Zeta'] = np.zeros((Nt, Nx, Ny)) + self.diss_coeffs['Kappa'] = np.zeros((Nt, Nx, Ny, n_dims)) + self.diss_coeffs['Eta'] = np.zeros((Nt, Nx, Ny, n_dims, n_dims)) + + self.vars = self.filter_vars + self.vars.update(self.meso_vars) + self.vars.update(self.deriv_vars) + self.vars.update(self.diss_residuals) + self.vars.update(self.diss_vars) + self.vars.update(self.diss_coeffs) + + def p_from_EoS(self, rho, n): + """ + Calculate pressure from EoS using rho (energy density) and n (number density) + """ + p = (self.coefficients['Gamma']-1)*(rho-n) + return p + + def filter_micro_variables(self): + """ + 'Spatially' average required variables from the micromodel w.r.t. + the observers that have been found. + """ + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + self.filter_vars['BC'][h,i,j] = self.Filter.filter_var_point('BC', self.meso_vars['U_coords'][h,i,j], + self.meso_vars['U'][h,i,j]) + self.filter_vars['SET'][h,i,j] = self.Filter.filter_var_point('SET', self.meso_vars['U_coords'][h,i,j], + self.meso_vars['U'][h,i,j]) + + # self.filter_vars['n'][h,i,j] =\ + # self.Filter.filter_prim_var(self.meso_vars['U_coords'][h,i,j], self.meso_vars['U'][h,i,j], 'n') + # self.filter_vars['SET'][h,i,j] =\ + # self.Filter.filter_struc(self.meso_vars['U_coords'][h,i,j], self.meso_vars['U'][h,i,j], 'SET') + + # Should be able to convert here to only 1 filter function... (not prim/struct) + # for filter_var_str in self.filter_var_strs: + # filter_args = [(coord, U, filter_var_str) for coord, U in zip(self.U_coords, self.Us)] + # with Pool(2) as p: + # self.micro_vars[micro_var_str] = p.starmap(self.Filter.filter_var, filter_args) + # self.micro_vars[micro_var_str] = p.starmap(self.Filter.filter_prim_var, filter_args) + + def calculate_derivatives(self): + """ + Calculate the required derivatives of MesoModel variables for + constructing the non-ideal terms. + """ + for nonlocal_var_str in self.nonlocal_var_strs: + self.calculate_time_derivatives(nonlocal_var_str) + self.calculate_x_derivatives(nonlocal_var_str) + self.calculate_y_derivatives(nonlocal_var_str) + + def calculate_time_derivatives(self, nonlocal_var_str): + deriv_var_str = 'dt'+nonlocal_var_str + stencil = self.cen_FO_stencil + samples = [-1,0,1] + for h in range(self.domain_vars['Nt']): + if h == 0: + stencil = self.fw_FO_stencil + samples = [0,1] + if h == (self.domain_vars['Nt']-1): + stencil = self.fw_FO_stencil + samples = [-1,0] + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + for s in range(len(samples)): + self.deriv_vars[deriv_var_str][h,i,j] \ + += (stencil[s]*self.meso_vars[nonlocal_var_str][h+samples[s],i,j]) / self.domain_vars['dT'] + + def calculate_x_derivatives(self, nonlocal_var_str): + deriv_var_str = 'dx'+nonlocal_var_str + stencil = self.cen_FO_stencil + samples = [-1,0,1] + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + if i == 0: + stencil = self.fw_FO_stencil + samples = [0,1] + if i == (self.domain_vars['Nx']-1): + stencil = self.fw_FO_stencil + samples = [-1,0] + for j in range(self.domain_vars['Ny']): + for s in range(len(samples)): + # print((stencil[s]*self.meso_vars[nonlocal_var_str][h,i+samples[s],j]) / self.domain_vars['dX']) + self.deriv_vars[deriv_var_str][h,i,j] \ + += (stencil[s]*self.meso_vars[nonlocal_var_str][h,i+samples[s],j]) / self.domain_vars['dX'] + + def calculate_y_derivatives(self, nonlocal_var_str): + deriv_var_str = 'dy'+nonlocal_var_str + stencil = self.cen_FO_stencil + samples = [-1,0,1] + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + if j == 0: + stencil = self.fw_FO_stencil + samples = [0,1] + if j == (self.domain_vars['Ny']-1): + stencil = self.fw_FO_stencil + samples = [-1,0] + for s in range(len(samples)): + self.deriv_vars[deriv_var_str][h,i,j] \ + += (stencil[s]*self.meso_vars[nonlocal_var_str][h,i,j+samples[s]]) / self.domain_vars['dY'] + + def calculate_dissipative_residuals(self, h, i, j): + """ + Returns the inferred (residual) values of the + dissipative terms from the filtered MicroModel SET. + + Parameters + ---------- + indices : TYPE + DESCRIPTION. + + Returns + ------- + Pi_res : scalar float + Bulk viscosity + q_res : (d+1) vector of floats + Heat flux. + pi_res : (d+1)x(d+1) tensor of floats + Shear viscosity. + + """ + # Move this + # h, i, j = indices + # Filter the scalar fields + BC = self.filter_vars['BC'][h,i,j] + N = np.sqrt(-Base.Mink_dot(BC, BC)) + self.meso_vars['N'][h,i,j] = N + Id_SET = self.filter_vars['SET'][h,i,j] + U = self.meso_vars['U'][h,i,j] + + # Do required projections of SET + h_mu_nu = Base.orthogonal_projector(U, self.metric) + rho_res = Base.project_tensor(U,U,Id_SET) + + # Set Meso temperature from filtered quantities using EoS + p_tilde = self.p_from_EoS(rho_res, N) + T_tilde = p_tilde/N + self.meso_vars['T~'][h,i,j] = T_tilde + + # Calculate dissipative residual with tensor manipulation + q_res = np.einsum('ij,i,jk',Id_SET,U,h_mu_nu) + tau_res = np.einsum('ij,ik,jl',Id_SET,h_mu_nu,h_mu_nu) + tau_trace = np.trace(tau_res) + Pi_res = tau_trace - p_tilde + pi_res = tau_res - np.dot((p_tilde + Pi_res),h_mu_nu) + + # print(Pi_res, q_res, pi_res) + + self.diss_residuals['Pi'][h,i,j] = Pi_res + self.diss_residuals['q'][h,i,j] = q_res + self.diss_residuals['pi'][h,i,j] = pi_res + + def calculate_dissipative_variables(self, h, i, j): + """ + Calculates the non-ideal, dissipation terms (without coefficeints) + for the non-ideal MesoModel SET. + + Parameters + ---------- + indices : list of ints + Indices of data-point. + coord : list of floats + Coordinates in (t,x,y). + + Returns + ------- + Decomposition of the observer velocity: + + Theta : float (scalar) + Divergence (isotropic). + omega : vector of floats + Transverse momentum. + sigma : (d+1)x(d+1) tensor of floats + Symmetric, trace-free. + + """ + # h, i, j = indices + + T = self.meso_vars['T~'][h, i, j] + dtT = self.deriv_vars['dtT~'][h, i, j] + dxT = self.deriv_vars['dxT~'][h, i, j] + dyT = self.deriv_vars['dyT~'][h, i, j] + + # U = self.meso_vars['U'][h,i,j][:] + Ut, Ux, Uy = self.meso_vars['U'][h,i,j][:] + + # print(Ut, Ux, Uy) + + # dtU = self.deriv_vars['dtU'][h,i,j] + dtUt, dtUx, dtUy = self.deriv_vars['dtU'][h,i,j][:] + dxUt, dxUx, dxUy = self.deriv_vars['dxU'][h,i,j][:] + dyUt, dyUx, dyUy = self.deriv_vars['dyU'][h,i,j][:] + + # Need to do this with Einsum... + Theta = dtUt + dxUx + dyUy + a = np.array([Ut*dtUt + Ux*dxUt + Uy*dyUt, Ut*dtUx + Ux*dxUx + Uy*dyUx, Ut*dtUy + Ux*dxUy + Uy*dyUy]) + + Omega = np.array([dtT, dxT, dyT]) + np.multiply(T,a) + Sigma = np.array([[2*dtUt - (2/3)*Theta, dtUx + dxUt, dtUy + dyUt],\ + [dxUt + dtUx, 2*dxUx - (2/3)*Theta, dxUy + dyUx], + [dyUt + dtUy, dyUx + dxUy, 2*dyUy - (2/3)*Theta]]) + + self.diss_vars['Theta'][h,i,j] = Theta + self.diss_vars['Omega'][h,i,j] = Omega + self.diss_vars['Sigma'][h,i,j] = Sigma + + def calculate_dissipative_coefficients(self): + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + # parallel_args = [(h, i, j) for h in range(Nt) for i in range(Nx) for j in range(Ny)] + # # print(parallel_args) + # with Pool(2) as p: + # p.starmap(self.calculate_dissipative_residuals, parallel_args) + # p.starmap(self.calculate_dissipative_variables, parallel_args) + + # self.diss_coeffs['Zeta'] = -self.diss_residuals['Pi'] / self.diss_vars['Theta'] + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + self.calculate_dissipative_residuals(h,i,j) + + # Derivative calculations need to be done after all residuals + self.calculate_derivatives() + for h in range(self.domain_vars['Nt']): + for i in range(self.domain_vars['Nx']): + for j in range(self.domain_vars['Ny']): + self.calculate_dissipative_variables(h,i,j) + self.diss_coeffs['Zeta'][h,i,j] = -self.diss_residuals['Pi'][h,i,j] / self.diss_vars['Theta'][h,i,j] + for k in range(self.n_dims): + self.diss_coeffs['Kappa'][h,i,j,k] = -self.diss_residuals['q'][h,i,j,k] / self.diss_vars['Omega'][h,i,j,k] + for l in range(self.n_dims): + self.diss_coeffs['Eta'][h,i,j,k,l] = -self.diss_residuals['pi'][h,i,j,k,l] / self.diss_vars['Sigma'][h,i,j,k,l] + + for diss_coeff_str in self.diss_coeff_strs: + coeffs_handle = open(diss_coeff_str+'.pickle', 'wb') + pickle.dump(self.diss_coeffs[diss_coeff_str], coeffs_handle, protocol=pickle.HIGHEST_PROTOCOL) + +class resMHD2D(object): + """ + Idea: alternative meso_models (e.g. with different closures scheme) will + only have to change the method model_residuals and list of non_local_vars + (possibly but less likely decompose_structure_gridpoint) + """ + def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): + """ + ADD DESCRIPTION OF THIS + """ + self.micro_model = micro_model + self.find_obs = find_obs + self.filter = filter + self.spatial_dims = 2 + self.interp_method = interp_method + + self.domain_int_strs = ('Nt','Nx','Ny') + self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") + self.domain_array_strs = ("T","X","Y","Points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) + for var in self.domain_vars: + self.domain_vars[var] = [] + + # This is the Levi-Civita symbol, not tensor, so be careful when using it + self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ + for k in range(3)]for j in range(3) ] for i in range(3) ]) + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + self.filter_vars_strs = ['U', 'U_errors', 'U_success'] + self.filter_vars = dict.fromkeys(self.filter_vars_strs) + for var in self.filter_vars: + var = [] + + self.meso_structures_strs = ['SETfl', 'SETem', 'BC', 'Fab'] + self.meso_structures = dict.fromkeys(self.meso_structures_strs) + for var in self.meso_structures: + self.meso_structures[var] = [] + + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde', 'T_tilde'] + self.meso_vectors_strs = ['u_tilde', 'q_res', 'e_tilde', 'j', 'J_tilde'] + self.meso_r2tensors_strs = ['pi_res'] + self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs + self.meso_vars = dict.fromkeys(self.meso_vars_strs) + + self.coefficient_strs = ["Gamma"] + self.coefficients = dict.fromkeys(self.coefficient_strs) + self.coefficients['Gamma'] = 4.0/3.0 + + # Dictionary with stencils and coefficients for finite differencing + self.differencing = dict.fromkeys((1, 2)) + self.differencing[1] = dict.fromkeys(['fw', 'bw', 'cen']) + self.differencing[1]['fw'] = {'coefficients' : [-1, 1] , 'stencil' : [0, 1]} + self.differencing[1]['bw'] = {'coefficients' : [1, -1] , 'stencil' : [0, -1]} + self.differencing[1]['cen'] = {'coefficients' : [-1/2., 0, 1/2.] , 'stencil' : [-1, 0, 1]} + self.differencing[2] = dict.fromkeys(['fw', 'bw', 'cen']) + self.differencing[2]['fw'] = {'coefficients' : [-3/2., 2., -1/2.] , 'stencil' : [0, 1, 2]} + self.differencing[2]['bw'] = {'coefficients' : [3/2., -2., 1/2.] , 'stencil' : [0, -1, -2]} + self.differencing[2]['cen'] = {'coefficients' : [1/12., -2/3., 0., 2/3., -1/12.] , 'stencil' : [-2, -1, 0, 1, 2]} + + + # dictionary with non-local quantities (keys must match one of meso_vars or structure) + self.nonlocal_vars_strs = ['u_tilde', 'Fab', 'T_tilde'] + # self.nonlocal_vars_strs = ['u_tilde', 'Fab'] + Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] + self.deriv_vars = dict.fromkeys(Dstrs) + + # Additional, closure-scheme specific vars must be added appropriately using model_residuals() + + # self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs + # Run some compatibility test... + compatible = True + error = '' + if self.spatial_dims != micro_model.get_spatial_dims(): + compatible = False + error += '\nError: different dimensions.' + for struct in self.meso_structures_strs: + if struct not in self.micro_model.get_structures_strs(): + compatible = False + error += f'\nError: {struct} not in micro_model!' + + if not compatible: + print("Meso and Micro models are incompatible:"+error) + + def get_all_var_strs(self): + return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) + + def get_gridpoints(self): + """ + Pretty self-explanatory + """ + return self.domain_vars['Points'] + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + var: str corresponding to meso_structure, meso_vars or deriv_vars + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is not meso_structures/meso_vars or deriv_vars of the model. + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + if var in self.meso_structures: + return interpn(self.domain_vars['Points'], self.meso_structures[var], point, method = self.interp_method)[0] + elif var in self.meso_vars: + return interpn(self.domain_vars['Points'], self.meso_vars[var], point, method = self.interp_method)[0] + elif var in self.deriv_vars: + return interpn(self.domain_vars['Points'], self.deriv_vars[var], point, method = self.interp_method)[0] + else: + print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) + + @multimethod + def get_var_gridpoint(self, var: str, h: object, i: object, j: object): + """ + Returns variable corresponding to input 'var' at gridpoint + identified by h,i,j + + Parameters: + ----------- + var: string corresponding to structure, meso_vars or deriv_vars + + h,i,j: int + integers corresponding to position on the grid. + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest + grid-point to input 'point'. + + Notes: + ------ + This method is useful e.g. for plotting the raw data. + """ + if var in self.meso_structures: + return self.meso_structures[var][h,i,j] + elif var in self.meso_vars: + return self.meso_vars[var][h,i,j] + elif var in self.deriv_vars: + return self.deriv_vars[var][h,i,j] + else: + print("{} is not a variable of resMHD_2D!".format(var)) + return None + + @multimethod + def get_var_gridpoint(self, var: str, point: object): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + vars: string corresponding to structure, meso_vars or deriv_vars + + point: list of 2+1 floats + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest + grid-point to input 'point'. + + Notes: + ------ + This method should be used in case using interpolated values becomes + too expensive. + """ + indices = Base.find_nearest_cell(point, self.domain_vars['Points']) + if var in self.meso_structures: + return self.meso_structures[var][tuple(indices)] + elif var in self.meso_vars: + return self.meso_vars[var][tuple(indices)] + elif var in self.deriv_vars: + return self.deriv_vars[var][tuple(indices)] + else: + print("{} is not a variable of resMHD_2D!".format(var)) + return None + + def get_model_name(self): + return 'resMHD2D' + + def set_find_obs_method(self, find_obs): + self.find_obs = find_obs + + def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): + """ + Builds the meso_model grid using the micro_model grid points contained in + the input patch (defined via 'patch_bdrs'). The method allows for coarse graining + the grid in the spatial directions ONLY. + Then store the info about the meso grid and set up arrays of definite rank and size + for the quantities needed later. + + Parameters: + ----------- + patch_bdrs: list of lists of two floats, + [[tmin, tmax],[xmin,xmax],[ymin,ymax]] + + coarse_factor: integer + + Notes: + ------ + If the patch_bdrs are larger than micro_grid, the method will not set-up the meso_grid, + and an error message is printed. This is extra safety measure! + """ + + # Is the patch within the micro_model domain? + conditions = patch_bdrs[0][0] < self.micro_model.domain_vars['tmin'] or \ + patch_bdrs[0][1] > self.micro_model.domain_vars['tmax'] or \ + patch_bdrs[1][0] < self.micro_model.domain_vars['xmin'] or \ + patch_bdrs[1][1] > self.micro_model.domain_vars['xmax'] or \ + patch_bdrs[2][0] < self.micro_model.domain_vars['ymin'] or \ + patch_bdrs[2][1] > self.micro_model.domain_vars['ymax'] + + if conditions: + print('Error: the input region for filtering is larger than micro_model domain!') + return None + + #Find the nearest cell to input patch bdrs + patch_min = [patch_bdrs[0][0], patch_bdrs[1][0], patch_bdrs[2][0]] + patch_max = [patch_bdrs[0][1], patch_bdrs[1][1], patch_bdrs[2][1]] + idx_mins = Base.find_nearest_cell(patch_min, self.micro_model.domain_vars['points']) + idx_maxs = Base.find_nearest_cell(patch_max, self.micro_model.domain_vars['points']) + + # Set meso_grid spacings + self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] + self.domain_vars['Dx'] = self.micro_model.domain_vars['dx'] * coarse_factor + self.domain_vars['Dy'] = self.micro_model.domain_vars['dy'] * coarse_factor + + # Building the meso_grid + h, i, j = idx_mins[0], idx_mins[1], idx_mins[2] + while h <= idx_maxs[0]: + t = self.micro_model.domain_vars['t'][h] + self.domain_vars['T'].append(t) + h += 1 + while i <= idx_maxs[1]: + x = self.micro_model.domain_vars['x'][i] + self.domain_vars['X'].append(x) + i += coarse_factor + while j <= idx_maxs[2]: + y = self.micro_model.domain_vars['y'][j] + self.domain_vars['Y'].append(y) + j += coarse_factor + + # Saving the info about the meso_grid + self.domain_vars['Points'] = [self.domain_vars['T'], self.domain_vars['X'], self.domain_vars['Y']] + self.domain_vars['Tmin'] = np.amin(self.domain_vars['T']) + self.domain_vars['Xmin'] = np.amin(self.domain_vars['X']) + self.domain_vars['Ymin'] = np.amin(self.domain_vars['Y']) + self.domain_vars['Tmax'] = np.amax(self.domain_vars['T']) + self.domain_vars['Xmax'] = np.amax(self.domain_vars['X']) + self.domain_vars['Ymax'] = np.amax(self.domain_vars['Y']) + self.domain_vars['Nt'] = len(self.domain_vars['T']) + self.domain_vars['Nx'] = len(self.domain_vars['X']) + self.domain_vars['Ny'] = len(self.domain_vars['Y']) + + # Setup arrays for structures + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + self.meso_structures['BC'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.meso_structures['SETfl'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.meso_structures['SETem'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.meso_structures['Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + + # Setup arrays for meso_vars + for str in self.meso_scalars_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny)) + for str in self.meso_vectors_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + for str in self.meso_r2tensors_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + + # Setup arrays for filter_vars + self.filter_vars['U'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) + self.filter_vars['U_success'] = dict() + + + # Setup arrays for derivatives of the model. + # THOMAS'S WAY (SAVE THEM AS TENSORS) + self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.deriv_vars['D_Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1, self.spatial_dims+1)) + self.deriv_vars['D_T_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + + # MARCUS'S WAY + # for str in self.non_local_vars_strs: + # if str in self.meso_vars: + # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_vars[str])}) + # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_vars[str])}) + # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_vars[str])}) + # if str in self.meso_structures: + # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_structures[str])}) + # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) + # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) + + def find_observers(self): + """ + Method to compute filtering observers at grid points built with setup_meso_grid. + The observers found (and relative errors) are saved in the dictionary self.filter_vars. + Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices + on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) + + Notes: + ------ + Requires setup_meso_grid() and setup_variables() to be called first. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,x,y] + sol = self.find_obs.find_observer(point) + if sol[0]: + self.filter_vars['U'][h,i,j] = sol[1] + self.filter_vars['U_errors'][h,i,j] = sol[2] + self.filter_vars['U_success'].update({(h,i,j) : True}) + + if not sol[0]: + self.filter_vars['U_success'].update({(h,i,j) : False}) + print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) + + def filter_micro_variables(self): + """ + Filter all meso_model structures AND micro pressure at the grid_points built with setup_meso_grid(). + This method relies on filter_var_point implemented separately for the filter, e.g. spatial_box_filter + + Parameters: + ----------- + + Notes: + ------ + Requires find_observers() and setup_meso_grid() to be called first. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t, x, y] + if self.filter_vars['U_success'][h,i,j]: + obs = self.filter_vars['U'][h,i,j] + for struct in self.meso_structures: + self.meso_structures[struct][h,i,j] = self.filter.filter_var_point(struct, point, obs) + self.meso_vars['p_filt'][h,i,j] = self.filter.filter_var_point('p', point, obs) + else: + print('Could not filter at {}: observer not found.'.format(point)) + + def p_from_EOS(self, eps, n): + """ + Compute pressure from Gamma-law EoS + """ + return (self.coefficients['Gamma']-1)*eps*n + + def decompose_structures_gridpoint(self, h, i, j): + """ + Decompose the fluid part of SET as well as Fab at grid point (h,i,j) + + Parameters: + ----------- + h, i, j: integers + the indices on the grid where the decomposition is performed. + + Returns: + -------- + None + + Notes: + ------ + Decomposition of BC, fluid SET and Faraday tensor. The EM part of SET is decomposed later + via non local operations (same story for the charge current). + + """ + # Computing the Favre density and velocity + n_t = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) + u_t = np.multiply(1 / n_t, self.meso_structures['BC'][h,i,j]) + T_ab = self.meso_structures['SETfl'][h,i,j,:,:] # Remember this is rank (2,0) + + # Computing the decomposition at each point + eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, self.metric, self.metric, T_ab) + h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) # This is a rank (1,1) tensor, i.e. a real projector. + q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) + s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, T_ab) + s = np.einsum('ii',s_ab) + p_t = self.p_from_EOS(eps_t, n_t) + + + # Storing the decomposition with appropriate names. + self.meso_vars['n_tilde'][h,i,j] = n_t + self.meso_vars['u_tilde'][h,i,j,:] = u_t + self.meso_vars['eps_tilde'][h,i,j] = eps_t + self.meso_vars['q_res'][h,i,j,:] = q_a + self.meso_vars['pi_res'][h,i,j,:,:] = s_ab - np.multiply(s / self.spatial_dims , self.metric +np.outer(u_t,u_t)) + self.meso_vars['p_tilde'][h,i,j] = p_t + self.meso_vars['Pi_res'][h,i,j] = s - p_t + self.meso_vars['eos_res'][h,i,j] = self.meso_vars['p_filt'][h,i,j] - p_t + self.meso_vars['T_tilde'][h,i,j] = p_t / n_t + + # Repeat for the Faraday tensor (which is rank (0,2)) + Fab = self.meso_structures['Fab'][h,i,j] + self.meso_vars['e_tilde'][h,i,j,:] = np.einsum('ij,j->i',Fab, u_t) + self.meso_vars['b_tilde'][h,i,j] = np.einsum('ijk,i,jl,km,lm', self.Levi3D, u_t, self.metric, self.metric, Fab) + + # Corresponding 3+1 d formulae: only change in B + # np.einsum('ijkl,i,km,ln,mn->i', self.Levi4D, u_t, self.metric, self.metric, Fab) + + def decompose_structures(self): + """ + Decompose structures at all points on the meso_grid where observers could be found. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,y,x] + if self.filter_vars['U_success'][h,i,j]: + self.decompose_structures_gridpoint(h,i,j) + else: + print('Structures not decomposed at {}: observer could not be found.'.format(point)) + + def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): + """ + Calculate partial derivative in the input 'direction' of the variable corresponding to 'nonlocal_var_str' + at the position on the grid identified by indices h,i,j. The order of the differencing scheme + can also be specified, default to 1. + + Parameters: + ----------- + nonlocal_var_str: string + quantity to be taken derivate of, must be in self.nonlocal_vars_strs() + + h, i ,j: integers + indices for point on the meso_grid + + direction: integer < self.spatial_dim + 1 + + order: integer, defatult to 1 + order of the differencing scheme + + Returns: + -------- + Finite-differenced quantity at (h,i,j) + + Notes: + ------ + The method returns the value instead of storing it, so that these can be + rearranged as preferred later, that is in calculate_derivatives() + """ + + if direction > self.spatial_dims: + print('Directions are numbered from 0 up to {}'.format(self.spatial_dims)) + return None + + if order > len(self.differencing): + print('Maximum order implemented is {}: continuing with it.'.format(len(self.differencing))) + order = len(self.differencing) + + # Forward, backward or centered differencing? + k = [h,i,j][direction] + N = len(self.domain_vars['Points'][direction]) + + if k in [l for l in range(order)]: + coefficients = self.differencing[order]['fw']['coefficients'] + stencil = self.differencing[order]['fw']['stencil'] + elif k in [N-1-l for l in range(order)]: + coefficients = self.differencing[order]['bw']['coefficients'] + stencil = self.differencing[order]['bw']['stencil'] + else: + coefficients = self.differencing[order]['cen']['coefficients'] + stencil = self.differencing[order]['cen']['stencil'] + + temp = 0 + for s, sample in enumerate(stencil): + idxs = [h,i,j] + idxs[direction] += sample + temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.get_var_gridpoint(nonlocal_var_str, *idxs)) + return temp + + def calculate_derivatives(self): + """ + Compute all the derivatives of the quantities corresponding to nonlocal_vars_strs, for all + gridpoints on the meso-grid. + + Notes: + ------ + The derived quantities are stored as 'tensors' as follows: + 1st three indices refer to the position on the grid + + 4th index refers to the directionality of the derivative + + last indices (1 or 2) correspond to the components of the quantity to be derived + + The index corresponding to the derivative is covariant, i.e. down. + + Example: + + Fab [h,i,j,a,b] : h,i,j grid; a,b, spacetime components + + D_Fab[h,i,j,c,a,b]: h,i,j grid; c direction of derivative; a,b as for Fab + + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,y,x] + if self.filter_vars['U_success'][h,i,j]: + for dir in range(self.spatial_dims+1): + for str in self.nonlocal_vars_strs: + dstr = 'D_' + str + self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivatives_gridpoint(str, h, i, j, dir) + else: + print('Derivatives not calculated at {}: observer could not be found.'.format(point)) + + def closure_ingredients_gridpoint(self, h, i, j): + """ + Decompose quantities obatined via non_local operations (i.e. derivatives) as needed + for the closure scheme. This is done at gridpoint (h,i,j) and then relevant values are + returned + + Parameters: + ----------- + h, i, j: integers + the indices of the gridpoint + + Returns: + -------- + list of strings: names of the quantities computed + + list of nd.arrays with quantities computed + + Notes: + ------ + The ingredients for the closure scheme are model specific: it makes sense that the relevant + dictionary is set up not on construction. + + UNDER CONSTRUCTION: THE EM PART NEEDS MORE THINKING + + Rename: closure_ingredients_gridpoint() ?? + Coefficients can be computed here directly, but more general models will require more work. + Do not split this, at least for now + """ + # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS + u_t = self.meso_vars['u_tilde'][h,i,j] + u_t_cov = np.einsum('ij,j->i', self.metric, u_t) + Nabla_Fab = self.deriv_vars['D_Fab'][h,i,j] + + # CLOSURE INGREDIENTS: FLUID BIT - WORKING WITH (2,0) + nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor + nabla_u = np.einsum('ij,jk->ik', self.metric, nabla_u) #this is a rank (2,0) tensor + acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector + + # The following two quantities should be identically zero, but they won't be due to numerical errors. + # Computing them and removing to project velocity gradients + unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector + acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) + Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ + np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor + h_ab = self.metric + np.einsum('i,j->ij', u_t, u_t) + exp_t = np.einsum('ii->',Daub) + shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/self.spatial_dims * exp_t, h_ab) + vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + + + # CLOSURE INGREDIENTS: HEAT FLUX + nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] + projector = np.zeros((3,3)) + np.fill_diagonal(projector, 1) + projector += np.einsum('i,j->ij',u_t_cov, u_t) + DaT = np.einsum('ij,j->i', projector, nabla_T) + Theta_tilde = DaT + np.multiply(self.meso_vars['T_tilde'][h,i,j], np.einsum('ij,j->i', self.metric, acc_t)) + + # CLOSURE INGREDIENTS: EM PART - DO YOU NEED MORE? + # current = np.einsum('ij, kl, kjl->i', self.metric, self.metric, DFab) + # sigma_t = np.einsum('i,ij,j->',current, self.metric, u_t) + # J_t = current - np.multiply(sigma_t, u_t) + # # self.meso_vars['j'][h,i,j,:] = current ?? + + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'Theta_tilde'] + closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] + return closure_vars_strs, closure_vars + + def closure_ingredients(self): + """ + Wrapper of the corresponding gridpoint method. + Set up the dictionary for the closure variables not yet defined. + Loop over the grid and store the decomposed variables. + """ + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][h,i,j]: + keys, values = self.closure_ingredients_gridpoint(h,i,j) + # try-except block to extend the meso_vars dictionary + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) + shape = values[idx].shape + self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + finally: + self.meso_vars[key][h,i,j] = values[idx] + + def EL_style_closure_gridpoint(self, h, i, j): + """ + Compute bulk, shear via scalarization + PA trick, thermal conductivity later. + At a point. + Returns: coefficient(s) at a point. + """ + coefficients_names=[] + coefficients=[] + + # CALCULATING BULK VISCOUS COEFF + zeta = self.meso_vars['Pi_res'][h,i,j] / self.meso_vars['exp_tilde'][h,i,j] + coefficients_names.append('zeta') + coefficients.append(zeta) + + # CALCULATING SHEAR VISCOUS COEFF + pi_res_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['pi_res'][h,i,j], self.meso_vars['pi_res'][h,i,j], self.metric, self.metric) + shear_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['shear_tilde'][h,i,j], self.meso_vars['shear_tilde'][h,i,j], self.metric, self.metric) + eta = pi_res_sq/shear_sq + # Compute sign of eta by looking at the PA of pi_res with higher eigenvalue. + # Change shear to the eigenbasis of pi_res (using that the matrix is unitary as pi_res is sym) + pi_eig, pi_eigv = np.linalg.eigh(self.meso_vars['pi_res'][h,i,j]) + transformed_shear = np.einsum('ji,jk,kl', pi_eigv, self.meso_vars['shear_tilde'][h,i,j], pi_eigv) + abs_pi_eig = np.abs(pi_eig) + pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) + eta = eta * np.sign(pi_eig[-1] / transformed_shear[pos,pos]) + coefficients_names.append('eta') + coefficients.append(eta) + + # CALCULATING THE HEAT CONDUCTIVITIY + q_res = self.meso_vars['q_res'][h,i,j] + Theta_t = self.meso_vars['Theta_tilde'][h,i,j] + q_res_sq = np.einsum('i,ij,j', q_res, self.metric, q_res) + Theta_t_sq = np.einsum('i,ij,j', Theta_t, self.metric, Theta_t) + cos_angle = np.einsum('i,ij,j', q_res, self.metric, Theta_t) / (q_res_sq * Theta_t_sq) + sign = np.sign(cos_angle) + kappa = sign * np.sqrt(q_res_sq / Theta_t_sq) + coefficients_names.append('kappa') + coefficients.append(kappa) + + return coefficients_names, coefficients + + def EL_style_closure(self): + """ + Wrapper of EL_style_closure_gridpoint() + """ + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][h,i,j]: + keys, values = self.EL_style_closure_gridpoint(h,i,j) + # try-except block to extend the meso_vars dictionary with the dissipative coefficients + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) + shape = values[idx].shape + self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + finally: + self.meso_vars[key][h,i,j] = values[idx] + + def EL_style_closure_regression(self, CoefficientAnalysis): + """ + Takes in a list of correlation quantities (strings? The regressors needed are computed + previously via closure ingredients) + Pass the dataset to a CoefficientAnalysis instance. + Plot the correlations with the regressors and perform the regression using methods from a Coefficient + Analysis instance. + """ + # FICTITIOUS RANGES AND POINTS TO TEST TRIM_DATA ROUTINE + ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ + [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ + [self.domain_vars['Ymin'], self.domain_vars['Ymax']]] + points = self.domain_vars['Points'] + + # TESTING SCALAR REGRESSION ROUTINE + # y = self.meso_vars['zeta'] + # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) + # print(*stats_result) + + # TESTING TENSOR REGRESSION ROUTINE + # y=self.meso_vars['q_res'] + # X=[self.meso_vars['e_tilde'], self.meso_vars['u_tilde']] + # # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, components=[(0,),(2,)]) + # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, add_intercept=False) + # for i in range(len(stats_result)): + # print('{}-th component: \n {} \n\n'.format(i, stats_result[i])) + + # TESTING CORRELATION VISUALIZATION ROUTINES + # x = self.meso_vars['n_tilde'] + # y = self.meso_vars['eps_tilde'] + # labels = ['n_tilde','eps_tilde'] + # g1=CoefficientAnalysis.visualize_correlation(x, y, xlabel=labels[0], ylabel=labels[1]) + # fig=g1.figure + # fig.savefig('Single_correlation_plot.pdf', format='pdf') + + # data=[self.meso_vars['eta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['eta', 'b_tilde', 'eps_tilde', 'n_tilde'] + # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) + + # data=[self.meso_vars['zeta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['zeta', 'b_tilde', 'eps_tilde', 'n_tilde'] + # gbis= CoefficientAnalysis.visualize_many_correlations(data, labels) + # # plt.savefig('Many_correlations_plot.pdf', format='pdf') + # plt.show() + + # TESTING THE CHECK REGRESSORS ROUTINE: + # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['b_tilde', 'eps_tilde', 'n_tilde'] + # g1=CoefficientAnalysis.visualize_many_correlations(data, labels) + # comps, g2 = CoefficientAnalysis.PCA_find_regressors_subset(data, ranges = ranges, model_points = points) + # g2.fig.suptitle("Correlation between PC (standardized data)") + # g2.fig.subplots_adjust(top=0.94) + # print(comps) + # plt.show() + + + +if __name__ == '__main__': + + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + + + CPU_start_time = time.process_time() + meso_model = resMHD2D(micro_model, find_obs, filter) + + # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.5],[0.4, 0.6]],1) + meso_model.setup_meso_grid([[1.501, 1.502],[0.3, 0.35],[0.4, 0.45]],1) + # print(meso_model.domain_vars['Points']) + meso_model.find_observers() + meso_model.filter_micro_variables() + + print('Filter stage ended') + # meso_model.decompose_structures_gridpoint(1,1,1) + meso_model.decompose_structures() + + # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) + meso_model.calculate_derivatives() + + # meso_model.closure_ingredients_gridpoint(1,1,0) + meso_model.closure_ingredients() + # print('Derivatives and Decomposition stage ended') + + # meso_model.EL_style_closure_gridpoint(1,2,3) + meso_model.EL_style_closure() + # regressor = CoefficientsAnalysis() + # meso_model.EL_style_closure_regression(regressor) + + # print('Total time is {}'.format(time.process_time() - CPU_start_time)) + diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py new file mode 100644 index 0000000..6daf245 --- /dev/null +++ b/master_files/MicroModels.py @@ -0,0 +1,756 @@ +# -*- coding: utf-8 -*- +""" +Created on Fri Mar 31 10:00:02 2023 + +@author: Thomas +""" + +import numpy as np +import math +import time + +from scipy.interpolate import interpn +from multimethod import multimethod + +from FileReaders import * +from system.BaseFunctionality import * + + +# These are the symbols, so be careful when using these to construct vectors! +# levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ +# for k in range(3)]for j in range(3) ] for i in range(3) ]) + +# levi4D = np.array([[[[ np.sign(i - j) * np.sign(j - k) * np.sign(k - l) * np.sign(i - l) \ +# for l in range(4)] for k in range(4) ] for j in range(4)] for i in range(4)]) + + +class IdealMHD_2D(object): + """ + ADD SHORT DESCRIPTION OF THE CLASS AND ITS METHODS + """ + def __init__(self, interp_method = "linear"): + """ + Sets up the variables and dictionaries, strings correspond to + those used in METHOD + + Parameters + ---------- + interp_method: str + optional method to be used by interpn + """ + self.spatial_dims = 2 + self.interp_method = interp_method + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + # This is the Levi-Civita symbol, not tensor, so be careful when using it + self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ + for k in range(3)]for j in range(3) ] for i in range(3) ]) + + #Dictionary for grid: info and points + self.domain_int_strs = ('nt','nx','ny') + self.domain_float_strs = ("tmin","tmax","xmin","xmax","ymin","ymax","dt","dx","dy") + self.domain_array_strs = ("t","x","y","points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) + for str in self.domain_vars: + self.domain_vars[str] = [] + + #Dictionary for primitive var + self.prim_strs = ("vx","vy","n","p", "Bz") + self.prim_vars = dict.fromkeys(self.prim_strs) + for str in self.prim_strs: + self.prim_vars[str] = [] + + #Dictionary for auxiliary var + self.aux_strs = ("W","h","bz","bsq", "e") + self.aux_vars = dict.fromkeys(self.aux_strs) + for str in self.aux_strs: + self.aux_vars[str] = [] + + #Dictionary for structures + self.structures_strs = ("BC", "SETfl", "SETem", "Fab") + self.structures = dict.fromkeys(self.structures_strs) + for str in self.structures_strs: + self.structures[str] = [] + + def get_spatial_dims(self): + return self.spatial_dims + + def get_model_name(self): + return 'IdealMHD_2D' + + def get_domain_strs(self): + return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs + + def get_prim_strs(self): + return self.prim_strs + + def get_aux_strs(self): + return self.aux_strs + + def get_structures_strs(self): + return self.structures_strs + + def get_all_var_strs(self): + return self.get_prim_strs() + self.get_aux_strs() + self.get_structures_strs() + + def get_gridpoints(self): + """ + Pretty self-explanatory + """ + return self.domain_vars['points'] + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + var: str corresponding to primitive, auxiliary or structre variable + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + if var in self.get_prim_strs(): + return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] + elif var in self.get_aux_strs(): + return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] + elif var in self.get_structures_strs(): + return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] + else: + print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + + @multimethod + def get_var_gridpoint(self, var: str, h: object, i: object, j: object): + """ + Returns variable corresponding to input 'var' at gridpoint identified by h,i,j + + Parameters: + ----------- + var: str + String corresponding to primitive, auxiliary or structure variable + + h,i,j: int + integers corresponding to position on the grid. + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. + + Notes: + ------ + This method is useful e.g. for plotting the raw data. + """ + if var in self.get_prim_strs(): + return self.prim_vars[var][h,i,j] + elif var in self.get_aux_strs(): + return self.aux_vars[var][h,i,j] + elif var in self.get_structures_strs(): + return self.structures[var][h,i,j] + else: + print('{} is not a variable of model {}'.format(var, self.get_model_name())) + return None + + @multimethod + def get_var_gridpoint(self, var: str, point: object): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + vars: string corresponding to primitive, auxiliary or structure variable + + point: list of 2+1 floats + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. + + Notes: + ------ + This method should be used in case using interpolated values + becomes too expensive. + """ + indices = Base.find_nearest_cell(point, self.domain_vars['points']) + if var in self.get_prim_strs(): + return self.prim_vars[var][tuple(indices)] + elif var in self.get_aux_strs(): + return self.aux_vars[var][tuple(indices)] + elif var in self.get_structures_strs(): + return self.structures[var][tuple(indices)] + else: + print(f"{var} is not a variable of IdealMHD_2D!") + return None + + def setup_structures(self): + """ + Set up the structures (i.e baryon (mass) current BC, Stress-Energy and Faraday tensors + + Notes: + ------ + Structures are built as multi-dim np.arrays, with the first three indices referring + to the grid, while the last one or two refer to space-time components. + + The Faraday tensor is stored as a fully co-variant tensor (both indices down) + The stress-energy tensor is stored as a fully contra-variant tensor (both indices up) + """ + self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["SETfl"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + self.structures["SETem"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + self.structures["Fab"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + + for h in range(self.domain_vars['nt']): + for i in range(self.domain_vars['nx']): + for j in range(self.domain_vars['ny']): + vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ + self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j]]) + + fibr_b = self.aux_vars['bz'][h,i,j] + + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) + + self.structures['SETfl'][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + + self.prim_vars['p'][h,i,j] * self.metric + + self.structures['SETem'][h,i,j:,:] = (fibr_b**2) * np.outer(vel_vec, vel_vec) + (fibr_b/2) * self.metric + + self.structures['Fab'][h,i,j,:,:] = np.tensordot(self.Levi3D, vel_vec, axes = ([2,0])) * fibr_b + + + ################################ + # CORRESPONDING 3+1-d FORMULAE + ################################ + + # vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ + # self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j], self.aux_vars['W'][h,i,j] * self.prim_vars['vz'][h,i,j]]) + + # fibr_b = [0, self.aux_vars['bx'][h,i,j], self.aux_vars['by'][h,i,j], self.aux_vars['bz'][h,i,j]] + + # self.structures['Fab'][h,i,j,:,:] = np.tensordot(np.tensordot(self.Levi4D, vel_vec, axes = ([2,0])), fibr_b, axes= ([2,0])) + + # self.structures['SETem'][h,i,j,:,:] = Base.Mink_dot(fibr_b, fibr_b) * np.outer(vel_vec, vel_vec) +\ + # (Base.Mink_dot(fibr_b, fibr_b)/2) * self.metric -\ + # np.multiply(1/2., np.outer(fibr_b, fibr_b) ) + + + # This might be useful/needed + # self.all_vars = self.prim_vars + # self.all_vars.update(self.aux_vars) + # self.all_vars.update(self.structures) + + self.vars = self.prim_vars + self.vars.update(self.aux_vars) + self.vars.update(self.structures) + + +class IdealHydro_2D(object): + + def __init__(self, interp_method = "linear"): + """ + Sets up the variables and dictionaries, strings correspond to + those used in METHOD + + Parameters + ---------- + interp_method: str + optional method to be used by interpn + """ + self.spatial_dims = 2 + self.interp_method = interp_method + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + # This is the Levi-Civita symbol, not tensor, so be careful when using it + self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ + for k in range(3)]for j in range(3) ] for i in range(3) ]) + + #Dictionary for grid: info and points + self.domain_int_strs = ('nt','nx','ny') + self.domain_float_strs = ("tmin","tmax","xmin","xmax","ymin","ymax","dt","dx","dy") + self.domain_array_strs = ("t","x","y","points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs + self.domain_array_strs) + for str in self.domain_vars: + self.domain_vars[str] = [] + + #Dictionary for primitive var + self.prim_strs = ("v1","v2","rho","p","n") + self.prim_vars = dict.fromkeys(self.prim_strs) + for str in self.prim_strs: + self.prim_vars[str] = [] + + #Dictionary for auxiliary var + self.aux_strs = ("W","h","T") + self.aux_vars = dict.fromkeys(self.aux_strs) + for str in self.aux_strs: + self.aux_vars[str] = [] + + #Dictionary for structures + self.structures_strs = ("BC","SET") + self.structures = dict.fromkeys(self.structures_strs) + for str in self.structures_strs: + self.structures[str] = [] + + #Dictionary for all vars + self.var_strs = self.prim_strs + self.aux_strs + self.structures_strs + # self.vars = self.prim_vars + # self.vars.update(self.aux_vars) + # self.vars.update(self.structures) + + self.all_var_strs = self.prim_strs + self.aux_strs + self.structures_strs + + def get_model_name(self): + return 'IdealHydro_2D' + + def get_spatial_dims(self): + return self.spatial_dims + + def get_domain_strs(self): + return self.domain_info_strs + self.domain_points_strs + + def get_prim_strs(self): + return list(self.prim_vars.keys()) + + def get_aux_strs(self): + return list(self.aux_vars.keys()) + + def get_structures_strs(self): + return list(self.structures.keys()) + + def get_all_var_strs(self): + return list(self.prim_vars.keys()) + list(self.aux_vars_vars.keys()) + list(self.structures.keys()) + + def setup_structures(self): + """ + Set up the structures (i.e baryon current, SET) + + Structures are built as multi-dim np.arrays, with the first three indices referring + to the grid, while the last one or two refer to space-time components. + """ + self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + + for h in range(self.domain_vars['nt']): + for i in range(self.domain_vars['nx']): + for j in range(self.domain_vars['ny']): + vel = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['v1'][h,i,j] ,\ + self.aux_vars['W'][h,i,j] * self.prim_vars['v2'][h,i,j]]) + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel ) + + + self.structures["SET"][h,i,j,:,:] = (self.prim_vars["rho"][h,i,j] + self.prim_vars["p"][h,i,j]) * \ + np.outer(vel,vel) + self.prim_vars["p"][h,i,j] * self.metric + + self.vars = self.prim_vars + self.vars.update(self.aux_vars) + self.vars.update(self.structures) + + def get_interpol_prim(self, var_names, point): + """ + Returns the interpolated variable at the point + + Parameters + ---------- + vars : list of strings + strings have to be in to prim_vars keys + point : list of floats + ordered coordinates: t,x,y + Return + ------ + list of floats corresponding to string of vars + + Notes + ----- + Interpolation raises a ValueError when out of grid boundaries. + """ + res = [] + + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.prim_vars[var_name], point, method = self.interp_method)[0]) #, bounds_error = False) + except KeyError: + print(f"{var_name} does not belong to the primitive variables of the micro_model!") + return res + + def get_interpol_aux(self, var_names, point): + """ + Returns the interpolated variable at the point + + Parameters + ---------- + vars : list of strings + strings have to be in to aux_vars keys + point : list of floats + ordered coordinates: t,x,y + Return + ------ + list of floats corresponding to string of vars + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + res = [] + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.aux_vars[var_name], point, method = self.interp_method)[0]) + except KeyError: + print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") + return res + + def get_interpol_struct(self, var_name, point): + """ + Returns the interpolated structure at the point + + Parameters + ---------- + var : str corresponding to one of the structures + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Array with the interpolated values of the var structure + Empty list if var is not a structure in the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + res = [] + if var_name == "BC": + res = np.zeros(len(self.structures[var_name][:,0,0,0])) + for a in range(len(self.structures[var_name][:,0,0,0])): + res[a] = interpn(self.domain_vars["points"], self.structures[var_name][a,:,:,:], point, method = self.interp_method)[0] + elif var_name == "SET": + res = np.zeros((len(self.structures[var_name][:,0,0,0,0]),len(self.structures[var][0,:,0,0,0]))) + for a in range(len(self.structures[var_name][:,0,0,0,0])): + for b in range(len(self.structures[var_name][0,:,0,0,0])): + res[a,b] = interpn(self.domain_vars["points"],self.structures[var_name][a,b,:,:,:], point, method = self.interp_method)[0] + else: + print(f"{var} does not belong to the structures in the micro_model") + return res + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + vars : str corresponding to primitive, auxiliary or structre variable + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + if var in self.get_prim_strs(): + return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] + elif var in self.get_aux_strs(): + return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] + elif var in self.get_structures_strs(): + return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] + else: + print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + + + + """ + Returns the interpolated structure at the point + Parameters + ---------- + var : str corresponding to one of the structures + + point : list of floats + ordered coordinates: t,x,y + Return + ------ + Array with the interpolated values of any var + Empty list if var is not a structure in the micro_model + Notes + ----- + Interpolation gives errors when applied to boundary + """ + res = [] + for var_name in var_names: + try: + res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) + except KeyError: + print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") + return res + + +class IdealHD_2D(object): + """ + CUT AND PAST OF IdealMHD_2D --> removing the magnetic bits + """ + def __init__(self, interp_method = "linear"): + """ + Sets up the variables and dictionaries, strings correspond to + those used in METHOD + + Parameters + ---------- + interp_method: str + optional method to be used by interpn + """ + self.spatial_dims = 2 + self.interp_method = interp_method + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + #Dictionary for grid: info and points + self.domain_int_strs = ('nt','nx','ny') + self.domain_float_strs = ("tmin","tmax","xmin","xmax","ymin","ymax","dt","dx","dy") + self.domain_array_strs = ("t","x","y","points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) + for str in self.domain_vars: + self.domain_vars[str] = [] + + #Dictionary for primitive var + self.prim_strs = ("vx","vy","n","p") + self.prim_vars = dict.fromkeys(self.prim_strs) + for str in self.prim_strs: + self.prim_vars[str] = [] + + #Dictionary for auxiliary var + self.aux_strs = ("W", "h", "e") + self.aux_vars = dict.fromkeys(self.aux_strs) + for str in self.aux_strs: + self.aux_vars[str] = [] + + #Dictionary for structures + self.structures_strs = ("BC", "SET") + self.structures = dict.fromkeys(self.structures_strs) + for str in self.structures_strs: + self.structures[str] = [] + + def get_spatial_dims(self): + return self.spatial_dims + + def get_model_name(self): + return 'Ideal Hydro (2+1d)' + + def get_domain_strs(self): + return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs + + def get_prim_strs(self): + return self.prim_strs + + def get_aux_strs(self): + return self.aux_strs + + def get_structures_strs(self): + return self.structures_strs + + def get_all_var_strs(self): + return self.get_prim_strs() + self.get_aux_strs() + self.get_structures_strs() + + def get_gridpoints(self): + """ + Pretty self-explanatory + """ + return self.domain_vars['points'] + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + var: str corresponding to primitive, auxiliary or structre variable + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + + if var in self.get_prim_strs(): + return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] + elif var in self.get_aux_strs(): + return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] + elif var in self.get_structures_strs(): + return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] + else: + print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') + + @multimethod + def get_var_gridpoint(self, var: str, h: object, i: object, j: object): + """ + Returns variable corresponding to input 'var' at gridpoint identified by h,i,j + + Parameters: + ----------- + var: str + String corresponding to primitive, auxiliary or structure variable + + h,i,j: int + integers corresponding to position on the grid. + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. + + Notes: + ------ + This method is useful e.g. for plotting the raw data. + """ + if var in self.get_prim_strs(): + return self.prim_vars[var][h,i,j] + elif var in self.get_aux_strs(): + return self.aux_vars[var][h,i,j] + elif var in self.get_structures_strs(): + return self.structures[var][h,i,j] + else: + print('{} is not a variable of model {}'.format(var, self.get_model_name())) + return None + + @multimethod + def get_var_gridpoint(self, var: str, point: object): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + vars: string corresponding to primitive, auxiliary or structure variable + + point: list of 2+1 floats + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. + + Notes: + ------ + This method should be used in case using interpolated values + becomes too expensive. + """ + indices = Base.find_nearest_cell(point, self.domain_vars['points']) + if var in self.get_prim_strs(): + return self.prim_vars[var][tuple(indices)] + elif var in self.get_aux_strs(): + return self.aux_vars[var][tuple(indices)] + elif var in self.get_structures_strs(): + return self.structures[var][tuple(indices)] + else: + print(f"{var} is not a variable of IdealMHD_2D!") + return None + + def setup_structures(self): + """ + Set up the structures (i.e baryon (mass) current BC, Stress-Energy and Faraday tensors + + Notes: + ------ + Structures are built as multi-dim np.arrays, with the first three indices referring + to the grid, while the last one or two refer to space-time components. + + The Faraday tensor is stored as a fully co-variant tensor (both indices down) + The stress-energy tensor is stored as a fully contra-variant tensor (both indices up) + """ + self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) + + for h in range(self.domain_vars['nt']): + for i in range(self.domain_vars['nx']): + for j in range(self.domain_vars['ny']): + vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ + self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j]]) + + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) + + self.structures['SET'][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + + self.prim_vars['p'][h,i,j] * self.metric + + + + + self.vars = self.prim_vars + self.vars.update(self.aux_vars) + self.vars.update(self.structures) + + + +# TC +if __name__ == '__main__': + + CPU_start_time = time.process_time() + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + + point = [1.502,0.4,0.2] + # vars = ['SETfl', 'BC', 'Fab', 'SETem'] + vars = ['BC'] + for var in vars: + res = micro_model.get_interpol_var(var, point) + res2 = micro_model.get_var_gridpoint(var, point) + print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') + +# MH +# if __name__ == '__main__': + +# CPU_start_time = time.process_time() + +# FileReader = METHOD_HDF5('./Data/Testing/') +# # micro_model = IdealMHD_2D() +# micro_model = IdealHydro_2D() +# FileReader.read_in_data(micro_model) +# micro_model.setup_structures() + +# res = micro_model.get_interpol_var(['v1','rho'],[10.0,0.3,0.2]) +# print(type(res),"\n", res) + +# res = micro_model.get_interpol_var(['T','W'],[10.0,0.3,0.2]) +# print(type(res),"\n", res) + +# # res = micro_model.get_interpol_var(['bar_vel','SET'],[10.0,0.3,0.2]) +# # print(type(res),"\n", res) +# # res = micro_model.get_interpol_aux(["b0","bx"],[0.5, 0.3,0.2]) +# # print(type(res),"\n", res) + +# # res = micro_model.get_interpol_struct("SET",[2.5, 0.3, 0.2]) +# # print(res.shape, res[0],'\n') + +# # print( micro_model.get_domain_strs() ) + +# CPU_end_time = time.process_time() +# CPU_time = CPU_end_time - CPU_start_time +# print(f'The CPU time is {CPU_time} seconds') \ No newline at end of file diff --git a/master_files/Visualization.py b/master_files/Visualization.py new file mode 100644 index 0000000..954ee24 --- /dev/null +++ b/master_files/Visualization.py @@ -0,0 +1,342 @@ +# -*- coding: utf-8 -*- +""" +Created on Mon Jun 5 03:14:43 2023 + +@author: marcu +""" + + +import matplotlib.pyplot as plt +import numpy as np +import h5py +from mpl_toolkits.axes_grid1 import make_axes_locatable +from scipy.interpolate import interpn +from system.BaseFunctionality import * + +from MicroModels import * +from MesoModels import * +from Filters import * + +class Plotter_2D(object): + + def __init__(self, screen_size = [11.97, 8.36]): + """ + Parameters: + ----------- + screen_size = list of 2 floats + width and height of the screen: used to rescale plots' size. + """ + self.subplots_dims = {1 : (1,1), + 2 : (1,2), + 3 : (1,3), + 4 : (2,2), + 5 : (2,3), + 6 : (2,3)} + + self.screen_size = np.array(screen_size) + + def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, method= 'raw_data', component_indices=()): + """ + Retrieves the required data from model to plot a variable defined by + var_str over coordinates t, x_range, y_range, either from the model's + raw data or by interpolating between the model's raw data over the coords. + + Parameters + ---------- + model : Micro or Meso Model + var_str : str + Must match a variable of the model. + t : float + time coordinate (defines the foliation). + x_range : list of 2 floats: x_start and x_end + defines range of x coordinates within foliation. + y_range : list of 2 floats: y_start and y_end + defines range of y coordinates within foliation. + interp_dims : tuple of integers + defines the number of points to interpolate at in x and y directions. + method : str + currently either raw_data or interpolate. + component_indices : tuple + the indices of the component to pick out if the variable is a vector/tensor. + + Returns + ------- + data_to_plot : numpy array of floats + the (2D) data to be plotted by plt.imshow(). + extent: list of floats + + + Notes: + ------ + Logic: if method is raw_data, then no interp_dims are needed. + Better to have 'raw_data' and no interp_dims = None as default + """ + if var_str in model.get_all_var_strs(): + + # Block to check component_indices passed are compatible with shape of var to be plotted. + st1 = len(component_indices) + st2 = len(model.get_var_gridpoint(var_str, 0, 0, 0).shape) + compatible = st1==st2 + if not compatible: + if st2 == 0 : + print('WARNING: {} is a scalar but you passed some indices, ignoring this and moving on.'.format(var_str)) + component_indices = () + elif st2 != 0: + print('WARNING: {} is a tensor but you passed more/fewer indices than required. Retrieving the "first component"!'.format(var_str)) + component_indices = tuple([0 for _ in range(st2)]) + + extent = [*x_range, *y_range] + + if method == 'interpolate': + compatible = interp_dims != None and len(interp_dims) ==2 + if not compatible: + print('Error: when using (linearly spaced) interpolated data, you must' +\ + 'specify # points in each direction! Exiting.') + return None + + nx, ny = interp_dims[:] + xs, ys = np.linspace(x_range[0], x_range[1], nx), np.linspace(y_range[0], y_range[1], ny) + data_to_plot = np.zeros((nx, ny)) + + for i in range(nx): + for j in range(ny): + point = [t, xs[i], ys[j]] + data_to_plot[i, j] = model.get_interpol_var(var_str, point)[component_indices] + + elif method == 'raw_data': + start_indices = Base.find_nearest_cell([t, x_range[0], y_range[0]], model.get_gridpoints()) + end_indices = Base.find_nearest_cell([t, x_range[1], y_range[1]], model.get_gridpoints()) + + h = start_indices[0] + i_s, i_f = start_indices[1], end_indices[1] + j_s, j_f = start_indices[2], end_indices[2] + + data_shape = (i_f+ 1- i_s, j_f+ 1- j_s) + data_to_plot = np.zeros(data_shape) + + for i in range(i_f+ 1 - i_s): + for j in range(j_f+ 1 - j_s): + data_to_plot[i,j] = model.get_var_gridpoint(var_str, h, i, j)[component_indices] + + else: + print('Data method is not a valid choice! Must be interpolate or raw_data.') + return None + + # return data_to_plot, points + return data_to_plot, extent + + else: + print(f'{var_str} is not a plottable variable of the model!') + return None + + def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=[()]): + """ + Plot variable(s) from model, defined by var_strs, over coordinates + t, x_range, y_range. Either from the model's raw data or by interpolating + between the model's raw data over the coords. + + Parameters + ---------- + model : Micro or Meso Model + var_strs : list of str + Must match entries in the models' 'vars' dictionary. + t : float + time coordinate (defines the foliation). + x_range : list of 2 floats: x_start and x_end + defines range of x coordinates within foliation. + y_range : list of 2 floats: y_start and y_end + defines range of y coordinates within foliation. + interp_dims : tuple of integers + defines the number of points to interpolate at in x and y directions. + method : str + currently either raw_data or . + components_indices : list of tuple(s) + the indices of the components to pick out if the variables are vectors/tensors. + Can be omitted if all variables are scalars, otherwise must be a list + of tuples matching the length of var_strs that corresponds with each + variable in the list. + + Output + ------- + Plots the (2D) data using imshow. Note that the plotting data's coordinates + may not perfectly match the input coordinates if method=raw_data as + nearest-cell data is used where the input coordinates do not coincide + with the model's raw data coordinates. + + """ + n_plots = len(var_strs) + n_rows, n_cols = self.subplots_dims[n_plots] + + # Block to determine adaptively the figsize. + figsizes = {1 : (1/3.,1/3.), + 2 : (2/3.,1/3.), + 3 : (1,1/3.), + 4 : (2/3.,2/3.), + 5 : (1,2/3.), + 6 : (1,2/3.)} + for item in figsizes: + figsizes[item] = tuple(figsizes[item] * self.screen_size) + figsize = figsizes[n_plots] + + + fig, axes = plt.subplots(n_rows,n_cols, figsize=figsize) + if n_plots == 1: + axes = [axes] + else: + axes = axes.flatten() + + for var_str, component_indices, ax in zip(var_strs, components_indices, axes): + # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + im = ax.imshow(data_to_plot, extent=extent) + divider = make_axes_locatable(ax) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + ax.set_title(var_str) + ax.set_xlabel(r'$y$') # WHY? + ax.set_ylabel(r'$x$') + + fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), t), fontsize = 12) + fig.tight_layout() + # plt.show() + return fig + + def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp_dims=None, method='raw_data', component_indices=(), diff_plot=True): + """ + Plot a variable from a number of models. If 2 models are given, a third + plot of the difference will be automatically plotted, too. If 'raw_data' + is chosen as the method, will check to see if the data points in the model + lie at the same coordinates, which they must. + + Parameters + ---------- + models : Micro or Meso Models + var_sts : str + Must match entries in the models' 'vars' dictionary. + t : float + time coordinate (defines the foliation). + x_range : list of 2 floats: x_start and x_end + defines range of x coordinates within foliation. + y_range : list of 2 floats: y_start and y_end + defines range of y coordinates within foliation. + interp_dims : tuple of integers + defines the number of points to interpolate at in x and y directions. + method : str + currently either raw_data or interpolate. + component_indices : tuple + the indices of the component to pick out if the variable is a vector/tensor. + + Output + ------- + Plots the (2D) data using imshow. Note that the plotting data's coordinates + may not perfectly match the input coordinates if method=raw_data as + nearest-cell data is used where the input coordinates do not coincide + with the model's raw data coordinates. + + """ + n_cols = len(models) + if not n_cols == 2: + diff_plot = False # Only plot difference of 2 models... + n_rows = 1 + if diff_plot: + n_cols+=1 + + # Block to determine adaptively the figsize. + figsize = np.array([1,2/3.]) * self.screen_size + + fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=figsize) + + for model, ax in zip(models, axes.flatten()): + # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) + im = ax.imshow(data_to_plot, extent=extent) + divider = make_axes_locatable(ax) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + ax.set_title(model.get_model_name()) + ax.set_xlabel(r'$y$') + ax.set_ylabel(r'$x$') + + if diff_plot: + ax = axes.flatten()[-1] + # data_to_plot1, points1 = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) + # data_to_plot2, points2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices) + data_to_plot1, extent = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) + data_to_plot2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices)[0] + # if len(points1) != len(points2): + # diff_plot = False + # pass + # for t_points1, t_points2 in zip(points1, points2): + # print(t_points1, t_points2) + # if len(t_points1) != t_points2.shape: + # diff_plot = False + # continue + # if not np.allclose(t_points1, t_points2): + # diff_plot = False + # if diff_plot: + try: + # extent = [points1[2][0],points1[2][-1],points1[1][0],points1[1][-1]] + im = ax.imshow(data_to_plot1 - data_to_plot2, extent=extent) + divider = make_axes_locatable(ax) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + ax.set_title('Models difference') + ax.set_xlabel(r'$y$') + ax.set_ylabel(r'$x$') + except(ValueError): + print(f"Cannot plot the difference between {var_str} in the two " + "models. Likely due to the data coordinates not coinciding.") + fig.suptitle('Contrasting {} against different models'.format(var_str)) + fig.tight_layout() + # plt.show() + return fig + + + +if __name__ == '__main__': + + FileReader = METHOD_HDF5('./Data/test_res100/') + micro_model = IdealMHD_2D() + FileReader.read_in_data(micro_model) + micro_model.setup_structures() + + + visualizer = Plotter_2D([11.97, 8.36]) + + # TESTING GET_VAR_DATA + ###################### + # var = 'BC' + # components = (0,2) + # data1= visualizer.get_var_data(micro_model, var, 1.502, [0.3, 0.4], [0.3,0.4], component_indices=components)[0] + # data, extent= visualizer.get_var_data(micro_model, var, 1.502, [0.3, 0.4], [0.3,0.4], component_indices=components, method='interpolate', interp_dims=(20,20)) + # print(extent) + + # TESTING PLOT_VARS + ################### + # vars = ['BC', 'vx', 'vy', 'Bz', 'p', 'W'] + # components = [(0,), (), (), (), (), ()] + # model = micro_model + # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], components_indices=components) + # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], method = 'interpolate', interp_dims=(100,100), components_indices=components) + + + # TESTING PLOT_VAR_MODEL_COMPARISON + ################################### + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + meso_model = resMHD2D(micro_model, find_obs, filter) + ranges = [0.3, 0.4] + meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) + meso_model.find_observers() + meso_model.filter_micro_variables() + + var = 'BC' + component = (0,) + models = [micro_model, meso_model] + smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries + # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ + # method='interpolate', interp_dims=(30,30), component_indices=component) + visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, component_indices=component) + diff --git a/master_files/system/BaseFunctionality.py b/master_files/system/BaseFunctionality.py new file mode 100644 index 0000000..c410d74 --- /dev/null +++ b/master_files/system/BaseFunctionality.py @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- +""" +Created on Wed Nov 2 17:21:02 2022 + +@author: mjh1n20 +""" + +from multiprocessing import Process, Pool +import os +import numpy as np +#import matplotlib.pyplot as plt +import pickle +from timeit import default_timer as timer +import h5py +from scipy.interpolate import interpn +from scipy.optimize import root, minimize +#from mpl_toolkits.mplot3d import Axes3D +from scipy.integrate import solve_ivp, quad +import cProfile, pstats, io +import math + +class Base(object): + + @staticmethod + def Mink_dot(vec1, vec2): + """ + Parameters: + ----------- + vec1, vec2 : list of floats (or np.arrays) + + Return: + ------- + mink-dot (cartesian) in 1+n dim + """ + if len(vec1) != len(vec2): + print("The two vectors passed to Mink_dot are not of same dimension!") + + dot = -vec1[0]*vec2[0] + for i in range(1,len(vec1)): + dot += vec1[i] * vec2[i] + return dot + + @staticmethod + def get_rel_vel(spatial_vels): + """ + Build unit vectors starting from spatial components + Needed as this will enter the minimization procedure + + Parameters: + ---------- + spatial_vels: list of floats + + Returns: + -------- + list of floats: the d+1 vector, normalized wrt Mink metric + """ + W = 1 / np.sqrt(1-np.sum(spatial_vels**2)) + return W * np.insert(spatial_vels,0,1.0) + + @staticmethod + def project_tensor(vector1_wrt, vector2_wrt, to_project): + """ + """ + return np.inner(vector1_wrt,np.inner(vector2_wrt,to_project)) + + + @staticmethod + def orthogonal_projector(u, metric): + """ + Returns: + -------- + Orthogonal projector wrt vector u + + Notes: + ------ + The vector u must be time-like + """ + return metric + np.outer(u,u) + + + """ + A pair of functions that work in conjuction (thank you stack overflow). + find_nearest returns the closest value to 'value' in 'array', + find_nearest_cell then takes this closest value and returns its indices. + Should now work for any dimensional data. + """ + @staticmethod + def find_nearest(array, value): + """ + Returns closest value to input 'value' in 'array' + + Parameters: + ----------- + array: np.array of shape (n,) + + value: float + + Returns: + -------- + float + + Note: + ----- + To be used together with find_nearest_cell. + """ + idx = np.searchsorted(array, value, side="left") + if idx > 0 and (idx == len(array) or math.fabs(value - array[idx-1]) < math.fabs(value - array[idx])): + return array[idx-1] + else: + return array[idx] + + @staticmethod + def find_nearest_cell(point, points): + """ + Use find nearest to find closest value in a list of input 'points' to + input 'point'. + + Parameters: + ----------- + point: list of d+1 float + + points: list of lists of d+1 floats + + Returns: + -------- + List of d+1 indices corresponding to closest value to point in points + """ + if len(points) != len(point): + print("find_nearest_cell: The length of the coordinate vector\ + does not match the length of the coordinates.") + positions = [] + for dim in range(len(point)): + positions.append(Base.find_nearest(points[dim], point[dim])) + return [np.where(points[i] == positions[i])[0][0] for i in range(len(positions))] + + def profile(self, fnc): + """A decorator that uses cProfile to profile a function""" + def inner(*args, **kwargs): + pr = cProfile.Profile() + pr.enable() + retval = fnc(*args, **kwargs) + pr.disable() + s = io.StringIO() + sortby = 'cumulative' + ps = pstats.Stats(pr, stream=s).sort_stats(sortby) + ps.print_stats() + print(s.getvalue()) + return retval + return inner From 4403aa27ad9ae40c8e6c8046fcbee92b52e5101c Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 17 Nov 2023 16:08:35 +0100 Subject: [PATCH 030/111] TC updates to Visualization.py --- master_files/Filters.py | 2 +- master_files/MesoModels.py | 723 +++++++++++++++++++++++++++++++++- master_files/Visualization.py | 184 +++++---- 3 files changed, 826 insertions(+), 83 deletions(-) diff --git a/master_files/Filters.py b/master_files/Filters.py index 501bb58..9c40b60 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -433,7 +433,7 @@ def find_observers_ranges(self, num_points, ranges, flux_str = "gauss" ): class FindObs_drift_root(object): """ - Class for computing the observer by minimizing the (micro) baryon current + Class for computing the observer by root-finding on the net baryon current drift over a box. Work in any dimension, read on construction from the micro-model. """ diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 7992df3..f6fedd3 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -7,7 +7,6 @@ import numpy as np from scipy.interpolate import interpn -import pickle from multiprocessing import Process, Pool from multimethod import multimethod @@ -1103,12 +1102,722 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # print(comps) # plt.show() +class resHD2D(object): + """ + CUT AND PAST OF resMHD_2D --> removing the magnetic bits + """ + def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): + """ + ADD DESCRIPTION OF THIS + """ + self.micro_model = micro_model + self.find_obs = find_obs + self.filter = filter + self.spatial_dims = 2 + self.interp_method = interp_method + + self.domain_int_strs = ('Nt','Nx','Ny') + self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") + self.domain_array_strs = ("T","X","Y","Points") + self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) + for var in self.domain_vars: + self.domain_vars[var] = [] + + # This is the Levi-Civita symbol, not tensor, so be careful when using it + self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ + for k in range(3)]for j in range(3) ] for i in range(3) ]) + + self.metric = np.zeros((3,3)) + self.metric[0,0] = -1 + self.metric[1,1] = self.metric[2,2] = +1 + + self.filter_vars_strs = ['U', 'U_errors', 'U_success'] + self.filter_vars = dict.fromkeys(self.filter_vars_strs) + for var in self.filter_vars: + var = [] + + self.meso_structures_strs = ['SET', 'BC'] + self.meso_structures = dict.fromkeys(self.meso_structures_strs) + for var in self.meso_structures: + self.meso_structures[var] = [] + + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'sigma_tilde', 'T_tilde'] + self.meso_vectors_strs = ['u_tilde', 'q_res'] + self.meso_r2tensors_strs = ['pi_res'] + self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs + self.meso_vars = dict.fromkeys(self.meso_vars_strs) + + self.coefficient_strs = ["Gamma"] + self.coefficients = dict.fromkeys(self.coefficient_strs) + self.coefficients['Gamma'] = 4.0/3.0 + + # Dictionary with stencils and coefficients for finite differencing + self.differencing = dict.fromkeys((1, 2)) + self.differencing[1] = dict.fromkeys(['fw', 'bw', 'cen']) + self.differencing[1]['fw'] = {'coefficients' : [-1, 1] , 'stencil' : [0, 1]} + self.differencing[1]['bw'] = {'coefficients' : [1, -1] , 'stencil' : [0, -1]} + self.differencing[1]['cen'] = {'coefficients' : [-1/2., 0, 1/2.] , 'stencil' : [-1, 0, 1]} + self.differencing[2] = dict.fromkeys(['fw', 'bw', 'cen']) + self.differencing[2]['fw'] = {'coefficients' : [-3/2., 2., -1/2.] , 'stencil' : [0, 1, 2]} + self.differencing[2]['bw'] = {'coefficients' : [3/2., -2., 1/2.] , 'stencil' : [0, -1, -2]} + self.differencing[2]['cen'] = {'coefficients' : [1/12., -2/3., 0., 2/3., -1/12.] , 'stencil' : [-2, -1, 0, 1, 2]} + + + # dictionary with non-local quantities (keys must match one of meso_vars or structure) + self.nonlocal_vars_strs = ['u_tilde', 'T_tilde'] + # self.nonlocal_vars_strs = ['u_tilde', 'Fab'] + Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] + self.deriv_vars = dict.fromkeys(Dstrs) + + # Additional, closure-scheme specific vars must be added appropriately using model_residuals() + + # self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs + # Run some compatibility test... + compatible = True + error = '' + if self.spatial_dims != micro_model.get_spatial_dims(): + compatible = False + error += '\nError: different dimensions.' + for struct in self.meso_structures_strs: + if struct not in self.micro_model.get_structures_strs(): + compatible = False + error += f'\nError: {struct} not in micro_model!' + + if not compatible: + print("Meso and Micro models are incompatible:"+error) + + def get_all_var_strs(self): + return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) + \ + list(self.filter_vars.keys()) + + def get_gridpoints(self): + """ + Pretty self-explanatory + """ + return self.domain_vars['Points'] + + def get_interpol_var(self, var, point): + """ + Returns the interpolated variables at the point. + + Parameters + ---------- + var: str corresponding to meso_structure, meso_vars or deriv_vars + + point : list of floats + ordered coordinates: t,x,y + + Return + ------ + Interpolated values/arrays corresponding to variable. + Empty list if none of the variables is not meso_structures/meso_vars or deriv_vars of the model. + + Notes + ----- + Interpolation gives errors when applied to boundary + """ + if var in self.meso_structures: + return interpn(self.domain_vars['Points'], self.meso_structures[var], point, method = self.interp_method)[0] + elif var in self.meso_vars: + return interpn(self.domain_vars['Points'], self.meso_vars[var], point, method = self.interp_method)[0] + elif var in self.deriv_vars: + return interpn(self.domain_vars['Points'], self.deriv_vars[var], point, method = self.interp_method)[0] + else: + print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) + + @multimethod + def get_var_gridpoint(self, var: str, h: object, i: object, j: object): + """ + Returns variable corresponding to input 'var' at gridpoint + identified by h,i,j + + Parameters: + ----------- + var: string corresponding to structure, meso_vars or deriv_vars + + h,i,j: int + integers corresponding to position on the grid. + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest + grid-point to input 'point'. + + Notes: + ------ + This method is useful e.g. for plotting the raw data. + """ + if var in self.meso_structures: + return self.meso_structures[var][h,i,j] + elif var in self.meso_vars: + return self.meso_vars[var][h,i,j] + elif var in self.deriv_vars: + return self.deriv_vars[var][h,i,j] + elif var in self.filter_vars: + return self.filter_vars[var][h,i,j] + else: + print("{} is not a variable of resHD_2D!".format(var)) + return None + + @multimethod + def get_var_gridpoint(self, var: str, point: object): + """ + Returns variable corresponding to input 'var' at gridpoint + closest to input 'point'. + + Parameters: + ----------- + vars: string corresponding to structure, meso_vars or deriv_vars + + point: list of 2+1 floats + + Returns: + -------- + Values or arrays corresponding to variable evaluated at the closest + grid-point to input 'point'. + + Notes: + ------ + This method should be used in case using interpolated values becomes + too expensive. + """ + indices = Base.find_nearest_cell(point, self.domain_vars['Points']) + if var in self.meso_structures: + return self.meso_structures[var][tuple(indices)] + elif var in self.meso_vars: + return self.meso_vars[var][tuple(indices)] + elif var in self.deriv_vars: + return self.deriv_vars[var][tuple(indices)] + elif var in self.filter_vars: + return self.filter_vars[var][h,i,j] + else: + print("{} is not a variable of resHD_2D!".format(var)) + return None + + def get_model_name(self): + return 'resHD2D' + + def set_find_obs_method(self, find_obs): + self.find_obs = find_obs + + def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): + """ + Builds the meso_model grid using the micro_model grid points contained in + the input patch (defined via 'patch_bdrs'). The method allows for coarse graining + the grid in the spatial directions ONLY. + Then store the info about the meso grid and set up arrays of definite rank and size + for the quantities needed later. + + Parameters: + ----------- + patch_bdrs: list of lists of two floats, + [[tmin, tmax],[xmin,xmax],[ymin,ymax]] + + coarse_factor: integer + + Notes: + ------ + If the patch_bdrs are larger than micro_grid, the method will not set-up the meso_grid, + and an error message is printed. This is extra safety measure! + """ + + # Is the patch within the micro_model domain? + conditions = patch_bdrs[0][0] < self.micro_model.domain_vars['tmin'] or \ + patch_bdrs[0][1] > self.micro_model.domain_vars['tmax'] or \ + patch_bdrs[1][0] < self.micro_model.domain_vars['xmin'] or \ + patch_bdrs[1][1] > self.micro_model.domain_vars['xmax'] or \ + patch_bdrs[2][0] < self.micro_model.domain_vars['ymin'] or \ + patch_bdrs[2][1] > self.micro_model.domain_vars['ymax'] + + if conditions: + print('Error: the input region for filtering is larger than micro_model domain!') + return None + + #Find the nearest cell to input patch bdrs + patch_min = [patch_bdrs[0][0], patch_bdrs[1][0], patch_bdrs[2][0]] + patch_max = [patch_bdrs[0][1], patch_bdrs[1][1], patch_bdrs[2][1]] + idx_mins = Base.find_nearest_cell(patch_min, self.micro_model.domain_vars['points']) + idx_maxs = Base.find_nearest_cell(patch_max, self.micro_model.domain_vars['points']) + + # Set meso_grid spacings + self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] + self.domain_vars['Dx'] = self.micro_model.domain_vars['dx'] * coarse_factor + self.domain_vars['Dy'] = self.micro_model.domain_vars['dy'] * coarse_factor + + # Building the meso_grid + h, i, j = idx_mins[0], idx_mins[1], idx_mins[2] + while h <= idx_maxs[0]: + t = self.micro_model.domain_vars['t'][h] + self.domain_vars['T'].append(t) + h += 1 + while i <= idx_maxs[1]: + x = self.micro_model.domain_vars['x'][i] + self.domain_vars['X'].append(x) + i += coarse_factor + while j <= idx_maxs[2]: + y = self.micro_model.domain_vars['y'][j] + self.domain_vars['Y'].append(y) + j += coarse_factor + + # Saving the info about the meso_grid + self.domain_vars['Points'] = [self.domain_vars['T'], self.domain_vars['X'], self.domain_vars['Y']] + self.domain_vars['Tmin'] = np.amin(self.domain_vars['T']) + self.domain_vars['Xmin'] = np.amin(self.domain_vars['X']) + self.domain_vars['Ymin'] = np.amin(self.domain_vars['Y']) + self.domain_vars['Tmax'] = np.amax(self.domain_vars['T']) + self.domain_vars['Xmax'] = np.amax(self.domain_vars['X']) + self.domain_vars['Ymax'] = np.amax(self.domain_vars['Y']) + self.domain_vars['Nt'] = len(self.domain_vars['T']) + self.domain_vars['Nx'] = len(self.domain_vars['X']) + self.domain_vars['Ny'] = len(self.domain_vars['Y']) + + # Setup arrays for structures + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + self.meso_structures['BC'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.meso_structures['SET'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + + # Setup arrays for meso_vars + for str in self.meso_scalars_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny)) + for str in self.meso_vectors_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + for str in self.meso_r2tensors_strs: + self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + + # Setup arrays for filter_vars + self.filter_vars['U'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) + self.filter_vars['U_success'] = dict() + + + # Setup arrays for derivatives of the model. + # THOMAS'S WAY (SAVE THEM AS TENSORS) + self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) + self.deriv_vars['D_T_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + + # MARCUS'S WAY + # for str in self.non_local_vars_strs: + # if str in self.meso_vars: + # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_vars[str])}) + # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_vars[str])}) + # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_vars[str])}) + # if str in self.meso_structures: + # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_structures[str])}) + # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) + # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) + + def find_observers(self): + """ + Method to compute filtering observers at grid points built with setup_meso_grid. + The observers found (and relative errors) are saved in the dictionary self.filter_vars. + Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices + on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) + + Notes: + ------ + Requires setup_meso_grid() and setup_variables() to be called first. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,x,y] + sol = self.find_obs.find_observer(point) + if sol[0]: + self.filter_vars['U'][h,i,j] = sol[1] + self.filter_vars['U_errors'][h,i,j] = sol[2] + self.filter_vars['U_success'].update({(h,i,j) : True}) + + if not sol[0]: + self.filter_vars['U_success'].update({(h,i,j) : False}) + print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) + + def filter_micro_variables(self): + """ + Filter all meso_model structures AND micro pressure at the grid_points built with setup_meso_grid(). + This method relies on filter_var_point implemented separately for the filter, e.g. spatial_box_filter + + Parameters: + ----------- + + Notes: + ------ + Requires find_observers() and setup_meso_grid() to be called first. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t, x, y] + if self.filter_vars['U_success'][h,i,j]: + obs = self.filter_vars['U'][h,i,j] + for struct in self.meso_structures: + self.meso_structures[struct][h,i,j] = self.filter.filter_var_point(struct, point, obs) + self.meso_vars['p_filt'][h,i,j] = self.filter.filter_var_point('p', point, obs) + else: + print('Could not filter at {}: observer not found.'.format(point)) + + def p_from_EOS(self, eps, n): + """ + Compute pressure from Gamma-law EoS + """ + return (self.coefficients['Gamma']-1)*eps*n + + def decompose_structures_gridpoint(self, h, i, j): + """ + Decompose the fluid part of SET as well as Fab at grid point (h,i,j) + + Parameters: + ----------- + h, i, j: integers + the indices on the grid where the decomposition is performed. + + Returns: + -------- + None + + Notes: + ------ + Decomposition of BC, fluid SET and Faraday tensor. The EM part of SET is decomposed later + via non local operations (same story for the charge current). + + """ + # Computing the Favre density and velocity + n_t = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) + u_t = np.multiply(1 / n_t, self.meso_structures['BC'][h,i,j]) + T_ab = self.meso_structures['SET'][h,i,j,:,:] # Remember this is rank (2,0) + + # Computing the decomposition at each point + eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, self.metric, self.metric, T_ab) + h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) # This is a rank (1,1) tensor, i.e. a real projector. + q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) + s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, T_ab) + s = np.einsum('ii',s_ab) + p_t = self.p_from_EOS(eps_t, n_t) + + + # Storing the decomposition with appropriate names. + self.meso_vars['n_tilde'][h,i,j] = n_t + self.meso_vars['u_tilde'][h,i,j,:] = u_t + self.meso_vars['eps_tilde'][h,i,j] = eps_t + self.meso_vars['q_res'][h,i,j,:] = q_a + self.meso_vars['pi_res'][h,i,j,:,:] = s_ab - np.multiply(s / self.spatial_dims , self.metric +np.outer(u_t,u_t)) + self.meso_vars['p_tilde'][h,i,j] = p_t + self.meso_vars['Pi_res'][h,i,j] = s - p_t + self.meso_vars['eos_res'][h,i,j] = self.meso_vars['p_filt'][h,i,j] - p_t + self.meso_vars['T_tilde'][h,i,j] = p_t / n_t + + def decompose_structures(self): + """ + Decompose structures at all points on the meso_grid where observers could be found. + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,y,x] + if self.filter_vars['U_success'][h,i,j]: + self.decompose_structures_gridpoint(h,i,j) + else: + print('Structures not decomposed at {}: observer could not be found.'.format(point)) + + def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): + """ + Calculate partial derivative in the input 'direction' of the variable corresponding to 'nonlocal_var_str' + at the position on the grid identified by indices h,i,j. The order of the differencing scheme + can also be specified, default to 1. + + Parameters: + ----------- + nonlocal_var_str: string + quantity to be taken derivate of, must be in self.nonlocal_vars_strs() + + h, i ,j: integers + indices for point on the meso_grid + + direction: integer < self.spatial_dim + 1 + + order: integer, defatult to 1 + order of the differencing scheme + + Returns: + -------- + Finite-differenced quantity at (h,i,j) + + Notes: + ------ + The method returns the value instead of storing it, so that these can be + rearranged as preferred later, that is in calculate_derivatives() + """ + + if direction > self.spatial_dims: + print('Directions are numbered from 0 up to {}'.format(self.spatial_dims)) + return None + + if order > len(self.differencing): + print('Maximum order implemented is {}: continuing with it.'.format(len(self.differencing))) + order = len(self.differencing) + + # Forward, backward or centered differencing? + k = [h,i,j][direction] + N = len(self.domain_vars['Points'][direction]) + + if k in [l for l in range(order)]: + coefficients = self.differencing[order]['fw']['coefficients'] + stencil = self.differencing[order]['fw']['stencil'] + elif k in [N-1-l for l in range(order)]: + coefficients = self.differencing[order]['bw']['coefficients'] + stencil = self.differencing[order]['bw']['stencil'] + else: + coefficients = self.differencing[order]['cen']['coefficients'] + stencil = self.differencing[order]['cen']['stencil'] + + temp = 0 + for s, sample in enumerate(stencil): + idxs = [h,i,j] + idxs[direction] += sample + temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.get_var_gridpoint(nonlocal_var_str, *idxs)) + return temp + + def calculate_derivatives(self): + """ + Compute all the derivatives of the quantities corresponding to nonlocal_vars_strs, for all + gridpoints on the meso-grid. + + Notes: + ------ + The derived quantities are stored as 'tensors' as follows: + 1st three indices refer to the position on the grid + + 4th index refers to the directionality of the derivative + + last indices (1 or 2) correspond to the components of the quantity to be derived + + The index corresponding to the derivative is covariant, i.e. down. + + Example: + + Fab [h,i,j,a,b] : h,i,j grid; a,b, spacetime components + + D_Fab[h,i,j,c,a,b]: h,i,j grid; c direction of derivative; a,b as for Fab + + """ + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): + point = [t,y,x] + if self.filter_vars['U_success'][h,i,j]: + for dir in range(self.spatial_dims+1): + for str in self.nonlocal_vars_strs: + dstr = 'D_' + str + self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivatives_gridpoint(str, h, i, j, dir) + else: + print('Derivatives not calculated at {}: observer could not be found.'.format(point)) + + def closure_ingredients_gridpoint(self, h, i, j): + """ + Decompose quantities obatined via non_local operations (i.e. derivatives) as needed + for the closure scheme. This is done at gridpoint (h,i,j) and then relevant values are + returned + + Parameters: + ----------- + h, i, j: integers + the indices of the gridpoint + + Returns: + -------- + list of strings: names of the quantities computed + + list of nd.arrays with quantities computed + + Notes: + ------ + The ingredients for the closure scheme are model specific: it makes sense that the relevant + dictionary is set up not on construction. + + UNDER CONSTRUCTION: THE EM PART NEEDS MORE THINKING + + Rename: closure_ingredients_gridpoint() ?? + Coefficients can be computed here directly, but more general models will require more work. + Do not split this, at least for now + """ + # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS + u_t = self.meso_vars['u_tilde'][h,i,j] + u_t_cov = np.einsum('ij,j->i', self.metric, u_t) + Nabla_Fab = self.deriv_vars['D_Fab'][h,i,j] + + # CLOSURE INGREDIENTS: FLUID BIT - WORKING WITH (2,0) + nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor + nabla_u = np.einsum('ij,jk->ik', self.metric, nabla_u) #this is a rank (2,0) tensor + acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector + + # The following two quantities should be identically zero, but they won't be due to numerical errors. + # Computing them and removing to project velocity gradients + unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector + acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) + Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ + np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor + h_ab = self.metric + np.einsum('i,j->ij', u_t, u_t) + exp_t = np.einsum('ii->',Daub) + shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/self.spatial_dims * exp_t, h_ab) + vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + + + # CLOSURE INGREDIENTS: HEAT FLUX + nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] + projector = np.zeros((3,3)) + np.fill_diagonal(projector, 1) + projector += np.einsum('i,j->ij',u_t_cov, u_t) + DaT = np.einsum('ij,j->i', projector, nabla_T) + Theta_tilde = DaT + np.multiply(self.meso_vars['T_tilde'][h,i,j], np.einsum('ij,j->i', self.metric, acc_t)) + + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'Theta_tilde'] + closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] + return closure_vars_strs, closure_vars + + def closure_ingredients(self): + """ + Wrapper of the corresponding gridpoint method. + Set up the dictionary for the closure variables not yet defined. + Loop over the grid and store the decomposed variables. + """ + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][h,i,j]: + keys, values = self.closure_ingredients_gridpoint(h,i,j) + # try-except block to extend the meso_vars dictionary + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) + shape = values[idx].shape + self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + finally: + self.meso_vars[key][h,i,j] = values[idx] + + def EL_style_closure_gridpoint(self, h, i, j): + """ + Compute bulk, shear via scalarization + PA trick, thermal conductivity later. + At a point. + Returns: coefficient(s) at a point. + """ + coefficients_names=[] + coefficients=[] + + # CALCULATING BULK VISCOUS COEFF + zeta = self.meso_vars['Pi_res'][h,i,j] / self.meso_vars['exp_tilde'][h,i,j] + coefficients_names.append('zeta') + coefficients.append(zeta) + + # CALCULATING SHEAR VISCOUS COEFF + pi_res_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['pi_res'][h,i,j], self.meso_vars['pi_res'][h,i,j], self.metric, self.metric) + shear_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['shear_tilde'][h,i,j], self.meso_vars['shear_tilde'][h,i,j], self.metric, self.metric) + eta = pi_res_sq/shear_sq + # Compute sign of eta by looking at the PA of pi_res with higher eigenvalue. + # Change shear to the eigenbasis of pi_res (using that the matrix is unitary as pi_res is sym) + pi_eig, pi_eigv = np.linalg.eigh(self.meso_vars['pi_res'][h,i,j]) + transformed_shear = np.einsum('ji,jk,kl', pi_eigv, self.meso_vars['shear_tilde'][h,i,j], pi_eigv) + abs_pi_eig = np.abs(pi_eig) + pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) + eta = eta * np.sign(pi_eig[-1] / transformed_shear[pos,pos]) + coefficients_names.append('eta') + coefficients.append(eta) + + # CALCULATING THE HEAT CONDUCTIVITIY + q_res = self.meso_vars['q_res'][h,i,j] + Theta_t = self.meso_vars['Theta_tilde'][h,i,j] + q_res_sq = np.einsum('i,ij,j', q_res, self.metric, q_res) + Theta_t_sq = np.einsum('i,ij,j', Theta_t, self.metric, Theta_t) + cos_angle = np.einsum('i,ij,j', q_res, self.metric, Theta_t) / (q_res_sq * Theta_t_sq) + sign = np.sign(cos_angle) + kappa = sign * np.sqrt(q_res_sq / Theta_t_sq) + coefficients_names.append('kappa') + coefficients.append(kappa) + + return coefficients_names, coefficients + + def EL_style_closure(self): + """ + Wrapper of EL_style_closure_gridpoint() + """ + Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + if self.filter_vars['U_success'][h,i,j]: + keys, values = self.EL_style_closure_gridpoint(h,i,j) + # try-except block to extend the meso_vars dictionary with the dissipative coefficients + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) + shape = values[idx].shape + self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) + finally: + self.meso_vars[key][h,i,j] = values[idx] + + def EL_style_closure_regression(self, CoefficientAnalysis): + """ + Takes in a list of correlation quantities (strings? The regressors needed are computed + previously via closure ingredients) + Pass the dataset to a CoefficientAnalysis instance. + Plot the correlations with the regressors and perform the regression using methods from a Coefficient + Analysis instance. + """ + # FICTITIOUS RANGES AND POINTS TO TEST TRIM_DATA ROUTINE + ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ + [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ + [self.domain_vars['Ymin'], self.domain_vars['Ymax']]] + points = self.domain_vars['Points'] + + # TESTING SCALAR REGRESSION ROUTINE + # y = self.meso_vars['zeta'] + # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) + # print(*stats_result) + + # TESTING TENSOR REGRESSION ROUTINE + # y=self.meso_vars['q_res'] + # X=[self.meso_vars['e_tilde'], self.meso_vars['u_tilde']] + # # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, components=[(0,),(2,)]) + # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, add_intercept=False) + # for i in range(len(stats_result)): + # print('{}-th component: \n {} \n\n'.format(i, stats_result[i])) + + # TESTING CORRELATION VISUALIZATION ROUTINES + # x = self.meso_vars['n_tilde'] + # y = self.meso_vars['eps_tilde'] + # labels = ['n_tilde','eps_tilde'] + # g1=CoefficientAnalysis.visualize_correlation(x, y, xlabel=labels[0], ylabel=labels[1]) + # fig=g1.figure + # fig.savefig('Single_correlation_plot.pdf', format='pdf') + + # data=[self.meso_vars['eta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['eta', 'b_tilde', 'eps_tilde', 'n_tilde'] + # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) + + # data=[self.meso_vars['zeta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['zeta', 'b_tilde', 'eps_tilde', 'n_tilde'] + # gbis= CoefficientAnalysis.visualize_many_correlations(data, labels) + # # plt.savefig('Many_correlations_plot.pdf', format='pdf') + # plt.show() + # TESTING THE CHECK REGRESSORS ROUTINE: + # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] + # labels=['b_tilde', 'eps_tilde', 'n_tilde'] + # g1=CoefficientAnalysis.visualize_many_correlations(data, labels) + # comps, g2 = CoefficientAnalysis.PCA_find_regressors_subset(data, ranges = ranges, model_points = points) + # g2.fig.suptitle("Correlation between PC (standardized data)") + # g2.fig.subplots_adjust(top=0.94) + # print(comps) + # plt.show() if __name__ == '__main__': - FileReader = METHOD_HDF5('./Data/test_res100/') + FileReader = METHOD_HDF5('/Users/thomas/Dropbox/Work/projects/Filtering/Data/test_res100/') micro_model = IdealMHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() @@ -1125,19 +1834,19 @@ def EL_style_closure_regression(self, CoefficientAnalysis): meso_model.find_observers() meso_model.filter_micro_variables() - print('Filter stage ended') + # print('Filter stage ended') # meso_model.decompose_structures_gridpoint(1,1,1) - meso_model.decompose_structures() + # meso_model.decompose_structures() # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) - meso_model.calculate_derivatives() + # meso_model.calculate_derivatives() # meso_model.closure_ingredients_gridpoint(1,1,0) - meso_model.closure_ingredients() + # meso_model.closure_ingredients() # print('Derivatives and Decomposition stage ended') # meso_model.EL_style_closure_gridpoint(1,2,3) - meso_model.EL_style_closure() + # meso_model.EL_style_closure() # regressor = CoefficientsAnalysis() # meso_model.EL_style_closure_regression(regressor) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 954ee24..a7c03ec 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -26,13 +26,14 @@ def __init__(self, screen_size = [11.97, 8.36]): screen_size = list of 2 floats width and height of the screen: used to rescale plots' size. """ - self.subplots_dims = {1 : (1,1), + self.plot_vars_subplots_dims = {1 : (1,1), 2 : (1,2), 3 : (1,3), 4 : (2,2), 5 : (2,3), 6 : (2,3)} - + + self.screen_size = np.array(screen_size) def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, method= 'raw_data', component_indices=()): @@ -129,7 +130,7 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me print(f'{var_str} is not a plottable variable of the model!') return None - def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=[()]): + def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=None): #[()]): """ Plot variable(s) from model, defined by var_strs, over coordinates t, x_range, y_range. Either from the model's raw data or by interpolating @@ -165,7 +166,7 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth """ n_plots = len(var_strs) - n_rows, n_cols = self.subplots_dims[n_plots] + n_rows, n_cols = self.plot_vars_subplots_dims[n_plots] # Block to determine adaptively the figsize. figsizes = {1 : (1/3.,1/3.), @@ -184,6 +185,10 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth axes = [axes] else: axes = axes.flatten() + + if not components_indices: + print('No list of components indices passed: setting this to an empty list.') + components_indices = [ () for _ in range(len(var_strs))] for var_str, component_indices, ax in zip(var_strs, components_indices, axes): # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) @@ -193,7 +198,10 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title(var_str) + title = var_str + if component_indices != (): + title = title + ", {}-component".format(component_indices) + ax.set_title(title) ax.set_xlabel(r'$y$') # WHY? ax.set_ylabel(r'$x$') @@ -202,7 +210,7 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth # plt.show() return fig - def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp_dims=None, method='raw_data', component_indices=(), diff_plot=True): + def plot_var_model_comparison(self, models, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=None, diff_plot=False): """ Plot a variable from a number of models. If 2 models are given, a third plot of the difference will be automatically plotted, too. If 'raw_data' @@ -211,9 +219,9 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp Parameters ---------- - models : Micro or Meso Models - var_sts : str - Must match entries in the models' 'vars' dictionary. + models : list of Micro or Meso Models + var_strs : list of lists of strings + each sublist must match entries in the models' 'vars' dictionary. t : float time coordinate (defines the foliation). x_range : list of 2 floats: x_start and x_end @@ -224,8 +232,9 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp defines the number of points to interpolate at in x and y directions. method : str currently either raw_data or interpolate. - component_indices : tuple - the indices of the component to pick out if the variable is a vector/tensor. + component_indices : list of list of tuples + each tuple identifies the indices of the component to pick out if the variable + is a vector/tensor. Output ------- @@ -235,60 +244,81 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp with the model's raw data coordinates. """ + if len(var_strs) != len(models): + print("I need a list of vars to plot per model. Check!") + return None + num_vars_1st_model = len(var_strs[0]) + for i in range(1, len(var_strs)): + if len(var_strs[i]) != num_vars_1st_model: + print("The number of variables per model must be the same. Exiting.") + return None + n_cols = len(models) - if not n_cols == 2: - diff_plot = False # Only plot difference of 2 models... - n_rows = 1 if diff_plot: - n_cols+=1 - - # Block to determine adaptively the figsize. - figsize = np.array([1,2/3.]) * self.screen_size + if len(models)!=2: + print("Can plot the difference between TWO models, not more.") + else: + n_cols+=1 - fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=figsize) + n_rows = len(var_strs[0]) + if len(var_strs[0])>3: + print("This function is meant to compare up to 3 vars. Plotting the first three.") + n_rows = 3 - for model, ax in zip(models, axes.flatten()): - # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] - data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - im = ax.imshow(data_to_plot, extent=extent) - divider = make_axes_locatable(ax) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title(model.get_model_name()) - ax.set_xlabel(r'$y$') - ax.set_ylabel(r'$x$') + if not components_indices: + print('No list of components indices passed: setting this to an empty list.') + empty_tuple_components = [ () for _ in range(len(var_strs[0]))] + components_indices = [] + for i in range(len(models)): + components_indices.append(empty_tuple_components) - if diff_plot: - ax = axes.flatten()[-1] - # data_to_plot1, points1 = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) - # data_to_plot2, points2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices) - data_to_plot1, extent = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) - data_to_plot2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices)[0] - # if len(points1) != len(points2): - # diff_plot = False - # pass - # for t_points1, t_points2 in zip(points1, points2): - # print(t_points1, t_points2) - # if len(t_points1) != t_points2.shape: - # diff_plot = False - # continue - # if not np.allclose(t_points1, t_points2): - # diff_plot = False - # if diff_plot: - try: - # extent = [points1[2][0],points1[2][-1],points1[1][0],points1[1][-1]] - im = ax.imshow(data_to_plot1 - data_to_plot2, extent=extent) - divider = make_axes_locatable(ax) + # Block to determine adaptively the figsize. + # figsize = np.array([1,2/3.]) * self.screen_size + figsize = self.screen_size + # fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=figsize) + fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize, squeeze=False) + + for j in range(len(models)): + for i in range(n_rows): + data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, interp_dims, method, components_indices[j][i]) + # im=axes[i,j].imshow(data_to_plot, extent) + im=axes[i,j].imshow(data_to_plot, extent=extent) + divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title('Models difference') - ax.set_xlabel(r'$y$') - ax.set_ylabel(r'$x$') - except(ValueError): - print(f"Cannot plot the difference between {var_str} in the two " + title = models[j].get_model_name() + "\n"+var_strs[j][i] + if components_indices[j][i] != (): + title += " {}-component".format(components_indices[j][i]) + axes[i,j].set_title(title) + axes[i,j].set_xlabel(r'$y$') + axes[i,j].set_ylabel(r'$x$') + + + if diff_plot and len(models)==2: + try: + for i in range(len(var_strs[0])): + data1, extent1 = self.get_var_data(models[0], var_strs[0][i], t, x_range, y_range, interp_dims, method, components_indices[0][i]) + data2, extent2 = self.get_var_data(models[1], var_strs[1][i], t, x_range, y_range, interp_dims, method, components_indices[1][i]) + if extent1 != extent2: + print("Cannot plot the difference between the vars: data not aligned.") + continue + im = axes[i,2].imshow(data1-data2, extent=extent1) + divider = make_axes_locatable(axes[i,2]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + axes[i,2].set_title('Models difference') + axes[i,2].set_xlabel(r'$y$') + axes[i,2].set_ylabel(r'$x$') + except (ValueError): + print(f"Cannot plot the difference between {var_strs} in the two "+\ "models. Likely due to the data coordinates not coinciding.") - fig.suptitle('Contrasting {} against different models'.format(var_str)) + + models_names = [model.get_model_name() for model in models] + suptitle = "Comparing " + for i in range(len(models_names)): + suptitle += models_names[i] + " " + suptitle += "models." + fig.suptitle(suptitle) fig.tight_layout() # plt.show() return fig @@ -296,14 +326,16 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp if __name__ == '__main__': - - FileReader = METHOD_HDF5('./Data/test_res100/') + + FileReader = METHOD_HDF5('../Data/test_res100/') micro_model = IdealMHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() - visualizer = Plotter_2D([11.97, 8.36]) + # visualizer = Plotter_2D([11.97, 8.36]) + + # print('Finished initializing') # TESTING GET_VAR_DATA ###################### @@ -324,19 +356,21 @@ def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp # TESTING PLOT_VAR_MODEL_COMPARISON ################################### - find_obs = FindObs_drift_root(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - meso_model = resMHD2D(micro_model, find_obs, filter) - ranges = [0.3, 0.4] - meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) - meso_model.find_observers() - meso_model.filter_micro_variables() - - var = 'BC' - component = (0,) - models = [micro_model, meso_model] - smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries - # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ - # method='interpolate', interp_dims=(30,30), component_indices=component) - visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, component_indices=component) + # find_obs = FindObs_drift_root(micro_model, 0.001) + # filter = spatial_box_filter(micro_model, 0.003) + # meso_model = resMHD2D(micro_model, find_obs, filter) + # ranges = [0.2, 0.25] + # meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) + # meso_model.find_observers() + # meso_model.filter_micro_variables() + + # print("Finished filtering") + + # vars = [['W'],['BC']] + # components = [[()],[(0,)]] + # models = [micro_model, meso_model] + # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries + # # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ + # # method='interpolate', interp_dims=(30,30), component_indices=component) + # visualizer.plot_var_model_comparison(models, vars, 1.502, smaller_ranges, smaller_ranges, components_indices=components, diff_plot=True) From 3548a174f75fc342ed3c56b4d98ed278ebe7fffa Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 17 Nov 2023 16:10:31 +0100 Subject: [PATCH 031/111] Resctructuring folder --- Analysis.py | 503 -------- FileReaders.py | 216 ---- Filters.py | 1088 ---------------- MesoModels.py | 1145 ----------------- MicroModels.py | 554 -------- Visualization.py | 340 ----- system/BaseFunctionality.py | 149 --- .../BaseFunctionality.cpython-36.pyc | Bin 6767 -> 0 bytes 8 files changed, 3995 deletions(-) delete mode 100644 Analysis.py delete mode 100644 FileReaders.py delete mode 100644 Filters.py delete mode 100644 MesoModels.py delete mode 100644 MicroModels.py delete mode 100644 Visualization.py delete mode 100644 system/BaseFunctionality.py delete mode 100644 system/__pycache__/BaseFunctionality.cpython-36.pyc diff --git a/Analysis.py b/Analysis.py deleted file mode 100644 index 4ceb68c..0000000 --- a/Analysis.py +++ /dev/null @@ -1,503 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Tue Jan 24 18:02:05 2023 - -@author: marcu -""" - -# USE !SCIKIT LEARN INSTEAD? IT'S THE PACKAGE FOR MACHINE LEARNING SO! - -import matplotlib.pyplot as plt -import numpy as np -import pandas as pd -import h5py -import pickle -import seaborn as sns -import warnings -from sklearn.linear_model import LinearRegression -from sklearn.decomposition import PCA -import statsmodels.api as sm - -from system.BaseFunctionality import * -from MicroModels import * -from FileReaders import * -from Filters import * -from Visualization import * -from MesoModels import * - - - -class CoefficientsAnalysis(object): - """ - Class containing a number of methods for performing statistical analysis on gridded data. - Methods include: regression, visualizing correlations, PCAs - """ - def __init__(self): #, visualizer, spatial_dims): - """ - Nothing as yet... - - Returns - ------- - Also nothing... - - """ - # self.visualizer=visualizer # need to re-structure this... or do I - # Do you really need the visualizer here? I don't think so! - # self.spatial_dims=spatial_dims - - def trim_data(self, data, ranges, model_points): - """ - Takes input gridded data 'data' (grid points given by input 'model_points') - and trim this to lie within ranges. - - Parameters: - ----------- - data: ndarray - gridded data, shape must be compatible with model_points - - ranges: list of lists of 2 floats - the min and max in each direction - - model_points: list of list of floats - - Returns: - -------- - data trimmed to within ranges - """ - if not (len(ranges)==len(model_points) and len(ranges) == len(data.shape)): - print('Check: i) data incompatible with ranges or ii) ranges incompatible with model_points. Skipping!') - return data - else: - mins = [i[0] for i in ranges] - maxs = [i[1] for i in ranges] - start_indices = Base.find_nearest_cell(mins, model_points) - end_indices = Base.find_nearest_cell(maxs, model_points) - - IdxsToRemove = [] - num_points = [len(model_points[i]) for i in range(len(model_points))] - for i in range(len(ranges)): - IdxsToRemove.append([ j for j in range(num_points[i]) if j < start_indices[i] or j > end_indices[i]]) - - for i in range(len(ranges)): - data = np.delete(data, IdxsToRemove[i], axis=i) - - return data - - def scalar_regression(self, y, X, ranges = None, model_points = None, weights=None, add_intercept=False): - """ - Routine to perform ordinary or weighted (multivariate) regression on some gridded data. - - Parameters: - ----------- - y: ndarray of gridded scalar data - the "measurements" of the dependent quantity - - X: list of ndarrays of gridded data (treated as indep scalars) - the "data" for the regressors - - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - - weights: ndarray of gridded weights - - add_intercept: bool - whether to extemd the dataset to account for a constant offset in the - regression model. - - Returns: - -------- - list of fitted parameters - list of std errors associated with the fitted parameters, - (i.e. parameter * std error of the corresponding regressor.) - - Notes: - ------ - Gridded data and ranges are chosen at the model level, and passed here - as parameters, so that this external method will not change/access any - model quantity. - - Works in any dimensions! - - Intercept must be added manually: for the future, add this via true/false block. - """ - # CHECKING ALIGNMENT OF PASSED DATA - dep_shape = np.shape(y) - n_reg = len(X) - for i in range(n_reg): - if np.shape(X[i]) != dep_shape: - print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') - X.remove(X[i]) - - if len(X)==0: - print('None of the passed regressors is compatible with dep data. Exiting.') - return None - - if weights: - if np.shape(y) != np.shape(weights): - print(f'The weights passed are not not aligned with data, setting these to 1') - weights=np.ones(y.shape) - - # TRIMMING THE DATA TO WITHIN RANGES - if ranges != None and model_points != None: - print('Trimming dataset for regression') - Y = self.trim_data(y, ranges, model_points) - if weights: - Weights = self.trim_data(weights, ranges, model_points) - XX = [] - for x in X: - XX.append(self.trim_data(x, ranges, model_points)) - - # FLATTENING + FITTING - Y = Y.flatten() - if weights: - Weights = Weights.flatten() - - for i in range(len(X)): - XX[i]=XX[i].flatten() - - if add_intercept: - const = np.ones(Y.shape) - XX.insert(0,const) - n_reg = len(XX) - n_data = len(Y) - XX= np.reshape(XX, (n_data, n_reg)) - - # VERSION USING STATSMODELS - # if weights: - # model=sm.WLS(y, Xfl, W) - # result= model.fit() - # return result.params, result.bse - # else: - # model=sm.OLS(y, Xfl) - # result=model.fit() - # return result.params, result.bse - - # VERSION USING SKLEARN: - model=LinearRegression(fit_intercept=False) - if weights: - model.fit(XX,Y,sample_weight=weights) - else: - model.fit(XX,Y) - Y_hat = model.predict(XX) - res = Y - Y_hat - res_sum_of_sq = np.dot(res,res) - sigma_res_sq = res_sum_of_sq / (n_data-n_reg) - var_beta = np.linalg.inv(np.einsum('ji,jl->il',XX,XX)) * sigma_res_sq - bse = [var_beta[i,i]**0.5 for i in range(n_reg)] - - result = [(model.coef_[i], bse[i]) for i in range(n_reg)] - return result - - def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_points=None, components=None, weights=None, add_intercept=False): - """ - Wrapper of scalar_regression: if no components is passed, perform regression on each tensor component - independently, otherwise only on a subset of these. All the component-wise results are returned. - - Parameters: - ----------- - y: ndarray of gridded tensorial data - the "measurements" of the dependent quantity - - X: list of ndarrays of gridded data - the "data" for the regressors - - spatial_dims: int - - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - - components: list of tuples - the tensor components to be considered for regression - - add_intercept: bool - whether to extemd the dataset to account for a constant offset in the - regression model. - - Returns: - -------- - list of fitted parameters - list of std errors associated with the fitted parameters, - (i.e. parameter * std error of the corresponding regressor.) - - Notes: - ------ - Weights are taken as the same for each component here. - For future: do you need to change this? - """ - # CHECKING THE RANK OF DATA AND REGRESSORS ARE COMPATIBLE - l=[0 for i in range(spatial_dims+1)] - dep_shape = y[tuple(l)].shape - for i, x in enumerate(X): - if x[tuple(l)].shape != dep_shape: - print('The {}-th regressor shape {} is not compatible with dependent shape {}. Removing it!'.format(i, x[0,0,0].shape, dep_shape)) - X.remove(x) - if len(X)==0: - print('None of the passed regressors is compatible with dep data. Exiting.') - return None - - # PREPARING COMPONENTS TO LOOP OVER: user-provided or all? - if components: - for c in components: - if len(c) != len(dep_shape): - print(f'The components indices passed {c} are not compatible with data. Ignoring this and moving on!') - components.remove(c) - else: - components = [] - for i in np.ndindex(dep_shape): - components.append(i) - - # RESHAPING: now grid indices come last, needed for flattening! - tot_shape=y.shape - reshaping=[tot_shape[i] for i in range(spatial_dims+1)] - reshaping=tuple(list(dep_shape)+reshaping) - y = y.reshape(reshaping) - for i in range(len(X)): - X[i]= X[i].reshape(reshaping) - - # SCALAR REGRESSION ON EACH COMPONENT - results = [] - for c in components: - yc = y[c] - Xc = [ x[c] for x in X] - results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) - return results - - def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None): - """ - Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. - Possibility to cut data to lie within ranges. - - Parameters: - ----------- - x,y: nd.arrays - must be of the same shape - - xlabel, ylabel: strs - labels for the quantites plotted - - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - - - Returns: - -------- - Instance of JointGrid. Figure can then be accessed at the model level - for further adjustments. - """ - if x.shape != y.shape: - print('Cannot check correlation: data is misaligned!') - return None - - if ranges != None and model_points != None: - print('Trimming dataset for correlation plot') - X = self.trim_data(x, ranges, model_points) - Y = self.trim_data(y, ranges, model_points) - - X=X.flatten() - Y=Y.flatten() - - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') - warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') - g=sns.JointGrid() - sns.scatterplot(x=X, y=Y, ax=g.ax_joint) - sns.kdeplot(x=X, ax=g.ax_marg_x) - sns.histplot(y=Y, ax=g.ax_marg_y, kde=True) - g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) - g.fig.tight_layout() - return g - - def visualize_many_correlations(self, data, labels, ranges=None, model_points=None): - """ - Method that returns an instance of PairGrid of correlation plots for a list of vars. - Plotted is: - daigonal: univariate histogram (with kde superimposed) - upper triangle: scatteplots - lower triangle: bivariate kde(s) - - Possibility to cut data to lie within ranges. - - Parameters: - ----------- - data: list of nd.arrays - must be of the same shape - - labels: list of strs - labels for the quantites plotted - - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - - - Returns: - -------- - Instance of PairGrid. Figure can then be accessed at the model level - for further adjustments. - """ - ref_shape=data[0].shape - Idx_to_delete = [] - for i in range(1,len(data)): - if data[i].shape != ref_shape: - print(f'Cannot use {labels[i]} data: misaligned with first array. Removing it.') - Idx_to_delete.append(i) - - if len(Idx_to_delete) >= 1: - data=list(np.delete(data, Idx_to_delete, axis=0)) - - if ranges != None and model_points != None: - print('Trimming dataset for correlation plot') - for i in range(len(data)): - data[i]=self.trim_data(data[i], ranges, model_points) - - Data = [] - for i in range(len(data)): - Data.append(data[i].flatten()) - - Data=np.column_stack(Data) - Data_df = pd.DataFrame(Data, columns=labels) - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') - warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') - g=sns.PairGrid(Data_df) - g.map_upper(sns.scatterplot) - g.map_lower(sns.kdeplot) - g.map_diag(sns.histplot, kde=True) - g.fig.tight_layout() - return g - - def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_wanted = 1.): - """ - Idea: pass a large list of quantities that are correlated with a residual/closure coeff. - Check if there is a smaller subset of principal components to be retained that are sufficient - to explain the enough of the observed variance in the dataset. - - AIM: linear dimensionality reductions for regressors - - Parameters: - ----------- - data: list of gridded data - - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - - var_wanted: float - should be a number between 0. and 1. : the percetage of variance to be retained. - - Returns: - -------- - comp_decomp: array of shape (n_var, n_comp) - Matrix whose columns contains the decomposition of the principal components - written in the basis of the untransformed (original) variables. - - g: result of self.visualize_many_correlations --> show the PCs are indeed uncorrelated. - - Notes: - ------ - - """ - # CHECKING AND PREPROCESSING THE DATA - ref_shape = data[0].shape - n_vars = len(data) - for i in range(1, n_vars): - if data[i].shape != ref_shape: - print(f'The {i}-th feature passed is not aligned with the first, removing {i}-th feature.') - data.remove(data[i]) - - if len(data)==0: - print('No two vars are compatible. Exiting.') - return None - - if ranges != None and model_points != None: - print('Trimming dataset for PCA analysis.') - for i in range(len(data)): - data[i] = self.trim_data(data[i], ranges, model_points) - - for i in range(n_vars): - data[i] = data[i].flatten() - - #STANDARDIZING DATA TO ZERO MEAN AND UNIT VARIANCE - Data = [] - for i in range(len(data)): - x = data[i] - mean = np.mean(x) - var = np.var(x) - y = np.array([x[j]-mean for j in range(len(x))]) - Data.append(y/var) - # Data.append(y) - Data = np.column_stack(tuple([Data[i] for i in range(n_vars)])) - - # HOW MANY PRINCIPAL COMPONENTS HAVE TO BE RETAINED? - pca_model = PCA().fit(Data) - n_comp=0 - exp_var_sum = np.cumsum(pca_model.explained_variance_ratio_) - var_captured = exp_var_sum[n_comp] - while var_captured < var_wanted: - n_comp +=1 - var_captured= exp_var_sum[n_comp] - n_comp = n_comp+1 - - # WORKING OUT THE COMPONENTS DECOMPOSITION - var2comp = pca_model.components_ # shape: n_comp * n_var - comp_decomp = [] - for i in range(n_comp): - # should be the inverse but var2comp is unitary, so transpose - comp_decomp.append(np.einsum('ij->ji', var2comp)[:,i]) - - # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEY'RE UNCORRELATED - # this block is temporary and will be removed later. - pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) - labels = [f'{i} comp' for i in range(n_comp)] - pc_for_plotting = [pc_profiles[:,i] for i in range(n_comp)] - g = self.visualize_many_correlations(pc_for_plotting, labels) - - return comp_decomp, g - - def PCA_find_regressors(): - """ - Idea pass both the residuals and a large list of quantities, identify which among these are - correlated with the residual. Do this by checking the scores of the residual (the first var - of the dataset, say) on the various principal components. - Then return the weights. This will then be used in regression (possibyly after checking they're - not correlated). - """ - pass - - # Not sure about these two methods. - def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ - interp_dims, method, y_component_indices, x_component_indices): - y_data_to_plot, points = \ - self.visualizer.get_var_data(model, y_var_str, t, x_range, y_range, interp_dims, method, y_component_indices) - x_data_to_plot, points = \ - self.visualizer.get_var_data(model, x_var_str, t, x_range, y_range, interp_dims, method, x_component_indices) - fig = plt.figure(figsize=(16,16)) - sns.jointplot(x=x_data_to_plot.flatten(), y=y_data_to_plot.flatten(), kind="hex", color="#4CB391") - plt.title(y_var_str+'('+x_var_str+')') - fig.tight_layout() - plt.show() - - def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, method, component_indices): - - data_to_plot, points = \ - self.visualizer.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - - # print(data_to_plot) - - fig = plt.figure(figsize=(16,16)) - sns.displot(data_to_plot) - plt.title(var_str) - fig.tight_layout() - plt.show() - - -if __name__ == '__main__': - pass diff --git a/FileReaders.py b/FileReaders.py deleted file mode 100644 index 1779599..0000000 --- a/FileReaders.py +++ /dev/null @@ -1,216 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Fri Mar 31 10:00:00 2023 - -@author: Thomas -""" - -import h5py -import glob -import numpy as np - -class METHOD_HDF5(object): - - def __init__(self, directory): - """ - Set up the list of files (from hdf5) and dictionary with dataset names - in the hdf5 file. - - Parameters - ---------- - directory: string - the filenames in the directory have to be incremental (sorted is used) - """ - - hdf5_filenames = sorted( glob.glob(directory+str('*.hdf5'))) - self.hdf5_files = [] - for filename in hdf5_filenames: - self.hdf5_files.append(h5py.File(filename,'r')) - self.num_files = len(self.hdf5_files) - - self.hdf5_keys = dict.fromkeys(list(self.hdf5_files[0].keys())) - for key in self.hdf5_keys: - self.hdf5_keys[key] = list(self.hdf5_files[0][key].keys()) - - def get_hdf5_keys(self): - return self.hdf5_keys - - def read_in_data(self, micro_model): - """ - Store data from files into micro_model - - Parameters - ---------- - micro_model: class MicroModel - strs in micromodel have to be the same as hdf5 files output from METHOD. - """ - - self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) - for prim_str in micro_model.get_prim_strs(): - if prim_str == "n": - self.translating_prims[prim_str] = "rho" - else: - self.translating_prims[prim_str] = prim_str - - for prim_var_str in micro_model.prim_vars: - try: - method_str = self.translating_prims[prim_var_str] - for counter in range(self.num_files): - micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+method_str][:] ) - # The [:] is for returning the arrays not the dataset - micro_model.prim_vars[prim_var_str] = np.array(micro_model.prim_vars[prim_var_str]) - except KeyError: - print(f'{method_str} is not in the hdf5 dataset: check Primitive/') - - - self.translating_aux = dict.fromkeys(micro_model.get_aux_strs()) - for aux_str in micro_model.get_aux_strs(): - self.translating_aux[aux_str] = aux_str - - for aux_var_str in micro_model.aux_vars: - try: - method_str = self.translating_aux[aux_var_str] - for counter in range(self.num_files): - micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+method_str][:] ) - micro_model.aux_vars[aux_var_str] = np.array(micro_model.aux_vars[aux_var_str]) - except KeyError: - print(f'{method_str} is not in the hdf5 dataset: check Auxiliary/') - - # As METHOD saves endTime, the time variables (and points) need to be dealt with separately - for dom_var_str in micro_model.domain_int_strs: - try: - if dom_var_str == 'nt': - pass - else: - micro_model.domain_vars[dom_var_str] = int( self.hdf5_files[0]['Domain/' + dom_var_str][:]) - except KeyError: - print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - - for dom_var_str in micro_model.domain_float_strs: - try: - if dom_var_str in ['tmin', 'tmax']: - pass - else: - micro_model.domain_vars[dom_var_str] = float( self.hdf5_files[0]['Domain/' + dom_var_str][:]) - except KeyError: - print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - - for dom_var_str in micro_model.domain_array_strs: - try: - if dom_var_str in ['t','points']: - pass - else: - micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] - except KeyError: - print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - - - micro_model.domain_vars['nt'] = self.num_files - for counter in range(self.num_files): - micro_model.domain_vars['t'].append( float(self.hdf5_files[counter]['Domain/endTime'][:])) - micro_model.domain_vars['t'] = np.array(micro_model.domain_vars['t']) - micro_model.domain_vars['tmin'] = np.amin(micro_model.domain_vars['t']) - micro_model.domain_vars['tmax'] = np.amax(micro_model.domain_vars['t']) - micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ - micro_model.domain_vars['y']] - - def read_in_data_HDF5_missing_xy(self, micro_model): - """ - Store data from files into micro_model - - Parameters - ---------- - micro_model: class MicroModel - strs in micromodel have to be the same as hdf5 files output from METHOD. - """ - - self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) - for prim_str in micro_model.get_prim_strs(): - if prim_str == "n": - self.translating_prims[prim_str] = "rho" - else: - self.translating_prims[prim_str] = prim_str - - for prim_var_str in micro_model.prim_vars: - try: - method_str = self.translating_prims[prim_var_str] - for counter in range(self.num_files): - micro_model.prim_vars[prim_var_str].append( self.hdf5_files[counter]["Primitive/"+method_str][:] ) - # The [:] is for returning the arrays not the dataset - micro_model.prim_vars[prim_var_str] = np.array(micro_model.prim_vars[prim_var_str]) - except KeyError: - print(f'{method_str} is not in the hdf5 dataset: check Primitive/') - - - self.translating_aux = dict.fromkeys(micro_model.get_aux_strs()) - for aux_str in micro_model.get_aux_strs(): - self.translating_aux[aux_str] = aux_str - - for aux_var_str in micro_model.aux_vars: - try: - method_str = self.translating_aux[aux_var_str] - for counter in range(self.num_files): - micro_model.aux_vars[aux_var_str].append( self.hdf5_files[counter]["Auxiliary/"+method_str][:] ) - micro_model.aux_vars[aux_var_str] = np.array(micro_model.aux_vars[aux_var_str]) - except KeyError: - print(f'{method_str} is not in the hdf5 dataset: check Auxiliary/') - - # As METHOD saves endTime, the time variables (and points) need to be dealt with separately - # Similar is for x,y which are not stored properly in by METHOD parallelSaveDataHDF5 - for dom_var_str in micro_model.domain_int_strs: - try: - if dom_var_str == 'nt': - pass - else: - micro_model.domain_vars[dom_var_str] = int( self.hdf5_files[0]['Domain/' + dom_var_str][:]) - except KeyError: - print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - - for dom_var_str in micro_model.domain_float_strs: - try: - if dom_var_str in ['tmin', 'tmax']: - pass - else: - micro_model.domain_vars[dom_var_str] = float( self.hdf5_files[0]['Domain/' + dom_var_str][:]) - except KeyError: - print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - - for dom_var_str in micro_model.domain_array_strs: - try: - if dom_var_str in ['t','points', 'x', 'y']: - pass - else: - micro_model.domain_vars[dom_var_str] = self.hdf5_files[0]['Domain/' + dom_var_str][:] - except KeyError: - print(f'{dom_var_str} is not in the hdf5 dataset: check Domain/') - - - micro_model.domain_vars['nt'] = self.num_files - for counter in range(self.num_files): - micro_model.domain_vars['t'].append( float(self.hdf5_files[counter]['Domain/endTime'][:])) - micro_model.domain_vars['t'] = np.array(micro_model.domain_vars['t']) - micro_model.domain_vars['tmin'] = np.amin(micro_model.domain_vars['t']) - micro_model.domain_vars['tmax'] = np.amax(micro_model.domain_vars['t']) - - micro_model.domain_vars['x'] = np.zeros(micro_model.domain_vars['nx']) - for i in range(len(micro_model.domain_vars['x'])): - micro_model.domain_vars['x'][i] = (micro_model.domain_vars['xmax']- micro_model.domain_vars['xmin']) / (2 * micro_model.domain_vars['nx']) + \ - i * micro_model.domain_vars['dx'] - - micro_model.domain_vars['y'] = np.zeros(micro_model.domain_vars['ny']) - for i in range(len(micro_model.domain_vars['y'])): - micro_model.domain_vars['y'][i] = (micro_model.domain_vars['ymax']- micro_model.domain_vars['ymin']) / (2 * micro_model.domain_vars['ny']) + \ - i * micro_model.domain_vars['dy'] - - micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ - micro_model.domain_vars['y']] - - -if __name__ == '__main__': - - from MicroModels import * - - FileReader = METHOD_HDF5('./Data/test_res100/') - MicroModel = IdealMHD_2D() - FileReader.read_in_data(MicroModel) - # FileReader.read_in_data_HDF5_missing_xy(MicroModel) diff --git a/Filters.py b/Filters.py deleted file mode 100644 index 501bb58..0000000 --- a/Filters.py +++ /dev/null @@ -1,1088 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Tue Mar 28 15:36:01 2023 - -@author: Thomas -""" - -import numpy as np -import time -import scipy.integrate as integrate -from scipy.optimize import minimize, root -from itertools import product -from system.BaseFunctionality import * - -from MicroModels import * -from FileReaders import * -from system.BaseFunctionality import * - -class FindObs_flux_min(object): - """ - Class for computing the observer by minimizing the (micro) baryon current - surface flux over a box. - Work in any dimension, read on construction from the micro-model. - """ - def __init__(self, micro_model, box_len): - """ - Parameters: - ----------- - micro_model: instance of class containing the microdata - - box_len: float, side of the box - """ - self.micro_model = micro_model - self.spatial_dims = micro_model.get_spatial_dims() - self.L = box_len - - self.flux_methods = { - "gauss" : self.flux_residual_Gauss , - "linear" : self.flux_residual , - "inbuilt" : self.flux_residual_ib - } - - def set_box_length(self, bl): - self.L = bl - - def get_tetrad_from_U(self, U): - """ - Build tetrad orthogonal to unit velocity with from complete velocity vector - - Parameters: - ----------- - U: list of d+1 floats, with d the number of spatial dimensions - - Return: - ------- - list of arrays: U + d unit vectors that complete it to a orthonormal basis - """ - if len(U) != 1+self.spatial_dims: - print('The dimension of U passed is not compatible with \ - micro_model dimensionality!') - return None - - es =[] - for _ in range(self.spatial_dims): - es.append(np.zeros(self.spatial_dims+1)) - for i in range(len(es)): - es[i][i+1] = 1 - tetrad = [U] - for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! - vec = vec + np.multiply(Base.Mink_dot(vec, U), U) - for j in range(i-1,-1,-1): - vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) - es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) - tetrad += [es[i]] - return tetrad - - def get_tetrad_from_vels(self, spatial_vels): - """ - Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels - - Parameters: - ----------- - spatial_vels: list of d floats, with d the number of spatial dimensions - - Return: - ------- - list of arrays: U + d unit vectors that complete it to a orthonormal basis - """ - if len(spatial_vels) != self.spatial_dims: - print('The number of spatial velocities passed is not compatible with \ - micro_model dimensionality!') - return None - - U = np.array(Base.get_rel_vel(spatial_vels)) - return self.get_tetrad_from_U(U) - - def flux_residual(self, spatial_vels, point, lin_spacing = 10): - """ - Compute the drift of baryons through the box built from spatial_vels. - First get the center of the 2*(d+1) faces of the (d+1)-box, then build coords - for points to sample the flux through each face. - Next, approximate the flux integral as a sum - - Parameters: - ----------- - spatial_vels: list of d (spatial dimension) floats, spatial coord of vel - - point: list of d+1 floats (t,x,y) for the box center - - lin_spacing: integer, lin_spacing**spatial_dim is the # of points used to - sample the flux through each face. - - Returns: - -------- - float: absolute flux - - Notes: - ------ - Much faster than method based on inbuilt dblquad. - """ - tetrad = self.get_tetrad_from_vels(spatial_vels) - flux = 0 - - xs = [] - for i in range(self.spatial_dims): - xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) - coords = [] - for element in product(*xs): - coords.append(np.array(element)) - - for vec in tetrad: - rem_vecs = [x for x in tetrad if not (x==vec).all()] - for i in range(2): - center = point + np.multiply( (-1)**i * self.L / 2, vec) - - surf_coords = [] - for coord in coords: - temp = center - for i in range(self.spatial_dims): - temp += np.multiply(coord[i], rem_vecs[i]) - surf_coords.append(temp) - - for coord in surf_coords: - flux += Base.Mink_dot(self.micro_model.get_interpol_var('BC', coord), vec) - - flux *= (self.L / lin_spacing) ** self.spatial_dims - return abs(flux) - - def flux_residual_Gauss(self, spatial_vels, point, order = 3): - """ - Alternative for computing manually the flux residual, using the Gauss - Legendre method. - - Parameters: - ----------- - spatial_vels: list of d (spatial dimension) floats, spatial coord of vel - - point: list of d+1 floats (t,x,y) for the box center - - order: integer, order of the Gauss-Legendre sampling method. - - Returns: - -------- - float: absolute flux - - Notes: - ------ - Much faster than method based on inbuilt dblquad. Should be more accurate - than the one based on linearly spaced points. - """ - ps1d = [] - ws1d = [] - if order == 3: - ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] - ws1d = [8./9. , 5./9. , 5./9.] - elif order == 4: - p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) - p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) - w1 = (18. + np.sqrt(30) ) / 36. - w2 = (18. - np.sqrt(30) ) / 36. - ps1d = [p1, -p1, p2, -p2] - ws1d = [w1, w1, w2, w2] - elif order == 5: - p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 - p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 - w1 = (322 + 13 * np.sqrt(70)) /900 - w2 = (322 - 13 * np.sqrt(70)) /900 - ps1d = [0, p1, -p1, p2, -p2] - ws1d = [128/225, w1, w1, w2, w2] - else: - print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") - return [] - - xs = [] - ws = [] - for _ in range(self.spatial_dims): - xs.append(ps1d) - ws.append(ws1d) - - coords = [] - for element in product(*xs): - coords.append(np.multiply(self.L/2, np.array(element) )) - - totws = [] - for element in product(*ws): - temp = 1. - for w in element: - temp *= w - totws.append(temp) - - tetrad = self.get_tetrad_from_vels(spatial_vels) - flux = 0. - for vec in tetrad: - rem_vecs = [x for x in tetrad if not (x==vec).all()] - for i in range(2): - center = point + np.multiply( (-1)**i * self.L / 2, vec) - - surf_coords = [] - for coord in coords: - temp = center - for i in range(self.spatial_dims): - temp += np.multiply(coord[i], rem_vecs[i]) - surf_coords.append(temp) - - for i, coord in enumerate(surf_coords): - Na = self.micro_model.get_interpol_var('BC',coord) - flux += Base.Mink_dot(Na, vec) * totws[i] - - flux *= (self.L /2 ) ** self.spatial_dims - return abs(flux) - - def flux_residual_ib(self, spatial_vels, point, abserr = 1e-7): - """ - Compute the flux residual using inbuilt method dblquad or tplquad - Based on function point_flux (nested definition) - - Parameters: - ----------- - Vs: list of d floats, spatial components of the velocity vec - - point: float, center of the box - - Returns: - -------- - tuple: absolute flux and error estimate - - Notes: - ------ - Vastly slower than method above. - As this is based on dblquad, this method works fine only if it's 2+1 dim - - Also, dblquad returns an estimate of the error. This info is presently discarded - in order to have the method return a scalar value so that this can be minimized - by find observer. - - """ - tetrad = self.get_tetrad_from_vels(spatial_vels) - flux = 0 - partial_flux = 0 - error = 0 - partial_error = 0 - - if self.spatial_dims == 2: - def point_flux(x, y , center, Vx, Vy, normal): - coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) - Na = self.micro_model.get_interpol_var('BC', coords) - flux = Base.Mink_dot(Na, normal) - return flux - - for vec in tetrad: - rem_vecs = [x for x in tetrad if not (x==vec).all()] - for i in range(2): - center = point + np.multiply( (-1)**i * self.L / 2, vec) - partial_flux, partial_error = integrate.dblquad(point_flux, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, \ - args = (center, rem_vecs[0], rem_vecs[1], vec), epsabs = abserr)[:] - flux += partial_flux - error += partial_error - return abs(flux) - - elif self.spatial_dims == 3: - def point_flux(x, y, z , center, Vx, Vy, Vz, normal): - coords = center + np.multiply(x, Vx) + np.multiply(y, Vy) + np.multiply(z, Vz) - Na = self.micro_model.get_interpol_var('BC', coords) - flux = Base.Mink_dot(Na, normal) - return flux - - for vec in tetrad: - rem_vecs = [x for x in tetrad if not (x==vec).all()] - for i in range(2): - center = point + np.multiply( (-1)**i * self.L / 2, vec) - partial_flux, partial_error = integrate.tplquad(point_flux, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2,\ - args = (center, rem_vecs[0], rem_vecs[1], vec), epsabs = abserr)[:] - flux += partial_flux - error += partial_error - return abs(flux) - - def find_observer(self, point, flux_str = "gauss", initial_guess = None): - """ - Key function: minimize the flux residual and find the observer at point. - - Parameters: - ----------- - point: list of spatial_dims+1 floats (t,x,y) - - flux_str: string, default to "gauss" - string for the method used to compute the flux, must be chosen within - ['gauss', 'linear', 'inbuilt'] - - initial_guess: DEPRECATED, list of d floats: the spatial velocities of the initial guess - if the number of spatial vels passed does not match the micro_model dimensionality, the - pointwise velocity is used instead. - - Returns: - -------- - Successful minimization: Boolean True, observer, error - Failed minimization: Boolean False, coordinates - - Notes: - ------ - To change the number of points to be used with linear or Gauss-Legendre methods, - or the absolute relative of the inbuilt method, change the default values of - the optional arguments in the corresponding methods. - """ - - guess = [] - if initial_guess is not None and len(initial_guess) == self.spatial_dims: - guess = initial_guess - else: - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) - for i in range(1, len(U)): - guess.append(U[i] / U[0]) - guess = np.array(guess) - - try: - sol = minimize(self.flux_methods[flux_str], x0 = guess, args = (point), bounds=((-0.8,0.8),(-0.8,0.8)),tol=1e-6) - if sol.success: - observer = Base.get_rel_vel(sol.x) - if sol.fun > 1e-5: - print(f'Warning: residual is large at {point}', sol.fun) - return sol.success, observer, sol.fun - if not sol.success: - return sol.success, point - except KeyError: - print(f"The method you want to use for computing the flux, {flux_str}, does not exist!") - return None - - def find_observers_points(self, points, flux_str = "gauss"): - """ - Key function: minimize the flux residual and find the observers for points. - flux_str determines the method to compute the flux residual. - - Parameters: - ----------- - points: list of spatial_dims+1 floats, ordered as (t,x,y) - - flux_str: string, default to "gauss" - string for the method used to compute the flux, must be chosen within - ['gauss', 'linear', 'inbuilt'] - - Returns: - -------- - list of: - 1) coordinates at which minimization is successful - 2) corresponding observers - 3) corresponding residual - - list of coordinates at which the minimization failed - - Notes: - ------ - To change the number of points to be used with linear or Gauss-Legendre methods, - change the default values of the optional arguments in the corresponding methods. - """ - observers = [] - errors = [] - success_coords = [] - failed_coords = [] - - for point in points: - sol = self.find_observer(point, flux_str) - if sol[0]: - observers.append(sol[1]) - errors.append(sol[2]) - success_coords.append(point) - - if not sol[0]: - failed_coords.append[point] - - return [success_coords, observers, errors] , failed_coords - - def find_observers_ranges(self, num_points, ranges, flux_str = "gauss" ): - """ - Key function: minimize the flux_residual and find the observers for points - in the ranges. flux_str determines the method to compute the flux residual. - - Parameters: - ----------- - num_points: list of integers - number of points to find observers at in each direction - - ranges: list of lists of two floats [[t_min,t_max], ...] - Define the coord ranges to find observers at - - flux_str: string, default to "gauss" - string for the method used to compute the flux, must be chosen within - ['gauss', 'linear', 'inbuilt'] - - Returns: - -------- - list of: - 1) coordinates at which minimization is successful - 2) corresponding observers - 3) corresponding residual - - list of coordinates at which the minimization failed - - Notes: - ------ - To change the number of points to be used with linear or Gauss-Legendre methods, - change the default values of the optional arguments in the corresponding methods. - """ - - list_of_coords = [] - for i in range(len(num_points)): - list_of_coords.append( np.linspace( ranges[i][0], ranges[i][-1] , num_points[i]) ) - - points = [] - for element in product(*list_of_coords): - points.append(np.array(element)) - - return self.find_observers_points(points, flux_str) - - -class FindObs_drift_root(object): - """ - Class for computing the observer by minimizing the (micro) baryon current - drift over a box. - Work in any dimension, read on construction from the micro-model. - """ - def __init__(self, micro_model, box_len): - """ - Parameters: - ----------- - micro_model: instance of micro_model, the micro data to be filtered - - box_len: float, side of the box for computing drift - """ - self.micro_model = micro_model - self.L = box_len - self.spatial_dims = micro_model.get_spatial_dims() - - self.drift_methods = { - "gauss" : self.drift_residual_gauss , - "linear" : self.drift_residual , - "inbuilt" : self.drift_residual_ib - } - - def set_box_len(self, box_len): - self.L = box_len - - def get_tetrad_from_vels(self, spatial_vels): - """ - Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels - - Parameters: - ----------- - spatial_vels: list of d floats, with d the number of spatial dimensions - - Return: - ------- - list of arrays: U + d unit vectors that complete it to a orthonormal basis - """ - if len(spatial_vels) != self.spatial_dims: - print('The number of spatial velocities passed is not compatible with \ - micro_model dimensionality!') - return None - U = Base.get_rel_vel(spatial_vels) - es =[] - for _ in range(self.spatial_dims): - es.append(np.zeros(self.spatial_dims+1)) - for i in range(len(es)): - es[i][i+1] = 1 - tetrad = [U] - for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! - vec = vec + np.multiply(Base.Mink_dot(vec, U), U) - for j in range(i-1,-1,-1): - vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) - es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) - tetrad += [es[i]] - return tetrad - - def drift_residual(self, spatial_vels, point, lin_spacing = 10): - """ - Compute the averaged baryon current over box, using linearly spaced sample points. - Then compute the drift with respect to each element in the triad completing U (built from - spatial vels) to a tetrad. - - Parameters: - ----------- - spatial_vels: list of d floats, the spatial vels of the observer - - point: list of d+1 floats, the coord of the center of the box - - lin_spacing: integer, number of points (in each direction) used for sampling the box - - Returns: - -------- - list of d floats, the drifts in each direction of the triad. - """ - tetrad = self.get_tetrad_from_vels(spatial_vels) - - xs = [] - for i in range(self.spatial_dims+1): - xs.append(np.linspace(-self.L /2 , self.L /2, lin_spacing)) - adapt_coords = [] - for element in product(*xs): - adapt_coords.append(np.array(element)) - - coords = [] - for coord in adapt_coords: - temp = np.array(point) - for i in range(self.spatial_dims+1): - temp += np.multiply(coord[i], tetrad[i]) - coords.append(temp) - - integral = np.zeros(1+self.spatial_dims) - for coord in coords: - DeltaV = (self.L / lin_spacing)**(1+self.spatial_dims) - integral += np.multiply(DeltaV, self.micro_model.get_interpol_var('BC', coord)) - - drifts = [] - for i in range(len(tetrad)-1): - drifts.append(Base.Mink_dot(integral, tetrad[i+1])) - - return drifts - - def drift_residual_gauss(self, spatial_vels, point, order = 3): - """ - Method to compute net drift residual through the box, using Gauss-Legendre method - - Parameters: - ----------- - spatial_vels: list of d floats, the spatial velocities of the observer - - point: list of d+1 floats ordered as (t,x,y,...), the centre of the box - - order: integer, the order of the Gauss-Legendre approximation - - Returns: - -------- - list of d floats, the average drift in each direction of the triad. - - """ - ps1d = [] - ws1d = [] - if order == 3: - ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] - ws1d = [8./9. , 5./9. , 5./9.] - elif order == 4: - p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) - p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) - w1 = (18. + np.sqrt(30) ) / 36. - w2 = (18. - np.sqrt(30) ) / 36. - ps1d = [p1, -p1, p2, -p2] - ws1d = [w1, w1, w2, w2] - elif order == 5: - p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 - p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 - w1 = (322 + 13 * np.sqrt(70)) /900 - w2 = (322 - 13 * np.sqrt(70)) /900 - ps1d = [0, p1, -p1, p2, -p2] - ws1d = [128/225, w1, w1, w2, w2] - else: - print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") - return [] - - xs = [] - ws = [] - for _ in range(self.spatial_dims+1): - xs.append(ps1d) - ws.append(ws1d) - - adapt_coords = [] - for element in product(*xs): - adapt_coords.append(np.multiply(self.L/2, np.array(element) )) - - totws = [] - for element in product(*ws): - temp = 1. - for w in element: - temp *= w - totws.append(temp) - - tetrad = self.get_tetrad_from_vels(spatial_vels) - coords = [] - for coord in adapt_coords: - temp = np.array(point) - for i in range(self.spatial_dims+1): - temp += np.multiply(coord[i], tetrad[i]) - coords.append(temp) - - integral = np.zeros(1+self.spatial_dims) - for i, coord in enumerate(coords): - integral += np.multiply(totws[i], self.micro_model.get_interpol_var('BC', coord)) - integral *= (self.L /2 ) ** (self.spatial_dims+1) - - drifts = [] - for i in range(len(tetrad)-1): - drifts.append(Base.Mink_dot(integral, tetrad[i+1])) - - return drifts - - def drift_residual_ib(self, spatial_vels, point, abserr = 1e-7): - """ - Method to compute net drift residual through the box, using Gauss-Legendre method - - Parameters: - ----------- - spatial_vels: list of d floats, the spatial velocities of the observer - - point: list of d+1 floats ordered as (t,x,y,...), the centre of the box - - abserr = optional float, absolute error to be passed to tplquad - - Returns: - -------- - list of d floats, the average drift in each direction of the triad. - """ - if self.spatial_dims != 2: - print('In built methods for integrals can be used only in 2+1 dimensions (i.e. tplquad). The 1+1 \ - dimensional case is not really interesting so not implemented.') - return None - else: - tetrad = self.get_tetrad_from_vels(spatial_vels) - integral = np.zeros(3) - error = np.zeros(3) - - for i in range(3): - def BC_point_value(t, x, y): - coord = point + np.multiply(t, tetrad[0]) + np.multiply(x, tetrad[1]) + np.multiply(y, tetrad[2]) - return self.micro_model.get_interpol_var('BC', coord)[i] - - integral[i], error[i] = integrate.tplquad(BC_point_value, -self.L / 2, self.L / 2, -self.L / 2, self.L / 2, \ - -self.L/2, self.L/2, args = (), epsabs = abserr)[:] - - drifts = [] - for i in range(len(tetrad)-1): - drifts.append(Base.Mink_dot(integral, tetrad[i+1])) - - return drifts - - def find_observer(self, point, drift_str = "gauss", initial_guess= None): - """ - Key method: use optimize.root to find the roots of the drift function. - - Parameters: - ----------- - point: list of d+1 floats, ordered (t,x,y,...), the coordinates of the box centre - - drift_str: string, the method used to compute the net drift - must be either 'gauss', 'linear', 'inbuilt' - - initial_guess: DEPRECATED, list of d floats: the spatial velocities of the initial guess - if the number of spatial vels passed does not match the micro_model dimensionality, the - pointwise velocity is used instead. - - Returns: - -------- - Successful root-finding: Boolean True, observer, avg error - Failed minimization: Boolean False, coordinates - - Notes: - ------ - Change the optional values of the corresponding drift methods to change - the number of points used to sample the d+1 box. - """ - guess = [] - if initial_guess is not None and len(initial_guess) == self.spatial_dims: - guess = initial_guess - else: - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) - for i in range(1, len(U)): - guess.append(U[i] / U[0]) - guess = np.array(guess) - - try: - sol = root(self.drift_methods[drift_str], x0 = guess, args = (point)) - if sol.success: - observer = Base.get_rel_vel(sol.x) - error = 0 - for i in range(len(sol.fun)): - error += sol.fun[i] - avg_error = error/ len(sol.fun) - if avg_error > 1e-5: - print(f'Warning: residual is large at {point}', avg_error) - return sol.success, observer, avg_error - if not sol.success: - return sol.success, point - except KeyError: - print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") - return None - - def find_observers_points(self, points, drift_str = "gauss"): - """ - Key method: use optimize.root to find the roots of the drift function. - - Parameters: - ----------- - points: list of spatial_dims+1 floats, ordered as (t,x,y) - - drift_str: string, default to "gauss" - string for the method used to compute the drifts, must be chosen within - ['gauss', 'linear', 'inbuilt'] - - Returns: - -------- - list of: - 1) coordinates at which root finding is successful - 2) corresponding observers - 3) corresponding residual - - list of coordinates at which root finding failed - - Notes: - ------ - Change the optional values of the corresponding drift methods to change - the number of points used to sample the d+1 box. - """ - observers = [] - avg_errors = [] - success_coords = [] - failed_coords = [] - - for point in points: - sol = self.find_observer(point, drift_str) - if sol[0]: - observers.append(sol[1]) - avg_errors.append(sol[2]) - success_coords.append(point) - - if not sol[0]: - failed_coords.append[point] - - return [success_coords, observers, avg_errors] , failed_coords - - def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): - """ - Key function: minimize the flux_residual and find the observers for points - in the ranges. drift_str determines the method to compute the flux residual. - - Parameters: - ----------- - num_points: list of integers - number of points to find observers at in each direction - - ranges: list of lists of two floats [[t_min,t_max], ...] - Define the coord ranges to find observers at - - drift_str: string, default to "gauss" - string for the method used to compute the flux, must be chosen within - ['gauss', 'linear', 'inbuilt'] - - Returns: - -------- - list of: - 1) coordinates at which root finding is successful - 2) corresponding observers - 3) corresponding residual - - list of coordinates at which the root finding failed - - Notes: - ------ - Change the optional values of the corresponding drift methods to change - the number of points used to sample the d+1 box. - """ - - list_of_coords = [] - for i in range(len(num_points)): - list_of_coords.append( np.linspace( ranges[i][0], ranges[i][-1] , num_points[i]) ) - - points = [] - for element in product(*list_of_coords): - points.append(np.array(element)) - - return self.find_observers_points(points, drift_str) - - -class spatial_box_filter(object): - """ - Class for box-filtering the variables of a micro_model. - Work in any dimensions, read on construction from the micro_model. - The observers for covariant filtering are computed separately and must be - passed when the fitlering methods are called. - """ - def __init__(self, micro_model, filter_width): - """ - Parameters: - ---------- - filter_width: float, width of the filter window in each direction - - micro_model: instance of micro_model class, so to have access to its structures - """ - self.micro_model = micro_model - self.spatial_dims = micro_model.get_spatial_dims() - self.filter_width = filter_width - - def set_filter_width(self, filter_width): - """ - Method to change the width of the filter. - - Parameters: - ----------- - filter_width: float - - Returns: - -------- - None - """ - self.filter_width = filter_width - - def set_micro_model(self, micro_model): - """ - Method to change the micro_model to be filtered. - - Parameters: - ----------- - micro_model: instance of micro_model class - - Returns: - -------- - None - """ - self.micro_model = micro_model - self.spatial_dims = micro_model.get_spatial_dims() - - def complete_U_tetrad(self, U): - """ - Build unit vectors orthogonal to observer U - U has to be normalized to -1. - - Parameters: - ----------- - U: np.array of shape (1+spatial_dims,) - - Returns: - list of spatial_dims numpy arrays of shape (1+spatial_dims,) - which complete U to a orthonormal tetrad - """ - es =[] - for _ in range(self.spatial_dims): - es.append(np.zeros(self.spatial_dims+1)) - for i in range(len(es)): - es[i][i+1] = 1. - triad = [] - for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! - vec = vec + np.multiply(Base.Mink_dot(vec, U), U) - for j in range(i-1,-1,-1): - vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) - es[i] = np.multiply(vec, 1/np.sqrt(Base.Mink_dot(vec, vec))) - triad += [es[i]] - return triad - - def filter_var_point(self, var_str, point, observer, sample_method = "gauss", num_points = 3): - """ - First complete the observer to a tetrad at the point. Then build coords for - sample points in the spatial directions adapted to observer. Then approximate the - filter integral as a Riemann sum. - - Parameters: - ----------- - var_str: string corresponding to a variable of the micro_model - - point: list of floats (t,x,y) - - observer: np.array of shape (1+spatial_dims,) . - - sample_method: string, either 'gauss' or 'linear'. Decide how to sample the d-volume for - filtering. - - num_points: integer, number of sample points in each spatial direction. - - Returns: - -------- - nd.array with shape of the variable, the box_filtered quantity. - - Notes: - ------ - Current version uses interpolated values. Should this be too expensive, change - get_interpol_var for get_var_gridpoint! - - Gauss quadrature gives more accurate results for low values of num-points, hence - reduce number of interpolations. - The alternative is to use the gridpoint method with linearly spaced sampling points. - """ - xs = [] - coords = [] - if sample_method == "linear": - for i in range(self.spatial_dims): - xs.append(np.linspace(-self.filter_width /2 , self.filter_width /2, num_points)) - for element in product(*xs): - coords.append(np.array(element)) - - vecs = self.complete_U_tetrad(observer) - sample_points = [] - for coord in coords: - temp = np.array(point) - for i in range(self.spatial_dims): - temp += np.multiply(coord[i], vecs[i]) - sample_points.append(temp) - - filtered_var = np.zeros(self.micro_model.get_interpol_var(var_str, point).shape) - for sample in sample_points: - filtered_var += self.micro_model.get_interpol_var(var_str, sample) - - return np.multiply(filtered_var, 1 / (num_points**self.spatial_dims)) - - elif sample_method == "gauss": - ps1d = [] - ws1d = [] - ws = [] - totws = [] - if num_points == 3: - ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] - ws1d = [8./9. , 5./9. , 5./9.] - elif num_points == 4: - p1 = np.sqrt(3./7 - 2/7 * np.sqrt(6/5)) - p2 = np.sqrt(3./7 + 2/7 * np.sqrt(6/5)) - w1 = (18. + np.sqrt(30) ) / 36. - w2 = (18. - np.sqrt(30) ) / 36. - ps1d = [p1, -p1, p2, -p2] - ws1d = [w1, w1, w2, w2] - elif num_points == 5: - p1 = np.sqrt(5 - 2* np.sqrt(10/7.))/3 - p2 = np.sqrt(5 + 2* np.sqrt(10/7.))/3 - w1 = (322 + 13 * np.sqrt(70)) /900 - w2 = (322 - 13 * np.sqrt(70)) /900 - ps1d = [0, p1, -p1, p2, -p2] - ws1d = [128/225, w1, w1, w2, w2] - else: - print("The method is implemented for Gauss-Legendre quadrature of order 3, 4 and 5 only!") - return None - - for _ in range(self.spatial_dims): - xs.append(ps1d) - ws.append(ws1d) - - for element in product(*xs): - coords.append(np.multiply(self.filter_width/2, np.array(element) )) - - for element in product(*ws): - temp = 1. - for w in element: - temp *= w - totws.append(temp) - - vecs = self.complete_U_tetrad(observer) - sample_points = [] - for coord in coords: - temp = np.array(point) - for i in range(self.spatial_dims): - temp += np.multiply(coord[i], vecs[i]) - sample_points.append(temp) - - filtered_var = np.zeros(self.micro_model.get_interpol_var(var_str, point).shape) - for i, sample in enumerate(sample_points): - filtered_var += totws[i] * self.micro_model.get_interpol_var(var_str, sample) - - return np.multiply(filtered_var, 1 / (2**self.spatial_dims)) - - else: - print("Sample methods to filter variable must be either 'gauss' or 'linear'! ") - return None - - def filter_var_manypoints(self, var_str, points, observers, sample_method = "gauss", num_points = 3): - """ - Method to filter a variable in the micro_model given a list of points and observers. - - Parameters: - ----------- - var_str: string corresponding to a variable in the micro_model - - points: list of N lists of floats, ordered as (t,x,y) - - observers: list of N np.array of shape (1 + spatial_dims,) - - sample_method, num_points: string, int passed to filter method at a point to choose how to - sample the d-volume for box filtering. - - Returns: - -------- - List with the filtered var (np.float or nd.array depending on var_str) at all points - - Notes: - ------ - Current version uses interpolated values. Should this be too expensive, change - get_interpol_var for get_var_gridpoint! - """ - if len(points) != len(observers): - print("The number of points and observers do not match!") - return [] - - filtered_var = [] - for i, point in enumerate(points): - filtered_var.append(self.filter_var_point(var_str, point, observers[i], sample_method = sample_method, num_points = num_points )) - - return filtered_var - - def filter_var_point_inbuilt(self, var_str, point, observer): - """ - Computed the filtered variable using the inbuilt scipy quad method and the interpolated - values of the quantity. - - Parameters: - ----------- - var_str: string, must correspond to a string in the micro_model - - point: list of 1+spatial_dims floats, center of the box-filter - - observer: nd.array of shape (1+spatial_dims,) - - Returns: - -------- - The filtered quantity at the point, or none if the dimensionality is more than 3+1. - - Notes: - ------ - Much slower than corresponding filter_var_manypoints_ip method, this has been implemented - to check the accuracy of the alternative methods. - """ - vecs = self.complete_U_tetrad(observer) - integrand = np.zeros(self.micro_model.get_interpol_var(var_str,point).shape) - error = np.zeros(self.micro_model.get_interpol_var(var_str,point).shape) - - if self.spatial_dims == 2: - def interpol_var_adapt_coord(x, y, *ind): - p = np.array(point) - p += np.multiply(x, vecs[0]) + np.multiply(y, vecs[1]) - return self.micro_model.get_interpol_var(var_str, p)[ind] - - for ind in np.ndindex(integrand.shape): - integrand[ind], error[ind] = integrate.dblquad(interpol_var_adapt_coord, -self.filter_width/2 , self.filter_width/2, -self.filter_width/2, self.filter_width/2, \ - args = (ind), epsabs = 1e-7 )[:] - - elif self.spatial_dims ==3: - def interpol_var_adapt_coord(self, x, y, z, *ind): - p = np.array(point) - p += np.multiply(x, vecs[0]) + np.multiply(y, vecs[1]) + np.multiply(z, vecs[2]) - return self.micro_model.get_interpol_var(var_str, p)[ind] - - for ind in np.ndindex(integrand.shape): - integrand[ind], error[ind] = integrate.tplquad(interpol_var_adapt_coord, -self.filter_width/2, self.filter_width/2, -self.filter_width/2, self.filter_width/2, \ - -self.filter_width/2, self.filter_width/2, args = (point), epsabs = 1e-7 )[:] - else: - print("The inbuilt method is implemented in 2+1 and 3+1 dims only!") - return None - - return np.multiply( integrand, 1 / (self.filter_width**self.spatial_dims)), error - -if __name__ == '__main__': - CPU_start_time = time.process_time() - - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() - FileReader.read_in_data(micro_model) - micro_model.setup_structures() - - find_obs = FindObs_drift_root(micro_model, 0.001) - # find_obs = FindObs_flux_min(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - - vars = ['BC','SETfl', 'Fab', 'SETem'] - points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] - observers = find_obs.find_observers_points(points)[0][1] - - for i in range(len(points)): - for var in vars: - CPU_start_time = time.process_time() - filtvar1 = filter.filter_var_point_inbuilt(var, points[i], observers[i]) - inbuilt_time = time.process_time() - CPU_start_time - - CPU_start_time = time.process_time() - filtvar2 = filter.filter_var_point(var, points[i], observers[i]) - gauss_time = time.process_time() - CPU_start_time - print(f"CPU time speed-up to filter {var} with Gauss method at {points[i]} is {inbuilt_time/gauss_time}. ") - print(f'Filtered quantity with Gauss is \n {filtvar2}') - print(f"Difference with method based on inbuilt integration is: \n {filtvar1[0] - filtvar2}") - print('\n************************\n') \ No newline at end of file diff --git a/MesoModels.py b/MesoModels.py deleted file mode 100644 index 7992df3..0000000 --- a/MesoModels.py +++ /dev/null @@ -1,1145 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Mon Mar 27 18:53:53 2023 - -@author: Marcus -""" - -import numpy as np -from scipy.interpolate import interpn -import pickle -from multiprocessing import Process, Pool -from multimethod import multimethod - -import matplotlib.pyplot as plt -import seaborn as sns -import scipy.stats as scst - -from system.BaseFunctionality import * -from MicroModels import * -from FileReaders import * -from Filters import * -from Visualization import * -from Analysis import * - -class NonIdealHydro2D(object): - - def __init__(self, MicroModel, ObsFinder, Filter, interp_method = "linear"): - self.MicroModel = MicroModel - self.ObsFinder = ObsFinder - self.Filter = Filter - self.spatial_dims = 2 - self.n_dims = self.spatial_dims + 1 - self.interp_method = interp_method - - self.domain_var_strs = ('Nt','Nx','Ny','dT','dX','dY','points') - self.domain_vars = dict.fromkeys(self.domain_var_strs) - - #Dictionary for 'local' variables, - #obtained from filtering the appropriate MicroModel variables - self.filter_var_strs = ("BC","SET") - self.filter_vars = dict.fromkeys(self.filter_var_strs) - - #Dictionary for MesoModel variables - self.meso_var_strs = ("U","U_coords","U_errors","T~","N") - self.meso_vars = dict.fromkeys(self.meso_var_strs) - - #Strings for 'non-local' variables - ones we need to take derivatives of - self.nonlocal_var_strs = ("U","T~") - - #Dictionary for derivative variables - calculated by finite differencing - self.deriv_var_strs = ("dtU","dxU","dyU","dtT~","dxT~","dyT~") - self.deriv_vars = dict.fromkeys(self.deriv_var_strs) - - #Dictionary for dissipative residuals - from the NonId-SET that we take - #projections of - self.diss_residual_strs = ("Pi","q","pi") - self.diss_residuals = dict.fromkeys(self.diss_residual_strs) - - self.diss_var_strs = ("Theta","Omega","Sigma") - self.diss_vars = dict.fromkeys(self.diss_residual_strs) - - self.diss_coeff_strs = ("Zeta", "Kappa", "Eta") - self.diss_coeffs = dict.fromkeys(self.diss_coeff_strs) - - self.coefficient_strs = ("Gamma") - self.coefficients = dict.fromkeys(self.diss_coeff_strs) - self.coefficients['Gamma'] = 4.0/3.0 - - #Dictionary for all vars - useful for e.g. plotting - self.all_var_strs = self.filter_var_strs + self.meso_var_strs + self.deriv_var_strs\ - + self.diss_residual_strs + self.diss_var_strs + self.diss_coeff_strs - - # Stencils for finite-differencing - self.cen_SO_stencil = [1/12, -2/3, 0, 2/3, -1/12] - self.cen_FO_stencil = [-1/2, 0, 1/2] - self.fw_FO_stencil = [-1, 1] - self.bw_FO_stencil = [-1, 1] - - self.metric = np.zeros((3,3)) - self.metric[0,0] = -1 - self.metric[1,1] = self.metric[2,2] = +1 - - # Run some compatability test... - compatible = True - if self.spatial_dims != self.MicroModel.spatial_dims: - compatible = False - for filter_var_str in self.filter_var_strs: - if not (filter_var_str in MicroModel.vars.keys()): - compatible = False - - if compatible: - print("Meso and Micro models are compatible!") - else: - print("Meso and Micro models are incompatible!") - - def get_all_var_strs(self): - return self.all_var_strs - - def get_model_name(self): - return 'NonIdealHydro2D' - - def find_observers(self, num_points, ranges):#, spacing): - # self.meso_vars['U_coords'], self.meso_vars['U'], self.meso_vars['U_errors'] = \ - # self.ObsFinder.find_observers(num_points, ranges, spacing)[0] - self.meso_vars['U_coords'], self.meso_vars['U'], self.meso_vars['U_errors'] = \ - self.ObsFinder.find_observers_ranges(num_points, ranges)[0] - self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] = num_points[:] - self.domain_vars['dT'] = (ranges[0][-1] - ranges[0][0]) / self.domain_vars['Nt'] - self.domain_vars['dX'] = (ranges[1][-1] - ranges[1][0]) / self.domain_vars['Nx'] - self.domain_vars['dY'] = (ranges[2][-1] - ranges[2][0]) / self.domain_vars['Ny'] - self.domain_vars['points'] = [np.linspace(ranges[0][0], ranges[0][-1], num_points[0]),\ - np.linspace(ranges[1][0], ranges[1][-1], num_points[1]),\ - np.linspace(ranges[2][0], ranges[2][-1], num_points[2])] - - def setup_variables(self): - Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] - n_dims = self.spatial_dims+1 - self.meso_vars['U_coords'] = np.array(self.meso_vars['U_coords']).reshape([Nt, Nx, Ny, n_dims]) - self.meso_vars['U'] = np.array(self.meso_vars['U']).reshape([Nt, Nx, Ny, n_dims]) - self.meso_vars['U_errors'] = np.array(self.meso_vars['U_errors']).reshape([Nt, Nx, Ny, 2]) - self.meso_vars['T~'] = np.zeros((Nt, Nx, Ny)) - self.meso_vars['N'] = np.zeros((Nt, Nx, Ny)) - - self.filter_vars['BC'] = np.zeros((Nt, Nx, Ny, n_dims)) - self.filter_vars['SET'] = np.zeros((Nt, Nx, Ny, n_dims,n_dims)) - - for nonlocal_var_str in self.nonlocal_var_strs: - self.deriv_vars['dt'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) - self.deriv_vars['dx'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) - self.deriv_vars['dy'+nonlocal_var_str] = np.zeros_like(self.meso_vars[nonlocal_var_str]) - - self.diss_residuals['Pi'] = np.zeros((Nt, Nx, Ny)) - self.diss_vars['Theta'] = np.zeros((Nt, Nx, Ny)) - self.diss_residuals['q'] = np.zeros((Nt, Nx, Ny, n_dims)) - self.diss_vars['Omega'] = np.zeros((Nt, Nx, Ny, n_dims)) - self.diss_residuals['pi'] = np.zeros((Nt, Nx, Ny, n_dims, n_dims)) - self.diss_vars['Sigma'] = np.zeros((Nt, Nx, Ny, n_dims, n_dims)) - - # Single value for each coefficient (per data point) for now... - self.diss_coeffs['Zeta'] = np.zeros((Nt, Nx, Ny)) - self.diss_coeffs['Kappa'] = np.zeros((Nt, Nx, Ny, n_dims)) - self.diss_coeffs['Eta'] = np.zeros((Nt, Nx, Ny, n_dims, n_dims)) - - self.vars = self.filter_vars - self.vars.update(self.meso_vars) - self.vars.update(self.deriv_vars) - self.vars.update(self.diss_residuals) - self.vars.update(self.diss_vars) - self.vars.update(self.diss_coeffs) - - def p_from_EoS(self, rho, n): - """ - Calculate pressure from EoS using rho (energy density) and n (number density) - """ - p = (self.coefficients['Gamma']-1)*(rho-n) - return p - - def filter_micro_variables(self): - """ - 'Spatially' average required variables from the micromodel w.r.t. - the observers that have been found. - """ - for h in range(self.domain_vars['Nt']): - for i in range(self.domain_vars['Nx']): - for j in range(self.domain_vars['Ny']): - self.filter_vars['BC'][h,i,j] = self.Filter.filter_var_point('BC', self.meso_vars['U_coords'][h,i,j], - self.meso_vars['U'][h,i,j]) - self.filter_vars['SET'][h,i,j] = self.Filter.filter_var_point('SET', self.meso_vars['U_coords'][h,i,j], - self.meso_vars['U'][h,i,j]) - - # self.filter_vars['n'][h,i,j] =\ - # self.Filter.filter_prim_var(self.meso_vars['U_coords'][h,i,j], self.meso_vars['U'][h,i,j], 'n') - # self.filter_vars['SET'][h,i,j] =\ - # self.Filter.filter_struc(self.meso_vars['U_coords'][h,i,j], self.meso_vars['U'][h,i,j], 'SET') - - # Should be able to convert here to only 1 filter function... (not prim/struct) - # for filter_var_str in self.filter_var_strs: - # filter_args = [(coord, U, filter_var_str) for coord, U in zip(self.U_coords, self.Us)] - # with Pool(2) as p: - # self.micro_vars[micro_var_str] = p.starmap(self.Filter.filter_var, filter_args) - # self.micro_vars[micro_var_str] = p.starmap(self.Filter.filter_prim_var, filter_args) - - def calculate_derivatives(self): - """ - Calculate the required derivatives of MesoModel variables for - constructing the non-ideal terms. - """ - for nonlocal_var_str in self.nonlocal_var_strs: - self.calculate_time_derivatives(nonlocal_var_str) - self.calculate_x_derivatives(nonlocal_var_str) - self.calculate_y_derivatives(nonlocal_var_str) - - def calculate_time_derivatives(self, nonlocal_var_str): - deriv_var_str = 'dt'+nonlocal_var_str - stencil = self.cen_FO_stencil - samples = [-1,0,1] - for h in range(self.domain_vars['Nt']): - if h == 0: - stencil = self.fw_FO_stencil - samples = [0,1] - if h == (self.domain_vars['Nt']-1): - stencil = self.fw_FO_stencil - samples = [-1,0] - for i in range(self.domain_vars['Nx']): - for j in range(self.domain_vars['Ny']): - for s in range(len(samples)): - self.deriv_vars[deriv_var_str][h,i,j] \ - += (stencil[s]*self.meso_vars[nonlocal_var_str][h+samples[s],i,j]) / self.domain_vars['dT'] - - def calculate_x_derivatives(self, nonlocal_var_str): - deriv_var_str = 'dx'+nonlocal_var_str - stencil = self.cen_FO_stencil - samples = [-1,0,1] - for h in range(self.domain_vars['Nt']): - for i in range(self.domain_vars['Nx']): - if i == 0: - stencil = self.fw_FO_stencil - samples = [0,1] - if i == (self.domain_vars['Nx']-1): - stencil = self.fw_FO_stencil - samples = [-1,0] - for j in range(self.domain_vars['Ny']): - for s in range(len(samples)): - # print((stencil[s]*self.meso_vars[nonlocal_var_str][h,i+samples[s],j]) / self.domain_vars['dX']) - self.deriv_vars[deriv_var_str][h,i,j] \ - += (stencil[s]*self.meso_vars[nonlocal_var_str][h,i+samples[s],j]) / self.domain_vars['dX'] - - def calculate_y_derivatives(self, nonlocal_var_str): - deriv_var_str = 'dy'+nonlocal_var_str - stencil = self.cen_FO_stencil - samples = [-1,0,1] - for h in range(self.domain_vars['Nt']): - for i in range(self.domain_vars['Nx']): - for j in range(self.domain_vars['Ny']): - if j == 0: - stencil = self.fw_FO_stencil - samples = [0,1] - if j == (self.domain_vars['Ny']-1): - stencil = self.fw_FO_stencil - samples = [-1,0] - for s in range(len(samples)): - self.deriv_vars[deriv_var_str][h,i,j] \ - += (stencil[s]*self.meso_vars[nonlocal_var_str][h,i,j+samples[s]]) / self.domain_vars['dY'] - - def calculate_dissipative_residuals(self, h, i, j): - """ - Returns the inferred (residual) values of the - dissipative terms from the filtered MicroModel SET. - - Parameters - ---------- - indices : TYPE - DESCRIPTION. - - Returns - ------- - Pi_res : scalar float - Bulk viscosity - q_res : (d+1) vector of floats - Heat flux. - pi_res : (d+1)x(d+1) tensor of floats - Shear viscosity. - - """ - # Move this - # h, i, j = indices - # Filter the scalar fields - BC = self.filter_vars['BC'][h,i,j] - N = np.sqrt(-Base.Mink_dot(BC, BC)) - self.meso_vars['N'][h,i,j] = N - Id_SET = self.filter_vars['SET'][h,i,j] - U = self.meso_vars['U'][h,i,j] - - # Do required projections of SET - h_mu_nu = Base.orthogonal_projector(U, self.metric) - rho_res = Base.project_tensor(U,U,Id_SET) - - # Set Meso temperature from filtered quantities using EoS - p_tilde = self.p_from_EoS(rho_res, N) - T_tilde = p_tilde/N - self.meso_vars['T~'][h,i,j] = T_tilde - - # Calculate dissipative residual with tensor manipulation - q_res = np.einsum('ij,i,jk',Id_SET,U,h_mu_nu) - tau_res = np.einsum('ij,ik,jl',Id_SET,h_mu_nu,h_mu_nu) - tau_trace = np.trace(tau_res) - Pi_res = tau_trace - p_tilde - pi_res = tau_res - np.dot((p_tilde + Pi_res),h_mu_nu) - - # print(Pi_res, q_res, pi_res) - - self.diss_residuals['Pi'][h,i,j] = Pi_res - self.diss_residuals['q'][h,i,j] = q_res - self.diss_residuals['pi'][h,i,j] = pi_res - - def calculate_dissipative_variables(self, h, i, j): - """ - Calculates the non-ideal, dissipation terms (without coefficeints) - for the non-ideal MesoModel SET. - - Parameters - ---------- - indices : list of ints - Indices of data-point. - coord : list of floats - Coordinates in (t,x,y). - - Returns - ------- - Decomposition of the observer velocity: - - Theta : float (scalar) - Divergence (isotropic). - omega : vector of floats - Transverse momentum. - sigma : (d+1)x(d+1) tensor of floats - Symmetric, trace-free. - - """ - # h, i, j = indices - - T = self.meso_vars['T~'][h, i, j] - dtT = self.deriv_vars['dtT~'][h, i, j] - dxT = self.deriv_vars['dxT~'][h, i, j] - dyT = self.deriv_vars['dyT~'][h, i, j] - - # U = self.meso_vars['U'][h,i,j][:] - Ut, Ux, Uy = self.meso_vars['U'][h,i,j][:] - - # print(Ut, Ux, Uy) - - # dtU = self.deriv_vars['dtU'][h,i,j] - dtUt, dtUx, dtUy = self.deriv_vars['dtU'][h,i,j][:] - dxUt, dxUx, dxUy = self.deriv_vars['dxU'][h,i,j][:] - dyUt, dyUx, dyUy = self.deriv_vars['dyU'][h,i,j][:] - - # Need to do this with Einsum... - Theta = dtUt + dxUx + dyUy - a = np.array([Ut*dtUt + Ux*dxUt + Uy*dyUt, Ut*dtUx + Ux*dxUx + Uy*dyUx, Ut*dtUy + Ux*dxUy + Uy*dyUy]) - - Omega = np.array([dtT, dxT, dyT]) + np.multiply(T,a) - Sigma = np.array([[2*dtUt - (2/3)*Theta, dtUx + dxUt, dtUy + dyUt],\ - [dxUt + dtUx, 2*dxUx - (2/3)*Theta, dxUy + dyUx], - [dyUt + dtUy, dyUx + dxUy, 2*dyUy - (2/3)*Theta]]) - - self.diss_vars['Theta'][h,i,j] = Theta - self.diss_vars['Omega'][h,i,j] = Omega - self.diss_vars['Sigma'][h,i,j] = Sigma - - def calculate_dissipative_coefficients(self): - Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] - # parallel_args = [(h, i, j) for h in range(Nt) for i in range(Nx) for j in range(Ny)] - # # print(parallel_args) - # with Pool(2) as p: - # p.starmap(self.calculate_dissipative_residuals, parallel_args) - # p.starmap(self.calculate_dissipative_variables, parallel_args) - - # self.diss_coeffs['Zeta'] = -self.diss_residuals['Pi'] / self.diss_vars['Theta'] - for h in range(self.domain_vars['Nt']): - for i in range(self.domain_vars['Nx']): - for j in range(self.domain_vars['Ny']): - self.calculate_dissipative_residuals(h,i,j) - - # Derivative calculations need to be done after all residuals - self.calculate_derivatives() - for h in range(self.domain_vars['Nt']): - for i in range(self.domain_vars['Nx']): - for j in range(self.domain_vars['Ny']): - self.calculate_dissipative_variables(h,i,j) - self.diss_coeffs['Zeta'][h,i,j] = -self.diss_residuals['Pi'][h,i,j] / self.diss_vars['Theta'][h,i,j] - for k in range(self.n_dims): - self.diss_coeffs['Kappa'][h,i,j,k] = -self.diss_residuals['q'][h,i,j,k] / self.diss_vars['Omega'][h,i,j,k] - for l in range(self.n_dims): - self.diss_coeffs['Eta'][h,i,j,k,l] = -self.diss_residuals['pi'][h,i,j,k,l] / self.diss_vars['Sigma'][h,i,j,k,l] - - for diss_coeff_str in self.diss_coeff_strs: - coeffs_handle = open(diss_coeff_str+'.pickle', 'wb') - pickle.dump(self.diss_coeffs[diss_coeff_str], coeffs_handle, protocol=pickle.HIGHEST_PROTOCOL) - -class resMHD2D(object): - """ - Idea: alternative meso_models (e.g. with different closures scheme) will - only have to change the method model_residuals and list of non_local_vars - (possibly but less likely decompose_structure_gridpoint) - """ - def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): - """ - ADD DESCRIPTION OF THIS - """ - self.micro_model = micro_model - self.find_obs = find_obs - self.filter = filter - self.spatial_dims = 2 - self.interp_method = interp_method - - self.domain_int_strs = ('Nt','Nx','Ny') - self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") - self.domain_array_strs = ("T","X","Y","Points") - self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) - for var in self.domain_vars: - self.domain_vars[var] = [] - - # This is the Levi-Civita symbol, not tensor, so be careful when using it - self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ - for k in range(3)]for j in range(3) ] for i in range(3) ]) - - self.metric = np.zeros((3,3)) - self.metric[0,0] = -1 - self.metric[1,1] = self.metric[2,2] = +1 - - self.filter_vars_strs = ['U', 'U_errors', 'U_success'] - self.filter_vars = dict.fromkeys(self.filter_vars_strs) - for var in self.filter_vars: - var = [] - - self.meso_structures_strs = ['SETfl', 'SETem', 'BC', 'Fab'] - self.meso_structures = dict.fromkeys(self.meso_structures_strs) - for var in self.meso_structures: - self.meso_structures[var] = [] - - self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'sigma_tilde', 'b_tilde', 'T_tilde'] - self.meso_vectors_strs = ['u_tilde', 'q_res', 'e_tilde', 'j', 'J_tilde'] - self.meso_r2tensors_strs = ['pi_res'] - self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs - self.meso_vars = dict.fromkeys(self.meso_vars_strs) - - self.coefficient_strs = ["Gamma"] - self.coefficients = dict.fromkeys(self.coefficient_strs) - self.coefficients['Gamma'] = 4.0/3.0 - - # Dictionary with stencils and coefficients for finite differencing - self.differencing = dict.fromkeys((1, 2)) - self.differencing[1] = dict.fromkeys(['fw', 'bw', 'cen']) - self.differencing[1]['fw'] = {'coefficients' : [-1, 1] , 'stencil' : [0, 1]} - self.differencing[1]['bw'] = {'coefficients' : [1, -1] , 'stencil' : [0, -1]} - self.differencing[1]['cen'] = {'coefficients' : [-1/2., 0, 1/2.] , 'stencil' : [-1, 0, 1]} - self.differencing[2] = dict.fromkeys(['fw', 'bw', 'cen']) - self.differencing[2]['fw'] = {'coefficients' : [-3/2., 2., -1/2.] , 'stencil' : [0, 1, 2]} - self.differencing[2]['bw'] = {'coefficients' : [3/2., -2., 1/2.] , 'stencil' : [0, -1, -2]} - self.differencing[2]['cen'] = {'coefficients' : [1/12., -2/3., 0., 2/3., -1/12.] , 'stencil' : [-2, -1, 0, 1, 2]} - - - # dictionary with non-local quantities (keys must match one of meso_vars or structure) - self.nonlocal_vars_strs = ['u_tilde', 'Fab', 'T_tilde'] - # self.nonlocal_vars_strs = ['u_tilde', 'Fab'] - Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] - self.deriv_vars = dict.fromkeys(Dstrs) - - # Additional, closure-scheme specific vars must be added appropriately using model_residuals() - - # self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs - # Run some compatibility test... - compatible = True - error = '' - if self.spatial_dims != micro_model.get_spatial_dims(): - compatible = False - error += '\nError: different dimensions.' - for struct in self.meso_structures_strs: - if struct not in self.micro_model.get_structures_strs(): - compatible = False - error += f'\nError: {struct} not in micro_model!' - - if not compatible: - print("Meso and Micro models are incompatible:"+error) - - def get_all_var_strs(self): - return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) - - def get_gridpoints(self): - """ - Pretty self-explanatory - """ - return self.domain_vars['Points'] - - def get_interpol_var(self, var, point): - """ - Returns the interpolated variables at the point. - - Parameters - ---------- - var: str corresponding to meso_structure, meso_vars or deriv_vars - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Interpolated values/arrays corresponding to variable. - Empty list if none of the variables is not meso_structures/meso_vars or deriv_vars of the model. - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - if var in self.meso_structures: - return interpn(self.domain_vars['Points'], self.meso_structures[var], point, method = self.interp_method)[0] - elif var in self.meso_vars: - return interpn(self.domain_vars['Points'], self.meso_vars[var], point, method = self.interp_method)[0] - elif var in self.deriv_vars: - return interpn(self.domain_vars['Points'], self.deriv_vars[var], point, method = self.interp_method)[0] - else: - print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) - - @multimethod - def get_var_gridpoint(self, var: str, h: object, i: object, j: object): - """ - Returns variable corresponding to input 'var' at gridpoint - identified by h,i,j - - Parameters: - ----------- - var: string corresponding to structure, meso_vars or deriv_vars - - h,i,j: int - integers corresponding to position on the grid. - - Returns: - -------- - Values or arrays corresponding to variable evaluated at the closest - grid-point to input 'point'. - - Notes: - ------ - This method is useful e.g. for plotting the raw data. - """ - if var in self.meso_structures: - return self.meso_structures[var][h,i,j] - elif var in self.meso_vars: - return self.meso_vars[var][h,i,j] - elif var in self.deriv_vars: - return self.deriv_vars[var][h,i,j] - else: - print("{} is not a variable of resMHD_2D!".format(var)) - return None - - @multimethod - def get_var_gridpoint(self, var: str, point: object): - """ - Returns variable corresponding to input 'var' at gridpoint - closest to input 'point'. - - Parameters: - ----------- - vars: string corresponding to structure, meso_vars or deriv_vars - - point: list of 2+1 floats - - Returns: - -------- - Values or arrays corresponding to variable evaluated at the closest - grid-point to input 'point'. - - Notes: - ------ - This method should be used in case using interpolated values becomes - too expensive. - """ - indices = Base.find_nearest_cell(point, self.domain_vars['Points']) - if var in self.meso_structures: - return self.meso_structures[var][tuple(indices)] - elif var in self.meso_vars: - return self.meso_vars[var][tuple(indices)] - elif var in self.deriv_vars: - return self.deriv_vars[var][tuple(indices)] - else: - print("{} is not a variable of resMHD_2D!".format(var)) - return None - - def get_model_name(self): - return 'resMHD2D' - - def set_find_obs_method(self, find_obs): - self.find_obs = find_obs - - def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): - """ - Builds the meso_model grid using the micro_model grid points contained in - the input patch (defined via 'patch_bdrs'). The method allows for coarse graining - the grid in the spatial directions ONLY. - Then store the info about the meso grid and set up arrays of definite rank and size - for the quantities needed later. - - Parameters: - ----------- - patch_bdrs: list of lists of two floats, - [[tmin, tmax],[xmin,xmax],[ymin,ymax]] - - coarse_factor: integer - - Notes: - ------ - If the patch_bdrs are larger than micro_grid, the method will not set-up the meso_grid, - and an error message is printed. This is extra safety measure! - """ - - # Is the patch within the micro_model domain? - conditions = patch_bdrs[0][0] < self.micro_model.domain_vars['tmin'] or \ - patch_bdrs[0][1] > self.micro_model.domain_vars['tmax'] or \ - patch_bdrs[1][0] < self.micro_model.domain_vars['xmin'] or \ - patch_bdrs[1][1] > self.micro_model.domain_vars['xmax'] or \ - patch_bdrs[2][0] < self.micro_model.domain_vars['ymin'] or \ - patch_bdrs[2][1] > self.micro_model.domain_vars['ymax'] - - if conditions: - print('Error: the input region for filtering is larger than micro_model domain!') - return None - - #Find the nearest cell to input patch bdrs - patch_min = [patch_bdrs[0][0], patch_bdrs[1][0], patch_bdrs[2][0]] - patch_max = [patch_bdrs[0][1], patch_bdrs[1][1], patch_bdrs[2][1]] - idx_mins = Base.find_nearest_cell(patch_min, self.micro_model.domain_vars['points']) - idx_maxs = Base.find_nearest_cell(patch_max, self.micro_model.domain_vars['points']) - - # Set meso_grid spacings - self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] - self.domain_vars['Dx'] = self.micro_model.domain_vars['dx'] * coarse_factor - self.domain_vars['Dy'] = self.micro_model.domain_vars['dy'] * coarse_factor - - # Building the meso_grid - h, i, j = idx_mins[0], idx_mins[1], idx_mins[2] - while h <= idx_maxs[0]: - t = self.micro_model.domain_vars['t'][h] - self.domain_vars['T'].append(t) - h += 1 - while i <= idx_maxs[1]: - x = self.micro_model.domain_vars['x'][i] - self.domain_vars['X'].append(x) - i += coarse_factor - while j <= idx_maxs[2]: - y = self.micro_model.domain_vars['y'][j] - self.domain_vars['Y'].append(y) - j += coarse_factor - - # Saving the info about the meso_grid - self.domain_vars['Points'] = [self.domain_vars['T'], self.domain_vars['X'], self.domain_vars['Y']] - self.domain_vars['Tmin'] = np.amin(self.domain_vars['T']) - self.domain_vars['Xmin'] = np.amin(self.domain_vars['X']) - self.domain_vars['Ymin'] = np.amin(self.domain_vars['Y']) - self.domain_vars['Tmax'] = np.amax(self.domain_vars['T']) - self.domain_vars['Xmax'] = np.amax(self.domain_vars['X']) - self.domain_vars['Ymax'] = np.amax(self.domain_vars['Y']) - self.domain_vars['Nt'] = len(self.domain_vars['T']) - self.domain_vars['Nx'] = len(self.domain_vars['X']) - self.domain_vars['Ny'] = len(self.domain_vars['Y']) - - # Setup arrays for structures - Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] - self.meso_structures['BC'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) - self.meso_structures['SETfl'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - self.meso_structures['SETem'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - self.meso_structures['Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - - # Setup arrays for meso_vars - for str in self.meso_scalars_strs: - self.meso_vars[str] = np.zeros((Nt, Nx, Ny)) - for str in self.meso_vectors_strs: - self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) - for str in self.meso_r2tensors_strs: - self.meso_vars[str] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - - # Setup arrays for filter_vars - self.filter_vars['U'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) - self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) - self.filter_vars['U_success'] = dict() - - - # Setup arrays for derivatives of the model. - # THOMAS'S WAY (SAVE THEM AS TENSORS) - self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) - self.deriv_vars['D_Fab'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1, self.spatial_dims+1)) - self.deriv_vars['D_T_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) - - # MARCUS'S WAY - # for str in self.non_local_vars_strs: - # if str in self.meso_vars: - # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_vars[str])}) - # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_vars[str])}) - # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_vars[str])}) - # if str in self.meso_structures: - # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_structures[str])}) - # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) - # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) - - def find_observers(self): - """ - Method to compute filtering observers at grid points built with setup_meso_grid. - The observers found (and relative errors) are saved in the dictionary self.filter_vars. - Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices - on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) - - Notes: - ------ - Requires setup_meso_grid() and setup_variables() to be called first. - """ - for h, t in enumerate(self.domain_vars['T']): - for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): - point = [t,x,y] - sol = self.find_obs.find_observer(point) - if sol[0]: - self.filter_vars['U'][h,i,j] = sol[1] - self.filter_vars['U_errors'][h,i,j] = sol[2] - self.filter_vars['U_success'].update({(h,i,j) : True}) - - if not sol[0]: - self.filter_vars['U_success'].update({(h,i,j) : False}) - print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) - - def filter_micro_variables(self): - """ - Filter all meso_model structures AND micro pressure at the grid_points built with setup_meso_grid(). - This method relies on filter_var_point implemented separately for the filter, e.g. spatial_box_filter - - Parameters: - ----------- - - Notes: - ------ - Requires find_observers() and setup_meso_grid() to be called first. - """ - for h, t in enumerate(self.domain_vars['T']): - for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): - point = [t, x, y] - if self.filter_vars['U_success'][h,i,j]: - obs = self.filter_vars['U'][h,i,j] - for struct in self.meso_structures: - self.meso_structures[struct][h,i,j] = self.filter.filter_var_point(struct, point, obs) - self.meso_vars['p_filt'][h,i,j] = self.filter.filter_var_point('p', point, obs) - else: - print('Could not filter at {}: observer not found.'.format(point)) - - def p_from_EOS(self, eps, n): - """ - Compute pressure from Gamma-law EoS - """ - return (self.coefficients['Gamma']-1)*eps*n - - def decompose_structures_gridpoint(self, h, i, j): - """ - Decompose the fluid part of SET as well as Fab at grid point (h,i,j) - - Parameters: - ----------- - h, i, j: integers - the indices on the grid where the decomposition is performed. - - Returns: - -------- - None - - Notes: - ------ - Decomposition of BC, fluid SET and Faraday tensor. The EM part of SET is decomposed later - via non local operations (same story for the charge current). - - """ - # Computing the Favre density and velocity - n_t = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) - u_t = np.multiply(1 / n_t, self.meso_structures['BC'][h,i,j]) - T_ab = self.meso_structures['SETfl'][h,i,j,:,:] # Remember this is rank (2,0) - - # Computing the decomposition at each point - eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, self.metric, self.metric, T_ab) - h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) # This is a rank (1,1) tensor, i.e. a real projector. - q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) - s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, T_ab) - s = np.einsum('ii',s_ab) - p_t = self.p_from_EOS(eps_t, n_t) - - - # Storing the decomposition with appropriate names. - self.meso_vars['n_tilde'][h,i,j] = n_t - self.meso_vars['u_tilde'][h,i,j,:] = u_t - self.meso_vars['eps_tilde'][h,i,j] = eps_t - self.meso_vars['q_res'][h,i,j,:] = q_a - self.meso_vars['pi_res'][h,i,j,:,:] = s_ab - np.multiply(s / self.spatial_dims , self.metric +np.outer(u_t,u_t)) - self.meso_vars['p_tilde'][h,i,j] = p_t - self.meso_vars['Pi_res'][h,i,j] = s - p_t - self.meso_vars['eos_res'][h,i,j] = self.meso_vars['p_filt'][h,i,j] - p_t - self.meso_vars['T_tilde'][h,i,j] = p_t / n_t - - # Repeat for the Faraday tensor (which is rank (0,2)) - Fab = self.meso_structures['Fab'][h,i,j] - self.meso_vars['e_tilde'][h,i,j,:] = np.einsum('ij,j->i',Fab, u_t) - self.meso_vars['b_tilde'][h,i,j] = np.einsum('ijk,i,jl,km,lm', self.Levi3D, u_t, self.metric, self.metric, Fab) - - # Corresponding 3+1 d formulae: only change in B - # np.einsum('ijkl,i,km,ln,mn->i', self.Levi4D, u_t, self.metric, self.metric, Fab) - - def decompose_structures(self): - """ - Decompose structures at all points on the meso_grid where observers could be found. - """ - for h, t in enumerate(self.domain_vars['T']): - for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): - point = [t,y,x] - if self.filter_vars['U_success'][h,i,j]: - self.decompose_structures_gridpoint(h,i,j) - else: - print('Structures not decomposed at {}: observer could not be found.'.format(point)) - - def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): - """ - Calculate partial derivative in the input 'direction' of the variable corresponding to 'nonlocal_var_str' - at the position on the grid identified by indices h,i,j. The order of the differencing scheme - can also be specified, default to 1. - - Parameters: - ----------- - nonlocal_var_str: string - quantity to be taken derivate of, must be in self.nonlocal_vars_strs() - - h, i ,j: integers - indices for point on the meso_grid - - direction: integer < self.spatial_dim + 1 - - order: integer, defatult to 1 - order of the differencing scheme - - Returns: - -------- - Finite-differenced quantity at (h,i,j) - - Notes: - ------ - The method returns the value instead of storing it, so that these can be - rearranged as preferred later, that is in calculate_derivatives() - """ - - if direction > self.spatial_dims: - print('Directions are numbered from 0 up to {}'.format(self.spatial_dims)) - return None - - if order > len(self.differencing): - print('Maximum order implemented is {}: continuing with it.'.format(len(self.differencing))) - order = len(self.differencing) - - # Forward, backward or centered differencing? - k = [h,i,j][direction] - N = len(self.domain_vars['Points'][direction]) - - if k in [l for l in range(order)]: - coefficients = self.differencing[order]['fw']['coefficients'] - stencil = self.differencing[order]['fw']['stencil'] - elif k in [N-1-l for l in range(order)]: - coefficients = self.differencing[order]['bw']['coefficients'] - stencil = self.differencing[order]['bw']['stencil'] - else: - coefficients = self.differencing[order]['cen']['coefficients'] - stencil = self.differencing[order]['cen']['stencil'] - - temp = 0 - for s, sample in enumerate(stencil): - idxs = [h,i,j] - idxs[direction] += sample - temp += np.multiply( coefficients[s] / self.domain_vars['Dx'], self.get_var_gridpoint(nonlocal_var_str, *idxs)) - return temp - - def calculate_derivatives(self): - """ - Compute all the derivatives of the quantities corresponding to nonlocal_vars_strs, for all - gridpoints on the meso-grid. - - Notes: - ------ - The derived quantities are stored as 'tensors' as follows: - 1st three indices refer to the position on the grid - - 4th index refers to the directionality of the derivative - - last indices (1 or 2) correspond to the components of the quantity to be derived - - The index corresponding to the derivative is covariant, i.e. down. - - Example: - - Fab [h,i,j,a,b] : h,i,j grid; a,b, spacetime components - - D_Fab[h,i,j,c,a,b]: h,i,j grid; c direction of derivative; a,b as for Fab - - """ - for h, t in enumerate(self.domain_vars['T']): - for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): - point = [t,y,x] - if self.filter_vars['U_success'][h,i,j]: - for dir in range(self.spatial_dims+1): - for str in self.nonlocal_vars_strs: - dstr = 'D_' + str - self.deriv_vars[dstr][h,i,j,dir] = self.calculate_derivatives_gridpoint(str, h, i, j, dir) - else: - print('Derivatives not calculated at {}: observer could not be found.'.format(point)) - - def closure_ingredients_gridpoint(self, h, i, j): - """ - Decompose quantities obatined via non_local operations (i.e. derivatives) as needed - for the closure scheme. This is done at gridpoint (h,i,j) and then relevant values are - returned - - Parameters: - ----------- - h, i, j: integers - the indices of the gridpoint - - Returns: - -------- - list of strings: names of the quantities computed - - list of nd.arrays with quantities computed - - Notes: - ------ - The ingredients for the closure scheme are model specific: it makes sense that the relevant - dictionary is set up not on construction. - - UNDER CONSTRUCTION: THE EM PART NEEDS MORE THINKING - - Rename: closure_ingredients_gridpoint() ?? - Coefficients can be computed here directly, but more general models will require more work. - Do not split this, at least for now - """ - # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS - u_t = self.meso_vars['u_tilde'][h,i,j] - u_t_cov = np.einsum('ij,j->i', self.metric, u_t) - Nabla_Fab = self.deriv_vars['D_Fab'][h,i,j] - - # CLOSURE INGREDIENTS: FLUID BIT - WORKING WITH (2,0) - nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor - nabla_u = np.einsum('ij,jk->ik', self.metric, nabla_u) #this is a rank (2,0) tensor - acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector - - # The following two quantities should be identically zero, but they won't be due to numerical errors. - # Computing them and removing to project velocity gradients - unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector - acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) - Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ - np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor - h_ab = self.metric + np.einsum('i,j->ij', u_t, u_t) - exp_t = np.einsum('ii->',Daub) - shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/self.spatial_dims * exp_t, h_ab) - vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) - - - # CLOSURE INGREDIENTS: HEAT FLUX - nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] - projector = np.zeros((3,3)) - np.fill_diagonal(projector, 1) - projector += np.einsum('i,j->ij',u_t_cov, u_t) - DaT = np.einsum('ij,j->i', projector, nabla_T) - Theta_tilde = DaT + np.multiply(self.meso_vars['T_tilde'][h,i,j], np.einsum('ij,j->i', self.metric, acc_t)) - - # CLOSURE INGREDIENTS: EM PART - DO YOU NEED MORE? - # current = np.einsum('ij, kl, kjl->i', self.metric, self.metric, DFab) - # sigma_t = np.einsum('i,ij,j->',current, self.metric, u_t) - # J_t = current - np.multiply(sigma_t, u_t) - # # self.meso_vars['j'][h,i,j,:] = current ?? - - closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'Theta_tilde'] - closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] - return closure_vars_strs, closure_vars - - def closure_ingredients(self): - """ - Wrapper of the corresponding gridpoint method. - Set up the dictionary for the closure variables not yet defined. - Loop over the grid and store the decomposed variables. - """ - Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] - for h in range(Nt): - for i in range(Nx): - for j in range(Ny): - if self.filter_vars['U_success'][h,i,j]: - keys, values = self.closure_ingredients_gridpoint(h,i,j) - # try-except block to extend the meso_vars dictionary - for idx, key in enumerate(keys): - try: - self.meso_vars[key] - except KeyError: - print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) - shape = values[idx].shape - self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) - finally: - self.meso_vars[key][h,i,j] = values[idx] - - def EL_style_closure_gridpoint(self, h, i, j): - """ - Compute bulk, shear via scalarization + PA trick, thermal conductivity later. - At a point. - Returns: coefficient(s) at a point. - """ - coefficients_names=[] - coefficients=[] - - # CALCULATING BULK VISCOUS COEFF - zeta = self.meso_vars['Pi_res'][h,i,j] / self.meso_vars['exp_tilde'][h,i,j] - coefficients_names.append('zeta') - coefficients.append(zeta) - - # CALCULATING SHEAR VISCOUS COEFF - pi_res_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['pi_res'][h,i,j], self.meso_vars['pi_res'][h,i,j], self.metric, self.metric) - shear_sq = np.einsum('ij,kl,ik,jl->', self.meso_vars['shear_tilde'][h,i,j], self.meso_vars['shear_tilde'][h,i,j], self.metric, self.metric) - eta = pi_res_sq/shear_sq - # Compute sign of eta by looking at the PA of pi_res with higher eigenvalue. - # Change shear to the eigenbasis of pi_res (using that the matrix is unitary as pi_res is sym) - pi_eig, pi_eigv = np.linalg.eigh(self.meso_vars['pi_res'][h,i,j]) - transformed_shear = np.einsum('ji,jk,kl', pi_eigv, self.meso_vars['shear_tilde'][h,i,j], pi_eigv) - abs_pi_eig = np.abs(pi_eig) - pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) - eta = eta * np.sign(pi_eig[-1] / transformed_shear[pos,pos]) - coefficients_names.append('eta') - coefficients.append(eta) - - # CALCULATING THE HEAT CONDUCTIVITIY - q_res = self.meso_vars['q_res'][h,i,j] - Theta_t = self.meso_vars['Theta_tilde'][h,i,j] - q_res_sq = np.einsum('i,ij,j', q_res, self.metric, q_res) - Theta_t_sq = np.einsum('i,ij,j', Theta_t, self.metric, Theta_t) - cos_angle = np.einsum('i,ij,j', q_res, self.metric, Theta_t) / (q_res_sq * Theta_t_sq) - sign = np.sign(cos_angle) - kappa = sign * np.sqrt(q_res_sq / Theta_t_sq) - coefficients_names.append('kappa') - coefficients.append(kappa) - - return coefficients_names, coefficients - - def EL_style_closure(self): - """ - Wrapper of EL_style_closure_gridpoint() - """ - Nt, Nx, Ny = self.domain_vars['Nt'], self.domain_vars['Nx'], self.domain_vars['Ny'] - for h in range(Nt): - for i in range(Nx): - for j in range(Ny): - if self.filter_vars['U_success'][h,i,j]: - keys, values = self.EL_style_closure_gridpoint(h,i,j) - # try-except block to extend the meso_vars dictionary with the dissipative coefficients - for idx, key in enumerate(keys): - try: - self.meso_vars[key] - except KeyError: - print('The key {} does not belong to meso_vars yet, adding it!'.format(key)) - shape = values[idx].shape - self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) - finally: - self.meso_vars[key][h,i,j] = values[idx] - - def EL_style_closure_regression(self, CoefficientAnalysis): - """ - Takes in a list of correlation quantities (strings? The regressors needed are computed - previously via closure ingredients) - Pass the dataset to a CoefficientAnalysis instance. - Plot the correlations with the regressors and perform the regression using methods from a Coefficient - Analysis instance. - """ - # FICTITIOUS RANGES AND POINTS TO TEST TRIM_DATA ROUTINE - ranges = [[self.domain_vars['Tmin'], self.domain_vars['Tmax']],\ - [self.domain_vars['Xmin'], self.domain_vars['Xmax']],\ - [self.domain_vars['Ymin'], self.domain_vars['Ymax']]] - points = self.domain_vars['Points'] - - # TESTING SCALAR REGRESSION ROUTINE - # y = self.meso_vars['zeta'] - # X = [self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - # stats_result = CoefficientAnalysis.scalar_regression(y, X, ranges, points) - # print(*stats_result) - - # TESTING TENSOR REGRESSION ROUTINE - # y=self.meso_vars['q_res'] - # X=[self.meso_vars['e_tilde'], self.meso_vars['u_tilde']] - # # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, components=[(0,),(2,)]) - # stats_result = CoefficientAnalysis.tensor_components_regression(y, X, 2,ranges, points, add_intercept=False) - # for i in range(len(stats_result)): - # print('{}-th component: \n {} \n\n'.format(i, stats_result[i])) - - # TESTING CORRELATION VISUALIZATION ROUTINES - # x = self.meso_vars['n_tilde'] - # y = self.meso_vars['eps_tilde'] - # labels = ['n_tilde','eps_tilde'] - # g1=CoefficientAnalysis.visualize_correlation(x, y, xlabel=labels[0], ylabel=labels[1]) - # fig=g1.figure - # fig.savefig('Single_correlation_plot.pdf', format='pdf') - - # data=[self.meso_vars['eta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - # labels=['eta', 'b_tilde', 'eps_tilde', 'n_tilde'] - # g2=CoefficientAnalysis.visualize_many_correlations(data, labels) - - # data=[self.meso_vars['zeta'], self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - # labels=['zeta', 'b_tilde', 'eps_tilde', 'n_tilde'] - # gbis= CoefficientAnalysis.visualize_many_correlations(data, labels) - # # plt.savefig('Many_correlations_plot.pdf', format='pdf') - # plt.show() - - # TESTING THE CHECK REGRESSORS ROUTINE: - # data=[self.meso_vars['b_tilde'], self.meso_vars['eps_tilde'], self.meso_vars['n_tilde']] - # labels=['b_tilde', 'eps_tilde', 'n_tilde'] - # g1=CoefficientAnalysis.visualize_many_correlations(data, labels) - # comps, g2 = CoefficientAnalysis.PCA_find_regressors_subset(data, ranges = ranges, model_points = points) - # g2.fig.suptitle("Correlation between PC (standardized data)") - # g2.fig.subplots_adjust(top=0.94) - # print(comps) - # plt.show() - - - -if __name__ == '__main__': - - - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() - FileReader.read_in_data(micro_model) - micro_model.setup_structures() - find_obs = FindObs_drift_root(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - - - CPU_start_time = time.process_time() - meso_model = resMHD2D(micro_model, find_obs, filter) - - # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.5],[0.4, 0.6]],1) - meso_model.setup_meso_grid([[1.501, 1.502],[0.3, 0.35],[0.4, 0.45]],1) - # print(meso_model.domain_vars['Points']) - meso_model.find_observers() - meso_model.filter_micro_variables() - - print('Filter stage ended') - # meso_model.decompose_structures_gridpoint(1,1,1) - meso_model.decompose_structures() - - # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) - meso_model.calculate_derivatives() - - # meso_model.closure_ingredients_gridpoint(1,1,0) - meso_model.closure_ingredients() - # print('Derivatives and Decomposition stage ended') - - # meso_model.EL_style_closure_gridpoint(1,2,3) - meso_model.EL_style_closure() - # regressor = CoefficientsAnalysis() - # meso_model.EL_style_closure_regression(regressor) - - # print('Total time is {}'.format(time.process_time() - CPU_start_time)) - diff --git a/MicroModels.py b/MicroModels.py deleted file mode 100644 index 3656499..0000000 --- a/MicroModels.py +++ /dev/null @@ -1,554 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Fri Mar 31 10:00:02 2023 - -@author: Thomas -""" - -import numpy as np -import math -import time - -from scipy.interpolate import interpn -from multimethod import multimethod - -from FileReaders import * -from system.BaseFunctionality import * - - -# These are the symbols, so be careful when using these to construct vectors! -# levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ -# for k in range(3)]for j in range(3) ] for i in range(3) ]) - -# levi4D = np.array([[[[ np.sign(i - j) * np.sign(j - k) * np.sign(k - l) * np.sign(i - l) \ -# for l in range(4)] for k in range(4) ] for j in range(4)] for i in range(4)]) - - -class IdealMHD_2D(object): - """ - ADD SHORT DESCRIPTION OF THE CLASS AND ITS METHODS - """ - def __init__(self, interp_method = "linear"): - """ - Sets up the variables and dictionaries, strings correspond to - those used in METHOD - - Parameters - ---------- - interp_method: str - optional method to be used by interpn - """ - self.spatial_dims = 2 - self.interp_method = interp_method - - self.metric = np.zeros((3,3)) - self.metric[0,0] = -1 - self.metric[1,1] = self.metric[2,2] = +1 - - # This is the Levi-Civita symbol, not tensor, so be careful when using it - self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ - for k in range(3)]for j in range(3) ] for i in range(3) ]) - - #Dictionary for grid: info and points - self.domain_int_strs = ('nt','nx','ny') - self.domain_float_strs = ("tmin","tmax","xmin","xmax","ymin","ymax","dt","dx","dy") - self.domain_array_strs = ("t","x","y","points") - self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs+self.domain_array_strs) - for str in self.domain_vars: - self.domain_vars[str] = [] - - #Dictionary for primitive var - self.prim_strs = ("vx","vy","n","p", "Bz") - self.prim_vars = dict.fromkeys(self.prim_strs) - for str in self.prim_strs: - self.prim_vars[str] = [] - - #Dictionary for auxiliary var - self.aux_strs = ("W","h","bz","bsq", "e") - self.aux_vars = dict.fromkeys(self.aux_strs) - for str in self.aux_strs: - self.aux_vars[str] = [] - - #Dictionary for structures - self.structures_strs = ("BC", "SETfl", "SETem", "Fab") - self.structures = dict.fromkeys(self.structures_strs) - for str in self.structures_strs: - self.structures[str] = [] - - def get_spatial_dims(self): - return self.spatial_dims - - def get_model_name(self): - return 'IdealMHD_2D' - - def get_domain_strs(self): - return self.domain_int_strs + self.domain_float_strs + self.domain_array_strs - - def get_prim_strs(self): - return self.prim_strs - - def get_aux_strs(self): - return self.aux_strs - - def get_structures_strs(self): - return self.structures_strs - - def get_all_var_strs(self): - return self.get_prim_strs() + self.get_aux_strs() + self.get_structures_strs() - - def get_gridpoints(self): - """ - Pretty self-explanatory - """ - return self.domain_vars['points'] - - def get_interpol_var(self, var, point): - """ - Returns the interpolated variables at the point. - - Parameters - ---------- - var: str corresponding to primitive, auxiliary or structre variable - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Interpolated values/arrays corresponding to variable. - Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - - if var in self.get_prim_strs(): - return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] - elif var in self.get_aux_strs(): - return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] - elif var in self.get_structures_strs(): - return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] - else: - print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') - - @multimethod - def get_var_gridpoint(self, var: str, h: object, i: object, j: object): - """ - Returns variable corresponding to input 'var' at gridpoint identified by h,i,j - - Parameters: - ----------- - var: str - String corresponding to primitive, auxiliary or structure variable - - h,i,j: int - integers corresponding to position on the grid. - - Returns: - -------- - Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. - - Notes: - ------ - This method is useful e.g. for plotting the raw data. - """ - if var in self.get_prim_strs(): - return self.prim_vars[var][h,i,j] - elif var in self.get_aux_strs(): - return self.aux_vars[var][h,i,j] - elif var in self.get_structures_strs(): - return self.structures[var][h,i,j] - else: - print('{} is not a variable of model {}'.format(var, self.get_model_name())) - return None - - @multimethod - def get_var_gridpoint(self, var: str, point: object): - """ - Returns variable corresponding to input 'var' at gridpoint - closest to input 'point'. - - Parameters: - ----------- - vars: string corresponding to primitive, auxiliary or structure variable - - point: list of 2+1 floats - - Returns: - -------- - Values or arrays corresponding to variable evaluated at the closest grid-point to input 'point'. - - Notes: - ------ - This method should be used in case using interpolated values - becomes too expensive. - """ - indices = Base.find_nearest_cell(point, self.domain_vars['points']) - if var in self.get_prim_strs(): - return self.prim_vars[var][tuple(indices)] - elif var in self.get_aux_strs(): - return self.aux_vars[var][tuple(indices)] - elif var in self.get_structures_strs(): - return self.structures[var][tuple(indices)] - else: - print(f"{var} is not a variable of IdealMHD_2D!") - return None - - def setup_structures(self): - """ - Set up the structures (i.e baryon (mass) current BC, Stress-Energy and Faraday tensors - - Notes: - ------ - Structures are built as multi-dim np.arrays, with the first three indices referring - to the grid, while the last one or two refer to space-time components. - - The Faraday tensor is stored as a fully co-variant tensor (both indices down) - The stress-energy tensor is stored as a fully contra-variant tensor (both indices up) - """ - self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) - self.structures["SETfl"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) - self.structures["SETem"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) - self.structures["Fab"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) - - for h in range(self.domain_vars['nt']): - for i in range(self.domain_vars['nx']): - for j in range(self.domain_vars['ny']): - vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ - self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j]]) - - fibr_b = self.aux_vars['bz'][h,i,j] - - self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) - - self.structures['SETfl'][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ - + self.prim_vars['p'][h,i,j] * self.metric - - self.structures['SETem'][h,i,j:,:] = (fibr_b**2) * np.outer(vel_vec, vel_vec) + (fibr_b/2) * self.metric - - self.structures['Fab'][h,i,j,:,:] = np.tensordot(self.Levi3D, vel_vec, axes = ([2,0])) * fibr_b - - - ################################ - # CORRESPONDING 3+1-d FORMULAE - ################################ - - # vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ - # self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j], self.aux_vars['W'][h,i,j] * self.prim_vars['vz'][h,i,j]]) - - # fibr_b = [0, self.aux_vars['bx'][h,i,j], self.aux_vars['by'][h,i,j], self.aux_vars['bz'][h,i,j]] - - # self.structures['Fab'][h,i,j,:,:] = np.tensordot(np.tensordot(self.Levi4D, vel_vec, axes = ([2,0])), fibr_b, axes= ([2,0])) - - # self.structures['SETem'][h,i,j,:,:] = Base.Mink_dot(fibr_b, fibr_b) * np.outer(vel_vec, vel_vec) +\ - # (Base.Mink_dot(fibr_b, fibr_b)/2) * self.metric -\ - # np.multiply(1/2., np.outer(fibr_b, fibr_b) ) - - - # This might be useful/needed - # self.all_vars = self.prim_vars - # self.all_vars.update(self.aux_vars) - # self.all_vars.update(self.structures) - - self.vars = self.prim_vars - self.vars.update(self.aux_vars) - self.vars.update(self.structures) - - -class IdealHydro_2D(object): - - def __init__(self, interp_method = "linear"): - """ - Sets up the variables and dictionaries, strings correspond to - those used in METHOD - - Parameters - ---------- - interp_method: str - optional method to be used by interpn - """ - self.spatial_dims = 2 - self.interp_method = interp_method - - self.metric = np.zeros((3,3)) - self.metric[0,0] = -1 - self.metric[1,1] = self.metric[2,2] = +1 - - # This is the Levi-Civita symbol, not tensor, so be careful when using it - self.Levi3D = np.array([[[ np.sign(i-j) * np.sign(j- k) * np.sign(k-i) \ - for k in range(3)]for j in range(3) ] for i in range(3) ]) - - #Dictionary for grid: info and points - self.domain_int_strs = ('nt','nx','ny') - self.domain_float_strs = ("tmin","tmax","xmin","xmax","ymin","ymax","dt","dx","dy") - self.domain_array_strs = ("t","x","y","points") - self.domain_vars = dict.fromkeys(self.domain_int_strs+self.domain_float_strs + self.domain_array_strs) - for str in self.domain_vars: - self.domain_vars[str] = [] - - #Dictionary for primitive var - self.prim_strs = ("v1","v2","rho","p","n") - self.prim_vars = dict.fromkeys(self.prim_strs) - for str in self.prim_strs: - self.prim_vars[str] = [] - - #Dictionary for auxiliary var - self.aux_strs = ("W","h","T") - self.aux_vars = dict.fromkeys(self.aux_strs) - for str in self.aux_strs: - self.aux_vars[str] = [] - - #Dictionary for structures - self.structures_strs = ("BC","SET") - self.structures = dict.fromkeys(self.structures_strs) - for str in self.structures_strs: - self.structures[str] = [] - - #Dictionary for all vars - self.var_strs = self.prim_strs + self.aux_strs + self.structures_strs - # self.vars = self.prim_vars - # self.vars.update(self.aux_vars) - # self.vars.update(self.structures) - - self.all_var_strs = self.prim_strs + self.aux_strs + self.structures_strs - - def get_model_name(self): - return 'IdealHydro_2D' - - def get_spatial_dims(self): - return self.spatial_dims - - def get_domain_strs(self): - return self.domain_info_strs + self.domain_points_strs - - def get_prim_strs(self): - return list(self.prim_vars.keys()) - - def get_aux_strs(self): - return list(self.aux_vars.keys()) - - def get_structures_strs(self): - return list(self.structures.keys()) - - def get_all_var_strs(self): - return list(self.prim_vars.keys()) + list(self.aux_vars_vars.keys()) + list(self.structures.keys()) - - def setup_structures(self): - """ - Set up the structures (i.e baryon current, SET) - - Structures are built as multi-dim np.arrays, with the first three indices referring - to the grid, while the last one or two refer to space-time components. - """ - self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) - self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) - - for h in range(self.domain_vars['nt']): - for i in range(self.domain_vars['nx']): - for j in range(self.domain_vars['ny']): - vel = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['v1'][h,i,j] ,\ - self.aux_vars['W'][h,i,j] * self.prim_vars['v2'][h,i,j]]) - self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel ) - - - self.structures["SET"][h,i,j,:,:] = (self.prim_vars["rho"][h,i,j] + self.prim_vars["p"][h,i,j]) * \ - np.outer(vel,vel) + self.prim_vars["p"][h,i,j] * self.metric - - self.vars = self.prim_vars - self.vars.update(self.aux_vars) - self.vars.update(self.structures) - - def get_interpol_prim(self, var_names, point): - """ - Returns the interpolated variable at the point - - Parameters - ---------- - vars : list of strings - strings have to be in to prim_vars keys - point : list of floats - ordered coordinates: t,x,y - Return - ------ - list of floats corresponding to string of vars - - Notes - ----- - Interpolation raises a ValueError when out of grid boundaries. - """ - res = [] - - for var_name in var_names: - try: - res.append( interpn(self.domain_vars["points"], self.prim_vars[var_name], point, method = self.interp_method)[0]) #, bounds_error = False) - except KeyError: - print(f"{var_name} does not belong to the primitive variables of the micro_model!") - return res - - def get_interpol_aux(self, var_names, point): - """ - Returns the interpolated variable at the point - - Parameters - ---------- - vars : list of strings - strings have to be in to aux_vars keys - point : list of floats - ordered coordinates: t,x,y - Return - ------ - list of floats corresponding to string of vars - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - - res = [] - for var_name in var_names: - try: - res.append( interpn(self.domain_vars["points"], self.aux_vars[var_name], point, method = self.interp_method)[0]) - except KeyError: - print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") - return res - - def get_interpol_struct(self, var_name, point): - """ - Returns the interpolated structure at the point - - Parameters - ---------- - var : str corresponding to one of the structures - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Array with the interpolated values of the var structure - Empty list if var is not a structure in the micro_model - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - res = [] - if var_name == "BC": - res = np.zeros(len(self.structures[var_name][:,0,0,0])) - for a in range(len(self.structures[var_name][:,0,0,0])): - res[a] = interpn(self.domain_vars["points"], self.structures[var_name][a,:,:,:], point, method = self.interp_method)[0] - elif var_name == "SET": - res = np.zeros((len(self.structures[var_name][:,0,0,0,0]),len(self.structures[var][0,:,0,0,0]))) - for a in range(len(self.structures[var_name][:,0,0,0,0])): - for b in range(len(self.structures[var_name][0,:,0,0,0])): - res[a,b] = interpn(self.domain_vars["points"],self.structures[var_name][a,b,:,:,:], point, method = self.interp_method)[0] - else: - print(f"{var} does not belong to the structures in the micro_model") - return res - - def get_interpol_var(self, var, point): - """ - Returns the interpolated variables at the point. - - Parameters - ---------- - vars : str corresponding to primitive, auxiliary or structre variable - - point : list of floats - ordered coordinates: t,x,y - - Return - ------ - Interpolated values/arrays corresponding to variable. - Empty list if none of the variables is a primitive, auxiliary o structure of the micro_model - - Notes - ----- - Interpolation gives errors when applied to boundary - """ - - if var in self.get_prim_strs(): - return interpn(self.domain_vars['points'], self.prim_vars[var], point, method = self.interp_method)[0] - elif var in self.get_aux_strs(): - return interpn(self.domain_vars['points'], self.aux_vars[var], point, method = self.interp_method)[0] - elif var in self.get_structures_strs(): - return interpn(self.domain_vars['points'], self.structures[var], point, method = self.interp_method)[0] - else: - print(f'{var} is not a primitive, auxiliary variable or structure of the micro_model!!') - - - - """ - Returns the interpolated structure at the point - Parameters - ---------- - var : str corresponding to one of the structures - - point : list of floats - ordered coordinates: t,x,y - Return - ------ - Array with the interpolated values of any var - Empty list if var is not a structure in the micro_model - Notes - ----- - Interpolation gives errors when applied to boundary - """ - res = [] - for var_name in var_names: - try: - res.append( interpn(self.domain_vars["points"], self.vars[var_name], point, method = self.interp_method)[0]) - except KeyError: - print(f"{var_name} does not belong to the auxiliary variables of the micro_model!") - return res - - -# TC -if __name__ == '__main__': - - CPU_start_time = time.process_time() - - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() - FileReader.read_in_data(micro_model) - micro_model.setup_structures() - - point = [1.502,0.4,0.2] - # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - vars = ['BC'] - for var in vars: - res = micro_model.get_interpol_var(var, point) - res2 = micro_model.get_var_gridpoint(var, point) - print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') - -# MH -# if __name__ == '__main__': - -# CPU_start_time = time.process_time() - -# FileReader = METHOD_HDF5('./Data/Testing/') -# # micro_model = IdealMHD_2D() -# micro_model = IdealHydro_2D() -# FileReader.read_in_data(micro_model) -# micro_model.setup_structures() - -# res = micro_model.get_interpol_var(['v1','rho'],[10.0,0.3,0.2]) -# print(type(res),"\n", res) - -# res = micro_model.get_interpol_var(['T','W'],[10.0,0.3,0.2]) -# print(type(res),"\n", res) - -# # res = micro_model.get_interpol_var(['bar_vel','SET'],[10.0,0.3,0.2]) -# # print(type(res),"\n", res) -# # res = micro_model.get_interpol_aux(["b0","bx"],[0.5, 0.3,0.2]) -# # print(type(res),"\n", res) - -# # res = micro_model.get_interpol_struct("SET",[2.5, 0.3, 0.2]) -# # print(res.shape, res[0],'\n') - -# # print( micro_model.get_domain_strs() ) - -# CPU_end_time = time.process_time() -# CPU_time = CPU_end_time - CPU_start_time -# print(f'The CPU time is {CPU_time} seconds') \ No newline at end of file diff --git a/Visualization.py b/Visualization.py deleted file mode 100644 index b5d719e..0000000 --- a/Visualization.py +++ /dev/null @@ -1,340 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Mon Jun 5 03:14:43 2023 - -@author: marcu -""" - - -import matplotlib.pyplot as plt -import numpy as np -import h5py -from mpl_toolkits.axes_grid1 import make_axes_locatable -from scipy.interpolate import interpn -from system.BaseFunctionality import * - -from MicroModels import * -from MesoModels import * -from Filters import * - -class Plotter_2D(object): - - def __init__(self, screen_size = [11.97, 8.36]): - """ - Parameters: - ----------- - screen_size = list of 2 floats - width and height of the screen: used to rescale plots' size. - """ - self.subplots_dims = {1 : (1,1), - 2 : (1,2), - 3 : (1,3), - 4 : (2,2), - 5 : (2,3), - 6 : (2,3)} - - self.screen_size = np.array(screen_size) - - def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, method= 'raw_data', component_indices=()): - """ - Retrieves the required data from model to plot a variable defined by - var_str over coordinates t, x_range, y_range, either from the model's - raw data or by interpolating between the model's raw data over the coords. - - Parameters - ---------- - model : Micro or Meso Model - var_str : str - Must match a variable of the model. - t : float - time coordinate (defines the foliation). - x_range : list of 2 floats: x_start and x_end - defines range of x coordinates within foliation. - y_range : list of 2 floats: y_start and y_end - defines range of y coordinates within foliation. - interp_dims : tuple of integers - defines the number of points to interpolate at in x and y directions. - method : str - currently either raw_data or interpolate. - component_indices : tuple - the indices of the component to pick out if the variable is a vector/tensor. - - Returns - ------- - data_to_plot : numpy array of floats - the (2D) data to be plotted by plt.imshow(). - extent: list of floats - - - Notes: - ------ - Logic: if method is raw_data, then no interp_dims are needed. - Better to have 'raw_data' and no interp_dims = None as default - """ - if var_str in model.get_all_var_strs(): - - # Block to check component_indices passed are compatible with shape of var to be plotted. - st1 = len(component_indices) - st2 = len(model.get_var_gridpoint(var_str, 0, 0, 0).shape) - compatible = st1==st2 - if not compatible: - if st2 == 0 : - print('WARNING: {} is a scalar but you passed some indices, ignoring this and moving on.'.format(var_str)) - component_indices = () - elif st2 != 0: - print('WARNING: {} is a tensor but you passed more/fewer indices than required. Retrieving the "first component"!'.format(var_str)) - component_indices = tuple([0 for _ in range(st2)]) - - extent = [*x_range, *y_range] - - if method == 'interpolate': - compatible = interp_dims != None and len(interp_dims) ==2 - if not compatible: - print('Error: when using (linearly spaced) interpolated data, you must \ - specify # points in each direction! Exiting.') - return None - - nx, ny = interp_dims[:] - xs, ys = np.linspace(x_range[0], x_range[1], nx), np.linspace(y_range[0], y_range[1], ny) - data_to_plot = np.zeros((nx, ny)) - - for i in range(nx): - for j in range(ny): - point = [t, xs[i], ys[j]] - data_to_plot[i, j] = model.get_interpol_var(var_str, point)[component_indices] - - elif method == 'raw_data': - start_indices = Base.find_nearest_cell([t, x_range[0], y_range[0]], model.get_gridpoints()) - end_indices = Base.find_nearest_cell([t, x_range[1], y_range[1]], model.get_gridpoints()) - - h = start_indices[0] - i_s, i_f = start_indices[1], end_indices[1] - j_s, j_f = start_indices[2], end_indices[2] - - data_shape = (i_f+ 1- i_s, j_f+ 1- j_s) - data_to_plot = np.zeros(data_shape) - - for i in range(i_f+ 1 - i_s): - for j in range(j_f+ 1 - j_s): - data_to_plot[i,j] = model.get_var_gridpoint(var_str, h, i, j)[component_indices] - - else: - print('Data method is not a valid choice! Must be interpolate or raw_data.') - return None - - # return data_to_plot, points - return data_to_plot, extent - - else: - print(f'{var_str} is not a plottable variable of the model!') - return None - - def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=[()]): - """ - Plot variable(s) from model, defined by var_strs, over coordinates - t, x_range, y_range. Either from the model's raw data or by interpolating - between the model's raw data over the coords. - - Parameters - ---------- - model : Micro or Meso Model - var_strs : list of str - Must match entries in the models' 'vars' dictionary. - t : float - time coordinate (defines the foliation). - x_range : list of 2 floats: x_start and x_end - defines range of x coordinates within foliation. - y_range : list of 2 floats: y_start and y_end - defines range of y coordinates within foliation. - interp_dims : tuple of integers - defines the number of points to interpolate at in x and y directions. - method : str - currently either raw_data or interpolate. - components_indices : list of tuple(s) - the indices of the components to pick out if the variables are vectors/tensors. - Can be omitted if all variables are scalars, otherwise must be a list - of tuples matching the length of var_strs that corresponds with each - variable in the list. - - Output - ------- - Plots the (2D) data using imshow. Note that the plotting data's coordinates - may not perfectly match the input coordinates if method=raw_data as - nearest-cell data is used where the input coordinates do not coincide - with the model's raw data coordinates. - - """ - n_plots = len(var_strs) - n_rows, n_cols = self.subplots_dims[n_plots] - - # Block to determine adaptively the figsize. - figsizes = {1 : (1/3.,1/3.), - 2 : (2/3.,1/3.), - 3 : (1,1/3.), - 4 : (2/3.,2/3.), - 5 : (1,2/3.), - 6 : (1,2/3.)} - for item in figsizes: - figsizes[item] = tuple(figsizes[item] * self.screen_size) - figsize = figsizes[n_plots] - - - fig, axes = plt.subplots(n_rows,n_cols, figsize=figsize) - if n_plots == 1: - axes = [axes] - else: - axes = axes.flatten() - - for var_str, component_indices, ax in zip(var_strs, components_indices, axes): - # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] - data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - im = ax.imshow(data_to_plot, extent=extent) - divider = make_axes_locatable(ax) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title(var_str) - ax.set_xlabel(r'$y$') # WHY? - ax.set_ylabel(r'$x$') - - fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), t), fontsize = 12) - fig.tight_layout() - plt.show() - - def plot_var_model_comparison(self, models, var_str, t, x_range, y_range, interp_dims=None, method='raw_data', component_indices=(), diff_plot=True): - """ - Plot a variable from a number of models. If 2 models are given, a third - plot of the difference will be automatically plotted, too. If 'raw_data' - is chosen as the method, will check to see if the data points in the model - lie at the same coordinates, which they must. - - Parameters - ---------- - models : Micro or Meso Models - var_sts : str - Must match entries in the models' 'vars' dictionary. - t : float - time coordinate (defines the foliation). - x_range : list of 2 floats: x_start and x_end - defines range of x coordinates within foliation. - y_range : list of 2 floats: y_start and y_end - defines range of y coordinates within foliation. - interp_dims : tuple of integers - defines the number of points to interpolate at in x and y directions. - method : str - currently either raw_data or interpolate. - component_indices : tuple - the indices of the component to pick out if the variable is a vector/tensor. - - Output - ------- - Plots the (2D) data using imshow. Note that the plotting data's coordinates - may not perfectly match the input coordinates if method=raw_data as - nearest-cell data is used where the input coordinates do not coincide - with the model's raw data coordinates. - - """ - n_cols = len(models) - if not n_cols == 2: - diff_plot = False # Only plot difference of 2 models... - n_rows = 1 - if diff_plot: - n_cols+=1 - - # Block to determine adaptively the figsize. - figsize = np.array([1,2/3.]) * self.screen_size - - fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=figsize) - - for model, ax in zip(models, axes.flatten()): - # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] - data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - im = ax.imshow(data_to_plot, extent=extent) - divider = make_axes_locatable(ax) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title(model.get_model_name()) - ax.set_xlabel(r'$y$') - ax.set_ylabel(r'$x$') - - if diff_plot: - ax = axes.flatten()[-1] - # data_to_plot1, points1 = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) - # data_to_plot2, points2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices) - data_to_plot1, extent = self.get_var_data(models[0], var_str, t, x_range, y_range, interp_dims, method, component_indices) - data_to_plot2 = self.get_var_data(models[1], var_str, t, x_range, y_range, interp_dims, method, component_indices)[0] - # if len(points1) != len(points2): - # diff_plot = False - # pass - # for t_points1, t_points2 in zip(points1, points2): - # print(t_points1, t_points2) - # if len(t_points1) != t_points2.shape: - # diff_plot = False - # continue - # if not np.allclose(t_points1, t_points2): - # diff_plot = False - # if diff_plot: - try: - # extent = [points1[2][0],points1[2][-1],points1[1][0],points1[1][-1]] - im = ax.imshow(data_to_plot1 - data_to_plot2, extent=extent) - divider = make_axes_locatable(ax) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - ax.set_title('Models difference') - ax.set_xlabel(r'$y$') - ax.set_ylabel(r'$x$') - except(ValueError): - print(f"Cannot plot the difference between {var_str} in the two " - "models. Likely due to the data coordinates not coinciding.") - fig.suptitle('Contrasting {} against different models'.format(var_str)) - fig.tight_layout() - plt.show() - - - -if __name__ == '__main__': - - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() - FileReader.read_in_data(micro_model) - micro_model.setup_structures() - - - visualizer = Plotter_2D([11.97, 8.36]) - - # TESTING GET_VAR_DATA - ###################### - # var = 'BC' - # components = (0,2) - # data1= visualizer.get_var_data(micro_model, var, 1.502, [0.3, 0.4], [0.3,0.4], component_indices=components)[0] - # data, extent= visualizer.get_var_data(micro_model, var, 1.502, [0.3, 0.4], [0.3,0.4], component_indices=components, method='interpolate', interp_dims=(20,20)) - # print(extent) - - # TESTING PLOT_VARS - ################### - # vars = ['BC', 'vx', 'vy', 'Bz', 'p', 'W'] - # components = [(0,), (), (), (), (), ()] - # model = micro_model - # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], components_indices=components) - # visualizer.plot_vars(model, vars, 1.502, [0.01, 0.98], [0.01, 0.98], method = 'interpolate', interp_dims=(100,100), components_indices=components) - - - # TESTING PLOT_VAR_MODEL_COMPARISON - ################################### - find_obs = FindObs_drift_root(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - meso_model = resMHD2D(micro_model, find_obs, filter) - ranges = [0.3, 0.4] - meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) - meso_model.find_observers() - meso_model.filter_micro_variables() - - var = 'BC' - component = (0,) - models = [micro_model, meso_model] - smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries - # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ - # method='interpolate', interp_dims=(30,30), component_indices=component) - visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, component_indices=component) - diff --git a/system/BaseFunctionality.py b/system/BaseFunctionality.py deleted file mode 100644 index c410d74..0000000 --- a/system/BaseFunctionality.py +++ /dev/null @@ -1,149 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Created on Wed Nov 2 17:21:02 2022 - -@author: mjh1n20 -""" - -from multiprocessing import Process, Pool -import os -import numpy as np -#import matplotlib.pyplot as plt -import pickle -from timeit import default_timer as timer -import h5py -from scipy.interpolate import interpn -from scipy.optimize import root, minimize -#from mpl_toolkits.mplot3d import Axes3D -from scipy.integrate import solve_ivp, quad -import cProfile, pstats, io -import math - -class Base(object): - - @staticmethod - def Mink_dot(vec1, vec2): - """ - Parameters: - ----------- - vec1, vec2 : list of floats (or np.arrays) - - Return: - ------- - mink-dot (cartesian) in 1+n dim - """ - if len(vec1) != len(vec2): - print("The two vectors passed to Mink_dot are not of same dimension!") - - dot = -vec1[0]*vec2[0] - for i in range(1,len(vec1)): - dot += vec1[i] * vec2[i] - return dot - - @staticmethod - def get_rel_vel(spatial_vels): - """ - Build unit vectors starting from spatial components - Needed as this will enter the minimization procedure - - Parameters: - ---------- - spatial_vels: list of floats - - Returns: - -------- - list of floats: the d+1 vector, normalized wrt Mink metric - """ - W = 1 / np.sqrt(1-np.sum(spatial_vels**2)) - return W * np.insert(spatial_vels,0,1.0) - - @staticmethod - def project_tensor(vector1_wrt, vector2_wrt, to_project): - """ - """ - return np.inner(vector1_wrt,np.inner(vector2_wrt,to_project)) - - - @staticmethod - def orthogonal_projector(u, metric): - """ - Returns: - -------- - Orthogonal projector wrt vector u - - Notes: - ------ - The vector u must be time-like - """ - return metric + np.outer(u,u) - - - """ - A pair of functions that work in conjuction (thank you stack overflow). - find_nearest returns the closest value to 'value' in 'array', - find_nearest_cell then takes this closest value and returns its indices. - Should now work for any dimensional data. - """ - @staticmethod - def find_nearest(array, value): - """ - Returns closest value to input 'value' in 'array' - - Parameters: - ----------- - array: np.array of shape (n,) - - value: float - - Returns: - -------- - float - - Note: - ----- - To be used together with find_nearest_cell. - """ - idx = np.searchsorted(array, value, side="left") - if idx > 0 and (idx == len(array) or math.fabs(value - array[idx-1]) < math.fabs(value - array[idx])): - return array[idx-1] - else: - return array[idx] - - @staticmethod - def find_nearest_cell(point, points): - """ - Use find nearest to find closest value in a list of input 'points' to - input 'point'. - - Parameters: - ----------- - point: list of d+1 float - - points: list of lists of d+1 floats - - Returns: - -------- - List of d+1 indices corresponding to closest value to point in points - """ - if len(points) != len(point): - print("find_nearest_cell: The length of the coordinate vector\ - does not match the length of the coordinates.") - positions = [] - for dim in range(len(point)): - positions.append(Base.find_nearest(points[dim], point[dim])) - return [np.where(points[i] == positions[i])[0][0] for i in range(len(positions))] - - def profile(self, fnc): - """A decorator that uses cProfile to profile a function""" - def inner(*args, **kwargs): - pr = cProfile.Profile() - pr.enable() - retval = fnc(*args, **kwargs) - pr.disable() - s = io.StringIO() - sortby = 'cumulative' - ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - ps.print_stats() - print(s.getvalue()) - return retval - return inner diff --git a/system/__pycache__/BaseFunctionality.cpython-36.pyc b/system/__pycache__/BaseFunctionality.cpython-36.pyc deleted file mode 100644 index f32359144c0b81bdbd369efdea91b1eae71366c0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6767 zcma)BON=8|6|Jvb{_A9%~W0&2p z${pL3rGOKW1%pPcvOxETOdp_B_=baDYq0;X{n|vQ(bLp zrY3UTOo=>grjct+qm?nU<1slihc>;LZxzgfXiGJVttoS=RWeJWo^DRJX3Uw^tT{WL zGiT0;_2$iakuR7FPf5&Rnb#$jaTZ6axy00~QYAY$lYN{!cHppb+beG(+i35W%j@N} zGw0UV&aJMO*H_oqv)PMwH`s3TbLCcJd(B&4&F*1aQbmc^Sl%c7mNmUSjuvQTImL*KOaa%J>j^3P>rW$C3Q& zXst`3WPuK4Bn3%j@=)uo26Cv4B`HjupfAgWIW~t@@IYc z6_D`0q2PC)HijguSk8jrJYZ}3kjT`-`LZ^Ft$O9O1X;-g)XzpPsr{f8%e@|NLJ4^u;=U7|(|n#)XeB?2!O)B7^)D z4j~TQU1w!??NDufklXKr*mIH=UIdve`2J+U_xJvbhnLV-nTph!zZU7-_UcZQvO68e zW0AJ~aHk(>*r5}te!wHe>qJJ=_56-q#js8r>g7iY>ml_cecDG*70V~~5S+jE<)&HF2xhtB?UB|+xX_^OJ=%g9VM zu)IiiBe@~?8j}ioqPec+S-W<#>-biUw_6s;>Qm_P2@xe_a7-}ZZle-rWHB&7sPCf^ z%3{0fPz#7L(DsG`Y*L4MQddPimDDv+he-E`iZh0z-Am1q-zeK80`Eqx}eVs-!s`9Y03o3qP24=P!~ zx=0QpxfjV0cOm{!#)CYOkzrAGw|Ai2Ps*9i{bF}gJj(cmc$Ib&7SSU1R@-gP{Ack2 zKIyL{XXTQ3??q{FLL9^0=TyZ}VM-b)If(C~lCW0^Ww2jK>Z+)#NnI0lcnD+mtAhP7 z9tYSDGZNz1YU(hB;{kgG({WfyX`lVdQTFp?e9r&He!hZ{IHNL6I7Q?%k%x(VmB<+) zXNeF-9FXPtqwEnU_7Vp9c@XI!Kaby)pJGB404*Mpl0?bBj!G!gW^+(x%I9Etdk+Rs zZ`de#B##u>-ld@iCczR5MFyRd&S2cm{hLxqZjibmm>@(Fa6N-nBem&x6+PBeZP%%; znMBtkm9+!@WsEqwQRA}f-L%mA3OaoQL;_~b$x9Qwn)D}e!q*OXeJI59LDjpq`}Cb*ONtV;fK^VV+2{#L;3eq&xB9rQ*Hl*Lv2qzNuX6 zA@w79)6`zFTHQN#Q+EQ3Iv5s+ccaY6Lh57mo%AQWIfT9>9e3_VASw<9zyo(<4o8Nv z8kg&EfsP)9#xKz43;kECWY&Dwo4{x8TFb6m*Me;)uuY9BW-6{7-OKdkTdm7>{Y4so zlQt#H299X3AjIyDLCMYzofG-<;!JQVOs-=eL!1s;6zqc-fyNL-DNFF=R| z19h8o8>Y7f-`wpq`}}LfTcOvnXhiZ$rs`O|NPW`kQK?U*{!wn0s%;N`NVf_P-wC+Q z+QiHl9ZQh*1!(j`po4kA)}y1VB=Pi+fubW2)|#Y@aEyUN#KT3XC^#svZPK6+^?YYW~HgCk&d0>6hv#;`UA)hEX#KasCH1}CPP^fj>%pP(Z21mFeO zB14S}1TFvaP=cixE&_nMcX3hB8anoz0oZ{ubxDFPAVZ(+ z-43%D^Cm6~QX>`S#Jc%m0W;EJJ}iusVG%Qwn+ksmW?gC+VKLNm(ntYZ8ap@8r=#y@ zNnh$vpNhWUBz=(1WR{Gc8%a-U(z7Y;oZFOw3@$jN`$mjkdj$2ddq6T0pBH(^v74NY zLE&j?*dRi19pfB%ll&Z!Cy0=MN2>)INBhC07Np zJ%R9M+VAq3@Umk+Ad~g1S!QRETo~*UlI47_As2b$QT(XF91do1f{7ST6c`z+t z;NX}<@G}@p`xFpxKWvxS(_I7~f}XxD_5Kqm2SoiJNypPP}m`P&($IqIHd6}x{WS0U-^B4y=ec~BA( zabQxFK8GF>(FG(S5a%EeWGfRtA`3**fnyobD%?HzMLD#_ArTsjunX!cg*oET2tqM= zqykd0_hFa;{{3M%)%&eL!e0+dy$`}1+I}&dh5#wyR9Fh9M=J1kW;h$ph+EA;F(C+Xg# z(2olO?M=VUQ+5$N~BhW;k6PcmW=8o`D9JMC)I=#`no|5ECfa! zoPMm#95_2RHdqd}?V#NC9lu;fjF-Yh<)B^ejEi=e;D&-aba>*SqtcM*_U@Y?q^2WW zV?w2&j3B;m0^dXV2QnOErJ+&_xhfa|WJVMM(N3TruD7=oI39?O(v`N65#zF<*2s-; z?SaC?H?}BS?ZTC{DFBBca|);6*sV%FGA5rZV$<~9QOfb`E#ztD`l6t??Z~(qP{i<= z=MY2l0~-M^{i?{aewznY+>{dn7FRg=){%jGf(XsvHXJL?%~uI+O{K%56c?S5?3*cC zWvd@49Uq^k*1T$b>J?fOL_;Q%WQb NiUB33R6>18{T~4Z7pMRL From ad93753623d473e0fa712bf5c38ed9fd39968e1d Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 17 Nov 2023 16:16:54 +0100 Subject: [PATCH 032/111] TC update to Visualization.py --- master_files/Visualization.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index a7c03ec..9c50b10 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -210,12 +210,12 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth # plt.show() return fig - def plot_var_model_comparison(self, models, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=None, diff_plot=False): + def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=None, diff_plot=False): """ - Plot a variable from a number of models. If 2 models are given, a third - plot of the difference will be automatically plotted, too. If 'raw_data' - is chosen as the method, will check to see if the data points in the model - lie at the same coordinates, which they must. + Plot variables from a number of models. If 2 models are given, a third + plot of the difference can be added too. + If 'raw_data' is chosen as the method, will check to see if the data points in the model + lie at the same coordinates, which they must. Parameters ---------- @@ -324,7 +324,6 @@ def plot_var_model_comparison(self, models, var_strs, t, x_range, y_range, inter return fig - if __name__ == '__main__': FileReader = METHOD_HDF5('../Data/test_res100/') From b714777527bc71876a2ace15182129de63abc5da Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 20 Nov 2023 10:51:26 +0100 Subject: [PATCH 033/111] TC changes to find observers and filter on a patch of mesogrid --- master_files/MesoModels.py | 146 ++++++++++++++++++++++++++++++++----- 1 file changed, 126 insertions(+), 20 deletions(-) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index f6fedd3..b45aeff 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1389,6 +1389,12 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): self.filter_vars['U_errors'] = np.zeros((Nt, Nx, Ny)) self.filter_vars['U_success'] = dict() + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + self.filter_vars['U_success'].update({(h,i,j): False}) + + # Setup arrays for derivatives of the model. # THOMAS'S WAY (SAVE THEM AS TENSORS) @@ -1406,20 +1412,68 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) - def find_observers(self): + def find_observers(self, t_range=None, x_range=None, y_range=None): """ Method to compute filtering observers at grid points built with setup_meso_grid. The observers found (and relative errors) are saved in the dictionary self.filter_vars. Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) + Parameters: + ----------- + t_range: list of two floats + ranges in the t-direction, used for selecting the sublist of points from meso-grid + + x_range, y_range: similar to above + Notes: ------ - Requires setup_meso_grid() and setup_variables() to be called first. - """ - for h, t in enumerate(self.domain_vars['T']): - for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): + Requires setup_meso_grid() to be called first. + """ + conditions_t = False + conditions_x = False + conditions_y = False + + # If not specified, find observers at all points on meso-grid. + patch_min = [self.domain_vars['Tmin'], self.domain_vars['Xmin'], self.domain_vars['Ymin']] + patch_max = [self.domain_vars['Tmax'], self.domain_vars['Xmax'], self.domain_vars['Ymax']] + + if t_range: + conditions_t = t_range[0] < self.domain_vars['Tmin'] or t_range[1] > self.domain_vars['Tmax'] + if conditions_t: + print('Error: the input t_range for finding observers is larger than meso-grid.') + else: + patch_min[0] = t_range[0] + patch_max[0] = t_range[1] + if x_range: + conditions_x = x_range[0] < self.domain_vars['Xmin'] or x_range[1] > self.domain_vars['Xmax'] + if conditions_x: + print('Error: the input x_range for finding observers is larger than meso-grid.') + else: + patch_min[1] = x_range[0] + patch_max[1] = x_range[1] + if y_range: + conditions_y = y_range[0] < self.domain_vars['Ymin'] or y_range[1] > self.domain_vars['Ymax'] + if conditions_y: + print('Error: the input y_range for finding observers is larger than meso-grid.') + else: + patch_min[2] = y_range[0] + patch_max[2] = y_range[1] + + conditions = conditions_t or conditions_x or conditions_y + if conditions: + print('Exiting from find observers routine.\n') + return None + else: + idx_mins = Base.find_nearest_cell(patch_min, self.domain_vars['Points']) + idx_maxs = Base.find_nearest_cell(patch_max, self.domain_vars['Points']) + + for h in range(idx_mins[0], idx_maxs[0]): + for i in range(idx_mins[1], idx_maxs[1]): + for j in range(idx_mins[2], idx_maxs[2]): + t = self.domain_vars['T'][h] + x = self.domain_vars['X'][i] + y = self.domain_vars['Y'][j] point = [t,x,y] sol = self.find_obs.find_observer(point) if sol[0]: @@ -1428,25 +1482,76 @@ def find_observers(self): self.filter_vars['U_success'].update({(h,i,j) : True}) if not sol[0]: - self.filter_vars['U_success'].update({(h,i,j) : False}) + # self.filter_vars['U_success'].update({(h,i,j) : False}) + # No need to update the dictionary as this has been initialized to False everywhere. print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) - def filter_micro_variables(self): + def filter_micro_variables(self, t_range=None, x_range=None, y_range=None): """ - Filter all meso_model structures AND micro pressure at the grid_points built with setup_meso_grid(). + Filter all meso_model structures AND micro pressure within the input ranges. + If no range is provided in one direction, routine will try and filter at all points + on the meso-grid. Note this would require the grid to be set up wisely so to avoid + problems at the boundaries. + This method relies on filter_var_point implemented separately for the filter, e.g. spatial_box_filter Parameters: ----------- + t_range: list of two floats + ranges in the t-direction, used for selecting the sublist of points + + x_range, y_range: similar to above. Notes: ------ - Requires find_observers() and setup_meso_grid() to be called first. - """ - for h, t in enumerate(self.domain_vars['T']): - for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): - point = [t, x, y] + Requires setup_meso_grid() to be called first. + Also find_observers() should be called first, although not doing so won't crash it. + """ + conditions_t = False + conditions_x = False + conditions_y = False + + # If not specified, try with all points on meso-grid. + patch_min = [self.domain_vars['Tmin'], self.domain_vars['Xmin'], self.domain_vars['Ymin']] + patch_max = [self.domain_vars['Tmax'], self.domain_vars['Xmax'], self.domain_vars['Ymax']] + + if t_range: + conditions_t = t_range[0] < self.domain_vars['Tmin'] or t_range[1] > self.domain_vars['Tmax'] + if conditions_t: + print('Error: the input t_range for filtering is larger than meso-grid.') + else: + patch_min[0] = t_range[0] + patch_max[0] = t_range[1] + if x_range: + conditions_x = x_range[0] < self.domain_vars['Xmin'] or x_range[1] > self.domain_vars['Xmax'] + if conditions_x: + print('Error: the input x_range for filtering is larger than meso-grid.') + else: + patch_min[1] = x_range[0] + patch_max[1] = x_range[1] + if y_range: + conditions_y = y_range[0] < self.domain_vars['Ymin'] or y_range[1] > self.domain_vars['Ymax'] + if conditions_y: + print('Error: the input y_range for filtering is larger than meso-grid.') + else: + patch_min[2] = y_range[0] + patch_max[2] = y_range[1] + + conditions = conditions_t or conditions_x or conditions_y + if conditions: + print('Exiting from filtering routine.\n') + return None + else: + idx_mins = Base.find_nearest_cell(patch_min, self.domain_vars['Points']) + idx_maxs = Base.find_nearest_cell(patch_max, self.domain_vars['Points']) + + for h in range(idx_mins[0], idx_maxs[0]): + for i in range(idx_mins[1], idx_maxs[1]): + for j in range(idx_mins[2], idx_maxs[2]): + t = self.domain_vars['T'][h] + x = self.domain_vars['X'][i] + y = self.domain_vars['Y'][j] + point = [t,x,y] if self.filter_vars['U_success'][h,i,j]: obs = self.filter_vars['U'][h,i,j] for struct in self.meso_structures: @@ -1818,7 +1923,7 @@ def EL_style_closure_regression(self, CoefficientAnalysis): FileReader = METHOD_HDF5('/Users/thomas/Dropbox/Work/projects/Filtering/Data/test_res100/') - micro_model = IdealMHD_2D() + micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() find_obs = FindObs_drift_root(micro_model, 0.001) @@ -1826,13 +1931,14 @@ def EL_style_closure_regression(self, CoefficientAnalysis): CPU_start_time = time.process_time() - meso_model = resMHD2D(micro_model, find_obs, filter) + meso_model = resHD2D(micro_model, find_obs, filter) # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.5],[0.4, 0.6]],1) - meso_model.setup_meso_grid([[1.501, 1.502],[0.3, 0.35],[0.4, 0.45]],1) + meso_model.setup_meso_grid([[1.501, 1.502],[0.29, 0.36],[0.39, 0.46]]) # print(meso_model.domain_vars['Points']) - meso_model.find_observers() - meso_model.filter_micro_variables() + meso_model.find_observers(x_range = [0.3, 0.34], y_range=[0.4, 0.44]) + meso_model.filter_micro_variables(x_range=[0.3, 0.34], y_range = [0.4, 0.45]) + # print('Filter stage ended') # meso_model.decompose_structures_gridpoint(1,1,1) From 7ee7d89192707c32719b9b0e865a79c6bbbf7172 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 20 Nov 2023 10:52:06 +0100 Subject: [PATCH 034/111] TC update to micro:storing also bar_vel --- master_files/MicroModels.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index 6daf245..605d3ee 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -546,7 +546,7 @@ def __init__(self, interp_method = "linear"): self.aux_vars[str] = [] #Dictionary for structures - self.structures_strs = ("BC", "SET") + self.structures_strs = ("BC", "SET", "bar_vel") self.structures = dict.fromkeys(self.structures_strs) for str in self.structures_strs: self.structures[str] = [] @@ -684,6 +684,7 @@ def setup_structures(self): The stress-energy tensor is stored as a fully contra-variant tensor (both indices up) """ self.structures["BC"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) + self.structures["bar_vel"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3)) self.structures["SET"] = np.zeros((self.domain_vars['nt'],self.domain_vars['nx'],self.domain_vars['ny'],3,3)) for h in range(self.domain_vars['nt']): @@ -691,6 +692,8 @@ def setup_structures(self): for j in range(self.domain_vars['ny']): vel_vec = np.array([self.aux_vars['W'][h,i,j],self.aux_vars['W'][h,i,j] * self.prim_vars['vx'][h,i,j] ,\ self.aux_vars['W'][h,i,j] * self.prim_vars['vy'][h,i,j]]) + + self.structures['bar_vel'][h,i,j,:] = vel_vec self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) @@ -711,14 +714,16 @@ def setup_structures(self): CPU_start_time = time.process_time() - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() + FileReader = METHOD_HDF5('../Data/test_res100/') + micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() + print('Structure strs: {}'.format(micro_model.get_structures_strs())) + point = [1.502,0.4,0.2] # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - vars = ['BC'] + vars = ['BC', 'bar_vel', 'n'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) From 6bfad57620f5f36d43d3b68cdeee0c3a4f04ea33 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 22 Nov 2023 17:31:10 +0100 Subject: [PATCH 035/111] Fixing bug in get_var_data --- master_files/Visualization.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 9c50b10..98e1e74 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -86,7 +86,7 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me print('WARNING: {} is a tensor but you passed more/fewer indices than required. Retrieving the "first component"!'.format(var_str)) component_indices = tuple([0 for _ in range(st2)]) - extent = [*x_range, *y_range] + # extent = [*x_range, *y_range] if method == 'interpolate': compatible = interp_dims != None and len(interp_dims) ==2 @@ -99,6 +99,9 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me xs, ys = np.linspace(x_range[0], x_range[1], nx), np.linspace(y_range[0], y_range[1], ny) data_to_plot = np.zeros((nx, ny)) + points = [t, xs, ys] + extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + for i in range(nx): for j in range(ny): point = [t, xs[i], ys[j]] @@ -112,12 +115,16 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me i_s, i_f = start_indices[1], end_indices[1] j_s, j_f = start_indices[2], end_indices[2] - data_shape = (i_f+ 1- i_s, j_f+ 1- j_s) + gridpoints = model.get_gridpoints() + points = [gridpoints[0][h], gridpoints[1][i_s:i_f+1], gridpoints[2][j_s:j_f+1]] + extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + + data_shape = (i_f - i_s + 1, j_f - j_s + 1) data_to_plot = np.zeros(data_shape) - for i in range(i_f+ 1 - i_s): - for j in range(j_f+ 1 - j_s): - data_to_plot[i,j] = model.get_var_gridpoint(var_str, h, i, j)[component_indices] + for i in range(i_f - i_s + 1): + for j in range(j_f - j_s + 1): + data_to_plot[i,j] = model.get_var_gridpoint(var_str, h, i + i_s, j + j_s)[component_indices] else: print('Data method is not a valid choice! Must be interpolate or raw_data.') @@ -281,7 +288,6 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int for j in range(len(models)): for i in range(n_rows): data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, interp_dims, method, components_indices[j][i]) - # im=axes[i,j].imshow(data_to_plot, extent) im=axes[i,j].imshow(data_to_plot, extent=extent) divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) From 03a9bc66e7e0121f5fe27f06135aa865a086b302 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 27 Nov 2023 16:17:38 +0100 Subject: [PATCH 036/111] Update to Visualization: plotting rel difference --- master_files/Visualization.py | 63 +++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 13 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 98e1e74..2ff28f5 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -129,7 +129,6 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me else: print('Data method is not a valid choice! Must be interpolate or raw_data.') return None - # return data_to_plot, points return data_to_plot, extent @@ -217,7 +216,8 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth # plt.show() return fig - def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=None, diff_plot=False): + def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', \ + components_indices=None, diff_plot=False, rel_diff=False): """ Plot variables from a number of models. If 2 models are given, a third plot of the difference can be added too. @@ -242,6 +242,10 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int component_indices : list of list of tuples each tuple identifies the indices of the component to pick out if the variable is a vector/tensor. + diff_plot: bool + Whether to add a column to show difference between models + rel_diff: bool + Whether to plot the absolute or relative difference between models Output ------- @@ -266,6 +270,11 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int print("Can plot the difference between TWO models, not more.") else: n_cols+=1 + if rel_diff: + if len(models)!=2: + print("Can plot the difference between TWO models, not more.") + else: + n_cols+=1 n_rows = len(var_strs[0]) if len(var_strs[0])>3: @@ -308,7 +317,8 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int if extent1 != extent2: print("Cannot plot the difference between the vars: data not aligned.") continue - im = axes[i,2].imshow(data1-data2, extent=extent1) + data_to_plot = data1 - data2 + im = axes[i,2].imshow(data_to_plot, extent=extent1) divider = make_axes_locatable(axes[i,2]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -319,10 +329,37 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int print(f"Cannot plot the difference between {var_strs} in the two "+\ "models. Likely due to the data coordinates not coinciding.") + + if rel_diff and len(models)==2: + try: + for i in range(len(var_strs[0])): + data1, extent1 = self.get_var_data(models[0], var_strs[0][i], t, x_range, y_range, interp_dims, method, components_indices[0][i]) + data2, extent2 = self.get_var_data(models[1], var_strs[1][i], t, x_range, y_range, interp_dims, method, components_indices[1][i]) + if extent1 != extent2: + print("Cannot plot the difference between the vars: data not aligned.") + continue + ar_mean = (np.abs(data1) + np.abs(data2))/2 + data_to_plot = np.abs(data1 -data2)/ar_mean + if diff_plot: + column = 3 + else: + column = 2 + im = axes[i,column].imshow(data_to_plot, extent=extent1) + divider = make_axes_locatable(axes[i,column]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + axes[i,column].set_title('Relative difference') + axes[i,column].set_xlabel(r'$y$') + axes[i,column].set_ylabel(r'$x$') + except (ValueError): + print(f"Cannot plot the difference between {var_strs} in the two "+\ + "models. Likely due to the data coordinates not coinciding.") + + models_names = [model.get_model_name() for model in models] suptitle = "Comparing " for i in range(len(models_names)): - suptitle += models_names[i] + " " + suptitle += models_names[i] + ", " suptitle += "models." fig.suptitle(suptitle) fig.tight_layout() @@ -330,6 +367,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int return fig + if __name__ == '__main__': FileReader = METHOD_HDF5('../Data/test_res100/') @@ -338,9 +376,8 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int micro_model.setup_structures() - # visualizer = Plotter_2D([11.97, 8.36]) - - # print('Finished initializing') + visualizer = Plotter_2D([11.97, 8.36]) + print('Finished initializing') # TESTING GET_VAR_DATA ###################### @@ -365,17 +402,17 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int # filter = spatial_box_filter(micro_model, 0.003) # meso_model = resMHD2D(micro_model, find_obs, filter) # ranges = [0.2, 0.25] - # meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=2) + # meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=1) # meso_model.find_observers() # meso_model.filter_micro_variables() # print("Finished filtering") - # vars = [['W'],['BC']] - # components = [[()],[(0,)]] + # vars = [['BC'],['BC']] + # components = [[(0,)],[(0,)]] # models = [micro_model, meso_model] - # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries + # # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries # # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ # # method='interpolate', interp_dims=(30,30), component_indices=component) - # visualizer.plot_var_model_comparison(models, vars, 1.502, smaller_ranges, smaller_ranges, components_indices=components, diff_plot=True) - + # visualizer.plot_vars_models_comparison(models, vars, 1.502, ranges, ranges, components_indices=components, diff_plot=True, rel_diff = False) + # plt.show() From 87cbb0269b396d3e27cc665e2f62e216eee09a92 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 30 Nov 2023 11:53:36 +0100 Subject: [PATCH 037/111] Fixing issue with smaller ranges --- master_files/MesoModels.py | 134 ++++++++-------------------------- master_files/Visualization.py | 43 +++++------ 2 files changed, 54 insertions(+), 123 deletions(-) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index b45aeff..9c27a44 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1412,7 +1412,7 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) - def find_observers(self, t_range=None, x_range=None, y_range=None): + def find_observers(self): """ Method to compute filtering observers at grid points built with setup_meso_grid. The observers found (and relative errors) are saved in the dictionary self.filter_vars. @@ -1430,50 +1430,9 @@ def find_observers(self, t_range=None, x_range=None, y_range=None): ------ Requires setup_meso_grid() to be called first. """ - conditions_t = False - conditions_x = False - conditions_y = False - - # If not specified, find observers at all points on meso-grid. - patch_min = [self.domain_vars['Tmin'], self.domain_vars['Xmin'], self.domain_vars['Ymin']] - patch_max = [self.domain_vars['Tmax'], self.domain_vars['Xmax'], self.domain_vars['Ymax']] - - if t_range: - conditions_t = t_range[0] < self.domain_vars['Tmin'] or t_range[1] > self.domain_vars['Tmax'] - if conditions_t: - print('Error: the input t_range for finding observers is larger than meso-grid.') - else: - patch_min[0] = t_range[0] - patch_max[0] = t_range[1] - if x_range: - conditions_x = x_range[0] < self.domain_vars['Xmin'] or x_range[1] > self.domain_vars['Xmax'] - if conditions_x: - print('Error: the input x_range for finding observers is larger than meso-grid.') - else: - patch_min[1] = x_range[0] - patch_max[1] = x_range[1] - if y_range: - conditions_y = y_range[0] < self.domain_vars['Ymin'] or y_range[1] > self.domain_vars['Ymax'] - if conditions_y: - print('Error: the input y_range for finding observers is larger than meso-grid.') - else: - patch_min[2] = y_range[0] - patch_max[2] = y_range[1] - - conditions = conditions_t or conditions_x or conditions_y - if conditions: - print('Exiting from find observers routine.\n') - return None - else: - idx_mins = Base.find_nearest_cell(patch_min, self.domain_vars['Points']) - idx_maxs = Base.find_nearest_cell(patch_max, self.domain_vars['Points']) - - for h in range(idx_mins[0], idx_maxs[0]): - for i in range(idx_mins[1], idx_maxs[1]): - for j in range(idx_mins[2], idx_maxs[2]): - t = self.domain_vars['T'][h] - x = self.domain_vars['X'][i] - y = self.domain_vars['Y'][j] + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): point = [t,x,y] sol = self.find_obs.find_observer(point) if sol[0]: @@ -1486,7 +1445,7 @@ def find_observers(self, t_range=None, x_range=None, y_range=None): # No need to update the dictionary as this has been initialized to False everywhere. print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) - def filter_micro_variables(self, t_range=None, x_range=None, y_range=None): + def filter_micro_variables(self): """ Filter all meso_model structures AND micro pressure within the input ranges. If no range is provided in one direction, routine will try and filter at all points @@ -1507,50 +1466,9 @@ def filter_micro_variables(self, t_range=None, x_range=None, y_range=None): Requires setup_meso_grid() to be called first. Also find_observers() should be called first, although not doing so won't crash it. """ - conditions_t = False - conditions_x = False - conditions_y = False - - # If not specified, try with all points on meso-grid. - patch_min = [self.domain_vars['Tmin'], self.domain_vars['Xmin'], self.domain_vars['Ymin']] - patch_max = [self.domain_vars['Tmax'], self.domain_vars['Xmax'], self.domain_vars['Ymax']] - - if t_range: - conditions_t = t_range[0] < self.domain_vars['Tmin'] or t_range[1] > self.domain_vars['Tmax'] - if conditions_t: - print('Error: the input t_range for filtering is larger than meso-grid.') - else: - patch_min[0] = t_range[0] - patch_max[0] = t_range[1] - if x_range: - conditions_x = x_range[0] < self.domain_vars['Xmin'] or x_range[1] > self.domain_vars['Xmax'] - if conditions_x: - print('Error: the input x_range for filtering is larger than meso-grid.') - else: - patch_min[1] = x_range[0] - patch_max[1] = x_range[1] - if y_range: - conditions_y = y_range[0] < self.domain_vars['Ymin'] or y_range[1] > self.domain_vars['Ymax'] - if conditions_y: - print('Error: the input y_range for filtering is larger than meso-grid.') - else: - patch_min[2] = y_range[0] - patch_max[2] = y_range[1] - - conditions = conditions_t or conditions_x or conditions_y - if conditions: - print('Exiting from filtering routine.\n') - return None - else: - idx_mins = Base.find_nearest_cell(patch_min, self.domain_vars['Points']) - idx_maxs = Base.find_nearest_cell(patch_max, self.domain_vars['Points']) - - for h in range(idx_mins[0], idx_maxs[0]): - for i in range(idx_mins[1], idx_maxs[1]): - for j in range(idx_mins[2], idx_maxs[2]): - t = self.domain_vars['T'][h] - x = self.domain_vars['X'][i] - y = self.domain_vars['Y'][j] + for h, t in enumerate(self.domain_vars['T']): + for i, x in enumerate(self.domain_vars['X']): + for j, y in enumerate(self.domain_vars['Y']): point = [t,x,y] if self.filter_vars['U_success'][h,i,j]: obs = self.filter_vars['U'][h,i,j] @@ -1587,7 +1505,7 @@ def decompose_structures_gridpoint(self, h, i, j): """ # Computing the Favre density and velocity n_t = np.sqrt(-Base.Mink_dot(self.meso_structures['BC'][h,i,j], self.meso_structures['BC'][h,i,j])) - u_t = np.multiply(1 / n_t, self.meso_structures['BC'][h,i,j]) + u_t = np.multiply(1. / n_t, self.meso_structures['BC'][h,i,j]) T_ab = self.meso_structures['SET'][h,i,j,:,:] # Remember this is rank (2,0) # Computing the decomposition at each point @@ -1610,13 +1528,20 @@ def decompose_structures_gridpoint(self, h, i, j): self.meso_vars['eos_res'][h,i,j] = self.meso_vars['p_filt'][h,i,j] - p_t self.meso_vars['T_tilde'][h,i,j] = p_t / n_t - def decompose_structures(self): + def decompose_structures(self, t_range=None, x_range=None, y_range=None): """ Decompose structures at all points on the meso_grid where observers could be found. + + Parameters: + ----------- + t_range: list of two floats + ranges in the t-direction, used for selecting the sublist of points + + x_range, y_range: similar to above. """ for h, t in enumerate(self.domain_vars['T']): for i, x in enumerate(self.domain_vars['X']): - for j, y in enumerate(self.domain_vars['Y']): + for j, y in enumerate(self.domain_vars['Y']): point = [t,y,x] if self.filter_vars['U_success'][h,i,j]: self.decompose_structures_gridpoint(h,i,j) @@ -1933,19 +1858,24 @@ def EL_style_closure_regression(self, CoefficientAnalysis): CPU_start_time = time.process_time() meso_model = resHD2D(micro_model, find_obs, filter) - # meso_model.setup_meso_grid([[1.501, 1.504],[0.3, 0.5],[0.4, 0.6]],1) - meso_model.setup_meso_grid([[1.501, 1.502],[0.29, 0.36],[0.39, 0.46]]) - # print(meso_model.domain_vars['Points']) - meso_model.find_observers(x_range = [0.3, 0.34], y_range=[0.4, 0.44]) - meso_model.filter_micro_variables(x_range=[0.3, 0.34], y_range = [0.4, 0.45]) + # print(micro_model.domain_vars['tmin'], micro_model.domain_vars['tmax']) + + t_range = [1.502, 1.505] + x_range = [0.29, 0.36] + y_range = [0.39, 0.46] + + meso_model.setup_meso_grid([t_range, x_range, y_range]) + meso_model.find_observers() + meso_model.filter_micro_variables() - # print('Filter stage ended') - # meso_model.decompose_structures_gridpoint(1,1,1) - # meso_model.decompose_structures() + print('Filter stage ended') + # # meso_model.decompose_structures_gridpoint(1,1,1) + meso_model.decompose_structures() + print('Decomposition stage ended') # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) - # meso_model.calculate_derivatives() + meso_model.calculate_derivatives() # meso_model.closure_ingredients_gridpoint(1,1,0) # meso_model.closure_ingredients() diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 2ff28f5..348cc9d 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -291,8 +291,9 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int # Block to determine adaptively the figsize. # figsize = np.array([1,2/3.]) * self.screen_size figsize = self.screen_size - # fig, axes = plt.subplots(n_rows,n_cols,sharex='row',sharey='col',figsize=figsize) - fig, axes = plt.subplots(n_rows, n_cols, figsize=figsize, squeeze=False) + # gridspec_dict = {'wspace' : 0.3, 'hspace': 0.3} + # fig, axes = plt.subplots(n_rows, n_cols, squeeze=False, gridspec_kw=gridspec_dict, figsize=figsize) + fig, axes = plt.subplots(n_rows, n_cols, squeeze=False, figsize=figsize) for j in range(len(models)): for i in range(n_rows): @@ -363,7 +364,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int suptitle += "models." fig.suptitle(suptitle) fig.tight_layout() - # plt.show() + # plt.subplot_tool() return fig @@ -398,21 +399,21 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int # TESTING PLOT_VAR_MODEL_COMPARISON ################################### - # find_obs = FindObs_drift_root(micro_model, 0.001) - # filter = spatial_box_filter(micro_model, 0.003) - # meso_model = resMHD2D(micro_model, find_obs, filter) - # ranges = [0.2, 0.25] - # meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=1) - # meso_model.find_observers() - # meso_model.filter_micro_variables() - - # print("Finished filtering") - - # vars = [['BC'],['BC']] - # components = [[(0,)],[(0,)]] - # models = [micro_model, meso_model] - # # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries - # # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ - # # method='interpolate', interp_dims=(30,30), component_indices=component) - # visualizer.plot_vars_models_comparison(models, vars, 1.502, ranges, ranges, components_indices=components, diff_plot=True, rel_diff = False) - # plt.show() + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + meso_model = resMHD2D(micro_model, find_obs, filter) + ranges = [0.2, 0.25] + meso_model.setup_meso_grid([[1.501, 1.503],ranges, ranges], coarse_factor=1) + meso_model.find_observers() + meso_model.filter_micro_variables() + + print("Finished filtering") + + vars = [['BC', 'BC', 'BC', 'BC'],['BC', 'BC' ,'BC' ,'BC']] + components = [[(0,), (0,), (0,), (0,)],[(0,), (0,), (0,), (0,)]] + models = [micro_model, meso_model] + # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries + # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ + # method='interpolate', interp_dims=(30,30), component_indices=component) + visualizer.plot_vars_models_comparison(models, vars, 1.502, ranges, ranges, components_indices=components, diff_plot=True, rel_diff = False) + plt.show() From ec5679cfad7270ab58c403f99fd8b3ec70ae5731 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 5 Dec 2023 18:39:36 +0100 Subject: [PATCH 038/111] First attempt to parallelize routines --- master_files/Filters.py | 222 ++++++++++++++++++++++++++++++++---- master_files/MicroModels.py | 108 +++++++++++------- 2 files changed, 263 insertions(+), 67 deletions(-) diff --git a/master_files/Filters.py b/master_files/Filters.py index 9c40b60..06e293a 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -698,7 +698,7 @@ def find_observer(self, point, drift_str = "gauss", initial_guess= None): return sol.success, point except KeyError: print(f"The method you want to use for computing the flux, {drift_str}, does not exist!") - return None + return None def find_observers_points(self, points, drift_str = "gauss"): """ @@ -786,6 +786,128 @@ def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): return self.find_observers_points(points, drift_str) +class FindObs_root_parallel(object): + """ + AN ATTEMPT TO PARALLELIZE THE FINDING OBSERVERS + WHEN FINISHED: NEED RE-ADJUSTING METHODS WITHIN MESOMODEL. + """ + def __init__(self, micro_model, box_len): + """ + """ + self.micro_model = micro_model + self.L = box_len + + @staticmethod + def get_tetrad_from_vels(spatial_vels): + spatial_dims = len(spatial_vels) + U = Base.get_rel_vel(spatial_vels) + es =[] + for _ in range(spatial_dims): + es.append(np.zeros(spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1 + tetrad = [U] + for i, vec in enumerate(es): + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1 / np.sqrt(Base.Mink_dot(vec, vec))) + tetrad += [es[i]] + return tetrad + + @staticmethod + def initializer(spatial_dims, L): + global adapt_coords + global totws + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + xs = [] + ws = [] + for _ in range(spatial_dims+1): + xs.append(ps1d) + ws.append(ws1d) + + adapt_coords = [] + for element in product(*xs): + adapt_coords.append(np.multiply(L/2, np.array(element) )) + + totws = [] + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + # print('Initialized process in the pool', flush=True) + + def find_observer_Gauss(self, point): + """ + ATTEMPTS PASSING SELF AS ARGUMENT + """ + # The following vars are set up upon initialization by each process in the pool + global adapt_coords + global totws + + # Building the initial guess + guess = [] + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + for i in range(1, len(U)): + guess.append(U[i] / U[0]) + + # The function for the root-finding + def residual_gauss(spatial_vels, point, micro_model): + spatial_dims = micro_model.get_spatial_dims() + tetrad = FindObs_root_parallel.get_tetrad_from_vels(spatial_vels) + coords = [] + for coord in adapt_coords: + temp = np.array(point) + for i in range(spatial_dims+1): + temp += np.multiply(coord[i], tetrad[i]) + coords.append(temp) + + integral = np.zeros(1 + spatial_dims) + for i, coord in enumerate(coords): + integral += np.multiply(totws[i], micro_model.get_interpol_var('BC', coord)) + integral *= (self.L /2 ) ** (spatial_dims+1) + + drifts = [] + for i in range(len(tetrad)-1): + drifts.append(Base.Mink_dot(integral, tetrad[i+1])) + return drifts + + # root-finding the observer and returning it + sol = root(residual_gauss, x0 = guess, args = (point, self.micro_model)) + if sol.success: + observer = Base.get_rel_vel(sol.x) + avg_error = np.sum(sol.fun[1]) + avg_error /= len(sol.fun) + if avg_error > 1e-5: + print(f'Warning: residual is large at {point}: ', avg_error) + return sol.success, point, observer, avg_error + if not sol.success: + return sol.success, point + + def find_observer_parallel(self, points): + observers = [] + avg_errors = [] + success_coords = [] + failed_coords = [] + + spatial_dims = self.micro_model.get_spatial_dims() + L = self.L + with mp.Pool(initializer=FindObs_root_parallel.initializer, initargs=(spatial_dims,L,)) as pool: + # with mp.Pool() as pool: + print('Running with {} processes\n'.format(os.cpu_count()), flush=True) + for result in pool.map(self.find_observer_Gauss, points): + if (result[0] == True): + success_coords.append(result[1]) + observers.append(result[2]) + avg_errors.append(result[3]) + elif (result[0] == False): + failed_coords.append(result[1]) + + return [success_coords, observers, avg_errors] , failed_coords + + class spatial_box_filter(object): """ Class for box-filtering the variables of a micro_model. @@ -1058,31 +1180,81 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): return np.multiply( integrand, 1 / (self.filter_width**self.spatial_dims)), error if __name__ == '__main__': - CPU_start_time = time.process_time() - FileReader = METHOD_HDF5('./Data/test_res100/') - micro_model = IdealMHD_2D() + ######################################################## + # TESTING SERIAL IMPLEMENTATION + ######################################################## + # CPU_start_time = time.process_time() + + # FileReader = METHOD_HDF5('../Data/test_res100/') + # micro_model = IdealMHD_2D() + # FileReader.read_in_data(micro_model) + # micro_model.setup_structures() + + # find_obs = FindObs_drift_root(micro_model, 0.001) + # # find_obs = FindObs_flux_min(micro_model, 0.001) + # filter = spatial_box_filter(micro_model, 0.003) + + # # vars = ['BC','SETfl', 'Fab', 'SETem'] + # points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] + # observers = find_obs.find_observers_points(points)[0][1] + + # for i in range(len(points)): + # for var in vars: + # CPU_start_time = time.process_time() + # filtvar1 = filter.filter_var_point_inbuilt(var, points[i], observers[i]) + # inbuilt_time = time.process_time() - CPU_start_time + + # CPU_start_time = time.process_time() + # filtvar2 = filter.filter_var_point(var, points[i], observers[i]) + # gauss_time = time.process_time() - CPU_start_time + # print(f"CPU time speed-up to filter {var} with Gauss method at {points[i]} is {inbuilt_time/gauss_time}. ") + # print(f'Filtered quantity with Gauss is \n {filtvar2}') + # print(f"Difference with method based on inbuilt integration is: \n {filtvar1[0] - filtvar2}") + # print('\n************************\n') + + ######################################################## + # TESTING PARALLEL IMPLEMENTATION + ######################################################## + FileReader = METHOD_HDF5('../Data/test_res100/') + micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() - find_obs = FindObs_drift_root(micro_model, 0.001) - # find_obs = FindObs_flux_min(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - - vars = ['BC','SETfl', 'Fab', 'SETem'] - points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] - observers = find_obs.find_observers_points(points)[0][1] - - for i in range(len(points)): - for var in vars: - CPU_start_time = time.process_time() - filtvar1 = filter.filter_var_point_inbuilt(var, points[i], observers[i]) - inbuilt_time = time.process_time() - CPU_start_time - - CPU_start_time = time.process_time() - filtvar2 = filter.filter_var_point(var, points[i], observers[i]) - gauss_time = time.process_time() - CPU_start_time - print(f"CPU time speed-up to filter {var} with Gauss method at {points[i]} is {inbuilt_time/gauss_time}. ") - print(f'Filtered quantity with Gauss is \n {filtvar2}') - print(f"Difference with method based on inbuilt integration is: \n {filtvar1[0] - filtvar2}") - print('\n************************\n') \ No newline at end of file + # setting up the points for testing - pass all the points within a range + t_range = [1.502, 1.504] + x_range = [0.5, 0.6] + y_range = [0.5, 0.6] + + patch_min = [t_range[0], x_range[0], y_range[0]] + patch_max = [t_range[1], x_range[1], y_range[1]] + idx_mins = Base.find_nearest_cell(patch_min, micro_model.domain_vars['points']) + idx_maxs = Base.find_nearest_cell(patch_max, micro_model.domain_vars['points']) + + ts = micro_model.domain_vars['t'][idx_mins[0]:idx_maxs[0]] + xs = micro_model.domain_vars['x'][idx_mins[1]:idx_maxs[1]] + ys = micro_model.domain_vars['y'][idx_mins[2]:idx_maxs[2]] + + points = [] + for elem in product(ts,xs,ys): + points.append(list(elem)) + print('Number of points: {}'.format(len(points))) + + # Now find observers - serial version + start_time = time.perf_counter() + find_obs_serial = FindObs_drift_root(micro_model, 0.001) + observers = find_obs_serial.find_observers_points(points) + serial_time = time.perf_counter() - start_time + print('Serial time: {}\n'.format(serial_time)) + + # Now find observers - parallel version + start_time = time.perf_counter() + find_obs_parallel = FindObs_root_parallel(micro_model, 0.001) + point = points[0] + result, failed = find_obs_parallel.find_observer_parallel(points) + print('Number of points failed: {}'.format(len(failed))) + parallel_time = time.perf_counter() - start_time + print('Parallel time: {}\n'.format(parallel_time)) + print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) + + diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index 605d3ee..5cae663 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -8,6 +8,9 @@ import numpy as np import math import time +import os +import multiprocessing as mp +from itertools import product from scipy.interpolate import interpn from multimethod import multimethod @@ -707,7 +710,42 @@ def setup_structures(self): self.vars.update(self.aux_vars) self.vars.update(self.structures) - + #################################################### + # ATTEMPT TO PARALLELIZE SETUP STRUCTURES + #################################################### + + + # defining it statically means no self is passed on construction, but guarantees + # the method is the specific for this class! + @staticmethod + def setting_up(W, vx, vy, n, h, p, idxs): + bar_vel = [W, W * vx, W * vy] + BC = np.multiply(n, bar_vel) + metric = np.zeros((3,3)) + metric[0,0] = -1 + metric[1,1] = metric[2,2] = +1 + SET = n * h * np.outer(bar_vel, bar_vel) + p * metric + return bar_vel, BC, SET, idxs + + + def setup_structures_parallel(self): + args_to_pass = [] + for h in range(self.domain_vars['nt']): + for i in range(self.domain_vars['nx']): + for j in range(self.domain_vars['ny']): + args = (self.aux_vars['W'][h,i,j], self.prim_vars['vx'][h,i,j], self.prim_vars['vy'][h,i,j], \ + self.prim_vars['n'][h,i,j], self.aux_vars['h'][h,i,j], self.prim_vars['p'][h,i,j], (h,i,j)) + args_to_pass.append(args) + + # chunksize = [None, 1, len(args_to_pass)/ os.cpu_count()] + with mp.Manager() as manager: + print('Running with {} processes\n'.format(os.cpu_count())) + with manager.Pool(os.cpu_count()) as pool: + for result in pool.starmap(self.setting_up, args_to_pass): + bar_vel, BC, SET, grid_idxs = result[0], result[1], result[2], tuple(result[3]) + self.structures['bar_vel'][tuple(grid_idxs)] = bar_vel + self.structures['BC'][tuple(grid_idxs)] = BC + self.structures['SET'][tuple(grid_idxs)] = SET # TC if __name__ == '__main__': @@ -715,47 +753,33 @@ def setup_structures(self): CPU_start_time = time.process_time() FileReader = METHOD_HDF5('../Data/test_res100/') + # FileReader = METHOD_HDF5('../Data/res800/res800_t3') micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) - micro_model.setup_structures() - - print('Structure strs: {}'.format(micro_model.get_structures_strs())) - - point = [1.502,0.4,0.2] - # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - vars = ['BC', 'bar_vel', 'n'] - for var in vars: - res = micro_model.get_interpol_var(var, point) - res2 = micro_model.get_var_gridpoint(var, point) - print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') + # micro_model.setup_structures() -# MH -# if __name__ == '__main__': - -# CPU_start_time = time.process_time() - -# FileReader = METHOD_HDF5('./Data/Testing/') -# # micro_model = IdealMHD_2D() -# micro_model = IdealHydro_2D() -# FileReader.read_in_data(micro_model) -# micro_model.setup_structures() - -# res = micro_model.get_interpol_var(['v1','rho'],[10.0,0.3,0.2]) -# print(type(res),"\n", res) - -# res = micro_model.get_interpol_var(['T','W'],[10.0,0.3,0.2]) -# print(type(res),"\n", res) - -# # res = micro_model.get_interpol_var(['bar_vel','SET'],[10.0,0.3,0.2]) -# # print(type(res),"\n", res) -# # res = micro_model.get_interpol_aux(["b0","bx"],[0.5, 0.3,0.2]) -# # print(type(res),"\n", res) - -# # res = micro_model.get_interpol_struct("SET",[2.5, 0.3, 0.2]) -# # print(res.shape, res[0],'\n') - -# # print( micro_model.get_domain_strs() ) - -# CPU_end_time = time.process_time() -# CPU_time = CPU_end_time - CPU_start_time -# print(f'The CPU time is {CPU_time} seconds') \ No newline at end of file + #################################################### + # TESTING PARALLELIZED SETUP STRUCTURE + #################################################### + CPU_start_time = time.perf_counter() + micro_model.setup_structures() + serial_time= time.perf_counter() - CPU_start_time + print('Time taken serial: {}\n'.format(serial_time)) + + CPU_start_time = time.perf_counter() + micro_model.setup_structures_parallel() + parallel_time = time.perf_counter() - CPU_start_time + print('Time taken parallel: {}\n'.format(parallel_time)) + print('Speed up factor: {}\n'.format(serial_time/parallel_time)) + + #################################################### + # Comparing speed of gridpoint vs interpol routines + #################################################### + # print('Structure strs: {}'.format(micro_model.get_structures_strs())) + # point = [1.502,0.4,0.2] + # # vars = ['SETfl', 'BC', 'Fab', 'SETem'] + # vars = ['BC', 'bar_vel', 'n'] + # for var in vars: + # res = micro_model.get_interpol_var(var, point) + # res2 = micro_model.get_var_gridpoint(var, point) + # print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') \ No newline at end of file From 8362aa0774feb204d379069d4cb87eb55c6cf38a Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 6 Dec 2023 16:13:57 +0100 Subject: [PATCH 039/111] No gain in parallelizing setup_structures --- master_files/MicroModels.py | 90 ++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index 5cae663..6f18903 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -711,41 +711,40 @@ def setup_structures(self): self.vars.update(self.structures) #################################################### - # ATTEMPT TO PARALLELIZE SETUP STRUCTURES + # ATTEMPT TO PARALLELIZE SETUP STRUCTURES: NOT WORTHY + # FOR SETUP_STRUCTURES ROUTINE #################################################### - - - # defining it statically means no self is passed on construction, but guarantees - # the method is the specific for this class! - @staticmethod - def setting_up(W, vx, vy, n, h, p, idxs): - bar_vel = [W, W * vx, W * vy] - BC = np.multiply(n, bar_vel) - metric = np.zeros((3,3)) - metric[0,0] = -1 - metric[1,1] = metric[2,2] = +1 - SET = n * h * np.outer(bar_vel, bar_vel) + p * metric - return bar_vel, BC, SET, idxs + # # defining it statically means no self is passed on construction, but guarantees + # # the method is the specific for this class! + # @staticmethod + # def setting_up(W, vx, vy, n, h, p, idxs): + # bar_vel = [W, W * vx, W * vy] + # BC = np.multiply(n, bar_vel) + # metric = np.zeros((3,3)) + # metric[0,0] = -1 + # metric[1,1] = metric[2,2] = +1 + # SET = n * h * np.outer(bar_vel, bar_vel) + p * metric + # return bar_vel, BC, SET, idxs - def setup_structures_parallel(self): - args_to_pass = [] - for h in range(self.domain_vars['nt']): - for i in range(self.domain_vars['nx']): - for j in range(self.domain_vars['ny']): - args = (self.aux_vars['W'][h,i,j], self.prim_vars['vx'][h,i,j], self.prim_vars['vy'][h,i,j], \ - self.prim_vars['n'][h,i,j], self.aux_vars['h'][h,i,j], self.prim_vars['p'][h,i,j], (h,i,j)) - args_to_pass.append(args) - - # chunksize = [None, 1, len(args_to_pass)/ os.cpu_count()] - with mp.Manager() as manager: - print('Running with {} processes\n'.format(os.cpu_count())) - with manager.Pool(os.cpu_count()) as pool: - for result in pool.starmap(self.setting_up, args_to_pass): - bar_vel, BC, SET, grid_idxs = result[0], result[1], result[2], tuple(result[3]) - self.structures['bar_vel'][tuple(grid_idxs)] = bar_vel - self.structures['BC'][tuple(grid_idxs)] = BC - self.structures['SET'][tuple(grid_idxs)] = SET + # def setup_structures_parallel(self): + # args_to_pass = [] + # for h in range(self.domain_vars['nt']): + # for i in range(self.domain_vars['nx']): + # for j in range(self.domain_vars['ny']): + # args = (self.aux_vars['W'][h,i,j], self.prim_vars['vx'][h,i,j], self.prim_vars['vy'][h,i,j], \ + # self.prim_vars['n'][h,i,j], self.aux_vars['h'][h,i,j], self.prim_vars['p'][h,i,j], (h,i,j)) + # args_to_pass.append(args) + + # # chunksize = [None, 1, len(args_to_pass)/ os.cpu_count()] + # with mp.Manager() as manager: + # print('Running with {} processes\n'.format(os.cpu_count())) + # with manager.Pool(os.cpu_count()) as pool: + # for result in pool.starmap(self.setting_up, args_to_pass): + # bar_vel, BC, SET, grid_idxs = result[0], result[1], result[2], tuple(result[3]) + # self.structures['bar_vel'][tuple(grid_idxs)] = bar_vel + # self.structures['BC'][tuple(grid_idxs)] = BC + # self.structures['SET'][tuple(grid_idxs)] = SET # TC if __name__ == '__main__': @@ -758,6 +757,19 @@ def setup_structures_parallel(self): FileReader.read_in_data(micro_model) # micro_model.setup_structures() + #################################################### + # Comparing speed of gridpoint vs interpol routines + #################################################### + # print('Structure strs: {}'.format(micro_model.get_structures_strs())) + # point = [1.502,0.4,0.2] + # # vars = ['SETfl', 'BC', 'Fab', 'SETem'] + # vars = ['BC', 'bar_vel', 'n'] + # for var in vars: + # res = micro_model.get_interpol_var(var, point) + # res2 = micro_model.get_var_gridpoint(var, point) + # print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') + + #################################################### # TESTING PARALLELIZED SETUP STRUCTURE #################################################### @@ -770,16 +782,4 @@ def setup_structures_parallel(self): micro_model.setup_structures_parallel() parallel_time = time.perf_counter() - CPU_start_time print('Time taken parallel: {}\n'.format(parallel_time)) - print('Speed up factor: {}\n'.format(serial_time/parallel_time)) - - #################################################### - # Comparing speed of gridpoint vs interpol routines - #################################################### - # print('Structure strs: {}'.format(micro_model.get_structures_strs())) - # point = [1.502,0.4,0.2] - # # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - # vars = ['BC', 'bar_vel', 'n'] - # for var in vars: - # res = micro_model.get_interpol_var(var, point) - # res2 = micro_model.get_var_gridpoint(var, point) - # print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') \ No newline at end of file + print('Speed up factor: {}\n'.format(serial_time/parallel_time)) \ No newline at end of file From a38381075b06206f435488d92835ca1253f08718 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 6 Dec 2023 16:41:02 +0100 Subject: [PATCH 040/111] Parallelizing finding observers with multiprocessing --- master_files/Filters.py | 144 ++++++++++++++++++++++++++++++++----- master_files/MesoModels.py | 137 ++++++++++++++++++++++++++++------- 2 files changed, 239 insertions(+), 42 deletions(-) diff --git a/master_files/Filters.py b/master_files/Filters.py index 06e293a..e875816 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -788,17 +788,34 @@ def find_observers_ranges(self, num_points, ranges, drift_str = "gauss" ): class FindObs_root_parallel(object): """ - AN ATTEMPT TO PARALLELIZE THE FINDING OBSERVERS - WHEN FINISHED: NEED RE-ADJUSTING METHODS WITHIN MESOMODEL. + Parallel version (streamlined) of FindObs_drift_root + Currently: based on gauss-quadrature with order 3 + interpolation of quantities from micro_model. """ def __init__(self, micro_model, box_len): """ + Parameters: + ----------- + micro_model: instance of micro_model, the micro data to be filtered + + box_len: float, side of the box for computing drift """ self.micro_model = micro_model self.L = box_len @staticmethod def get_tetrad_from_vels(spatial_vels): + """ + Build tetrad orthogonal to unit velocity with spatial velocities spatial_vels + + Parameters: + ----------- + spatial_vels: list of d floats + + Return: + ------- + list of arrays: U + d unit vectors that complete it to a orthonormal basis + """ spatial_dims = len(spatial_vels) U = Base.get_rel_vel(spatial_vels) es =[] @@ -817,6 +834,21 @@ def get_tetrad_from_vels(spatial_vels): @staticmethod def initializer(spatial_dims, L): + """ + Initializer for processes in pool. + Build the adapted coordinates and weights: this is independent of specific point + so can be done once for all workers within a process managed by pool. + + Parameters: + ----------- + spatial dimensions: int + L: float + + Notes: + ------ + As this is built as static method, spatial_dims and L cannot be read + from the specific instance of class + """ global adapt_coords global totws ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] @@ -839,15 +871,37 @@ def initializer(spatial_dims, L): totws.append(temp) # print('Initialized process in the pool', flush=True) - def find_observer_Gauss(self, point): + def find_observer_Gauss(self, point_pos): """ - ATTEMPTS PASSING SELF AS ARGUMENT + CPU-bound task to be run in parralel. + The routine combines what has been split into many in the serial + version of this class. + + Parameters: + ----------- + point_pos: list + point_pos[0] contains the coordinates of point where to find observer + + point_pos[1] contains the position in a larger list passed to pool.map() + + Returns: + -------- + Successful root-finding: Boolean True, position of point in list passed to pool.map(), + observer, avg error + + Failed minimization: Boolean False, position of point in list passed to pool.map() + + Notes: + ------ + To be combined with method in MesoModel.find_obsevers_parallel() """ # The following vars are set up upon initialization by each process in the pool global adapt_coords global totws # Building the initial guess + point = point_pos[0] + pos_in_list_points = point_pos[1] guess = [] U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) for i in range(1, len(U)): @@ -882,30 +936,50 @@ def residual_gauss(spatial_vels, point, micro_model): avg_error /= len(sol.fun) if avg_error > 1e-5: print(f'Warning: residual is large at {point}: ', avg_error) - return sol.success, point, observer, avg_error + return sol.success, pos_in_list_points, observer, avg_error if not sol.success: - return sol.success, point + return sol.success, pos_in_list_points + + def find_observers_parallel(self, points): + """ + Method to run find_observer_Gauss in parallel on a list of points + + Parameters: + ----------- + points: list of d+1 float, d is the spatial dimension of micro_model + + Returns: + -------- - def find_observer_parallel(self, points): + successes: list of lists + successes[0]: positions of point in input list of points + successes[1]: observers found at successful points + successes[2]: average error + + failures: list (typically empty) + position of failed points in input list of points + """ observers = [] avg_errors = [] - success_coords = [] - failed_coords = [] + success_pos = [] + failed_pos = [] spatial_dims = self.micro_model.get_spatial_dims() L = self.L + args_for_pool = [ [points[i], i] for i in range(len(points))] + with mp.Pool(initializer=FindObs_root_parallel.initializer, initargs=(spatial_dims,L,)) as pool: # with mp.Pool() as pool: print('Running with {} processes\n'.format(os.cpu_count()), flush=True) - for result in pool.map(self.find_observer_Gauss, points): + for result in pool.map(self.find_observer_Gauss, args_for_pool): if (result[0] == True): - success_coords.append(result[1]) + success_pos.append(result[1]) observers.append(result[2]) avg_errors.append(result[3]) elif (result[0] == False): - failed_coords.append(result[1]) + failed_pos.append(result[1]) - return [success_coords, observers, avg_errors] , failed_coords + return [success_pos, observers, avg_errors] , failed_pos class spatial_box_filter(object): @@ -1179,6 +1253,44 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): return np.multiply( integrand, 1 / (self.filter_width**self.spatial_dims)), error + +class box_filter_parallel(object): + """ + UNFINISHED ATTEMPT TO PARALLELIZE THE FILTERING + WHEN FINISHED: NEED RE-ADJUSTING METHODS WITHIN MESOMODEL. + """ + def __init__(self, micro_model, filter_width): + self.micro_model = micro_model + self.spatial_dims = micro_model.get_spatial_dims() + self.filter_width = filter_width + + @staticmethod + def complete_U_tetrad(U, spatial_dims): + es =[] + for _ in range(spatial_dims): + es.append(np.zeros(spatial_dims+1)) + for i in range(len(es)): + es[i][i+1] = 1. + triad = [] + for i, vec in enumerate(es): #enumerate returns a tuple: so acts by value not reference! + vec = vec + np.multiply(Base.Mink_dot(vec, U), U) + for j in range(i-1,-1,-1): + vec = vec - np.multiply(Base.Mink_dot(vec, es[j]), es[j]) + es[i] = np.multiply(vec, 1/np.sqrt(Base.Mink_dot(vec, vec))) + triad += [es[i]] + return triad + + def filter_var(): + """ + Need thinking: The observer above is passed from an external class + Option 1) Pass it and assume this is provided. Require starmaps to pass both point and observer + Option 2) Make one single class with the streamlined methods + + Problem with option 2) is that the class above does not store the grid so. + """ + pass + + if __name__ == '__main__': ######################################################## @@ -1223,8 +1335,8 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): # setting up the points for testing - pass all the points within a range t_range = [1.502, 1.504] - x_range = [0.5, 0.6] - y_range = [0.5, 0.6] + x_range = [0.05, 0.95] + y_range = [0.05, 0.95] patch_min = [t_range[0], x_range[0], y_range[0]] patch_max = [t_range[1], x_range[1], y_range[1]] @@ -1251,7 +1363,7 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): start_time = time.perf_counter() find_obs_parallel = FindObs_root_parallel(micro_model, 0.001) point = points[0] - result, failed = find_obs_parallel.find_observer_parallel(points) + result, failed = find_obs_parallel.find_observers_parallel(points) print('Number of points failed: {}'.format(len(failed))) parallel_time = time.perf_counter() - start_time print('Parallel time: {}\n'.format(parallel_time)) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 9c27a44..77dad28 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1445,6 +1445,48 @@ def find_observers(self): # No need to update the dictionary as this has been initialized to False everywhere. print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) + def find_observers_parallel(self): + """ + Method to find observers at all points on meso-grid, parallelized version. + The observers found (and relative errors) are saved in the dictionary self.filter_vars. + Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices + on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) + + Notes: + ------ + This method relies on the routine find_obs.find_observers_parallel(). + So meso_class must be constructed passing parallelized class for finding observers. + """ + ts = self.domain_vars['T'] + xs = self.domain_vars['X'] + ys = self.domain_vars['Y'] + + t_idxs = np.arange(len(ts)) + x_idxs = np.arange(len(xs)) + y_idxs = np.arange(len(ys)) + + points = [] + for elem in product(ts,xs,ys): + points.append(list(elem)) + + indices_meso_grid = [] + for elem in product(t_idxs, x_idxs, y_idxs): + indices_meso_grid.append(elem) + + successes, failures = self.find_obs.find_observers_parallel(points) + + for i in range(len(successes[0])): + point_indxs_meso_grid = indices_meso_grid[successes[0][i]] + self.filter_vars['U'][point_indxs_meso_grid] = successes[1][i] + self.filter_vars['U_errors'][point_indxs_meso_grid] = successes[2][i] + self.filter_vars['U_success'].update({(point_indxs_meso_grid): True}) + + if len(failures)!=0: + print('Observers could not be found at the following points:\n') + for i in range(len(failures)): + failed_idxs_meso_grid = indices_meso_grid[failures[i]] + print('{}\n'.format(failed_idxs_meso_grid)) + def filter_micro_variables(self): """ Filter all meso_model structures AND micro pressure within the input ranges. @@ -1847,44 +1889,87 @@ def EL_style_closure_regression(self, CoefficientAnalysis): if __name__ == '__main__': - FileReader = METHOD_HDF5('/Users/thomas/Dropbox/Work/projects/Filtering/Data/test_res100/') + ######################################################## + # TESTING SERIAL IMPLEMENTATION + ######################################################## + # FileReader = METHOD_HDF5('/Users/thomas/Dropbox/Work/projects/Filtering/Data/test_res100/') + # micro_model = IdealHD_2D() + # FileReader.read_in_data(micro_model) + # micro_model.setup_structures() + # find_obs = FindObs_drift_root(micro_model, 0.001) + # filter = spatial_box_filter(micro_model, 0.003) + + + # CPU_start_time = time.process_time() + # meso_model = resHD2D(micro_model, find_obs, filter) + + # # print(micro_model.domain_vars['tmin'], micro_model.domain_vars['tmax']) + + # t_range = [1.502, 1.505] + # x_range = [0.29, 0.36] + # y_range = [0.39, 0.46] + + # meso_model.setup_meso_grid([t_range, x_range, y_range]) + # meso_model.find_observers() + # meso_model.filter_micro_variables() + + + # print('Filter stage ended') + # # # meso_model.decompose_structures_gridpoint(1,1,1) + # meso_model.decompose_structures() + + # print('Decomposition stage ended') + # # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) + # meso_model.calculate_derivatives() + + # # meso_model.closure_ingredients_gridpoint(1,1,0) + # # meso_model.closure_ingredients() + # # print('Derivatives and Decomposition stage ended') + + # # meso_model.EL_style_closure_gridpoint(1,2,3) + # # meso_model.EL_style_closure() + # # regressor = CoefficientsAnalysis() + # # meso_model.EL_style_closure_regression(regressor) + + # # print('Total time is {}'.format(time.process_time() - CPU_start_time)) + + + ######################################################## + # TESTING PARALLEL IMPLEMENTATION + ######################################################## + FileReader = METHOD_HDF5('../Data/test_res100/') micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() - find_obs = FindObs_drift_root(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) + t_range = [1.502, 1.504] + x_range = [0.05, 0.95] + y_range = [0.05, 0.95] - CPU_start_time = time.process_time() + start_time = time.perf_counter() + find_obs = FindObs_drift_root(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) meso_model = resHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([t_range, x_range, y_range]) - # print(micro_model.domain_vars['tmin'], micro_model.domain_vars['tmax']) - - t_range = [1.502, 1.505] - x_range = [0.29, 0.36] - y_range = [0.39, 0.46] + num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] + print(f'Testing parallelization with {num_points} points\n') - meso_model.setup_meso_grid([t_range, x_range, y_range]) meso_model.find_observers() - meso_model.filter_micro_variables() + serial_time = time.perf_counter() - start_time + print('Serial execution time: {}\n'.format(serial_time)) - - print('Filter stage ended') - # # meso_model.decompose_structures_gridpoint(1,1,1) - meso_model.decompose_structures() - print('Decomposition stage ended') - # print(meso_model.calculate_derivative_gridpoint('u_tilde', 0,1,1,0)) - meso_model.calculate_derivatives() + start_time = time.perf_counter() + find_obs = FindObs_root_parallel(micro_model, 0.001) + filter = spatial_box_filter(micro_model, 0.003) + meso_model = resHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([t_range, x_range, y_range]) + meso_model.find_observers_parallel() + parallel_time = time.perf_counter() - start_time + print('Parallel execution time: {}'.format(parallel_time)) + print('Speed-up factor: {}'.format(serial_time/parallel_time)) - # meso_model.closure_ingredients_gridpoint(1,1,0) - # meso_model.closure_ingredients() - # print('Derivatives and Decomposition stage ended') - # meso_model.EL_style_closure_gridpoint(1,2,3) - # meso_model.EL_style_closure() - # regressor = CoefficientsAnalysis() - # meso_model.EL_style_closure_regression(regressor) - # print('Total time is {}'.format(time.process_time() - CPU_start_time)) From 897a699fa89d5e50e99b34c1860d22f18639eb91 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 7 Dec 2023 11:48:24 +0100 Subject: [PATCH 041/111] Parallelizing filtering class --- master_files/Filters.py | 212 +++++++++++++++++++++++++++++++++---- master_files/MesoModels.py | 101 ++++++++++++++---- 2 files changed, 268 insertions(+), 45 deletions(-) diff --git a/master_files/Filters.py b/master_files/Filters.py index e875816..749734a 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -895,7 +895,7 @@ def find_observer_Gauss(self, point_pos): ------ To be combined with method in MesoModel.find_obsevers_parallel() """ - # The following vars are set up upon initialization by each process in the pool + # Declaring global vars set up by initializer global adapt_coords global totws @@ -907,7 +907,7 @@ def find_observer_Gauss(self, point_pos): for i in range(1, len(U)): guess.append(U[i] / U[0]) - # The function for the root-finding + # Routine to compute drift residual via Gauss-Legendre quadrature. def residual_gauss(spatial_vels, point, micro_model): spatial_dims = micro_model.get_spatial_dims() tetrad = FindObs_root_parallel.get_tetrad_from_vels(spatial_vels) @@ -928,14 +928,14 @@ def residual_gauss(spatial_vels, point, micro_model): drifts.append(Base.Mink_dot(integral, tetrad[i+1])) return drifts - # root-finding the observer and returning it + # observer: root of the residual gauss routine sol = root(residual_gauss, x0 = guess, args = (point, self.micro_model)) if sol.success: observer = Base.get_rel_vel(sol.x) avg_error = np.sum(sol.fun[1]) avg_error /= len(sol.fun) if avg_error > 1e-5: - print(f'Warning: residual is large at {point}: ', avg_error) + print(f'Warning: residual is large at {point}: ', avg_error, flush=True) return sol.success, pos_in_list_points, observer, avg_error if not sol.success: return sol.success, pos_in_list_points @@ -1256,16 +1256,37 @@ def interpol_var_adapt_coord(self, x, y, z, *ind): class box_filter_parallel(object): """ - UNFINISHED ATTEMPT TO PARALLELIZE THE FILTERING - WHEN FINISHED: NEED RE-ADJUSTING METHODS WITHIN MESOMODEL. + Parallel version (streamlined) of spatial_box_filter + Currently: based on gauss-quadrature with order 3 + interpolation of quantities from micro_model. """ def __init__(self, micro_model, filter_width): + """ + Constructor + + Parameters: + ----------- + micro_model: instance of a micro_model class + micro data to be filtered + + filter_width: float + """ self.micro_model = micro_model self.spatial_dims = micro_model.get_spatial_dims() self.filter_width = filter_width @staticmethod - def complete_U_tetrad(U, spatial_dims): + def complete_U_tetrad(U): + """ + Given a time-like vector in d+1 dims, build and return + d vectors that complete U to an ON basis. + + Notes: + ------ + This method is static: does not depend on instance vars of the class + But it makes sense to have it belong to the class nonetheless. + """ + spatial_dims = len(U)-1 es =[] for _ in range(spatial_dims): es.append(np.zeros(spatial_dims+1)) @@ -1280,15 +1301,137 @@ def complete_U_tetrad(U, spatial_dims): triad += [es[i]] return triad - def filter_var(): - """ - Need thinking: The observer above is passed from an external class - Option 1) Pass it and assume this is provided. Require starmaps to pass both point and observer - Option 2) Make one single class with the streamlined methods + @staticmethod + def initializer(spatial_dims, filter_width): + """ + Initializer for processes in pool. + Build the adapted coordinates and weights: this is independent of specific point + so can be done once for all workers within a process managed by pool. + + Currently: gauss-legendre quadrature of order 3. + """ + global abstract_coords + global totws + + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] + ws1d = [8./9. , 5./9. , 5./9.] + + xs = [] + ws = [] + for _ in range(spatial_dims): + xs.append(ps1d) + ws.append(ws1d) + + abstract_coords = [] + for element in product(*xs): + abstract_coords.append(np.multiply(filter_width/2, np.array(element) )) + + totws = [] + for element in product(*ws): + temp = 1. + for w in element: + temp *= w + totws.append(temp) + + def filter_vars_point_gauss(self, packed_args): + """ + CPU-bound task to be run in parralel. + The routine combines what has been split into many in the serial + version of this class. + + Parameters: + ----------- + packed_args: [[point, observer, [vars_to_be_filtered]], position] + + point: list of floats + the point at which filtering + + observer: nd.array + the observer wrt which filtering + + [vars_to_be_filtered]: list of strs + strings corresponding to vars in micro_model to be filtered + + position: integer + number of specific point etc in larger list passed to pool.map() + + Returns: + -------- + position: same as input + + filtered_vars: list containing vars corresponding to pass list, filtered + at point 'point' wrt observer 'observer' + + """ + # declaring global vars set up by initializer + global abstract_coords + global totws + + + # Unpacking arguments passed by pool.map + unpacked_args= packed_args[0] + pos = packed_args[1] + point = unpacked_args[0] + observer = unpacked_args[1] + vars_strs = unpacked_args[2] + + # From abstract to "real" coordinates + vecs = box_filter_parallel.complete_U_tetrad(observer) + sample_points = [] + for coord in abstract_coords: + temp = np.array(point) + for i in range(self.spatial_dims): + temp += np.multiply(coord[i], vecs[i]) + sample_points.append(temp) + + # Filtering the vars + filtered_vars = [] + for var in vars_strs: + filtered_var = np.zeros(self.micro_model.get_var_gridpoint(var,0,0,0).shape) + for i, sample in enumerate(sample_points): + filtered_var += totws[i] * self.micro_model.get_interpol_var(var, sample) + filtered_var = np.multiply(filtered_var, 1 / (2**self.spatial_dims)) + filtered_vars.append(filtered_var) + + # Different variables are returned as list (ordered as var_strs) + # Safer to return a dictionary: check redux in performance though. + return pos, filtered_vars + + def filter_vars_parallel(self, list_packed_args): + """ + Method to run filter_vars_point_gauss in parallel given a list of points, + observers and vars. - Problem with option 2) is that the class above does not store the grid so. - """ - pass + Parameters: + ----------- + + list_packed_args: [[point,observer, [vars_to_be_filtered]]] + + point: list of floats + + observer: nd.array + + [vars_to_be_filtered]: list of strs + each item must much a var in micromodel + + Returns: + -------- + position_in_list: list of integers + position in original list of points at which filtering + Required as pool.map not necessarily returns processes in order + + filtered_vars: list of lists containing filtered vars + """ + position_in_list = [] + filtered_vars = [] + args_for_pool = [ [list_packed_args[i], i] for i in range(len(list_packed_args))] + with mp.Pool(initializer=box_filter_parallel.initializer, initargs=(self.spatial_dims, self.filter_width)) as pool: + print('Running with {} processes\n'.format(os.cpu_count()), flush=True) + for result in pool.map(self.filter_vars_point_gauss, args_for_pool): + position_in_list.append(result[0]) + filtered_vars.append(result[1]) + + return position_in_list, filtered_vars if __name__ == '__main__': @@ -1307,7 +1450,7 @@ def filter_var(): # # find_obs = FindObs_flux_min(micro_model, 0.001) # filter = spatial_box_filter(micro_model, 0.003) - # # vars = ['BC','SETfl', 'Fab', 'SETem'] + # vars = ['BC','SETfl', 'Fab', 'SETem'] # points = [[1.502,0.3,0.5],[1.503,0.4,0.2]] # observers = find_obs.find_observers_points(points)[0][1] @@ -1350,14 +1493,15 @@ def filter_var(): points = [] for elem in product(ts,xs,ys): points.append(list(elem)) + print('Number of points: {}'.format(len(points))) - # Now find observers - serial version - start_time = time.perf_counter() - find_obs_serial = FindObs_drift_root(micro_model, 0.001) - observers = find_obs_serial.find_observers_points(points) - serial_time = time.perf_counter() - start_time - print('Serial time: {}\n'.format(serial_time)) + # # Now find observers - serial version + # start_time = time.perf_counter() + # find_obs_serial = FindObs_drift_root(micro_model, 0.001) + # observers = find_obs_serial.find_observers_points(points) + # serial_time = time.perf_counter() - start_time + # print('Serial time: {}\n'.format(serial_time)) # Now find observers - parallel version start_time = time.perf_counter() @@ -1367,6 +1511,28 @@ def filter_var(): print('Number of points failed: {}'.format(len(failed))) parallel_time = time.perf_counter() - start_time print('Parallel time: {}\n'.format(parallel_time)) - print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) + # print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) + observers = result[1] + vars = ['BC', 'SET'] + # Filtering serial + start_time = time.perf_counter() + serial_filter = spatial_box_filter(micro_model, 0.003) + for var in vars: + serial_filter.filter_var_manypoints(var, points, observers) + serial_time = time.perf_counter() - start_time + print('Finished filtering in serial, time-taken: {}'.format(serial_time)) + + # Preparing args for parallel filter routine + args_list = [] + for elem in zip(points, observers): + args_list.append([*elem, vars]) + + start_time = time.perf_counter() + parallel_filter = box_filter_parallel(micro_model, 0.003) + parallel_filter.filter_vars_parallel(args_list) + parallel_time = time.perf_counter() - start_time + print('Finished filtering in parallel, time-taken: {}'.format(parallel_time)) + print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) + diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 77dad28..d314d42 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1186,6 +1186,12 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): if not compatible: print("Meso and Micro models are incompatible:"+error) + def set_find_obs(self, find_obs): + self.find_obs = find_obs + + def set_filter(self, filter): + self.filter = filter + def get_all_var_strs(self): return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) + \ list(self.filter_vars.keys()) @@ -1419,13 +1425,6 @@ def find_observers(self): Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) - Parameters: - ----------- - t_range: list of two floats - ranges in the t-direction, used for selecting the sublist of points from meso-grid - - x_range, y_range: similar to above - Notes: ------ Requires setup_meso_grid() to be called first. @@ -1496,13 +1495,6 @@ def filter_micro_variables(self): This method relies on filter_var_point implemented separately for the filter, e.g. spatial_box_filter - Parameters: - ----------- - t_range: list of two floats - ranges in the t-direction, used for selecting the sublist of points - - x_range, y_range: similar to above. - Notes: ------ Requires setup_meso_grid() to be called first. @@ -1520,6 +1512,57 @@ def filter_micro_variables(self): else: print('Could not filter at {}: observer not found.'.format(point)) + def filter_micro_vars_parallel(self): + """ + Filter all meso_model structures AND micro pressure. + Routine will try and filter at all points on the meso-grid. + Note this would require the grid to be set up wisely so to avoid + problems at the boundaries. + + This method relies on filter_vars_parallel implemented separately for the + filter class, e.g. as in box_filter_parallel + + Notes: + ------ + Requires setup_meso_grid() to be called first. + Also find_observers() should be called first, although not doing so won't crash it. + """ + ts = self.domain_vars['T'] + xs = self.domain_vars['X'] + ys = self.domain_vars['Y'] + + t_idxs = np.arange(len(ts)) + x_idxs = np.arange(len(xs)) + y_idxs = np.arange(len(ys)) + + points = [] + for elem in product(ts,xs,ys): + points.append(list(elem)) + + indices_meso_grid = [] + for elem in product(t_idxs, x_idxs, y_idxs): + indices_meso_grid.append(elem) + + observers = [] + for elem in product(t_idxs, x_idxs, y_idxs): + if self.filter_vars['U_success'][elem]: + observers.append(self.filter_vars['U'][elem]) + else: + print('Observers are not computed on (parts of) the grid!') + return None + + vars = ['BC', 'SET', 'p'] + args_for_filtering_parallel = [] + for i in range(len(points)): + args_for_filtering_parallel.append([points[i], observers[i], vars]) + + positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel) + for i in range(len(positions)): + point_indxs_meso_grid = indices_meso_grid[positions[i]] + self.meso_structures['BC'][point_indxs_meso_grid] = filtered_vars[i][0] + self.meso_structures['SET'][point_indxs_meso_grid] = filtered_vars[i][1] + self.meso_vars['p_filt'][point_indxs_meso_grid] = filtered_vars[i][2] + def p_from_EOS(self, eps, n): """ Compute pressure from Gamma-law EoS @@ -1946,20 +1989,21 @@ def EL_style_closure_regression(self, CoefficientAnalysis): x_range = [0.05, 0.95] y_range = [0.05, 0.95] - start_time = time.perf_counter() + + # find obs - serial + # start_time = time.perf_counter() find_obs = FindObs_drift_root(micro_model, 0.001) filter = spatial_box_filter(micro_model, 0.003) meso_model = resHD2D(micro_model, find_obs, filter) meso_model.setup_meso_grid([t_range, x_range, y_range]) + # meso_model.find_observers() + # serial_time = time.perf_counter() - start_time + # print('Serial execution time: {}\n'.format(serial_time)) num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] print(f'Testing parallelization with {num_points} points\n') - meso_model.find_observers() - serial_time = time.perf_counter() - start_time - print('Serial execution time: {}\n'.format(serial_time)) - - + # fin obs - parallel start_time = time.perf_counter() find_obs = FindObs_root_parallel(micro_model, 0.001) filter = spatial_box_filter(micro_model, 0.003) @@ -1967,9 +2011,22 @@ def EL_style_closure_regression(self, CoefficientAnalysis): meso_model.setup_meso_grid([t_range, x_range, y_range]) meso_model.find_observers_parallel() parallel_time = time.perf_counter() - start_time - print('Parallel execution time: {}'.format(parallel_time)) - print('Speed-up factor: {}'.format(serial_time/parallel_time)) + print('Finished finding observers in parallel, execution time: {}\n'.format(parallel_time)) + # print('Speed-up factor: {}'.format(serial_time/parallel_time)) + # now filtering serial + start_time = time.perf_counter() + meso_model.filter_micro_variables() + serial_time = time.perf_counter() - start_time + print('Finished filtering in serial, time taken {}\n'.format(serial_time)) + # now filtering in parallel + parallel_filter = box_filter_parallel(micro_model, 0.003) + meso_model.set_filter(parallel_filter) + start_time = time.perf_counter() + meso_model.filter_micro_vars_parallel() + parallel_time = time.perf_counter() - start_time + print('Finished filtering in parallel, time taken {}\n'.format(parallel_time)) + print('Speed-up factor: {}'.format(serial_time/parallel_time)) From 10df804661d816f9da0082862d9970fb40e337ee Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Sat, 9 Dec 2023 11:17:43 +0100 Subject: [PATCH 042/111] Folder for Proof of Principle --- Proof_of_principle/config.txt | 6 + Proof_of_principle/multiple_filtering_meso.py | 58 +++++++ Proof_of_principle/pickling_meso_obs.py | 53 +++++++ Proof_of_principle/pickling_meso_setup.py | 64 ++++++++ Proof_of_principle/pickling_micro.py | 58 +++++++ Proof_of_principle/visualizing_meso.py | 117 ++++++++++++++ Proof_of_principle/visualizing_meso_manyFW.py | 144 ++++++++++++++++++ Proof_of_principle/visualizing_micro.py | 52 +++++++ Proof_of_principle/visualizing_obs.py | 47 ++++++ 9 files changed, 599 insertions(+) create mode 100644 Proof_of_principle/config.txt create mode 100644 Proof_of_principle/multiple_filtering_meso.py create mode 100644 Proof_of_principle/pickling_meso_obs.py create mode 100644 Proof_of_principle/pickling_meso_setup.py create mode 100644 Proof_of_principle/pickling_micro.py create mode 100644 Proof_of_principle/visualizing_meso.py create mode 100644 Proof_of_principle/visualizing_meso_manyFW.py create mode 100644 Proof_of_principle/visualizing_micro.py create mode 100644 Proof_of_principle/visualizing_obs.py diff --git a/Proof_of_principle/config.txt b/Proof_of_principle/config.txt new file mode 100644 index 0000000..28803bd --- /dev/null +++ b/Proof_of_principle/config.txt @@ -0,0 +1,6 @@ +JOB_ID ET +1 1.0 +2 2.0 +3 2.5 +4 3.0 +5 3.5 diff --git a/Proof_of_principle/multiple_filtering_meso.py b/Proof_of_principle/multiple_filtering_meso.py new file mode 100644 index 0000000..4d291a8 --- /dev/null +++ b/Proof_of_principle/multiple_filtering_meso.py @@ -0,0 +1,58 @@ +#!/bin/bash + +import sys +# import os +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +sys.path.append('/home/tc2m23/Filtering/master_files/') +import pickle +import time + +from FileReaders import * +from MicroModels import * +from Filters import * +from MesoModels import * + +if __name__ == '__main__': + + ET=sys.argv[1] + CPU_start_time = time.perf_counter() + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + MesoModelPickleLoadFile = directory + "observers_2dx_ET_" + ET + ".pickle" + with open(MesoModelPickleLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + time_taken = time.perf_counter()- CPU_start_time + print('Time taken to read data from pickle: {}'.format(time_taken)) + + # filter_widths_ratio = [8] + filter_widths_ratio = [2, 4, 8] + meso_dx = meso_model.domain_vars['Dx'] + filter = meso_model.filter + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + # saving_directory = "./" + + for i in range(len(filter_widths_ratio)): + CPU_start_time = time.perf_counter() + filter.set_filter_width(meso_dx * filter_widths_ratio[i]) + + # HOW LONG IT TAKES TO FILTER A VAR AT A POINT? + # vars = ['BC', 'SET', 'p'] + # for var in vars: + # h,l,j = 0, 200, 200 + # T, X, Y = meso_model.domain_vars['T'][h], meso_model.domain_vars['X'][l], meso_model.domain_vars['Y'][j] + # point = [T, X, Y] + # obs = meso_model.filter_vars['U'][h,l,j] + # filter.filter_var_point(var, point, obs) + # print('Filter var {}'.format(var)) + + meso_model.filter_micro_variables() + filename = "resHD_2D_ET_" + ET + "_FW_{}dx.pickle".format(int(filter_widths_ratio[i])) + MesoModelPickleDumpFile = saving_directory+filename + with open(MesoModelPickleDumpFile, 'wb') as filehandle: + pickle.dump(meso_model, filehandle) + time_taken = time.perf_counter() - CPU_start_time + print('Time taken to filter data (fw: {}): {}'.format(int(filter_widths_ratio[i]), time_taken)) + + + + + diff --git a/Proof_of_principle/pickling_meso_obs.py b/Proof_of_principle/pickling_meso_obs.py new file mode 100644 index 0000000..b7fc015 --- /dev/null +++ b/Proof_of_principle/pickling_meso_obs.py @@ -0,0 +1,53 @@ +import sys +# import os +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +sys.path.append('/home/tc2m23/Filtering/master_files/') +import pickle +import time + +from FileReaders import * +from MicroModels import * +from Filters import * +from MesoModels import * + + +if __name__ == '__main__': + + # READING THE MICRO-MODEL FROM PICKLE + CPU_start_time = time.perf_counter() + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + ET=str(sys.argv[1]) + MicroModelPickleLoadFile = directory + "IdealHD_2D_ET_"+ET+"_micro.pickle" + + with open(MicroModelPickleLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + + + # SETTING UP THE MESO-MODEL + num_snaps = 11 + box_len_ratio = 2. + central_slice_num = int(num_snaps/2) + t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num+1]] + box_len = box_len_ratio * micro_model.domain_vars['dx'] + + find_obs = FindObs_drift_root(micro_model, box_len) + filter = spatial_box_filter(micro_model, box_len) #THIS WILL BE CHANGED IN A LATER SCRIPT + meso_model = resHD2D(micro_model, find_obs, filter) + grid_bdrs = [0.03, 0.97] + meso_model.setup_meso_grid([t_bdrs, grid_bdrs, grid_bdrs]) + time_taken = time.perf_counter() - CPU_start_time + print('Read from micro, grid is set up, time taken: {}'.format(time_taken)) + + # ROOT-FINDING THE OBSERVERS + CPU_start_time = time.perf_counter() + # t, x, y = meso_model.domain_vars['T'][0], meso_model.domain_vars['X'][10], meso_model.domain_vars['Y'][10] + # point = [t,x,y] + # meso_model.find_obs.find_observer(point) + meso_model.find_observers() + time_taken = time.perf_counter() - CPU_start_time + print('Observers found, time taken: {}'.format(time_taken)) + + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + MesoModelPickleDumpFile = directory + "observers_2dx_ET_" + ET + ".pickle" + with open(MesoModelPickleDumpFile, 'wb') as filehandle: + pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/pickling_meso_setup.py b/Proof_of_principle/pickling_meso_setup.py new file mode 100644 index 0000000..7dd7f56 --- /dev/null +++ b/Proof_of_principle/pickling_meso_setup.py @@ -0,0 +1,64 @@ +import sys +# import os +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +# sys.path.append('/home/tc2m23/Filtering/master_files/') +import pickle +import time + +from FileReaders import * +from MicroModels import * +from Filters import * +from MesoModels import * + +if __name__ == '__main__': + + # READING MICRO DATA FORM PICKLE + CPU_start_time = time.perf_counter() + # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/pickle_files/400X400/" + directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/pickle_files/80X80/" + MicroModelPickleLoadFile = directory + "IdealHD_2D_ET_02_micro.pickle" + + with open(MicroModelPickleLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + time_taken = time.perf_counter() - CPU_start_time + print('Time taken to read file from pickle: {}'.format(time_taken)) + # print("Micro_model domain_vars keys: {}".format(micro_model.domain_vars.keys())) + + # FINDING THE OBSERVERS, FILTERING + CPU_start_time = time.perf_counter() + num_snaps = 21 + width_ratio_dx = 0.1 + central_slice_num = int(num_snaps/2) + t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num]] + width = width_ratio_dx * micro_model.domain_vars['dx'] + + # xm, xM = np.min(micro_model.domain_vars['x']), np.max(micro_model.domain_vars['x']) + # ym, yM = np.min(micro_model.domain_vars['y']), np.max(micro_model.domain_vars['y']) + # print('Micro_model grid extent - careful when setting up meso-grid: \nX: {}, {} \nY: {}, {}'.format(xm, xM, ym, yM)) + + find_obs = FindObs_drift_root(micro_model, width) + filter = spatial_box_filter(micro_model, width) + meso_model = resHD2D(micro_model, find_obs, filter) + # meso_model.setup_meso_grid([t_bdrs,[0.01,0.99],[0.01,0.99]]) + meso_model.setup_meso_grid([t_bdrs,[0.05,0.95],[0.05,0.95]]) + time_taken = time.perf_counter() - CPU_start_time + print('Grid is set up, time taken: {}'.format(time_taken)) + + CPU_start_time = time.perf_counter() + meso_model.find_observers() + time_taken = time.perf_counter() - CPU_start_time + print('Observers found, time taken: {}'.format(time_taken)) + + CPU_start_time = time.perf_counter() + meso_model.filter_micro_variables() + time_taken = time.perf_counter() - CPU_start_time + print('Filter stage ended, time taken: {}'.format(time_taken)) + + + # PICKLING THE FILTERED DATA CLASS + # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/pickle_files/400X400/" + directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/pickle_files/80X80/" + # MesoModelPickleDumpFile = directory + "resHD_2D_ET_02_meso.pickle" + MesoModelPickleDumpFile = directory + "resHD_2D_ET_02_all_grid.pickle" + with open(MesoModelPickleDumpFile, 'wb') as filehandle: + pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/pickling_micro.py b/Proof_of_principle/pickling_micro.py new file mode 100644 index 0000000..4a3db9b --- /dev/null +++ b/Proof_of_principle/pickling_micro.py @@ -0,0 +1,58 @@ +import sys +# import os +sys.path.append('/home/tc2m23/Filtering/master_files/') +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +import pickle +import time + +from FileReaders import * +from MicroModels import * + +if __name__ == '__main__': + + # READING DATA FORM CHECKPOINT, SETTING UP MICOR AND PICKLING + CPU_start_time = time.perf_counter() + # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/METHOD_output/80X80/ET_02" + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/METHOD_output/400X400/" + ET=str(sys.argv[1]) + names = directory + "ET_" + ET + print(names) + + FileReader = METHOD_HDF5(names) + micro_model = IdealHD_2D() + FileReader.read_in_data_HDF5_missing_xy(micro_model) + micro_model.setup_structures() + time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. + print('Time to set up micro model (ET: {}): {}'.format(ET, time_taken)) + + # Pickle save + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + CPU_start_time = time.perf_counter() + MicroModelPickleDumpFile = directory + "IdealHD_2D_ET_" + ET + "_micro.pickle" + with open(MicroModelPickleDumpFile, 'wb') as filehandle: + pickle.dump(micro_model, filehandle) + time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. + print('Time needed to pickle class: {}'.format(time_taken)) + + # READING AND PICKLING ALL CHECKPOINTS - FOR VISUALIZATION + # CPU_start_time = time.perf_counter() + # # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/METHOD_output/400X400/" + # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/METHOD_output/80X80/" + # FileReader = METHOD_HDF5(directory) + # micro_model = IdealHD_2D() + # FileReader.read_in_data_HDF5_missing_xy(micro_model) + # micro_model.setup_structures() + # time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. + # print('Time to set up micro model (up to 8ET): {}'.format(time_taken)) + + # # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/pickle_files/400X400/" + # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/pickle_files/80X80/" + # CPU_start_time = time.perf_counter() + # pickle_save = True + # MicroModelPickleDumpFile = directory + "IdealHD_2D_ET_ALL_micro.pickle" + # if pickle_save: + # with open(MicroModelPickleDumpFile, 'wb') as filehandle: + # pickle.dump(micro_model, filehandle) + # time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. + # print('Time needed to pickle class: {}'.format(time_taken)) + diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py new file mode 100644 index 0000000..b15c4db --- /dev/null +++ b/Proof_of_principle/visualizing_meso.py @@ -0,0 +1,117 @@ +import sys +# import os +sys.path.append('/home/tc2m23/Filtering/master_files/') +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +import pickle + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * + +if __name__ == '__main__': + + ############################################################### + # PRODUCE MANY DIFF PLOTS TO COMPARE MICRO AND MESO AT DIFFERENT + # ET AND DIFFERENT WIDTHS. FOR EACH ADD THE DIFFERENCE. + ############################################################### + + # Loading the models + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + ET = sys.argv[1] + # ET = "3.5" + MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET + "_micro.pickle" + with open(MicroModelLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + + num_snaps = 11 + central_slice_num = int(num_snaps/2.) + time_micro = micro_model.domain_vars['t'][central_slice_num] + + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + # saving_directory = "./" + # FW = ['2', '4', '8'] + # # FW = ['8'] + # if ET == "2.0": + # FW = ['2', '4'] + FW = ['2', '4'] + + for fw in FW: + MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + time_meso = meso_model.domain_vars['T'][0] + if time_meso != time_micro: + print("Slices of meso and micro model do not coincide. Careful!") + else: + print("Comparing data at same time-slice, hurray!") + + ranges_x = [0.03, 0.97] + ranges_y = [0.03, 0.97] + visualizer = Plotter_2D([11.97, 8.36]) + + ###################################################### + # CODE BLOCKS TO VISUALIZE SEPARATE QUANTITIES + ###################################################### + # # Plot for baryon current + # vars = [['BC', 'BC', 'BC'], ['BC', 'BC', 'BC']] + # models = [micro_model, meso_model] + # components = [[(0,), (1,), (2,)], [(0,), (1,), (2,)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + # fig.tight_layout() + # # plt.show() + # filename = "comparing_BC_ET_" + ET + "_FW_"+ fw + ".svg" + # plt.savefig(saving_directory + filename, format = "svg") + + + # # # Plots for SET - diag and off-diagonal + # vars = [['SET', 'SET', 'SET'], ['SET', 'SET', 'SET']] + # models = [micro_model, meso_model] + # components = [[(0,0), (1,1), (2,2)], [(0,0), (1,1), (2,2)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + # fig.tight_layout() + # # plt.show() + # filename = "comparing_SET_diagonal" + ET + "_FW_"+ fw + ".svg" + # plt.savefig(saving_directory + filename, format = "svg") + + # # vars = [['SET', 'SET', 'SET'], ['SET', 'SET', 'SET']] + # models = [micro_model, meso_model] + # components = [[(0,1), (0,2), (1,2)], [(0,1), (0,2), (1,2)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + # fig.tight_layout() + # # plt.show() + # filename = "comparing_SET_off_diag" + ET + "_FW_"+ fw + ".svg" + # plt.savefig(saving_directory + filename, format = "svg") + + + # # Plots to compare observer with Favre-velocity (need to decompose structures first!) + meso_model.decompose_structures() + vars = [['U', 'U', 'U'], ['u_tilde', 'u_tilde', 'u_tilde']] + models = [meso_model, meso_model] + components = [[(0,), (1,), (2,)], [(0,), (1,), (2,)]] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + fig.tight_layout() + filename = "comparing_obs_VS_Favre" + ET + "_FW_" + fw + ".svg" + plt.savefig(saving_directory + filename, format = 'svg') + + + ###################################################### + # CODE FOR SUMMARY PLOTS + ###################################################### + # vars = [['BC', 'SET'], ['BC', 'SET',]] + # models = [micro_model, meso_model] + # components = [[(0,), (0,0)], [(0,), (0,0)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + # fig.tight_layout() + # filename = "ET_"+ ET+ "_fw_" + fw + "_scaling.svg" + # plt.savefig(saving_directory + filename, format = 'svg') + + # vars = [['BC', 'SET'], ['BC', 'SET',]] + # models = [micro_model, meso_model] + # components = [[(2,), (0,2)], [(2,), (0,2)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + # fig.tight_layout() + # filename = "ET_"+ ET+ "_fw_" + fw + "_not_scaling.svg" + # plt.savefig(saving_directory + filename, format = 'svg') + \ No newline at end of file diff --git a/Proof_of_principle/visualizing_meso_manyFW.py b/Proof_of_principle/visualizing_meso_manyFW.py new file mode 100644 index 0000000..bbae684 --- /dev/null +++ b/Proof_of_principle/visualizing_meso_manyFW.py @@ -0,0 +1,144 @@ +import sys +# import os +sys.path.append('/home/tc2m23/Filtering/master_files/') +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +import pickle + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * + +if __name__ == '__main__': + + ############################################################### + # BLOCK TO COMPARE MICRO AND MESO WITH DIFF FILTERING SIZES + ############################################################### + # Loading the models + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + ET = sys.argv[1] + # ET = "1.0" + MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET + "_micro.pickle" + with open(MicroModelLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + + models = [micro_model] + FW = ['2', '4', '8'] + if ET == "2.0": + FW = ['2', '4'] + for fw in FW: + MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" + with open(MesoModelLoadFile, 'rb') as filehandle: + models.append(pickle.load(filehandle)) + + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + # saving_directory = "./" + + # Checking for alignment of slices + num_snaps = 11 + central_slice_num = int(num_snaps/2.) + time_micro = micro_model.domain_vars['t'][central_slice_num] + for i in range(1, len(models)): + time_meso = models[i].domain_vars['T'][0] + if time_meso != time_micro: + print('Careful: micro and meso slices appear not to be aligned!') + + + ranges_x = [0.04, 0.96] + ranges_y = [0.04, 0.96] + visualizer = Plotter_2D([11.97, 8.36]) + + # Plot for baryon current + to_be_plotted = ['BC', 'BC', 'BC'] + comp_to_plot = [(0,), (1,), (2,)] + vars = [to_be_plotted for _ in range(len(models))] + components = [comp_to_plot for _ in range(len(models))] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components) + fig.tight_layout() + # plt.show() + filename = "manyFW_BC_ET_" + ET + ".svg" + plt.savefig(saving_directory + filename, format = "svg") + + # plot for SET-diagonal + to_be_plotted = ['SET', 'SET', 'SET'] + comp_to_plot = [(0,0), (1,1), (2,2)] + vars = [to_be_plotted for _ in range(len(models))] + components = [comp_to_plot for _ in range(len(models))] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components) + fig.tight_layout() + # plt.show() + filename = "manyFW_SETdiag_ET_" + ET + ".svg" + plt.savefig(saving_directory + filename, format = "svg") + + # plot for SET off-diagonal + to_be_plotted = ['SET', 'SET', 'SET'] + comp_to_plot = [(0,1), (0,2), (1,2)] + vars = [to_be_plotted for _ in range(len(models))] + components = [comp_to_plot for _ in range(len(models))] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components) + fig.tight_layout() + # plt.show() + filename = "manyFW_SEToffdiag_ET_" + ET + ".svg" + plt.savefig(saving_directory + filename, format = "svg") + + + ############################################################### + # BLOCK TO PLOT THE DIFFERENCE BETWEEN TWO FILTER WIDTHS + ############################################################### + # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + # ET = sys.argv[1] + # models = [] + # FW = ['2', '8'] + # if ET == "2.0": + # FW = ['2', '4'] + # for fw in FW: + # MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" + # with open(MesoModelLoadFile, 'rb') as filehandle: + # models.append(pickle.load(filehandle)) + + # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + # # saving_directory = "./" + + # time_meso = models[0].domain_vars['T'][0] + # ranges_x = [0.04, 0.96] + # ranges_y = [0.04, 0.96] + # visualizer = Plotter_2D([11.97, 8.36]) + + # # Plot for baryon current + # to_be_plotted = ['BC', 'BC', 'BC'] + # comp_to_plot = [(0,), (1,), (2,)] + # vars = [to_be_plotted for _ in range(len(models))] + # components = [comp_to_plot for _ in range(len(models))] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True) + # # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, \ + # # method = "interpolate", interp_dims = (150, 150), diff_plot=True) + # fig.tight_layout() + # # plt.show() + # filename = "contrastFW_BC_ET_" + ET + ".svg" + # plt.savefig(saving_directory + filename, format = "svg") + + # # plot for SET-diagonal + # to_be_plotted = ['SET', 'SET', 'SET'] + # comp_to_plot = [(0,0), (1,1), (2,2)] + # vars = [to_be_plotted for _ in range(len(models))] + # components = [comp_to_plot for _ in range(len(models))] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True) + # # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, \ + # # method = "interpolate", interp_dims = (150, 150), diff_plot=True) + # fig.tight_layout() + # # plt.show() + # filename = "contrastFW_SETdiag_ET_" + ET + ".svg" + # plt.savefig(saving_directory + filename, format = "svg") + + # # plot for SET off-diagonal + # to_be_plotted = ['SET', 'SET', 'SET'] + # comp_to_plot = [(0,1), (0,2), (1,2)] + # vars = [to_be_plotted for _ in range(len(models))] + # components = [comp_to_plot for _ in range(len(models))] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True) + # # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, \ + # # method = "interpolate", interp_dims = (150, 150), diff_plot=True) + # fig.tight_layout() + # # plt.show() + # filename = "constrastFW_SEToffdiag_ET_" + ET + ".svg" + # plt.savefig(saving_directory + filename, format = "svg") \ No newline at end of file diff --git a/Proof_of_principle/visualizing_micro.py b/Proof_of_principle/visualizing_micro.py new file mode 100644 index 0000000..f2587ac --- /dev/null +++ b/Proof_of_principle/visualizing_micro.py @@ -0,0 +1,52 @@ +import sys +# import os +sys.path.append('/home/tc2m23/Filtering/master_files/') +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +import pickle + +from FileReaders import * +from MicroModels import * +from Visualization import * + +if __name__ == '__main__': + + # READING DATA + # CPU_start_time = time.perf_counter() + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + ET=str(sys.argv[1]) + MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET+ "_micro.pickle" + + with open(MicroModelLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + + ranges = [0.01, 0.99] + visualizer = Plotter_2D([11.97, 8.36]) + num_snaps = 11 + central_slice = int(num_snaps/2) + print("Plotting data from central slice, num: {}".format(central_slice)) + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + # saving_directory = "./" + + vars = ['BC', 'BC', 'BC', 'n', 'W', 'vx'] + components = [(0,), (1,), (2,), (), (), ()] + fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) + fig.tight_layout() + # plt.show() + filename = "micro_ET_"+ET+"_BC.svg" + plt.savefig(saving_directory + filename, format = "svg") + + # vars = ['SET', 'SET', 'SET', 'SET', 'SET', 'SET'] + # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + # fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) + # fig.tight_layout() + # # plt.show() + # filename = "micro_ET_"+ET+"_SET.svg" + # plt.savefig(saving_directory + filename, format = "svg") + + # vars = ['W', 'vx', 'vy', 'n', 'p', 'e'] + # fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges) + # fig.tight_layout() + # # plt.show() + # filename = "micro_ET_"+ET+"_prims.svg" + # plt.savefig(saving_directory + filename, format = "svg") + diff --git a/Proof_of_principle/visualizing_obs.py b/Proof_of_principle/visualizing_obs.py new file mode 100644 index 0000000..b8c3342 --- /dev/null +++ b/Proof_of_principle/visualizing_obs.py @@ -0,0 +1,47 @@ +#!/bin/bash + +import sys +# import os +sys.path.append('/home/tc2m23/Filtering/master_files/') +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +import pickle + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * + +if __name__ == '__main__': + + # Reading micro & meso models (with obs) from pickle + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + ET=str(sys.argv[1]) + MesoModelLoadFile = directory + "observers_2dx_ET_" + ET+ ".pickle" + MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET+"_micro.pickle" + + with open(MicroModelLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + num_snaps = 11 + central_slice_num = int(num_snaps/2.) + time_micro = micro_model.domain_vars['t'][central_slice_num] + time_meso = meso_model.domain_vars['T'][0] + if time_meso != time_micro: + print("Slices of meso and micro model do not coincide. Careful!") + else: + print("Comparing data at same time-slice, hurray!") + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + # saving_directory = "./" + + models = [micro_model, meso_model] + vars = [['bar_vel', 'bar_vel', 'bar_vel'],['U', 'U', 'U']] + components_indices= [[(0,),(1,),(2,)], [(0,), (1,), (2,)]] + ranges = [0.04, 0.97] + visualizer = Plotter_2D([11.97, 8.36]) + fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges, ranges, components_indices=components_indices, diff_plot=True, rel_diff=True) + fig.tight_layout() + filename="Meso_observers_2dx_ET_"+ET+".svg" + plt.savefig(saving_directory + filename, format="svg") \ No newline at end of file From 3407074939796486a60aa1b3905eda87f9bef365 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Sun, 10 Dec 2023 13:18:44 +0100 Subject: [PATCH 043/111] Config file for job array + slurm submit script --- Proof_of_principle/config.txt | 1 + Proof_of_principle/py_submit.slurm | 50 ++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 Proof_of_principle/py_submit.slurm diff --git a/Proof_of_principle/config.txt b/Proof_of_principle/config.txt index 28803bd..833c42b 100644 --- a/Proof_of_principle/config.txt +++ b/Proof_of_principle/config.txt @@ -1,3 +1,4 @@ +#Configuration file for job array JOB_ID ET 1 1.0 2 2.0 diff --git a/Proof_of_principle/py_submit.slurm b/Proof_of_principle/py_submit.slurm new file mode 100644 index 0000000..3f9edbd --- /dev/null +++ b/Proof_of_principle/py_submit.slurm @@ -0,0 +1,50 @@ +#!/bin/bash +#################################### +# JOB INFO: serial with array +#################################### + +#SBATCH --output=outputs/slurm-%A_%a.out +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --array=1-5 +#SBATCH --time=00:20:00 + + +#SBATCH --mail-type=begin # send email when job begins +#SBATCH --mail-type=fail +#SBATCH --mail-type=end # send email when job ends +#SBATCH --mail-user=tc1n19@soton.ac.uk + + +#################################### +# LOADING THE CONDA ENVIRONMENT +#################################### + +module load conda/py3-latest +# I get warnings to use 'conda deactivate' instead but that doesn't work +source deactivate +conda activate myenv + +#################################### +# RETRIEVING THE PARAMETERS FOR THE RUNS +#################################### + +ET=$(awk -v var=${SLURM_ARRAY_TASK_ID} '$1==var {print $2}' config.txt) +# response=$(conda list | grep h5py) +# echo "Response to conda list | grep h5py ${response}" +# echo "End time of task with ID ${SLURM_ARRAY_TASK_ID} is ${ET}." + + +#################################### +# LAUNCHING THE JOBS +##################################### + +#python3 -u main.py +# python -u RootVsMin.py +# python3 -u pickling_micro.py ${ET} +# python3 -u visualizing_micro.py ${ET} +# python3 -u pickling_meso_obs.py ${ET} +# python3 -u visualizing_obs.py ${ET} +# python3 -u multiple_filtering_meso.py ${ET} +python3 -u visualizing_meso.py ${ET} +# python3 -u visualizing_meso_manyFW.py ${ET} From 19fd30f72fe4ef8b9514df803bc451cb13ed9363 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 11 Dec 2023 09:33:58 +0100 Subject: [PATCH 044/111] script for testing parallel implementation --- master_files/py_submit_parallel.slurm | 32 +++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 master_files/py_submit_parallel.slurm diff --git a/master_files/py_submit_parallel.slurm b/master_files/py_submit_parallel.slurm new file mode 100644 index 0000000..194c114 --- /dev/null +++ b/master_files/py_submit_parallel.slurm @@ -0,0 +1,32 @@ +#!/bin/bash +#################################### +# JOB INFO: single node allocating max core available +#################################### + +#SBATCH --output=outputs/slurm-%A.out +#SBATCH --nodes=1 +#SBATCH --ntasks=40 +#SBATCH --time=00:30:00 + +#SBATCH --mail-type=begin +#SBATCH --mail-type=fail +#SBATCH --mail-type=end +#SBATCH --mail-user=celora@ice.csic.es + + +#################################### +# LOADING THE CONDA ENVIRONMENT +#################################### + +module load conda/py3-latest +# I get warnings to use 'conda deactivate' instead but that doesn't work +source deactivate +conda activate myenv + + +#################################### +# LAUNCHING THE JOBS +#################################### + +# python3 -u Filters.py +python3 -u MesoModels.py From a898fb9ebf488167116595483dac30ff24fc06d7 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 11 Dec 2023 09:37:37 +0100 Subject: [PATCH 045/111] script for testing parallel implementation --- master_files/py_submit_parallel.slurm | 1 + 1 file changed, 1 insertion(+) diff --git a/master_files/py_submit_parallel.slurm b/master_files/py_submit_parallel.slurm index 194c114..e7b15e2 100644 --- a/master_files/py_submit_parallel.slurm +++ b/master_files/py_submit_parallel.slurm @@ -6,6 +6,7 @@ #SBATCH --output=outputs/slurm-%A.out #SBATCH --nodes=1 #SBATCH --ntasks=40 +# #SBATCH -cpus-per-task #SBATCH --time=00:30:00 #SBATCH --mail-type=begin From cd3db402515118edc7dba0ed0e50df16f6861b20 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 11 Dec 2023 09:56:40 +0100 Subject: [PATCH 046/111] Config file for job array + slurm submit script --- Proof_of_principle/config.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Proof_of_principle/config.txt b/Proof_of_principle/config.txt index 833c42b..8e668b9 100644 --- a/Proof_of_principle/config.txt +++ b/Proof_of_principle/config.txt @@ -1,7 +1,8 @@ #Configuration file for job array JOB_ID ET 1 1.0 -2 2.0 -3 2.5 -4 3.0 -5 3.5 +2 1.5 +3 2.0 +4 2.5 +5 3.0 +6 3.5 From 5104f91b3f1e7adca74e95994675d24d6df26c75 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 12 Dec 2023 10:36:12 +0100 Subject: [PATCH 047/111] Adding set method to parallel filer --- master_files/FileReaders.py | 3 +-- master_files/Filters.py | 14 ++++++++++++++ master_files/MesoModels.py | 10 +++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/master_files/FileReaders.py b/master_files/FileReaders.py index 1779599..47c761b 100644 --- a/master_files/FileReaders.py +++ b/master_files/FileReaders.py @@ -20,8 +20,7 @@ def __init__(self, directory): ---------- directory: string the filenames in the directory have to be incremental (sorted is used) - """ - + """ hdf5_filenames = sorted( glob.glob(directory+str('*.hdf5'))) self.hdf5_files = [] for filename in hdf5_filenames: diff --git a/master_files/Filters.py b/master_files/Filters.py index 749734a..bf1b44b 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -1275,6 +1275,20 @@ def __init__(self, micro_model, filter_width): self.spatial_dims = micro_model.get_spatial_dims() self.filter_width = filter_width + def set_filter_width(self, filter_width): + """ + Method to change the width of the filter. + + Parameters: + ----------- + filter_width: float + + Returns: + -------- + None + """ + self.filter_width = filter_width + @staticmethod def complete_U_tetrad(U): """ diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index d314d42..d6a192e 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -2015,11 +2015,11 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # print('Speed-up factor: {}'.format(serial_time/parallel_time)) - # now filtering serial - start_time = time.perf_counter() - meso_model.filter_micro_variables() - serial_time = time.perf_counter() - start_time - print('Finished filtering in serial, time taken {}\n'.format(serial_time)) + # # now filtering serial + # start_time = time.perf_counter() + # meso_model.filter_micro_variables() + # serial_time = time.perf_counter() - start_time + # print('Finished filtering in serial, time taken {}\n'.format(serial_time)) # now filtering in parallel parallel_filter = box_filter_parallel(micro_model, 0.003) From a685bbaaa28825c75e726578f8532907df9d095d Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 12 Dec 2023 10:38:57 +0100 Subject: [PATCH 048/111] Redoing proof of principle in one go --- Proof_of_principle/pickling_meso_setup.py | 83 ++++++++++++++--------- Proof_of_principle/pickling_micro.py | 6 +- Proof_of_principle/visualizing_meso.py | 71 ++++++++++--------- Proof_of_principle/visualizing_micro.py | 54 ++++++++------- 4 files changed, 118 insertions(+), 96 deletions(-) diff --git a/Proof_of_principle/pickling_meso_setup.py b/Proof_of_principle/pickling_meso_setup.py index 7dd7f56..f49b7a5 100644 --- a/Proof_of_principle/pickling_meso_setup.py +++ b/Proof_of_principle/pickling_meso_setup.py @@ -1,7 +1,6 @@ import sys # import os -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') -# sys.path.append('/home/tc2m23/Filtering/master_files/') +sys.path.append('/home/tc2m23/Filtering/master_files/') import pickle import time @@ -14,51 +13,69 @@ # READING MICRO DATA FORM PICKLE CPU_start_time = time.perf_counter() - # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/pickle_files/400X400/" - directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/pickle_files/80X80/" - MicroModelPickleLoadFile = directory + "IdealHD_2D_ET_02_micro.pickle" + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + ET=str(sys.argv[1]) + MicroModelPickleLoadFile = directory + "HD_2D_ET_"+ET+"_micro.pickle" with open(MicroModelPickleLoadFile, 'rb') as filehandle: micro_model = pickle.load(filehandle) time_taken = time.perf_counter() - CPU_start_time print('Time taken to read file from pickle: {}'.format(time_taken)) - # print("Micro_model domain_vars keys: {}".format(micro_model.domain_vars.keys())) - # FINDING THE OBSERVERS, FILTERING + # SETTING UP THE MESO MODEL CPU_start_time = time.perf_counter() num_snaps = 21 - width_ratio_dx = 0.1 central_slice_num = int(num_snaps/2) - t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num]] - width = width_ratio_dx * micro_model.domain_vars['dx'] + t_bdrs = [micro_model.domain_vars['t'][central_slice_num-1], micro_model.domain_vars['t'][central_slice_num+1]] + x_bdrs = [0.02, 0.98] + y_bdrs = [0.02, 0.98] + # t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num]] + # x_bdrs = [0.2, 0.25] + # y_bdrs = [0.2, 0.25] - # xm, xM = np.min(micro_model.domain_vars['x']), np.max(micro_model.domain_vars['x']) - # ym, yM = np.min(micro_model.domain_vars['y']), np.max(micro_model.domain_vars['y']) - # print('Micro_model grid extent - careful when setting up meso-grid: \nX: {}, {} \nY: {}, {}'.format(xm, xM, ym, yM)) - - find_obs = FindObs_drift_root(micro_model, width) - filter = spatial_box_filter(micro_model, width) + box_len_ratio = 4. + box_len = box_len_ratio * micro_model.domain_vars['dx'] + # setting the width is needed for constructing the filter, and is changed later + width_ratio = 2. + width = width_ratio * micro_model.domain_vars['dx'] + find_obs = FindObs_root_parallel(micro_model, box_len) + filter = box_filter_parallel(micro_model, width) meso_model = resHD2D(micro_model, find_obs, filter) - # meso_model.setup_meso_grid([t_bdrs,[0.01,0.99],[0.01,0.99]]) - meso_model.setup_meso_grid([t_bdrs,[0.05,0.95],[0.05,0.95]]) + meso_model.setup_meso_grid([t_bdrs, x_bdrs, y_bdrs]) + + num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] + print('Number of points: {}'.format(num_points)) + time_taken = time.perf_counter() - CPU_start_time print('Grid is set up, time taken: {}'.format(time_taken)) - CPU_start_time = time.perf_counter() - meso_model.find_observers() - time_taken = time.perf_counter() - CPU_start_time - print('Observers found, time taken: {}'.format(time_taken)) + # FINDING THE OBSERVERS + start_time = time.perf_counter() + meso_model.find_observers_parallel() + time_taken = time.perf_counter() - start_time + print('Observers found in parallel: time taken= {}'.format(time_taken)) + + + # FILTERING WITH DIFFERENT WIDTHS + filter_width_ratios = [2., 4., 8.] + # filter_width_ratios = [2.] + filter = meso_model.filter + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + + for i in range(len(filter_width_ratios)): + # filtering in parallel + start_time = time.perf_counter() + filter.set_filter_width( micro_model.domain_vars['dx'] * filter_width_ratios[i] ) + meso_model.filter_micro_vars_parallel() + time_taken = time.perf_counter() - start_time + print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratios[i]), time_taken)) + + # Now saving the output + filename = "resHD_2D_ET_" + ET + "_FW_{}dx.pickle".format(int(filter_width_ratios[i])) + MesoModelPickleDumpFile = saving_directory+filename + with open(MesoModelPickleDumpFile, 'wb') as filehandle: + pickle.dump(meso_model, filehandle) + - CPU_start_time = time.perf_counter() - meso_model.filter_micro_variables() - time_taken = time.perf_counter() - CPU_start_time - print('Filter stage ended, time taken: {}'.format(time_taken)) - # PICKLING THE FILTERED DATA CLASS - # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/pickle_files/400X400/" - directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/pickle_files/80X80/" - # MesoModelPickleDumpFile = directory + "resHD_2D_ET_02_meso.pickle" - MesoModelPickleDumpFile = directory + "resHD_2D_ET_02_all_grid.pickle" - with open(MesoModelPickleDumpFile, 'wb') as filehandle: - pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/pickling_micro.py b/Proof_of_principle/pickling_micro.py index 4a3db9b..a53110b 100644 --- a/Proof_of_principle/pickling_micro.py +++ b/Proof_of_principle/pickling_micro.py @@ -13,7 +13,7 @@ # READING DATA FORM CHECKPOINT, SETTING UP MICOR AND PICKLING CPU_start_time = time.perf_counter() # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/METHOD_output/80X80/ET_02" - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/METHOD_output/400X400/" + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/" ET=str(sys.argv[1]) names = directory + "ET_" + ET print(names) @@ -26,9 +26,9 @@ print('Time to set up micro model (ET: {}): {}'.format(ET, time_taken)) # Pickle save - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" CPU_start_time = time.perf_counter() - MicroModelPickleDumpFile = directory + "IdealHD_2D_ET_" + ET + "_micro.pickle" + MicroModelPickleDumpFile = directory + "HD_2D_ET_" + ET + "_micro.pickle" with open(MicroModelPickleDumpFile, 'wb') as filehandle: pickle.dump(micro_model, filehandle) time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index b15c4db..6dde175 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -13,42 +13,39 @@ ############################################################### # PRODUCE MANY DIFF PLOTS TO COMPARE MICRO AND MESO AT DIFFERENT - # ET AND DIFFERENT WIDTHS. FOR EACH ADD THE DIFFERENCE. + # ENDTIME AND DIFFERENT WIDTHS. FOR EACH PLOT THE DIFFERENCE + # (ABSOLUTE AND RELATIVE). ############################################################### # Loading the models - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" ET = sys.argv[1] - # ET = "3.5" - MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET + "_micro.pickle" + MicroModelLoadFile = directory + "HD_2D_ET_" + ET + "_micro.pickle" with open(MicroModelLoadFile, 'rb') as filehandle: micro_model = pickle.load(filehandle) - num_snaps = 11 + num_snaps = 21 central_slice_num = int(num_snaps/2.) time_micro = micro_model.domain_vars['t'][central_slice_num] - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - # saving_directory = "./" - # FW = ['2', '4', '8'] - # # FW = ['8'] - # if ET == "2.0": - # FW = ['2', '4'] - FW = ['2', '4'] + # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/" + FW = ['2', '4', '8'] for fw in FW: MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - time_meso = meso_model.domain_vars['T'][0] + meso_central_slice = 1 + time_meso = meso_model.domain_vars['T'][meso_central_slice] if time_meso != time_micro: print("Slices of meso and micro model do not coincide. Careful!") else: print("Comparing data at same time-slice, hurray!") - ranges_x = [0.03, 0.97] - ranges_y = [0.03, 0.97] + ranges_x = [0.02, 0.98] + ranges_y = [0.02, 0.98] visualizer = Plotter_2D([11.97, 8.36]) ###################################################### @@ -86,32 +83,32 @@ # # Plots to compare observer with Favre-velocity (need to decompose structures first!) - meso_model.decompose_structures() - vars = [['U', 'U', 'U'], ['u_tilde', 'u_tilde', 'u_tilde']] - models = [meso_model, meso_model] - components = [[(0,), (1,), (2,)], [(0,), (1,), (2,)]] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - fig.tight_layout() - filename = "comparing_obs_VS_Favre" + ET + "_FW_" + fw + ".svg" - plt.savefig(saving_directory + filename, format = 'svg') + # meso_model.decompose_structures() + # vars = [['U', 'U', 'U'], ['u_tilde', 'u_tilde', 'u_tilde']] + # models = [meso_model, meso_model] + # components = [[(0,), (1,), (2,)], [(0,), (1,), (2,)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + # fig.tight_layout() + # filename = "comparing_obs_VS_Favre" + ET + "_FW_" + fw + ".svg" + # plt.savefig(saving_directory + filename, format = 'svg') ###################################################### # CODE FOR SUMMARY PLOTS ###################################################### - # vars = [['BC', 'SET'], ['BC', 'SET',]] - # models = [micro_model, meso_model] - # components = [[(0,), (0,0)], [(0,), (0,0)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - # fig.tight_layout() - # filename = "ET_"+ ET+ "_fw_" + fw + "_scaling.svg" - # plt.savefig(saving_directory + filename, format = 'svg') + vars = [['BC', 'SET'], ['BC', 'SET',]] + models = [micro_model, meso_model] + components = [[(0,), (0,0)], [(0,), (0,0)]] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + fig.tight_layout() + filename = "ET_"+ ET+ "_fw_" + fw + "_scaling.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') - # vars = [['BC', 'SET'], ['BC', 'SET',]] - # models = [micro_model, meso_model] - # components = [[(2,), (0,2)], [(2,), (0,2)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - # fig.tight_layout() - # filename = "ET_"+ ET+ "_fw_" + fw + "_not_scaling.svg" - # plt.savefig(saving_directory + filename, format = 'svg') + vars = [['BC', 'SET'], ['BC', 'SET',]] + models = [micro_model, meso_model] + components = [[(2,), (0,2)], [(2,), (0,2)]] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) + fig.tight_layout() + filename = "ET_"+ ET+ "_fw_" + fw + "_not_scaling.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') \ No newline at end of file diff --git a/Proof_of_principle/visualizing_micro.py b/Proof_of_principle/visualizing_micro.py index f2587ac..15413e5 100644 --- a/Proof_of_principle/visualizing_micro.py +++ b/Proof_of_principle/visualizing_micro.py @@ -11,42 +11,50 @@ if __name__ == '__main__': # READING DATA - # CPU_start_time = time.perf_counter() - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" ET=str(sys.argv[1]) - MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET+ "_micro.pickle" + # MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET+ "_micro.pickle" + MicroModelLoadFile = directory + "HD_2D_ET_" + ET+ "_micro.pickle" + with open(MicroModelLoadFile, 'rb') as filehandle: micro_model = pickle.load(filehandle) + # setting up ranges, picking central slice and choosing saving folder ranges = [0.01, 0.99] visualizer = Plotter_2D([11.97, 8.36]) - num_snaps = 11 + # num_snaps = 11 + num_snaps = 21 central_slice = int(num_snaps/2) print("Plotting data from central slice, num: {}".format(central_slice)) - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - # saving_directory = "./" + # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" + saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/" + + # Plotting the baryon current vars = ['BC', 'BC', 'BC', 'n', 'W', 'vx'] components = [(0,), (1,), (2,), (), (), ()] fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) fig.tight_layout() # plt.show() - filename = "micro_ET_"+ET+"_BC.svg" - plt.savefig(saving_directory + filename, format = "svg") - - # vars = ['SET', 'SET', 'SET', 'SET', 'SET', 'SET'] - # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - # fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) - # fig.tight_layout() - # # plt.show() - # filename = "micro_ET_"+ET+"_SET.svg" - # plt.savefig(saving_directory + filename, format = "svg") - - # vars = ['W', 'vx', 'vy', 'n', 'p', 'e'] - # fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges) - # fig.tight_layout() - # # plt.show() - # filename = "micro_ET_"+ET+"_prims.svg" - # plt.savefig(saving_directory + filename, format = "svg") + filename = "micro_ET_"+ET+"_BC.pdf" + plt.savefig(saving_directory + filename, format = "pdf") + + # Plotting the stress energy tensor + vars = ['SET', 'SET', 'SET', 'SET', 'SET', 'SET'] + components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) + fig.tight_layout() + # plt.show() + filename = "micro_ET_"+ET+"_SET.pdf" + plt.savefig(saving_directory + filename, format = "pdf") + + # plotting primitive quantities + vars = ['W', 'vx', 'vy', 'n', 'p', 'e'] + fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges) + fig.tight_layout() + # plt.show() + filename = "micro_ET_"+ET+"_prims.pdf" + plt.savefig(saving_directory + filename, format = "pdf") From 290bd396b6231591e703a169be1f488c48d9c3e0 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 12 Dec 2023 10:39:36 +0100 Subject: [PATCH 049/111] Removing deleted files from tracking --- Proof_of_principle/py_submit.slurm | 50 ------------------------------ 1 file changed, 50 deletions(-) delete mode 100644 Proof_of_principle/py_submit.slurm diff --git a/Proof_of_principle/py_submit.slurm b/Proof_of_principle/py_submit.slurm deleted file mode 100644 index 3f9edbd..0000000 --- a/Proof_of_principle/py_submit.slurm +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash -#################################### -# JOB INFO: serial with array -#################################### - -#SBATCH --output=outputs/slurm-%A_%a.out -#SBATCH --nodes=1 -#SBATCH --ntasks=1 -#SBATCH --array=1-5 -#SBATCH --time=00:20:00 - - -#SBATCH --mail-type=begin # send email when job begins -#SBATCH --mail-type=fail -#SBATCH --mail-type=end # send email when job ends -#SBATCH --mail-user=tc1n19@soton.ac.uk - - -#################################### -# LOADING THE CONDA ENVIRONMENT -#################################### - -module load conda/py3-latest -# I get warnings to use 'conda deactivate' instead but that doesn't work -source deactivate -conda activate myenv - -#################################### -# RETRIEVING THE PARAMETERS FOR THE RUNS -#################################### - -ET=$(awk -v var=${SLURM_ARRAY_TASK_ID} '$1==var {print $2}' config.txt) -# response=$(conda list | grep h5py) -# echo "Response to conda list | grep h5py ${response}" -# echo "End time of task with ID ${SLURM_ARRAY_TASK_ID} is ${ET}." - - -#################################### -# LAUNCHING THE JOBS -##################################### - -#python3 -u main.py -# python -u RootVsMin.py -# python3 -u pickling_micro.py ${ET} -# python3 -u visualizing_micro.py ${ET} -# python3 -u pickling_meso_obs.py ${ET} -# python3 -u visualizing_obs.py ${ET} -# python3 -u multiple_filtering_meso.py ${ET} -python3 -u visualizing_meso.py ${ET} -# python3 -u visualizing_meso_manyFW.py ${ET} From 926d3912fb8e948e048ec6f16fa1498c1ee4792e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 14 Dec 2023 11:50:30 +0100 Subject: [PATCH 050/111] Fixing minor bug in parallel routines --- master_files/Analysis.py | 4 +-- master_files/Filters.py | 36 ++++++++++++++++--------- master_files/MesoModels.py | 24 +++++++++-------- master_files/MicroModels.py | 38 +++++++++++++-------------- master_files/py_submit_parallel.slurm | 10 +++---- 5 files changed, 63 insertions(+), 49 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 4ceb68c..ba60c29 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -9,14 +9,14 @@ import matplotlib.pyplot as plt import numpy as np -import pandas as pd +# import pandas as pd import h5py import pickle import seaborn as sns import warnings from sklearn.linear_model import LinearRegression from sklearn.decomposition import PCA -import statsmodels.api as sm +# import statsmodels.api as sm from system.BaseFunctionality import * from MicroModels import * diff --git a/master_files/Filters.py b/master_files/Filters.py index bf1b44b..67b8896 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -7,7 +7,9 @@ import numpy as np import time +import os import scipy.integrate as integrate +import multiprocessing as mp from scipy.optimize import minimize, root from itertools import product from system.BaseFunctionality import * @@ -967,10 +969,14 @@ def find_observers_parallel(self, points): spatial_dims = self.micro_model.get_spatial_dims() L = self.L args_for_pool = [ [points[i], i] for i in range(len(points))] + + init = FindObs_root_parallel.initializer + initargs = (spatial_dims, L) + n_cpus = int(os.environ['SLURM_CPUS_PER_TASK']) - with mp.Pool(initializer=FindObs_root_parallel.initializer, initargs=(spatial_dims,L,)) as pool: + with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: # with mp.Pool() as pool: - print('Running with {} processes\n'.format(os.cpu_count()), flush=True) + print('Running with {} processes\n'.format(n_cpus), flush=True) for result in pool.map(self.find_observer_Gauss, args_for_pool): if (result[0] == True): success_pos.append(result[1]) @@ -1439,8 +1445,14 @@ def filter_vars_parallel(self, list_packed_args): position_in_list = [] filtered_vars = [] args_for_pool = [ [list_packed_args[i], i] for i in range(len(list_packed_args))] - with mp.Pool(initializer=box_filter_parallel.initializer, initargs=(self.spatial_dims, self.filter_width)) as pool: - print('Running with {} processes\n'.format(os.cpu_count()), flush=True) + + init = box_filter_parallel.initializer + initargs=(self.spatial_dims, self.filter_width) + n_cpus = int(os.environ['SLURM_CPUS_PER_TASK']) + + + with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: + print('Running with {} processes\n'.format(n_cpus), flush=True) for result in pool.map(self.filter_vars_point_gauss, args_for_pool): position_in_list.append(result[0]) filtered_vars.append(result[1]) @@ -1510,7 +1522,7 @@ def filter_vars_parallel(self, list_packed_args): print('Number of points: {}'.format(len(points))) - # # Now find observers - serial version + # Now find observers - serial version # start_time = time.perf_counter() # find_obs_serial = FindObs_drift_root(micro_model, 0.001) # observers = find_obs_serial.find_observers_points(points) @@ -1531,12 +1543,12 @@ def filter_vars_parallel(self, list_packed_args): observers = result[1] vars = ['BC', 'SET'] # Filtering serial - start_time = time.perf_counter() - serial_filter = spatial_box_filter(micro_model, 0.003) - for var in vars: - serial_filter.filter_var_manypoints(var, points, observers) - serial_time = time.perf_counter() - start_time - print('Finished filtering in serial, time-taken: {}'.format(serial_time)) + # start_time = time.perf_counter() + # serial_filter = spatial_box_filter(micro_model, 0.003) + # for var in vars: + # serial_filter.filter_var_manypoints(var, points, observers) + # serial_time = time.perf_counter() - start_time + # print('Finished filtering in serial, time-taken: {}'.format(serial_time)) # Preparing args for parallel filter routine args_list = [] @@ -1548,5 +1560,5 @@ def filter_vars_parallel(self, list_packed_args): parallel_filter.filter_vars_parallel(args_list) parallel_time = time.perf_counter() - start_time print('Finished filtering in parallel, time-taken: {}'.format(parallel_time)) - print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) + # print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index d6a192e..497249a 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -7,7 +7,7 @@ import numpy as np from scipy.interpolate import interpn -from multiprocessing import Process, Pool +import multiprocessing as mp from multimethod import multimethod import matplotlib.pyplot as plt @@ -1991,18 +1991,20 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # find obs - serial - # start_time = time.perf_counter() + start_time = time.perf_counter() find_obs = FindObs_drift_root(micro_model, 0.001) filter = spatial_box_filter(micro_model, 0.003) meso_model = resHD2D(micro_model, find_obs, filter) meso_model.setup_meso_grid([t_range, x_range, y_range]) - # meso_model.find_observers() - # serial_time = time.perf_counter() - start_time - # print('Serial execution time: {}\n'.format(serial_time)) num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] print(f'Testing parallelization with {num_points} points\n') + + meso_model.find_observers() + serial_time = time.perf_counter() - start_time + print('Serial execution time: {}\n'.format(serial_time)) + # fin obs - parallel start_time = time.perf_counter() find_obs = FindObs_root_parallel(micro_model, 0.001) @@ -2012,14 +2014,14 @@ def EL_style_closure_regression(self, CoefficientAnalysis): meso_model.find_observers_parallel() parallel_time = time.perf_counter() - start_time print('Finished finding observers in parallel, execution time: {}\n'.format(parallel_time)) - # print('Speed-up factor: {}'.format(serial_time/parallel_time)) + print('Speed-up factor: {}'.format(serial_time/parallel_time)) - # # now filtering serial - # start_time = time.perf_counter() - # meso_model.filter_micro_variables() - # serial_time = time.perf_counter() - start_time - # print('Finished filtering in serial, time taken {}\n'.format(serial_time)) + # now filtering serial + start_time = time.perf_counter() + meso_model.filter_micro_variables() + serial_time = time.perf_counter() - start_time + print('Finished filtering in serial, time taken {}\n'.format(serial_time)) # now filtering in parallel parallel_filter = box_filter_parallel(micro_model, 0.003) diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index 6f18903..0e28a23 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -755,31 +755,31 @@ def setup_structures(self): # FileReader = METHOD_HDF5('../Data/res800/res800_t3') micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) - # micro_model.setup_structures() + micro_model.setup_structures() #################################################### # Comparing speed of gridpoint vs interpol routines #################################################### - # print('Structure strs: {}'.format(micro_model.get_structures_strs())) - # point = [1.502,0.4,0.2] - # # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - # vars = ['BC', 'bar_vel', 'n'] - # for var in vars: - # res = micro_model.get_interpol_var(var, point) - # res2 = micro_model.get_var_gridpoint(var, point) - # print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') + print('Structure strs: {}'.format(micro_model.get_structures_strs())) + point = [1.502,0.4,0.2] + # vars = ['SETfl', 'BC', 'Fab', 'SETem'] + vars = ['BC', 'bar_vel', 'n'] + for var in vars: + res = micro_model.get_interpol_var(var, point) + res2 = micro_model.get_var_gridpoint(var, point) + print(f'{var}: \n {res} \n\n\n {res2} \n ********** \n ') #################################################### # TESTING PARALLELIZED SETUP STRUCTURE #################################################### - CPU_start_time = time.perf_counter() - micro_model.setup_structures() - serial_time= time.perf_counter() - CPU_start_time - print('Time taken serial: {}\n'.format(serial_time)) - - CPU_start_time = time.perf_counter() - micro_model.setup_structures_parallel() - parallel_time = time.perf_counter() - CPU_start_time - print('Time taken parallel: {}\n'.format(parallel_time)) - print('Speed up factor: {}\n'.format(serial_time/parallel_time)) \ No newline at end of file + # CPU_start_time = time.perf_counter() + # micro_model.setup_structures() + # serial_time= time.perf_counter() - CPU_start_time + # print('Time taken serial: {}\n'.format(serial_time)) + + # CPU_start_time = time.perf_counter() + # micro_model.setup_structures_parallel() + # parallel_time = time.perf_counter() - CPU_start_time + # print('Time taken parallel: {}\n'.format(parallel_time)) + # print('Speed up factor: {}\n'.format(serial_time/parallel_time)) \ No newline at end of file diff --git a/master_files/py_submit_parallel.slurm b/master_files/py_submit_parallel.slurm index e7b15e2..cde5275 100644 --- a/master_files/py_submit_parallel.slurm +++ b/master_files/py_submit_parallel.slurm @@ -4,9 +4,9 @@ #################################### #SBATCH --output=outputs/slurm-%A.out -#SBATCH --nodes=1 -#SBATCH --ntasks=40 -# #SBATCH -cpus-per-task +#SBATCH --nodes=1 +#SBATCH --ntasks=1 #number of MPI processes +#SBATCH --cpus-per-task=40 #number of threads per MPI process #SBATCH --time=00:30:00 #SBATCH --mail-type=begin @@ -29,5 +29,5 @@ conda activate myenv # LAUNCHING THE JOBS #################################### -# python3 -u Filters.py -python3 -u MesoModels.py +python3 -u Filters.py +# python3 -u MesoModels.py From 415884c7bae2948c202e728af525e884952c522a Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 14 Dec 2023 16:11:47 +0100 Subject: [PATCH 051/111] Update: number of processes is now chosen at the MesoModel level --- master_files/Filters.py | 28 ++-- master_files/MesoModels.py | 224 +++++++++++++++++++++----- master_files/py_submit_parallel.slurm | 4 +- 3 files changed, 209 insertions(+), 47 deletions(-) diff --git a/master_files/Filters.py b/master_files/Filters.py index 67b8896..6409066 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -942,7 +942,7 @@ def residual_gauss(spatial_vels, point, micro_model): if not sol.success: return sol.success, pos_in_list_points - def find_observers_parallel(self, points): + def find_observers_parallel(self, points, n_cpus): """ Method to run find_observer_Gauss in parallel on a list of points @@ -950,6 +950,9 @@ def find_observers_parallel(self, points): ----------- points: list of d+1 float, d is the spatial dimension of micro_model + n_cpus: int + number of processes + Returns: -------- @@ -960,6 +963,9 @@ def find_observers_parallel(self, points): failures: list (typically empty) position of failed points in input list of points + + there is no default value for n_cpus so that this has to be passed + explicitely in the tests below, or decided at the MesoModel level. """ observers = [] avg_errors = [] @@ -972,7 +978,6 @@ def find_observers_parallel(self, points): init = FindObs_root_parallel.initializer initargs = (spatial_dims, L) - n_cpus = int(os.environ['SLURM_CPUS_PER_TASK']) with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: # with mp.Pool() as pool: @@ -1417,7 +1422,7 @@ def filter_vars_point_gauss(self, packed_args): # Safer to return a dictionary: check redux in performance though. return pos, filtered_vars - def filter_vars_parallel(self, list_packed_args): + def filter_vars_parallel(self, list_packed_args, n_cpus): """ Method to run filter_vars_point_gauss in parallel given a list of points, observers and vars. @@ -1434,6 +1439,9 @@ def filter_vars_parallel(self, list_packed_args): [vars_to_be_filtered]: list of strs each item must much a var in micromodel + n_cpus: int + number of processes + Returns: -------- position_in_list: list of integers @@ -1441,6 +1449,9 @@ def filter_vars_parallel(self, list_packed_args): Required as pool.map not necessarily returns processes in order filtered_vars: list of lists containing filtered vars + + there is no default value for n_cpus so that this has to be passed + explicitely in the tests below, or decided at the MesoModel level. """ position_in_list = [] filtered_vars = [] @@ -1448,8 +1459,6 @@ def filter_vars_parallel(self, list_packed_args): init = box_filter_parallel.initializer initargs=(self.spatial_dims, self.filter_width) - n_cpus = int(os.environ['SLURM_CPUS_PER_TASK']) - with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: print('Running with {} processes\n'.format(n_cpus), flush=True) @@ -1504,8 +1513,8 @@ def filter_vars_parallel(self, list_packed_args): # setting up the points for testing - pass all the points within a range t_range = [1.502, 1.504] - x_range = [0.05, 0.95] - y_range = [0.05, 0.95] + x_range = [0.05, 0.15] + y_range = [0.05, 0.15] patch_min = [t_range[0], x_range[0], y_range[0]] patch_max = [t_range[1], x_range[1], y_range[1]] @@ -1530,10 +1539,11 @@ def filter_vars_parallel(self, list_packed_args): # print('Serial time: {}\n'.format(serial_time)) # Now find observers - parallel version + n_cpus = os.cpu_count() start_time = time.perf_counter() find_obs_parallel = FindObs_root_parallel(micro_model, 0.001) point = points[0] - result, failed = find_obs_parallel.find_observers_parallel(points) + result, failed = find_obs_parallel.find_observers_parallel(points, n_cpus) print('Number of points failed: {}'.format(len(failed))) parallel_time = time.perf_counter() - start_time print('Parallel time: {}\n'.format(parallel_time)) @@ -1557,7 +1567,7 @@ def filter_vars_parallel(self, list_packed_args): start_time = time.perf_counter() parallel_filter = box_filter_parallel(micro_model, 0.003) - parallel_filter.filter_vars_parallel(args_list) + parallel_filter.filter_vars_parallel(args_list, n_cpus) parallel_time = time.perf_counter() - start_time print('Finished filtering in parallel, time-taken: {}'.format(parallel_time)) # print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 497249a..00bb065 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1106,7 +1106,7 @@ class resHD2D(object): """ CUT AND PAST OF resMHD_2D --> removing the magnetic bits """ - def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): + def __init__(self, micro_model, find_obs, filter, interp_method = 'linear', local_or_slurm = True): """ ADD DESCRIPTION OF THIS """ @@ -1116,6 +1116,11 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): self.spatial_dims = 2 self.interp_method = interp_method + if local_or_slurm: + self.n_cpus = int(os.cpu_count()) + else: + self.n_cpus = int(os.environ['SLURM_CPUS_PER_TASK']) + self.domain_int_strs = ('Nt','Nx','Ny') self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") self.domain_array_strs = ("T","X","Y","Points") @@ -1192,6 +1197,12 @@ def set_find_obs(self, find_obs): def set_filter(self, filter): self.filter = filter + def set_n_cps(self, n_cpus): + """ + Set the number of processes to run in parallel on. + """ + self.n_cpus = n_cpus + def get_all_var_strs(self): return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) + \ list(self.filter_vars.keys()) @@ -1444,13 +1455,20 @@ def find_observers(self): # No need to update the dictionary as this has been initialized to False everywhere. print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) - def find_observers_parallel(self): + def find_observers_parallel(self, n_cpus = None): """ Method to find observers at all points on meso-grid, parallelized version. The observers found (and relative errors) are saved in the dictionary self.filter_vars. Set up the entry self.filter_vars['U_success'] as a dictionary with (tuples of) indices on the meso_grid as keys, and bool as values (true if the observer has been found, false otherwise) + Parameters: + ----------- + + n_cpus: int + If not passed explicitely this is set to self.n_cpus (initalized upon + construction) + Notes: ------ This method relies on the routine find_obs.find_observers_parallel(). @@ -1472,7 +1490,11 @@ def find_observers_parallel(self): for elem in product(t_idxs, x_idxs, y_idxs): indices_meso_grid.append(elem) - successes, failures = self.find_obs.find_observers_parallel(points) + num_processes = self.n_cpus + if n_cpus: + num_processes = n_cpus + + successes, failures = self.find_obs.find_observers_parallel(points, num_processes) for i in range(len(successes[0])): point_indxs_meso_grid = indices_meso_grid[successes[0][i]] @@ -1512,7 +1534,7 @@ def filter_micro_variables(self): else: print('Could not filter at {}: observer not found.'.format(point)) - def filter_micro_vars_parallel(self): + def filter_micro_vars_parallel(self, n_cpus = None): """ Filter all meso_model structures AND micro pressure. Routine will try and filter at all points on the meso-grid. @@ -1521,6 +1543,13 @@ def filter_micro_vars_parallel(self): This method relies on filter_vars_parallel implemented separately for the filter class, e.g. as in box_filter_parallel + + Parameters: + ----------- + + n_cpus: int + If not passed explicitely this is set to self.n_cpus (initalized upon + construction) Notes: ------ @@ -1556,7 +1585,11 @@ def filter_micro_vars_parallel(self): for i in range(len(points)): args_for_filtering_parallel.append([points[i], observers[i], vars]) - positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel) + num_processes = self.n_cpus + if n_cpus: + num_processes = n_cpus + + positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel, num_processes) for i in range(len(positions)): point_indxs_meso_grid = indices_meso_grid[positions[i]] self.meso_structures['BC'][point_indxs_meso_grid] = filtered_vars[i][0] @@ -1596,7 +1629,7 @@ def decompose_structures_gridpoint(self, h, i, j): # Computing the decomposition at each point eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, self.metric, self.metric, T_ab) h_ab = np.einsum('ij,jk->ik', self.metric + np.einsum('i,j->ij', u_t, u_t), self.metric) # This is a rank (1,1) tensor, i.e. a real projector. - q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) + q_a = np.einsum('ij,jk,kl,l->i',h_ab, T_ab, self.metric, u_t) # There might be missing a minus sign here? s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, T_ab) s = np.einsum('ii',s_ab) p_t = self.p_from_EOS(eps_t, n_t) @@ -1613,7 +1646,7 @@ def decompose_structures_gridpoint(self, h, i, j): self.meso_vars['eos_res'][h,i,j] = self.meso_vars['p_filt'][h,i,j] - p_t self.meso_vars['T_tilde'][h,i,j] = p_t / n_t - def decompose_structures(self, t_range=None, x_range=None, y_range=None): + def decompose_structures(self): """ Decompose structures at all points on the meso_grid where observers could be found. @@ -1633,6 +1666,78 @@ def decompose_structures(self, t_range=None, x_range=None, y_range=None): else: print('Structures not decomposed at {}: observer could not be found.'.format(point)) + @staticmethod + def p_Gamma_law(eps, n, Gamma): + return (Gamma-1)* eps*n + + @staticmethod + def decompose_structures_task(BC, SET , p_filt, h, i, j): + """ + work in progress + """ + # As this is staticmethod, no access to self.metric + metric = np.zeros((3,3)) + metric[0,0] = -1. + metric[1,1] = metric[2,2] = 1. + + # Computing the Favre density and velocity + n_t = np.sqrt(-Base.Mink_dot(BC, BC)) + u_t = np.multiply(1./ n_t, BC) + + # remember SET is a rank (2,0) tensor + # Computing the SET decomposition at each point + eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, metric, metric, SET) + h_ab = np.einsum('ij,jk->ik', metric + np.einsum('i,j->ij', u_t, u_t), metric) # This is a rank (1,1) tensor, i.e. a real projector. + q_a = np.einsum('ij,jk,kl,l->i', h_ab, SET, metric, u_t) # There might be missing a minus sign here + s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, SET) + s = np.einsum('ii',s_ab) + p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) + + # Additional quantities needed for residuals + Pi_res = s - p_t + s_ab_tracefree = s_ab - np.multiply(s/2., metric + np.einsum('i,j->ij', u_t, u_t)) + T_t = p_t/n_t + EOS_res = p_filt - p_t + + return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, EOS_res, T_t], [h,i,j] + + def decompose_structures_parallel(self, n_cpus = None): + """ + work in progress + + Parameters: + ----------- + n_cpus: int + If not passed explicitely this is set to the value on construction + """ + # Preparing arguments for pool + args_for_pool=[] + for h in range(len(self.domain_vars['T'])): + for i in range(len(self.domain_vars['X'])): + for j in range(len(self.domain_vars['Y'])): + BC = self.meso_structures['BC'][h,i,j] + SET = self.meso_structures['SET'][h,i,j] + p_filt = self.meso_vars['p_filt'][h,i,j] + args_for_pool.append((BC, SET, p_filt, h,i,j)) + + num_processes = self.n_cpus + if n_cpus: + num_processes = n_cpus + + with mp.Pool(processes=num_processes) as pool: + print('Running with {} processes\n'.format(num_processes), flush=True) + for result in pool.starmap(resHD2D.decompose_structures_task, args_for_pool): + h,i,j = result[1] + self.meso_vars['n_tilde'][h,i,j] = result[0][0] + self.meso_vars['u_tilde'][h,i,j,:] = result[0][1] + self.meso_vars['eps_tilde'][h,i,j] = result[0][2] + self.meso_vars['q_res'][h,i,j,:] = result[0][3] + self.meso_vars['pi_res'][h,i,j,:,:] = result[0][4] + self.meso_vars['p_tilde'][h,i,j] = result[0][5] + self.meso_vars['Pi_res'][h,i,j] = result[0][6] + self.meso_vars['eos_res'][h,i,j] = result[0][7] + self.meso_vars['T_tilde'][h,i,j] = result[0][8] + def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): """ Calculate partial derivative in the input 'direction' of the variable corresponding to 'nonlocal_var_str' @@ -1977,58 +2082,105 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # # print('Total time is {}'.format(time.process_time() - CPU_start_time)) + # ######################################################## + # # TESTING PARALLEL IMPLEMENTATION FOR FINDING OBS AND FILTER + # ######################################################## + # FileReader = METHOD_HDF5('../Data/test_res100/') + # micro_model = IdealHD_2D() + # FileReader.read_in_data(micro_model) + # micro_model.setup_structures() + + # t_range = [1.502, 1.504] + # x_range = [0.05, 0.95] + # y_range = [0.05, 0.95] + + + # # find obs - serial + # start_time = time.perf_counter() + # find_obs = FindObs_drift_root(micro_model, 0.001) + # filter = spatial_box_filter(micro_model, 0.003) + # meso_model = resHD2D(micro_model, find_obs, filter) + # meso_model.setup_meso_grid([t_range, x_range, y_range]) + + # num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] + # print(f'Testing parallelization with {num_points} points\n') + + # meso_model.find_observers() + # serial_time = time.perf_counter() - start_time + # print('Serial execution time: {}\n'.format(serial_time)) + + + # # fin obs - parallel + # start_time = time.perf_counter() + # find_obs = FindObs_root_parallel(micro_model, 0.001) + # filter = spatial_box_filter(micro_model, 0.003) + # meso_model = resHD2D(micro_model, find_obs, filter) + # meso_model.setup_meso_grid([t_range, x_range, y_range]) + # meso_model.find_observers_parallel() + # parallel_time = time.perf_counter() - start_time + # print('Finished finding observers in parallel, execution time: {}\n'.format(parallel_time)) + # print('Speed-up factor: {}'.format(serial_time/parallel_time)) + + + # # now filtering serial + # start_time = time.perf_counter() + # meso_model.filter_micro_variables() + # serial_time = time.perf_counter() - start_time + # print('Finished filtering in serial, time taken {}\n'.format(serial_time)) + + # # now filtering in parallel + # parallel_filter = box_filter_parallel(micro_model, 0.003) + # meso_model.set_filter(parallel_filter) + # start_time = time.perf_counter() + # meso_model.filter_micro_vars_parallel() + # parallel_time = time.perf_counter() - start_time + # print('Finished filtering in parallel, time taken {}\n'.format(parallel_time)) + # print('Speed-up factor: {}'.format(serial_time/parallel_time)) + + ######################################################## - # TESTING PARALLEL IMPLEMENTATION + # TESTING PARALLEL IMPLEMENTATION: meso routines ######################################################## - FileReader = METHOD_HDF5('../Data/test_res100/') + directory = '../Data/test_res100/' + FileReader = METHOD_HDF5(directory) micro_model = IdealHD_2D() FileReader.read_in_data(micro_model) micro_model.setup_structures() t_range = [1.502, 1.504] - x_range = [0.05, 0.95] - y_range = [0.05, 0.95] + x_range = [0.05, 0.15] + y_range = [0.05, 0.15] - - # find obs - serial - start_time = time.perf_counter() - find_obs = FindObs_drift_root(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - meso_model = resHD2D(micro_model, find_obs, filter) + # set up meso model and grid + find_obs = FindObs_root_parallel(micro_model, 0.001) + filter = box_filter_parallel(micro_model, 0.003) + meso_model = resHD2D(micro_model, find_obs, filter, local_or_slurm=False) meso_model.setup_meso_grid([t_range, x_range, y_range]) num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] print(f'Testing parallelization with {num_points} points\n') - - meso_model.find_observers() - serial_time = time.perf_counter() - start_time - print('Serial execution time: {}\n'.format(serial_time)) - # fin obs - parallel start_time = time.perf_counter() - find_obs = FindObs_root_parallel(micro_model, 0.001) - filter = spatial_box_filter(micro_model, 0.003) - meso_model = resHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([t_range, x_range, y_range]) meso_model.find_observers_parallel() parallel_time = time.perf_counter() - start_time print('Finished finding observers in parallel, execution time: {}\n'.format(parallel_time)) - print('Speed-up factor: {}'.format(serial_time/parallel_time)) + # now filtering in parallel + start_time = time.perf_counter() + meso_model.filter_micro_vars_parallel() + parallel_time = time.perf_counter() - start_time + print('Finished filtering in parallel, time taken {}\n'.format(parallel_time)) - # now filtering serial + # now testing serial vs parallel decomposition start_time = time.perf_counter() - meso_model.filter_micro_variables() + meso_model.decompose_structures() serial_time = time.perf_counter() - start_time - print('Finished filtering in serial, time taken {}\n'.format(serial_time)) + print('Finished decomposing in serial, time taken {}\n'.format(serial_time)) - # now filtering in parallel - parallel_filter = box_filter_parallel(micro_model, 0.003) - meso_model.set_filter(parallel_filter) start_time = time.perf_counter() - meso_model.filter_micro_vars_parallel() + meso_model.decompose_structures_parallel() parallel_time = time.perf_counter() - start_time - print('Finished filtering in parallel, time taken {}\n'.format(parallel_time)) + print('Finished decomposing in parallel, time taken {}\n'.format(parallel_time)) print('Speed-up factor: {}'.format(serial_time/parallel_time)) - + diff --git a/master_files/py_submit_parallel.slurm b/master_files/py_submit_parallel.slurm index cde5275..8fd07a4 100644 --- a/master_files/py_submit_parallel.slurm +++ b/master_files/py_submit_parallel.slurm @@ -29,5 +29,5 @@ conda activate myenv # LAUNCHING THE JOBS #################################### -python3 -u Filters.py -# python3 -u MesoModels.py +# python3 -u Filters.py +python3 -u MesoModels.py From a0f171e50bf3fcff0b4267e5593b284d968fb7c6 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 15 Dec 2023 18:22:02 +0100 Subject: [PATCH 052/111] Parallelizing meso-routines: decompose_structures + closure_ingredients + EL_style_closure --- Proof_of_principle/calibrating_meso.py | 88 ++++++ Proof_of_principle/pickling_meso_setup.py | 23 +- Proof_of_principle/pickling_micro.py | 13 +- master_files/Filters.py | 3 +- master_files/MesoModels.py | 311 +++++++++++++++++++++- 5 files changed, 407 insertions(+), 31 deletions(-) create mode 100644 Proof_of_principle/calibrating_meso.py diff --git a/Proof_of_principle/calibrating_meso.py b/Proof_of_principle/calibrating_meso.py new file mode 100644 index 0000000..ed6f463 --- /dev/null +++ b/Proof_of_principle/calibrating_meso.py @@ -0,0 +1,88 @@ +import sys +import os +# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +# sys.path.append('/home/hidra2/celora/Filtering/master_files/') +sys.path.append('/home/tc2m23/Filtering/master_files/') +import pickle +import time + +from FileReaders import * +from MicroModels import * +from Filters import * +from MesoModels import * + +if __name__ == '__main__': + + ######################################################### + # CODE FOR TESTING SPEED UP OF PARALLEL MESO ROUTINES + ######################################################### + # TESTING ON HIDRA/IRIDIS + ET = str(sys.argv[1]) + # x_ranges = [0.2,0.4] + # y_ranges = [0.2, 0.4] + directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + MesoModelPickleLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_8dx.pickle" + + # directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" + # MesoModelPickleLoadFile = directory + "rHD_2D_ET_"+ET+"_FW_8dx_x="+ str(x_ranges)+ "_y="+str(y_ranges)+".pickle" + + + with open(MesoModelPickleLoadFile, 'rb') as filehandle: + meso_model=pickle.load(filehandle) + + num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] + print('Working with {} points\n'.format(num_points)) + + n_cpus=40 + + # decomposing structures in serial + start_time = time.perf_counter() + meso_model.decompose_structures() + serial_time = time.perf_counter() - start_time + print('Meso structures decomposed in serial, time taken: {}\n'.format(serial_time)) + + # decomposing in parallel and comparing + start_time = time.perf_counter() + meso_model.decompose_structures_parallel(n_cpus=n_cpus) + parallel_time = time.perf_counter() - start_time + print('Meso structures decomposed in parallel, time taken: {}\n'.format(parallel_time)) + print('Speed-up factor: {}'.format(serial_time/parallel_time)) + + # compute derivatives serial + start_time = time.perf_counter() + meso_model.calculate_derivatives() + serial_time = time.perf_counter() - start_time + print('Calculated derivatives in serial, time taken: {}\n'.format(serial_time)) + + # computing the closure ingredients - serial + start_time = time.perf_counter() + meso_model.closure_ingredients() + serial_time = time.perf_counter() - start_time + print('Closure ingredients calculated in serial, time taken: {}\n'.format(serial_time)) + + # computing the closure ingredients in parallel and comparing + start_time = time.perf_counter() + meso_model.closure_ingredients_parallel(n_cpus = n_cpus) + parallel_time = time.perf_counter() - start_time + print('Closure ingredients calculated in parallel, time taken: {}\n'.format(parallel_time)) + print('Speed-up factor: {}'.format(serial_time/parallel_time)) + + # computing the coefficients - serial + start_time = time.perf_counter() + meso_model.EL_style_closure() + serial_time = time.perf_counter() - start_time + print('Dissipative coefficients calculated in serial, time taken: {}\n'.format(serial_time)) + + # computing the closure ingredients in parallel and comparing + start_time = time.perf_counter() + meso_model.closure_ingredients_parallel(n_cpus = n_cpus) + parallel_time = time.perf_counter() - start_time + print('Dissipative coefficients calculated in parallel, time taken: {}\n'.format(parallel_time)) + print('Speed-up factor: {}'.format(serial_time/parallel_time)) + + + + ########################################################## + # CODE FOR TESTING THAT THE TASKS DO WHAT THEY'RE MEANT TO + ########################################################## + # werk \ No newline at end of file diff --git a/Proof_of_principle/pickling_meso_setup.py b/Proof_of_principle/pickling_meso_setup.py index f49b7a5..238c621 100644 --- a/Proof_of_principle/pickling_meso_setup.py +++ b/Proof_of_principle/pickling_meso_setup.py @@ -1,6 +1,7 @@ import sys -# import os -sys.path.append('/home/tc2m23/Filtering/master_files/') +import os +sys.path.append('/home/hidra2/celora/Filtering/master_files/') +# sys.path.append('/home/tc2m23/Filtering/master_files/') import pickle import time @@ -13,7 +14,8 @@ # READING MICRO DATA FORM PICKLE CPU_start_time = time.perf_counter() - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" ET=str(sys.argv[1]) MicroModelPickleLoadFile = directory + "HD_2D_ET_"+ET+"_micro.pickle" @@ -27,8 +29,8 @@ num_snaps = 21 central_slice_num = int(num_snaps/2) t_bdrs = [micro_model.domain_vars['t'][central_slice_num-1], micro_model.domain_vars['t'][central_slice_num+1]] - x_bdrs = [0.02, 0.98] - y_bdrs = [0.02, 0.98] + x_bdrs = [0.2, 0.25] + y_bdrs = [0.2, 0.25] # t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num]] # x_bdrs = [0.2, 0.25] # y_bdrs = [0.2, 0.25] @@ -40,7 +42,7 @@ width = width_ratio * micro_model.domain_vars['dx'] find_obs = FindObs_root_parallel(micro_model, box_len) filter = box_filter_parallel(micro_model, width) - meso_model = resHD2D(micro_model, find_obs, filter) + meso_model = resHD2D(micro_model, find_obs, filter, local_or_slurm=False) meso_model.setup_meso_grid([t_bdrs, x_bdrs, y_bdrs]) num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] @@ -57,10 +59,11 @@ # FILTERING WITH DIFFERENT WIDTHS - filter_width_ratios = [2., 4., 8.] - # filter_width_ratios = [2.] + # filter_width_ratios = [2., 4., 8.] + filter_width_ratios = [8.] filter = meso_model.filter - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + saving_directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" for i in range(len(filter_width_ratios)): # filtering in parallel @@ -71,7 +74,7 @@ print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratios[i]), time_taken)) # Now saving the output - filename = "resHD_2D_ET_" + ET + "_FW_{}dx.pickle".format(int(filter_width_ratios[i])) + filename = "rHD_2D_ET_" + ET + "_FW_{}dx_x=" + str(x_bdrs) + "_y="+ str(y_bdrs) +".pickle".format(int(filter_width_ratios[i])) MesoModelPickleDumpFile = saving_directory+filename with open(MesoModelPickleDumpFile, 'wb') as filehandle: pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/pickling_micro.py b/Proof_of_principle/pickling_micro.py index a53110b..2fb2350 100644 --- a/Proof_of_principle/pickling_micro.py +++ b/Proof_of_principle/pickling_micro.py @@ -1,6 +1,7 @@ import sys # import os -sys.path.append('/home/tc2m23/Filtering/master_files/') +sys.path.append('/home/hidra2/celora/Filtering/master_files/') +# sys.path.append('/home/tc2m23/Filtering/master_files/') # sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') import pickle import time @@ -12,11 +13,10 @@ # READING DATA FORM CHECKPOINT, SETTING UP MICOR AND PICKLING CPU_start_time = time.perf_counter() - # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/METHOD_output/80X80/ET_02" - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/" + directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/METHOD_output/800X800/" + # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/" ET=str(sys.argv[1]) names = directory + "ET_" + ET - print(names) FileReader = METHOD_HDF5(names) micro_model = IdealHD_2D() @@ -26,9 +26,10 @@ print('Time to set up micro model (ET: {}): {}'.format(ET, time_taken)) # Pickle save - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" + saving_directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" + # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" CPU_start_time = time.perf_counter() - MicroModelPickleDumpFile = directory + "HD_2D_ET_" + ET + "_micro.pickle" + MicroModelPickleDumpFile = saving_directory + "HD_2D_ET_" + ET + "_micro.pickle" with open(MicroModelPickleDumpFile, 'wb') as filehandle: pickle.dump(micro_model, filehandle) time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. diff --git a/master_files/Filters.py b/master_files/Filters.py index 6409066..9262b59 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -975,7 +975,6 @@ def find_observers_parallel(self, points, n_cpus): spatial_dims = self.micro_model.get_spatial_dims() L = self.L args_for_pool = [ [points[i], i] for i in range(len(points))] - init = FindObs_root_parallel.initializer initargs = (spatial_dims, L) @@ -1546,7 +1545,7 @@ def filter_vars_parallel(self, list_packed_args, n_cpus): result, failed = find_obs_parallel.find_observers_parallel(points, n_cpus) print('Number of points failed: {}'.format(len(failed))) parallel_time = time.perf_counter() - start_time - print('Parallel time: {}\n'.format(parallel_time)) + print('Finished finding observers. Parallel time: {}\n'.format(parallel_time)) # print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 00bb065..b80245f 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1490,9 +1490,11 @@ def find_observers_parallel(self, n_cpus = None): for elem in product(t_idxs, x_idxs, y_idxs): indices_meso_grid.append(elem) - num_processes = self.n_cpus + num_processes = 0 if n_cpus: num_processes = n_cpus + else: + num_processes = self.n_cpus successes, failures = self.find_obs.find_observers_parallel(points, num_processes) @@ -1585,9 +1587,11 @@ def filter_micro_vars_parallel(self, n_cpus = None): for i in range(len(points)): args_for_filtering_parallel.append([points[i], observers[i], vars]) - num_processes = self.n_cpus + num_processes = 0 if n_cpus: num_processes = n_cpus + else: + num_processes = self.n_cpus positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel, num_processes) for i in range(len(positions)): @@ -1668,12 +1672,48 @@ def decompose_structures(self): @staticmethod def p_Gamma_law(eps, n, Gamma): + """ + staticmethod used by decompose_structures_task, that is the + parallel version decompose_structures_gridpoint + + Parameters: + ----------- + eps: float + the energy density of the fluid at a point + + n: float + the baryon number density of the fluid at a point + + Gamma: float + Gamma factor of the Gamma law + """ return (Gamma-1)* eps*n @staticmethod def decompose_structures_task(BC, SET , p_filt, h, i, j): """ - work in progress + Task to be executed in parallel: decomposing the structures at a point + + Parameters: + ----------- + + BC: np.array (3,) + the baryon current, rank: (1,0) + + SET: np.array (3,3) + the Stress-Energy tensor, rank (2,0) + + p_filt: float + the filtered pressure, scalar + + h,i,j: integers + indices of the corresponding gridpoint on meso-grid + + Returns: + -------- + (list, list): + first list contains the decomposition of structures at point + second list contains indices of point on mesogrid """ # As this is staticmethod, no access to self.metric metric = np.zeros((3,3)) @@ -1703,12 +1743,13 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): def decompose_structures_parallel(self, n_cpus = None): """ - work in progress + Routine to decompose structures on the entire grid in + parallel. Parameters: ----------- n_cpus: int - If not passed explicitely this is set to the value on construction + If not passed explicitely, the value set upon construction is used """ # Preparing arguments for pool args_for_pool=[] @@ -1720,9 +1761,11 @@ def decompose_structures_parallel(self, n_cpus = None): p_filt = self.meso_vars['p_filt'][h,i,j] args_for_pool.append((BC, SET, p_filt, h,i,j)) - num_processes = self.n_cpus + num_processes = 0 if n_cpus: num_processes = n_cpus + else: + num_processes = self.n_cpus with mp.Pool(processes=num_processes) as pool: print('Running with {} processes\n'.format(num_processes), flush=True) @@ -1859,12 +1902,11 @@ def closure_ingredients_gridpoint(self, h, i, j): Coefficients can be computed here directly, but more general models will require more work. Do not split this, at least for now """ - # COMPUTING THE MESO-CURRENT AND DECOMPOSING IT WRT FAVRE_OBS + # FAVRE_OBS u_t = self.meso_vars['u_tilde'][h,i,j] u_t_cov = np.einsum('ij,j->i', self.metric, u_t) - Nabla_Fab = self.deriv_vars['D_Fab'][h,i,j] - # CLOSURE INGREDIENTS: FLUID BIT - WORKING WITH (2,0) + # CLOSURE INGREDIENTS: FAVRE OBS DERIVATIVE DECOMPOSITION - WORKING WITH (2,0) nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] # This is a rank (1,1) tensor nabla_u = np.einsum('ij,jk->ik', self.metric, nabla_u) #this is a rank (2,0) tensor acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector @@ -1915,6 +1957,121 @@ def closure_ingredients(self): self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) finally: self.meso_vars[key][h,i,j] = values[idx] + + @staticmethod + def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): + """ + Task to decompose Favre obs + Temperature derivatives + These will be used in EL_style_closure to extract the + turbulent effective dissipative coefficients + + Parameters: + ----------- + u_t: np.array (3,) + the Favre-observer, rank: (1,0) + + nabla_u: np.array(3,3) + derivatives of Favre observer, rank: (1,1) + + T_t: float + The temperature from filtered EoS + + nabla_T: np.array (3,0) + Temperature derivatives, rank: (0,1) + + h,i,j: integers + the indices of point on meso-grid + + Returns: + -------- + (list, list, list): + first list contains the strings of the returned quantities + (needed as the meso-vars dictionary is extended appropriately) + + second list contains the corresponding np.arrays + + third list contains the indices on meso-grid + """ + # building blocks: this task is static so no access to self + spatial_dims = 2 + metric = np.zeros((3,3)) + metric[0,0] = -1 + metric[1,1] = metric[2,2] = 1 + + # CLOSURE INGREDIENTS: FAVRE OBS DERIVATIVE DECOMPOSITION - WORKING WITH (2,0) + u_t_cov = np.einsum('ij,j->i', metric, u_t) + nabla_u = np.einsum('ij,jk->ik', metric, nabla_u) #this is a rank (2,0) tensor + acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector + + # The following two quantities should be identically zero, but they won't be due to numerical errors. + # Computing them and removing to project velocity gradients + unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector + acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) + Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ + np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor + h_ab = metric + np.einsum('i,j->ij', u_t, u_t) + exp_t = np.einsum('ii->',Daub) + shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/spatial_dims * exp_t, h_ab) + vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + + # CLOSURE INGREDIENTS: HEAT FLUX + projector = np.zeros((3,3)) + np.fill_diagonal(projector, 1) + projector += np.einsum('i,j->ij',u_t_cov, u_t) + DaT = np.einsum('ij,j->i', projector, nabla_T) + Theta_tilde = DaT + np.multiply(T_t, np.einsum('ij,j->i', metric, acc_t)) + + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'Theta_tilde'] + closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] + return closure_vars_strs, closure_vars, [h,i,j] + + def closure_ingredients_parallel(self, n_cpus=None): + """ + Routine to compute and store the closure ingredients at all gridpoints in parallel + + Parameters: + ----------- + n_cpus: int + If not passed explicitely, the value set upon construction is used + + Notes: + ------ + worth thinking about merging this with decompose structures? + """ + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + args_for_pool = [] + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + u_t = self.meso_vars['u_tilde'][h,i,j] + nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] + T_t = self.meso_vars['T_tilde'][h,i,j] + nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] + args_for_pool.append((u_t, nabla_u, T_t, nabla_T, h, i, j)) + + num_processes = 0 + if n_cpus: + num_processes = n_cpus + else: + num_processes = self.n_cpus + + with mp.Pool(processes=num_processes) as pool: + print('Running with {} processes\n'.format(num_processes), flush=True) + for result in pool.starmap(resHD2D.closure_ingredients_task, args_for_pool): + keys, values, grid_idxs = result + # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key), flush=True) + shape = values[idx].shape + self.meso_vars.update({key : np.zeros(([Nt,Nx,Ny]+ list(shape)))}) + finally: + self.meso_vars[key][tuple(grid_idxs)] = values[idx] def EL_style_closure_gridpoint(self, h, i, j): """ @@ -1940,7 +2097,7 @@ def EL_style_closure_gridpoint(self, h, i, j): transformed_shear = np.einsum('ji,jk,kl', pi_eigv, self.meso_vars['shear_tilde'][h,i,j], pi_eigv) abs_pi_eig = np.abs(pi_eig) pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) - eta = eta * np.sign(pi_eig[-1] / transformed_shear[pos,pos]) + eta = eta * np.sign(pi_eig[pos] / transformed_shear[pos,pos]) coefficients_names.append('eta') coefficients.append(eta) @@ -1977,7 +2134,134 @@ def EL_style_closure(self): self.meso_vars.update({key: np.zeros(([Nt, Nx, Ny] + list(shape)))}) finally: self.meso_vars[key][h,i,j] = values[idx] - + + @staticmethod + def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): + """ + Task for computing the effective dissipative coefficients at all gridpoint + in parallel + + Parameters: + ----------- + Pi_res: float + Pressure residual using meso EoS + + exp: float + the Favre expansion rate + + pi_res: np.array (3,3) + the residual anisotropic stresses (should be trace-free) + + shear: np.array (3,3) + the shear matrix computed from Favre obs derivatives + + q_res: np.array (3,) + the residual heat-flux + + Theta: np.array (3,) + the temperature spatial gradients with acceleration term + + h,i,j: integers + grid indixes corresponding to point on grid + + Return: + ------- + (coeff names, coeff, [h,i,j]) + + """ + coefficients_names=[] + coefficients=[] + + metric = np.zeros((3,3)) + metric[0,0] = -1 + metric[1,1] = metric[2,2] = +1 + + # CALCULATING BULK VISCOUS COEFF + zeta = Pi_res / exp + coefficients_names.append('zeta') + coefficients.append(zeta) + + # CALCULATING SHEAR VISCOUS COEFF + pi_res_sq = np.einsum('ij,kl,ik,jl->', pi_res, pi_res, metric, metric) + shear_sq = np.einsum('ij,kl,ik,jl->', shear, shear, metric, metric) + eta = pi_res_sq/shear_sq + # Compute sign of eta by looking at the PA of pi_res with higher eigenvalue. + # Change shear to the eigenbasis of pi_res (using that the matrix is unitary as pi_res is sym) + pi_eig, pi_eigv = np.linalg.eigh(pi_res) + transformed_shear = np.einsum('ji,jk,kl', pi_eigv, shear, pi_eigv) + abs_pi_eig = np.abs(pi_eig) + pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) + eta = eta * np.sign(pi_eig[pos] / transformed_shear[pos,pos]) + coefficients_names.append('eta') + coefficients.append(eta) + + # CALCULATING THE HEAT CONDUCTIVITIY + q_res_sq = np.einsum('i,ij,j', q_res, metric, q_res) + Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) + cos_angle = np.einsum('i,ij,j', q_res, metric, Theta) / (q_res_sq * Theta_sq) + sign = np.sign(cos_angle) + kappa = sign * np.sqrt(q_res_sq / Theta_sq) + coefficients_names.append('kappa') + coefficients.append(kappa) + + return coefficients_names, coefficients, [h,i,j] + + def EL_style_closure_parallel(self, n_cpus=None): + """ + Routine to launch EL_style_closure_task in parallel across multiple gridpoints + + Parameters: + ----------- + n_cpus: int + if not passed, the value set upon construction is used + + Notes: + ------ + worth thinking about merging this with decompose_structures_task and closure_ingredients_task? + could compute just u_t at all points, so to be able to take derivatives of it + Then since you're accesing a single point, you could decompose, compute ingredients and extract coeff + in one go. Might give you a speed up: the loop for preparning arguments for pool is done once + and not twice/thrice, and the processes are opened/closed half the times. + """ + args_for_pool=[] + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + Pi_res = self.meso_vars['Pi_res'][h,i,j] + pi_res = self.meso_vars['pi_res'][h,i,j] + q_res = self.meso_vars['q_res'][h,i,j] + exp_t = self.meso_vars['exp_tilde'][h,i,j] + shear_t = self.meso_vars['shear_tilde'][h,i,j] + Theta_t = self.meso_vars['Theta_tilde'][h,i,j] + + args_for_pool.append((Pi_res, exp_t, pi_res, shear_t, q_res, Theta_t, h,i,j)) + + num_processes = 0 + if n_cpus: + num_processes = n_cpus + else: + num_processes = self.n_cpus + + with mp.Pool(processes=num_processes) as pool: + for result in pool.starmap(resHD2D.EL_style_closure_task, args_for_pool): + keys, values, grid_idxs = result + # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key), flush=True) + shape = values[idx].shape + self.meso_vars.update({key : np.zeros(([Nt,Nx,Ny]+ list(shape)))}) + finally: + self.meso_vars[key][tuple(grid_idxs)] = values[idx] + def EL_style_closure_regression(self, CoefficientAnalysis): """ Takes in a list of correlation quantities (strings? The regressors needed are computed @@ -2037,6 +2321,7 @@ def EL_style_closure_regression(self, CoefficientAnalysis): if __name__ == '__main__': + ######################################################## # TESTING SERIAL IMPLEMENTATION ######################################################## @@ -2083,7 +2368,7 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # ######################################################## - # # TESTING PARALLEL IMPLEMENTATION FOR FINDING OBS AND FILTER + # # TESTING PARALLEL IMPLEMENTATION: find obs + filter # ######################################################## # FileReader = METHOD_HDF5('../Data/test_res100/') # micro_model = IdealHD_2D() @@ -2154,7 +2439,7 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # set up meso model and grid find_obs = FindObs_root_parallel(micro_model, 0.001) filter = box_filter_parallel(micro_model, 0.003) - meso_model = resHD2D(micro_model, find_obs, filter, local_or_slurm=False) + meso_model = resHD2D(micro_model, find_obs, filter, local_or_slurm=True) meso_model.setup_meso_grid([t_range, x_range, y_range]) num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] From 00ddcb9b1e62cc807f73a5288997b8e96ad10ae8 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 10 Jan 2024 11:39:47 +0100 Subject: [PATCH 053/111] TC Updates to master_files --- Proof_of_principle/config.txt | 8 - Proof_of_principle/multiple_filtering_meso.py | 58 ------- Proof_of_principle/pickling_meso_obs.py | 53 ------- Proof_of_principle/pickling_micro.py | 59 ------- Proof_of_principle/visualizing_meso_manyFW.py | 144 ------------------ master_files/Filters.py | 4 +- master_files/MesoModels.py | 86 +++-------- 7 files changed, 24 insertions(+), 388 deletions(-) delete mode 100644 Proof_of_principle/config.txt delete mode 100644 Proof_of_principle/multiple_filtering_meso.py delete mode 100644 Proof_of_principle/pickling_meso_obs.py delete mode 100644 Proof_of_principle/pickling_micro.py delete mode 100644 Proof_of_principle/visualizing_meso_manyFW.py diff --git a/Proof_of_principle/config.txt b/Proof_of_principle/config.txt deleted file mode 100644 index 8e668b9..0000000 --- a/Proof_of_principle/config.txt +++ /dev/null @@ -1,8 +0,0 @@ -#Configuration file for job array -JOB_ID ET -1 1.0 -2 1.5 -3 2.0 -4 2.5 -5 3.0 -6 3.5 diff --git a/Proof_of_principle/multiple_filtering_meso.py b/Proof_of_principle/multiple_filtering_meso.py deleted file mode 100644 index 4d291a8..0000000 --- a/Proof_of_principle/multiple_filtering_meso.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash - -import sys -# import os -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') -sys.path.append('/home/tc2m23/Filtering/master_files/') -import pickle -import time - -from FileReaders import * -from MicroModels import * -from Filters import * -from MesoModels import * - -if __name__ == '__main__': - - ET=sys.argv[1] - CPU_start_time = time.perf_counter() - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - MesoModelPickleLoadFile = directory + "observers_2dx_ET_" + ET + ".pickle" - with open(MesoModelPickleLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) - time_taken = time.perf_counter()- CPU_start_time - print('Time taken to read data from pickle: {}'.format(time_taken)) - - # filter_widths_ratio = [8] - filter_widths_ratio = [2, 4, 8] - meso_dx = meso_model.domain_vars['Dx'] - filter = meso_model.filter - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - # saving_directory = "./" - - for i in range(len(filter_widths_ratio)): - CPU_start_time = time.perf_counter() - filter.set_filter_width(meso_dx * filter_widths_ratio[i]) - - # HOW LONG IT TAKES TO FILTER A VAR AT A POINT? - # vars = ['BC', 'SET', 'p'] - # for var in vars: - # h,l,j = 0, 200, 200 - # T, X, Y = meso_model.domain_vars['T'][h], meso_model.domain_vars['X'][l], meso_model.domain_vars['Y'][j] - # point = [T, X, Y] - # obs = meso_model.filter_vars['U'][h,l,j] - # filter.filter_var_point(var, point, obs) - # print('Filter var {}'.format(var)) - - meso_model.filter_micro_variables() - filename = "resHD_2D_ET_" + ET + "_FW_{}dx.pickle".format(int(filter_widths_ratio[i])) - MesoModelPickleDumpFile = saving_directory+filename - with open(MesoModelPickleDumpFile, 'wb') as filehandle: - pickle.dump(meso_model, filehandle) - time_taken = time.perf_counter() - CPU_start_time - print('Time taken to filter data (fw: {}): {}'.format(int(filter_widths_ratio[i]), time_taken)) - - - - - diff --git a/Proof_of_principle/pickling_meso_obs.py b/Proof_of_principle/pickling_meso_obs.py deleted file mode 100644 index b7fc015..0000000 --- a/Proof_of_principle/pickling_meso_obs.py +++ /dev/null @@ -1,53 +0,0 @@ -import sys -# import os -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') -sys.path.append('/home/tc2m23/Filtering/master_files/') -import pickle -import time - -from FileReaders import * -from MicroModels import * -from Filters import * -from MesoModels import * - - -if __name__ == '__main__': - - # READING THE MICRO-MODEL FROM PICKLE - CPU_start_time = time.perf_counter() - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - ET=str(sys.argv[1]) - MicroModelPickleLoadFile = directory + "IdealHD_2D_ET_"+ET+"_micro.pickle" - - with open(MicroModelPickleLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) - - - # SETTING UP THE MESO-MODEL - num_snaps = 11 - box_len_ratio = 2. - central_slice_num = int(num_snaps/2) - t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num+1]] - box_len = box_len_ratio * micro_model.domain_vars['dx'] - - find_obs = FindObs_drift_root(micro_model, box_len) - filter = spatial_box_filter(micro_model, box_len) #THIS WILL BE CHANGED IN A LATER SCRIPT - meso_model = resHD2D(micro_model, find_obs, filter) - grid_bdrs = [0.03, 0.97] - meso_model.setup_meso_grid([t_bdrs, grid_bdrs, grid_bdrs]) - time_taken = time.perf_counter() - CPU_start_time - print('Read from micro, grid is set up, time taken: {}'.format(time_taken)) - - # ROOT-FINDING THE OBSERVERS - CPU_start_time = time.perf_counter() - # t, x, y = meso_model.domain_vars['T'][0], meso_model.domain_vars['X'][10], meso_model.domain_vars['Y'][10] - # point = [t,x,y] - # meso_model.find_obs.find_observer(point) - meso_model.find_observers() - time_taken = time.perf_counter() - CPU_start_time - print('Observers found, time taken: {}'.format(time_taken)) - - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - MesoModelPickleDumpFile = directory + "observers_2dx_ET_" + ET + ".pickle" - with open(MesoModelPickleDumpFile, 'wb') as filehandle: - pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/pickling_micro.py b/Proof_of_principle/pickling_micro.py deleted file mode 100644 index 2fb2350..0000000 --- a/Proof_of_principle/pickling_micro.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys -# import os -sys.path.append('/home/hidra2/celora/Filtering/master_files/') -# sys.path.append('/home/tc2m23/Filtering/master_files/') -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') -import pickle -import time - -from FileReaders import * -from MicroModels import * - -if __name__ == '__main__': - - # READING DATA FORM CHECKPOINT, SETTING UP MICOR AND PICKLING - CPU_start_time = time.perf_counter() - directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/METHOD_output/800X800/" - # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/" - ET=str(sys.argv[1]) - names = directory + "ET_" + ET - - FileReader = METHOD_HDF5(names) - micro_model = IdealHD_2D() - FileReader.read_in_data_HDF5_missing_xy(micro_model) - micro_model.setup_structures() - time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. - print('Time to set up micro model (ET: {}): {}'.format(ET, time_taken)) - - # Pickle save - saving_directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" - # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" - CPU_start_time = time.perf_counter() - MicroModelPickleDumpFile = saving_directory + "HD_2D_ET_" + ET + "_micro.pickle" - with open(MicroModelPickleDumpFile, 'wb') as filehandle: - pickle.dump(micro_model, filehandle) - time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. - print('Time needed to pickle class: {}'.format(time_taken)) - - # READING AND PICKLING ALL CHECKPOINTS - FOR VISUALIZATION - # CPU_start_time = time.perf_counter() - # # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/METHOD_output/400X400/" - # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/METHOD_output/80X80/" - # FileReader = METHOD_HDF5(directory) - # micro_model = IdealHD_2D() - # FileReader.read_in_data_HDF5_missing_xy(micro_model) - # micro_model.setup_structures() - # time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. - # print('Time to set up micro model (up to 8ET): {}'.format(time_taken)) - - # # directory = "/scratch/tc2m23/NewData4Filtering/hydro/ideal/ET_2_4_6_8/10_dx_after/pickle_files/400X400/" - # directory = "/Users/thomas/Dropbox/Work/projects/Filtering/Data/ET_2_4_6_8/10_dx_after/pickle_files/80X80/" - # CPU_start_time = time.perf_counter() - # pickle_save = True - # MicroModelPickleDumpFile = directory + "IdealHD_2D_ET_ALL_micro.pickle" - # if pickle_save: - # with open(MicroModelPickleDumpFile, 'wb') as filehandle: - # pickle.dump(micro_model, filehandle) - # time_taken = int((time.perf_counter() - CPU_start_time) * 100)/100. - # print('Time needed to pickle class: {}'.format(time_taken)) - diff --git a/Proof_of_principle/visualizing_meso_manyFW.py b/Proof_of_principle/visualizing_meso_manyFW.py deleted file mode 100644 index bbae684..0000000 --- a/Proof_of_principle/visualizing_meso_manyFW.py +++ /dev/null @@ -1,144 +0,0 @@ -import sys -# import os -sys.path.append('/home/tc2m23/Filtering/master_files/') -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') -import pickle - -from FileReaders import * -from MicroModels import * -from MesoModels import * -from Visualization import * - -if __name__ == '__main__': - - ############################################################### - # BLOCK TO COMPARE MICRO AND MESO WITH DIFF FILTERING SIZES - ############################################################### - # Loading the models - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - ET = sys.argv[1] - # ET = "1.0" - MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET + "_micro.pickle" - with open(MicroModelLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) - - models = [micro_model] - FW = ['2', '4', '8'] - if ET == "2.0": - FW = ['2', '4'] - for fw in FW: - MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" - with open(MesoModelLoadFile, 'rb') as filehandle: - models.append(pickle.load(filehandle)) - - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - # saving_directory = "./" - - # Checking for alignment of slices - num_snaps = 11 - central_slice_num = int(num_snaps/2.) - time_micro = micro_model.domain_vars['t'][central_slice_num] - for i in range(1, len(models)): - time_meso = models[i].domain_vars['T'][0] - if time_meso != time_micro: - print('Careful: micro and meso slices appear not to be aligned!') - - - ranges_x = [0.04, 0.96] - ranges_y = [0.04, 0.96] - visualizer = Plotter_2D([11.97, 8.36]) - - # Plot for baryon current - to_be_plotted = ['BC', 'BC', 'BC'] - comp_to_plot = [(0,), (1,), (2,)] - vars = [to_be_plotted for _ in range(len(models))] - components = [comp_to_plot for _ in range(len(models))] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components) - fig.tight_layout() - # plt.show() - filename = "manyFW_BC_ET_" + ET + ".svg" - plt.savefig(saving_directory + filename, format = "svg") - - # plot for SET-diagonal - to_be_plotted = ['SET', 'SET', 'SET'] - comp_to_plot = [(0,0), (1,1), (2,2)] - vars = [to_be_plotted for _ in range(len(models))] - components = [comp_to_plot for _ in range(len(models))] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components) - fig.tight_layout() - # plt.show() - filename = "manyFW_SETdiag_ET_" + ET + ".svg" - plt.savefig(saving_directory + filename, format = "svg") - - # plot for SET off-diagonal - to_be_plotted = ['SET', 'SET', 'SET'] - comp_to_plot = [(0,1), (0,2), (1,2)] - vars = [to_be_plotted for _ in range(len(models))] - components = [comp_to_plot for _ in range(len(models))] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components) - fig.tight_layout() - # plt.show() - filename = "manyFW_SEToffdiag_ET_" + ET + ".svg" - plt.savefig(saving_directory + filename, format = "svg") - - - ############################################################### - # BLOCK TO PLOT THE DIFFERENCE BETWEEN TWO FILTER WIDTHS - ############################################################### - # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - # ET = sys.argv[1] - # models = [] - # FW = ['2', '8'] - # if ET == "2.0": - # FW = ['2', '4'] - # for fw in FW: - # MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" - # with open(MesoModelLoadFile, 'rb') as filehandle: - # models.append(pickle.load(filehandle)) - - # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - # # saving_directory = "./" - - # time_meso = models[0].domain_vars['T'][0] - # ranges_x = [0.04, 0.96] - # ranges_y = [0.04, 0.96] - # visualizer = Plotter_2D([11.97, 8.36]) - - # # Plot for baryon current - # to_be_plotted = ['BC', 'BC', 'BC'] - # comp_to_plot = [(0,), (1,), (2,)] - # vars = [to_be_plotted for _ in range(len(models))] - # components = [comp_to_plot for _ in range(len(models))] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True) - # # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, \ - # # method = "interpolate", interp_dims = (150, 150), diff_plot=True) - # fig.tight_layout() - # # plt.show() - # filename = "contrastFW_BC_ET_" + ET + ".svg" - # plt.savefig(saving_directory + filename, format = "svg") - - # # plot for SET-diagonal - # to_be_plotted = ['SET', 'SET', 'SET'] - # comp_to_plot = [(0,0), (1,1), (2,2)] - # vars = [to_be_plotted for _ in range(len(models))] - # components = [comp_to_plot for _ in range(len(models))] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True) - # # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, \ - # # method = "interpolate", interp_dims = (150, 150), diff_plot=True) - # fig.tight_layout() - # # plt.show() - # filename = "contrastFW_SETdiag_ET_" + ET + ".svg" - # plt.savefig(saving_directory + filename, format = "svg") - - # # plot for SET off-diagonal - # to_be_plotted = ['SET', 'SET', 'SET'] - # comp_to_plot = [(0,1), (0,2), (1,2)] - # vars = [to_be_plotted for _ in range(len(models))] - # components = [comp_to_plot for _ in range(len(models))] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True) - # # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, \ - # # method = "interpolate", interp_dims = (150, 150), diff_plot=True) - # fig.tight_layout() - # # plt.show() - # filename = "constrastFW_SEToffdiag_ET_" + ET + ".svg" - # plt.savefig(saving_directory + filename, format = "svg") \ No newline at end of file diff --git a/master_files/Filters.py b/master_files/Filters.py index 9262b59..37ea187 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -980,7 +980,7 @@ def find_observers_parallel(self, points, n_cpus): with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: # with mp.Pool() as pool: - print('Running with {} processes\n'.format(n_cpus), flush=True) + print('Finding observers in parallel with {} processes\n'.format(n_cpus), flush=True) for result in pool.map(self.find_observer_Gauss, args_for_pool): if (result[0] == True): success_pos.append(result[1]) @@ -1460,7 +1460,7 @@ def filter_vars_parallel(self, list_packed_args, n_cpus): initargs=(self.spatial_dims, self.filter_width) with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: - print('Running with {} processes\n'.format(n_cpus), flush=True) + print('Filtering in parallel with {} processes\n'.format(n_cpus), flush=True) for result in pool.map(self.filter_vars_point_gauss, args_for_pool): position_in_list.append(result[0]) filtered_vars.append(result[1]) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index b80245f..34607b7 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1106,7 +1106,7 @@ class resHD2D(object): """ CUT AND PAST OF resMHD_2D --> removing the magnetic bits """ - def __init__(self, micro_model, find_obs, filter, interp_method = 'linear', local_or_slurm = True): + def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): """ ADD DESCRIPTION OF THIS """ @@ -1116,11 +1116,6 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear', loca self.spatial_dims = 2 self.interp_method = interp_method - if local_or_slurm: - self.n_cpus = int(os.cpu_count()) - else: - self.n_cpus = int(os.environ['SLURM_CPUS_PER_TASK']) - self.domain_int_strs = ('Nt','Nx','Ny') self.domain_float_strs = ("Tmin","Tmax","Xmin","Xmax","Ymin","Ymax","Dt","Dx","Dy") self.domain_array_strs = ("T","X","Y","Points") @@ -1170,13 +1165,12 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear', loca # dictionary with non-local quantities (keys must match one of meso_vars or structure) self.nonlocal_vars_strs = ['u_tilde', 'T_tilde'] - # self.nonlocal_vars_strs = ['u_tilde', 'Fab'] Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] self.deriv_vars = dict.fromkeys(Dstrs) # Additional, closure-scheme specific vars must be added appropriately using model_residuals() - # self.all_var_strs = self.meso_vars_strs + Dstrs + self.meso_structures_strs + # Run some compatibility test... compatible = True error = '' @@ -1197,12 +1191,6 @@ def set_find_obs(self, find_obs): def set_filter(self, filter): self.filter = filter - def set_n_cps(self, n_cpus): - """ - Set the number of processes to run in parallel on. - """ - self.n_cpus = n_cpus - def get_all_var_strs(self): return list(self.meso_vars.keys()) + list(self.deriv_vars.keys()) + list(self.meso_structures.keys()) + \ list(self.filter_vars.keys()) @@ -1455,7 +1443,7 @@ def find_observers(self): # No need to update the dictionary as this has been initialized to False everywhere. print('Careful: obs could not be found at: ', self.domain_vars['Points'][h][i][j]) - def find_observers_parallel(self, n_cpus = None): + def find_observers_parallel(self, n_cpus): """ Method to find observers at all points on meso-grid, parallelized version. The observers found (and relative errors) are saved in the dictionary self.filter_vars. @@ -1466,8 +1454,7 @@ def find_observers_parallel(self, n_cpus = None): ----------- n_cpus: int - If not passed explicitely this is set to self.n_cpus (initalized upon - construction) + number of processes to run in parallel Notes: ------ @@ -1490,13 +1477,7 @@ def find_observers_parallel(self, n_cpus = None): for elem in product(t_idxs, x_idxs, y_idxs): indices_meso_grid.append(elem) - num_processes = 0 - if n_cpus: - num_processes = n_cpus - else: - num_processes = self.n_cpus - - successes, failures = self.find_obs.find_observers_parallel(points, num_processes) + successes, failures = self.find_obs.find_observers_parallel(points, n_cpus) for i in range(len(successes[0])): point_indxs_meso_grid = indices_meso_grid[successes[0][i]] @@ -1536,7 +1517,7 @@ def filter_micro_variables(self): else: print('Could not filter at {}: observer not found.'.format(point)) - def filter_micro_vars_parallel(self, n_cpus = None): + def filter_micro_vars_parallel(self, n_cpus): """ Filter all meso_model structures AND micro pressure. Routine will try and filter at all points on the meso-grid. @@ -1550,8 +1531,7 @@ def filter_micro_vars_parallel(self, n_cpus = None): ----------- n_cpus: int - If not passed explicitely this is set to self.n_cpus (initalized upon - construction) + number of processes for parallelization Notes: ------ @@ -1586,14 +1566,8 @@ def filter_micro_vars_parallel(self, n_cpus = None): args_for_filtering_parallel = [] for i in range(len(points)): args_for_filtering_parallel.append([points[i], observers[i], vars]) - - num_processes = 0 - if n_cpus: - num_processes = n_cpus - else: - num_processes = self.n_cpus - positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel, num_processes) + positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel, n_cpus) for i in range(len(positions)): point_indxs_meso_grid = indices_meso_grid[positions[i]] self.meso_structures['BC'][point_indxs_meso_grid] = filtered_vars[i][0] @@ -1741,7 +1715,7 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, EOS_res, T_t], [h,i,j] - def decompose_structures_parallel(self, n_cpus = None): + def decompose_structures_parallel(self, n_cpus): """ Routine to decompose structures on the entire grid in parallel. @@ -1749,7 +1723,7 @@ def decompose_structures_parallel(self, n_cpus = None): Parameters: ----------- n_cpus: int - If not passed explicitely, the value set upon construction is used + numbers of processes for parallelization """ # Preparing arguments for pool args_for_pool=[] @@ -1761,14 +1735,8 @@ def decompose_structures_parallel(self, n_cpus = None): p_filt = self.meso_vars['p_filt'][h,i,j] args_for_pool.append((BC, SET, p_filt, h,i,j)) - num_processes = 0 - if n_cpus: - num_processes = n_cpus - else: - num_processes = self.n_cpus - - with mp.Pool(processes=num_processes) as pool: - print('Running with {} processes\n'.format(num_processes), flush=True) + with mp.Pool(processes=n_cpus) as pool: + print('Running with {} processes\n'.format(n_cpus), flush=True) for result in pool.starmap(resHD2D.decompose_structures_task, args_for_pool): h,i,j = result[1] self.meso_vars['n_tilde'][h,i,j] = result[0][0] @@ -2025,14 +1993,14 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] return closure_vars_strs, closure_vars, [h,i,j] - def closure_ingredients_parallel(self, n_cpus=None): + def closure_ingredients_parallel(self, n_cpus): """ Routine to compute and store the closure ingredients at all gridpoints in parallel Parameters: ----------- n_cpus: int - If not passed explicitely, the value set upon construction is used + number of processes for parallelization Notes: ------ @@ -2052,14 +2020,8 @@ def closure_ingredients_parallel(self, n_cpus=None): nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] args_for_pool.append((u_t, nabla_u, T_t, nabla_T, h, i, j)) - num_processes = 0 - if n_cpus: - num_processes = n_cpus - else: - num_processes = self.n_cpus - - with mp.Pool(processes=num_processes) as pool: - print('Running with {} processes\n'.format(num_processes), flush=True) + with mp.Pool(processes=n_cpus) as pool: + print('Running with {} processes\n'.format(n_cpus), flush=True) for result in pool.starmap(resHD2D.closure_ingredients_task, args_for_pool): keys, values, grid_idxs = result # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True @@ -2206,14 +2168,15 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): return coefficients_names, coefficients, [h,i,j] - def EL_style_closure_parallel(self, n_cpus=None): + # this is to be modified + def EL_style_closure_parallel(self, n_cpus): """ Routine to launch EL_style_closure_task in parallel across multiple gridpoints Parameters: ----------- n_cpus: int - if not passed, the value set upon construction is used + number of processes for parallelization Notes: ------ @@ -2242,13 +2205,8 @@ def EL_style_closure_parallel(self, n_cpus=None): args_for_pool.append((Pi_res, exp_t, pi_res, shear_t, q_res, Theta_t, h,i,j)) - num_processes = 0 - if n_cpus: - num_processes = n_cpus - else: - num_processes = self.n_cpus - with mp.Pool(processes=num_processes) as pool: + with mp.Pool(processes=n_cpus) as pool: for result in pool.starmap(resHD2D.EL_style_closure_task, args_for_pool): keys, values, grid_idxs = result # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True @@ -2433,8 +2391,8 @@ def EL_style_closure_regression(self, CoefficientAnalysis): micro_model.setup_structures() t_range = [1.502, 1.504] - x_range = [0.05, 0.15] - y_range = [0.05, 0.15] + x_range = [0.05, 0.55] + y_range = [0.05, 0.55] # set up meso model and grid find_obs = FindObs_root_parallel(micro_model, 0.001) From 9b4630e0bb524ffc2665aba287ba8a897b2e561c Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 10 Jan 2024 11:40:45 +0100 Subject: [PATCH 054/111] TC cleaning up Proof of principle --- Proof_of_principle/pickling_meso_setup.py | 125 ++++++++--------- Proof_of_principle/visualizing_meso.py | 158 ++++++++-------------- Proof_of_principle/visualizing_micro.py | 76 +++++++---- Proof_of_principle/visualizing_obs.py | 51 ++++--- 4 files changed, 200 insertions(+), 210 deletions(-) diff --git a/Proof_of_principle/pickling_meso_setup.py b/Proof_of_principle/pickling_meso_setup.py index 238c621..f720a3e 100644 --- a/Proof_of_principle/pickling_meso_setup.py +++ b/Proof_of_principle/pickling_meso_setup.py @@ -1,9 +1,10 @@ import sys import os -sys.path.append('/home/hidra2/celora/Filtering/master_files/') -# sys.path.append('/home/tc2m23/Filtering/master_files/') +sys.path.append('../master_files/') import pickle import time +import configparser +import json from FileReaders import * from MicroModels import * @@ -11,73 +12,67 @@ from MesoModels import * if __name__ == '__main__': - - # READING MICRO DATA FORM PICKLE - CPU_start_time = time.perf_counter() - # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" - directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" - ET=str(sys.argv[1]) - MicroModelPickleLoadFile = directory + "HD_2D_ET_"+ET+"_micro.pickle" - - with open(MicroModelPickleLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) - time_taken = time.perf_counter() - CPU_start_time - print('Time taken to read file from pickle: {}'.format(time_taken)) - - # SETTING UP THE MESO MODEL - CPU_start_time = time.perf_counter() - num_snaps = 21 - central_slice_num = int(num_snaps/2) - t_bdrs = [micro_model.domain_vars['t'][central_slice_num-1], micro_model.domain_vars['t'][central_slice_num+1]] - x_bdrs = [0.2, 0.25] - y_bdrs = [0.2, 0.25] - # t_bdrs = [micro_model.domain_vars['t'][central_slice_num], micro_model.domain_vars['t'][central_slice_num]] - # x_bdrs = [0.2, 0.25] - # y_bdrs = [0.2, 0.25] - - box_len_ratio = 4. - box_len = box_len_ratio * micro_model.domain_vars['dx'] - # setting the width is needed for constructing the filter, and is changed later - width_ratio = 2. - width = width_ratio * micro_model.domain_vars['dx'] - find_obs = FindObs_root_parallel(micro_model, box_len) - filter = box_filter_parallel(micro_model, width) - meso_model = resHD2D(micro_model, find_obs, filter, local_or_slurm=False) - meso_model.setup_meso_grid([t_bdrs, x_bdrs, y_bdrs]) - - num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] - print('Number of points: {}'.format(num_points)) - - time_taken = time.perf_counter() - CPU_start_time - print('Grid is set up, time taken: {}'.format(time_taken)) - - # FINDING THE OBSERVERS - start_time = time.perf_counter() - meso_model.find_observers_parallel() - time_taken = time.perf_counter() - start_time - print('Observers found in parallel: time taken= {}'.format(time_taken)) - - - # FILTERING WITH DIFFERENT WIDTHS - # filter_width_ratios = [2., 4., 8.] - filter_width_ratios = [8.] - filter = meso_model.filter - # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" - saving_directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" - - for i in range(len(filter_width_ratios)): - # filtering in parallel + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + hdf5_directory = config['Directories']['hdf5_dir'] + filenames = hdf5_directory + FileReader = METHOD_HDF5(filenames) + num_snaps = FileReader.num_files + micro_model = IdealHD_2D() + FileReader.read_in_data_HDF5_missing_xy(micro_model) + micro_model.setup_structures() + print('Finished reading micro data from hdf5, structures also set up.') + + # SETTING UP THE MESO MODEL + CPU_start_time = time.perf_counter() + central_slice_num = int(num_snaps/2) + t_range = [micro_model.domain_vars['t'][central_slice_num-1], micro_model.domain_vars['t'][central_slice_num+1]] + + meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) + filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) + + x_range = meso_grid['x_range'] + y_range = meso_grid['y_range'] + box_len_ratio = float(filtering_options['box_len_ratio']) + filter_width_ratio = filtering_options['filter_width_ratio'] + + box_len = box_len_ratio * micro_model.domain_vars['dx'] + width = filter_width_ratio * micro_model.domain_vars['dx'] + find_obs = FindObs_root_parallel(micro_model, box_len) + filter = box_filter_parallel(micro_model, width) + meso_model = resHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([t_range, x_range, y_range]) + + time_taken = time.perf_counter() - CPU_start_time + print('Grid is set up, time taken: {}'.format(time_taken)) + num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] + print('Number of points: {}'.format(num_points)) + + # FINDING THE OBSERVERS AND FILTERING, THEN PICKLE SAVE + n_cpus = int(config['Meso_model_settings']['n_cpus']) + + start_time = time.perf_counter() + meso_model.find_observers_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Observers found in parallel: time taken= {}'.format(time_taken)) + start_time = time.perf_counter() - filter.set_filter_width( micro_model.domain_vars['dx'] * filter_width_ratios[i] ) - meso_model.filter_micro_vars_parallel() + meso_model.filter_micro_vars_parallel(n_cpus) time_taken = time.perf_counter() - start_time - print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratios[i]), time_taken)) + print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) - # Now saving the output - filename = "rHD_2D_ET_" + ET + "_FW_{}dx_x=" + str(x_bdrs) + "_y="+ str(y_bdrs) +".pickle".format(int(filter_width_ratios[i])) - MesoModelPickleDumpFile = saving_directory+filename + pickled_directory = config['Directories']['pickled_files_dir'] + filename = config['Pickled_filenames']['meso_pickled'] + "_FW_" +str(int(filter_width_ratio)) + "dx" + ".pickle" + MesoModelPickleDumpFile = pickled_directory + filename with open(MesoModelPickleDumpFile, 'wb') as filehandle: - pickle.dump(meso_model, filehandle) + pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index 6dde175..8016c9a 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -1,8 +1,9 @@ import sys # import os -sys.path.append('/home/tc2m23/Filtering/master_files/') -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +sys.path.append('../master_files/') import pickle +import configparser +import json from FileReaders import * from MicroModels import * @@ -12,103 +13,62 @@ if __name__ == '__main__': ############################################################### - # PRODUCE MANY DIFF PLOTS TO COMPARE MICRO AND MESO AT DIFFERENT - # ENDTIME AND DIFFERENT WIDTHS. FOR EACH PLOT THE DIFFERENCE - # (ABSOLUTE AND RELATIVE). + # PRODUCE PLOTS TO COMPARE MICRO AND MESO MODELS + # FOR EACH PLOT THE DIFFERENCE (ABSOLUTE AND RELATIVE). ############################################################### - - # Loading the models - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" - ET = sys.argv[1] - MicroModelLoadFile = directory + "HD_2D_ET_" + ET + "_micro.pickle" - with open(MicroModelLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) - - num_snaps = 21 + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + + MesoModelLoadFile = pickle_directory + meso_pickled_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + micro_model = meso_model.micro_model + + # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE + num_snaps = micro_model.domain_vars['nt'] central_slice_num = int(num_snaps/2.) time_micro = micro_model.domain_vars['t'][central_slice_num] - - # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/" - - FW = ['2', '4', '8'] - for fw in FW: - MesoModelLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_"+ fw +"dx.pickle" - with open(MesoModelLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) - - meso_central_slice = 1 - time_meso = meso_model.domain_vars['T'][meso_central_slice] - if time_meso != time_micro: - print("Slices of meso and micro model do not coincide. Careful!") - else: - print("Comparing data at same time-slice, hurray!") - - ranges_x = [0.02, 0.98] - ranges_y = [0.02, 0.98] - visualizer = Plotter_2D([11.97, 8.36]) - - ###################################################### - # CODE BLOCKS TO VISUALIZE SEPARATE QUANTITIES - ###################################################### - # # Plot for baryon current - # vars = [['BC', 'BC', 'BC'], ['BC', 'BC', 'BC']] - # models = [micro_model, meso_model] - # components = [[(0,), (1,), (2,)], [(0,), (1,), (2,)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - # fig.tight_layout() - # # plt.show() - # filename = "comparing_BC_ET_" + ET + "_FW_"+ fw + ".svg" - # plt.savefig(saving_directory + filename, format = "svg") - - - # # # Plots for SET - diag and off-diagonal - # vars = [['SET', 'SET', 'SET'], ['SET', 'SET', 'SET']] - # models = [micro_model, meso_model] - # components = [[(0,0), (1,1), (2,2)], [(0,0), (1,1), (2,2)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - # fig.tight_layout() - # # plt.show() - # filename = "comparing_SET_diagonal" + ET + "_FW_"+ fw + ".svg" - # plt.savefig(saving_directory + filename, format = "svg") - - # # vars = [['SET', 'SET', 'SET'], ['SET', 'SET', 'SET']] - # models = [micro_model, meso_model] - # components = [[(0,1), (0,2), (1,2)], [(0,1), (0,2), (1,2)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - # fig.tight_layout() - # # plt.show() - # filename = "comparing_SET_off_diag" + ET + "_FW_"+ fw + ".svg" - # plt.savefig(saving_directory + filename, format = "svg") - - - # # Plots to compare observer with Favre-velocity (need to decompose structures first!) - # meso_model.decompose_structures() - # vars = [['U', 'U', 'U'], ['u_tilde', 'u_tilde', 'u_tilde']] - # models = [meso_model, meso_model] - # components = [[(0,), (1,), (2,)], [(0,), (1,), (2,)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - # fig.tight_layout() - # filename = "comparing_obs_VS_Favre" + ET + "_FW_" + fw + ".svg" - # plt.savefig(saving_directory + filename, format = 'svg') - - - ###################################################### - # CODE FOR SUMMARY PLOTS - ###################################################### - vars = [['BC', 'SET'], ['BC', 'SET',]] - models = [micro_model, meso_model] - components = [[(0,), (0,0)], [(0,), (0,0)]] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - fig.tight_layout() - filename = "ET_"+ ET+ "_fw_" + fw + "_scaling.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - - vars = [['BC', 'SET'], ['BC', 'SET',]] - models = [micro_model, meso_model] - components = [[(2,), (0,2)], [(2,), (0,2)]] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges_x, ranges_y, components_indices = components, diff_plot=True, rel_diff=True) - fig.tight_layout() - filename = "ET_"+ ET+ "_fw_" + fw + "_not_scaling.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - \ No newline at end of file + time_meso = meso_model.domain_vars['T'][1] #Change this if meso_grid is not set up using three central micro_slices + if time_meso != time_micro: + print("Slices of meso and micro model do not coincide. Careful!") + else: + print("Comparing data at same time-slice, hurray!") + + # PLOT SETTINGS + plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) + x_range = plot_ranges['x_range'] + y_range = plot_ranges['y_range'] + saving_directory = config['Directories']['figures_dir'] + visualizer = Plotter_2D([11.97, 8.36]) + + # FINALLY PLOTTING + vars = [['BC', 'SET'], ['BC', 'SET']] + models = [micro_model, meso_model] + components = [[(0,), (0,0)], [(0,), (0,0)]] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, diff_plot=True, rel_diff=True) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filter_width = str(config['Figures_name_param']['filter_width']) + filename = "/ET_"+ time_for_filename+ "_fw_" + filter_width + "_scaling.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + + vars = [['BC', 'SET'], ['BC', 'SET',]] + models = [micro_model, meso_model] + components = [[(2,), (0,2)], [(2,), (0,2)]] + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, diff_plot=True, rel_diff=True) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filter_width = str(config['Figures_name_param']['filter_width']) + filename = "/ET_"+ time_for_filename+ "_fw_" + filter_width + "_not_scaling.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') diff --git a/Proof_of_principle/visualizing_micro.py b/Proof_of_principle/visualizing_micro.py index 15413e5..dd90797 100644 --- a/Proof_of_principle/visualizing_micro.py +++ b/Proof_of_principle/visualizing_micro.py @@ -1,7 +1,8 @@ import sys # import os -sys.path.append('/home/tc2m23/Filtering/master_files/') -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +sys.path.append('../master_files/') +import configparser +import json import pickle from FileReaders import * @@ -10,51 +11,70 @@ if __name__ == '__main__': - # READING DATA - # directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" - ET=str(sys.argv[1]) - # MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET+ "_micro.pickle" - MicroModelLoadFile = directory + "HD_2D_ET_" + ET+ "_micro.pickle" + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + config = configparser.ConfigParser() + config.read(sys.argv[1]) - with open(MicroModelLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) + # LOADING MICRO DATA FROM HDF5 OR PICKLE + micro_from_hdf5 = True - # setting up ranges, picking central slice and choosing saving folder - ranges = [0.01, 0.99] - visualizer = Plotter_2D([11.97, 8.36]) - # num_snaps = 11 - num_snaps = 21 - central_slice = int(num_snaps/2) - print("Plotting data from central slice, num: {}".format(central_slice)) - # saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/" + if micro_from_hdf5: + hdf5_directory = config['Directories']['hdf5_dir'] + filenames = hdf5_directory + FileReader = METHOD_HDF5(filenames) + num_snaps = FileReader.num_files + micro_model = IdealHD_2D() + FileReader.read_in_data_HDF5_missing_xy(micro_model) + micro_model.setup_structures() + print('Finished reading micro data from hdf5, structures also set up.') + + else: + pickle_directory = config['Directories']['pickled_files_dir'] + micro_pickled_filename = config['Filenames']['micro_pickled_filename'] + MicroModelLoadFile = pickle_directory + micro_pickled_filename + with open(MicroModelLoadFile, 'rb') as filehandle: + micro_model = pickle.load(filehandle) + + # PLOT SETTINGS + plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) + x_range = plot_ranges['x_range'] + y_range = plot_ranges['y_range'] + num_snaps = micro_model.domain_vars['nt'] + central_slice_num = int(num_snaps/2.) + plot_time = micro_model.domain_vars['t'][central_slice_num] + + saving_directory = config['Directories']['figures_dir'] + visualizer = Plotter_2D([11.97, 8.36]) + # FINALLY, PLOTTING # Plotting the baryon current vars = ['BC', 'BC', 'BC', 'n', 'W', 'vx'] components = [(0,), (1,), (2,), (), (), ()] - fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) + fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components) fig.tight_layout() - # plt.show() - filename = "micro_ET_"+ET+"_BC.pdf" + time_for_filename = str(round(plot_time,2)) + filename = "/micro_ET_" + time_for_filename + "_BC.pdf" plt.savefig(saving_directory + filename, format = "pdf") # Plotting the stress energy tensor vars = ['SET', 'SET', 'SET', 'SET', 'SET', 'SET'] components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges, components_indices = components) + fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components) fig.tight_layout() - # plt.show() - filename = "micro_ET_"+ET+"_SET.pdf" + time_for_filename = str(round(plot_time,2)) + filename = "/micro_ET_" + time_for_filename + "_SET.pdf" plt.savefig(saving_directory + filename, format = "pdf") # plotting primitive quantities vars = ['W', 'vx', 'vy', 'n', 'p', 'e'] - fig=visualizer.plot_vars(micro_model, vars, micro_model.domain_vars['t'][central_slice], ranges, ranges) + fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range) fig.tight_layout() - # plt.show() - filename = "micro_ET_"+ET+"_prims.pdf" + time_for_filename = str(round(plot_time,2)) + filename = "/micro_ET_" + time_for_filename + "_prims.pdf" plt.savefig(saving_directory + filename, format = "pdf") diff --git a/Proof_of_principle/visualizing_obs.py b/Proof_of_principle/visualizing_obs.py index b8c3342..c13a999 100644 --- a/Proof_of_principle/visualizing_obs.py +++ b/Proof_of_principle/visualizing_obs.py @@ -2,9 +2,10 @@ import sys # import os -sys.path.append('/home/tc2m23/Filtering/master_files/') -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') +sys.path.append('../master_files/') import pickle +import configparser +import json from FileReaders import * from MicroModels import * @@ -13,35 +14,49 @@ if __name__ == '__main__': - # Reading micro & meso models (with obs) from pickle - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/pickled_files/400X400/" - ET=str(sys.argv[1]) - MesoModelLoadFile = directory + "observers_2dx_ET_" + ET+ ".pickle" - MicroModelLoadFile = directory + "IdealHD_2D_ET_" + ET+"_micro.pickle" + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) - with open(MicroModelLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - num_snaps = 11 + micro_model = meso_model.micro_model + + # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE + num_snaps = micro_model.domain_vars['nt'] central_slice_num = int(num_snaps/2.) time_micro = micro_model.domain_vars['t'][central_slice_num] - time_meso = meso_model.domain_vars['T'][0] + time_meso = meso_model.domain_vars['T'][1] #Change this if meso_grid is not set up using three central micro_slices if time_meso != time_micro: print("Slices of meso and micro model do not coincide. Careful!") else: print("Comparing data at same time-slice, hurray!") - saving_directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_2_2.5_3_3.5/10dx_after/Figures/400X400/" - # saving_directory = "./" + + # PLOT SETTINGS + plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) + x_range = plot_ranges['x_range'] + y_range = plot_ranges['y_range'] + saving_directory = config['Directories']['figures_dir'] + visualizer = Plotter_2D([11.97, 8.36]) + + # FINALLY, PLOTTING models = [micro_model, meso_model] vars = [['bar_vel', 'bar_vel', 'bar_vel'],['U', 'U', 'U']] components_indices= [[(0,),(1,),(2,)], [(0,), (1,), (2,)]] - ranges = [0.04, 0.97] - visualizer = Plotter_2D([11.97, 8.36]) - fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, ranges, ranges, components_indices=components_indices, diff_plot=True, rel_diff=True) + fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices=components_indices, diff_plot=True, rel_diff=True) fig.tight_layout() - filename="Meso_observers_2dx_ET_"+ET+".svg" - plt.savefig(saving_directory + filename, format="svg") \ No newline at end of file + time_for_filename = str(round(time_meso,2)) + box_len = str(config['Figures_name_param']['box_length']) + filename="/Meso_obs_" + box_len + "dx_ET_"+ time_for_filename +".pdf" + plt.savefig(saving_directory + filename, format="pdf") \ No newline at end of file From e6bce26b95bbcb7178848fc3f50b9c15f841fe84 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 10 Jan 2024 11:42:18 +0100 Subject: [PATCH 055/111] Configuration files for PoP scripts --- Proof_of_principle/config_meso.txt | 11 +++++++++++ Proof_of_principle/config_visualizing.txt | 15 +++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 Proof_of_principle/config_meso.txt create mode 100644 Proof_of_principle/config_visualizing.txt diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt new file mode 100644 index 0000000..27a6593 --- /dev/null +++ b/Proof_of_principle/config_meso.txt @@ -0,0 +1,11 @@ +[Directories] +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_1.0 +pickled_files_dir = ./tests + +[Meso_model_settings] +meso_grid = {"x_range": [0.2, 0.22], "y_range": [0.2, 0.22]} +filtering_options = {"box_len_ratio": 2.0, "filter_width_ratio": 8.0 } +n_cpus = 40 + +[Pickled_filename] +meso_pickled = /resHD_2D_ET_1.0 \ No newline at end of file diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt new file mode 100644 index 0000000..66738db --- /dev/null +++ b/Proof_of_principle/config_visualizing.txt @@ -0,0 +1,15 @@ +[Directories] +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_1.0 +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 +figures_dir = ./tests + +[Filenames] +meso_pickled_filename = /resHD_2D_ET_1.0_FW_8dx.pickle +micro_pickled_filename = /HD_2D_ET_1.0_micro.pickle + +[Plot_settings] +plot_ranges = {"x_range": [0.2, 0.5], "y_range": [0.2, 0.5]} + +[Figures_name_param] +filter_width = 8.0 +box_length = 2.0 From 2a1d8c6a1f30565e028a78b92334d1bf84ae47d1 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 12 Jan 2024 17:52:42 +0100 Subject: [PATCH 056/111] Fixing bug in SET decomposition --- Proof_of_principle/calibrating_meso.py | 100 ++++++--------- Proof_of_principle/config_calibrating.txt | 14 +++ Proof_of_principle/config_meso.txt | 8 +- .../visualizing_decomposition.py | 118 ++++++++++++++++++ master_files/MesoModels.py | 6 +- 5 files changed, 176 insertions(+), 70 deletions(-) create mode 100644 Proof_of_principle/config_calibrating.txt create mode 100644 Proof_of_principle/visualizing_decomposition.py diff --git a/Proof_of_principle/calibrating_meso.py b/Proof_of_principle/calibrating_meso.py index ed6f463..9dda819 100644 --- a/Proof_of_principle/calibrating_meso.py +++ b/Proof_of_principle/calibrating_meso.py @@ -1,10 +1,9 @@ import sys -import os -# sys.path.append('/Users/thomas/Dropbox/Work/projects/Filtering/master_files') -# sys.path.append('/home/hidra2/celora/Filtering/master_files/') -sys.path.append('/home/tc2m23/Filtering/master_files/') +sys.path.append('../master_files/') import pickle import time +import configparser +import json from FileReaders import * from MicroModels import * @@ -12,77 +11,52 @@ from MesoModels import * if __name__ == '__main__': - - ######################################################### - # CODE FOR TESTING SPEED UP OF PARALLEL MESO ROUTINES - ######################################################### - # TESTING ON HIDRA/IRIDIS - ET = str(sys.argv[1]) - # x_ranges = [0.2,0.4] - # y_ranges = [0.2, 0.4] - directory = "/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800/" - MesoModelPickleLoadFile = directory + "resHD_2D_ET_" + ET + "_FW_8dx.pickle" - # directory = "/home/hidra2/celora/Data/KHIrandom/ideal_HD/ET_1_3.5_0.5/20dx/pickled_files/800X800_smaller/" - # MesoModelPickleLoadFile = directory + "rHD_2D_ET_"+ET+"_FW_8dx_x="+ str(x_ranges)+ "_y="+str(y_ranges)+".pickle" - + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() - with open(MesoModelPickleLoadFile, 'rb') as filehandle: - meso_model=pickle.load(filehandle) - - num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] - print('Working with {} points\n'.format(num_points)) - - n_cpus=40 + config = configparser.ConfigParser() + config.read(sys.argv[1]) - # decomposing structures in serial - start_time = time.perf_counter() - meso_model.decompose_structures() - serial_time = time.perf_counter() - start_time - print('Meso structures decomposed in serial, time taken: {}\n'.format(serial_time)) + # LOADING THE FILTERED MESO MODEL + directory = config['Directories']['pickled_files_dir'] + meso_setup_filename = config['Filenames']['meso_setup_pickle_name'] + MesoModelLoadFile = directory + meso_setup_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) - # decomposing in parallel and comparing - start_time = time.perf_counter() - meso_model.decompose_structures_parallel(n_cpus=n_cpus) - parallel_time = time.perf_counter() - start_time - print('Meso structures decomposed in parallel, time taken: {}\n'.format(parallel_time)) - print('Speed-up factor: {}'.format(serial_time/parallel_time)) - # compute derivatives serial - start_time = time.perf_counter() - meso_model.calculate_derivatives() - serial_time = time.perf_counter() - start_time - print('Calculated derivatives in serial, time taken: {}\n'.format(serial_time)) + Nt = meso_model.domain_vars['Nt'] + Nx = meso_model.domain_vars['Nx'] + Ny = meso_model.domain_vars['Ny'] - # computing the closure ingredients - serial - start_time = time.perf_counter() - meso_model.closure_ingredients() - serial_time = time.perf_counter() - start_time - print('Closure ingredients calculated in serial, time taken: {}\n'.format(serial_time)) + num_points = Nt * Nx * Ny + print(f'Number of meso gridpoints: {num_points}') + + # DECOMPOSING AND COMPUTING CLOSURE INGREDIENTS + n_cpus = int(config['Meso_model_settings']['n_cpus']) - # computing the closure ingredients in parallel and comparing start_time = time.perf_counter() - meso_model.closure_ingredients_parallel(n_cpus = n_cpus) - parallel_time = time.perf_counter() - start_time - print('Closure ingredients calculated in parallel, time taken: {}\n'.format(parallel_time)) - print('Speed-up factor: {}'.format(serial_time/parallel_time)) + meso_model.decompose_structures_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Finished decomposing meso structures in parallel, time taken: {}'.format(time_taken)) - # computing the coefficients - serial start_time = time.perf_counter() - meso_model.EL_style_closure() - serial_time = time.perf_counter() - start_time - print('Dissipative coefficients calculated in serial, time taken: {}\n'.format(serial_time)) + meso_model.calculate_derivatives() + time_taken = time.perf_counter() - start_time + print('Finished computing derivatives (serial), time taken: {}'.format(time_taken)) - # computing the closure ingredients in parallel and comparing start_time = time.perf_counter() - meso_model.closure_ingredients_parallel(n_cpus = n_cpus) - parallel_time = time.perf_counter() - start_time - print('Dissipative coefficients calculated in parallel, time taken: {}\n'.format(parallel_time)) - print('Speed-up factor: {}'.format(serial_time/parallel_time)) + meso_model.closure_ingredients_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Finished computing the closure ingredients in parallel, time taken: {}'.format(time_taken)) + # NOW SAVING + filename = str(config['Filenames']['meso_decomposed_pickle_name']) + MesoModelDumpFile = directory + filename + with open(MesoModelDumpFile, 'wb') as filehandle: + pickle.dump(meso_model, filehandle) - ########################################################## - # CODE FOR TESTING THAT THE TASKS DO WHAT THEY'RE MEANT TO - ########################################################## - # werk \ No newline at end of file diff --git a/Proof_of_principle/config_calibrating.txt b/Proof_of_principle/config_calibrating.txt new file mode 100644 index 0000000..0582044 --- /dev/null +++ b/Proof_of_principle/config_calibrating.txt @@ -0,0 +1,14 @@ +[Directories] +pickled_files_dir = ./tests +figures_dir = ./tests/figures + + +[Filenames] +meso_setup_pickle_name = /rHD2d_setup_small_ranges_FW_8dx.pickle +meso_decomposed_pickle_name = /rHD2d_decomposed_small_ranges_FW_8dx.pickle + +[Meso_model_settings] +n_cpus = 40 + +[Plot_settings] +plot_ranges = {"x_range": [0.2, 0.6], "y_range": [0.1, 0.4]} \ No newline at end of file diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index 27a6593..9a2d5ee 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -3,9 +3,9 @@ hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/8 pickled_files_dir = ./tests [Meso_model_settings] -meso_grid = {"x_range": [0.2, 0.22], "y_range": [0.2, 0.22]} -filtering_options = {"box_len_ratio": 2.0, "filter_width_ratio": 8.0 } +meso_grid = {"x_range": [0.2, 0.6], "y_range": [0.1, 0.4]} +filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 8.0 } n_cpus = 40 -[Pickled_filename] -meso_pickled = /resHD_2D_ET_1.0 \ No newline at end of file +[Pickled_filenames] +meso_pickled = /rHD2d_setup_small_ranges \ No newline at end of file diff --git a/Proof_of_principle/visualizing_decomposition.py b/Proof_of_principle/visualizing_decomposition.py new file mode 100644 index 0000000..8f3d83c --- /dev/null +++ b/Proof_of_principle/visualizing_decomposition.py @@ -0,0 +1,118 @@ +import sys +sys.path.append('../master_files/') +import pickle +import time +import configparser +import json + +from FileReaders import * +from MicroModels import * +from Filters import * +from MesoModels import * +from Visualization import * + +if __name__ == '__main__': + + + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING THE DECOMPOSED MESO MODEL + directory = config['Directories']['pickled_files_dir'] + meso_decomposed_filename = config['Filenames']['meso_decomposed_pickle_name'] + MesoModelLoadFile = directory + meso_decomposed_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + + ########################################################## + # CODE FOR TESTING THE ALGEBRAIC DECOMPOSITION OF SET + ########################################################## + + # WORKING OUT THE ALGEBRAIC CONSTRAINTS + heat_flux = meso_model.meso_vars['q_res'] + aniso_stresses = meso_model.meso_vars['pi_res'] + favre_vel = meso_model.meso_vars['u_tilde'] + + Nt = meso_model.domain_vars['Nt'] + Nx = meso_model.domain_vars['Nx'] + Ny = meso_model.domain_vars['Ny'] + + trace_free_constr = np.zeros((Nt, Nx, Ny)) + stress_ort_1st_constr = np.zeros((Nt, Nx, Ny)) + stress_ort_2ns_constr = np.zeros((Nt, Nx, Ny)) + heat_flux_ort_constr = np.zeros((Nt, Nx, Ny)) + + metric = meso_model.metric + + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + favre_vel_covector = np.einsum('ij,j', metric, favre_vel[h,i,j]) + + trace_free_constr[h,i,j] = np.einsum('ij,ji->', aniso_stresses[h,i,j], metric) + stress_ort_1st_constr[h,i,j] = np.einsum('i,ij->', favre_vel_covector, aniso_stresses[h,i,j]) + stress_ort_2ns_constr[h,i,j] = np.einsum('ij,j->', aniso_stresses[h,i,j], favre_vel_covector) + heat_flux_ort_constr[h,i,j] = np.einsum('i,i->', favre_vel_covector, heat_flux[h,i,j]) + + # Now choosing one particular slice: it should not matter which + slice_num = 0 + trace_free_check = trace_free_constr[slice_num,:,:] + stress_ort_1st_check = stress_ort_1st_constr[slice_num,:,:] + stress_ort_2nd_check = stress_ort_2ns_constr[slice_num,:,:] + heat_flux_ort_check = stress_ort_1st_constr[slice_num,:,:] + + # NOW PLOTTING ALL THESE QUANTITIES + fig, axes = plt.subplots(2,2) + + im = axes[0,0].imshow(trace_free_check) + divider = make_axes_locatable(axes[0,0]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[0,1].imshow(stress_ort_1st_check) + divider = make_axes_locatable(axes[0,1]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[1,0].imshow(stress_ort_2nd_check) + divider = make_axes_locatable(axes[1,0]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[1,1].imshow(heat_flux_ort_check) + divider = make_axes_locatable(axes[1,1]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + # Setting titles and labelling axes + axes[0,0].set_title(r'$\pi^a_a$') + axes[0,1].set_title(r'$u_a \pi^{ab}$') + axes[1,0].set_title(r'$\pi^{ab}u_{b}$') + axes[1,1].set_title(r'$q^a u_a$') + fig.suptitle('Checking algebraic constraints in decomposing filtered SET') + fig.tight_layout() + + for i in range(2): + for j in range(2): + axes[i,j].set_xlabel(r'$y$') + axes[i,j].set_ylabel(r'$x$') + + + saving_directory = config['Directories']['figures_dir'] + figurename = '/SET_algebraic_constraints.pdf' + plt.savefig(saving_directory + figurename, format = 'pdf') + + + ########################################################## + # CODE FOR TESTING THE ALGEBRAIC DECOMPOSITION OF SET + ########################################################## + + # shear = meso_model.domain_vars['shear_tilde'] + # acceleration = meso_model.domain_vars['acc_tilde'] \ No newline at end of file diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 34607b7..421ad0f 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1702,9 +1702,9 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): # Computing the SET decomposition at each point eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, metric, metric, SET) h_ab = np.einsum('ij,jk->ik', metric + np.einsum('i,j->ij', u_t, u_t), metric) # This is a rank (1,1) tensor, i.e. a real projector. - q_a = np.einsum('ij,jk,kl,l->i', h_ab, SET, metric, u_t) # There might be missing a minus sign here - s_ab = np.einsum('ij,kl,jl->ik',h_ab, h_ab, SET) - s = np.einsum('ii',s_ab) + q_a = - np.einsum('ij,jk,kl,l->i', h_ab, SET, metric, u_t) + s_ab = np.einsum('ij,kl,jl->ik', h_ab, h_ab, SET) + s = np.einsum('ij,ji->', s_ab, metric) p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) # Additional quantities needed for residuals From b82d45d3d71e6641ad20dfc9b6f8930cd2710f2b Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 15 Jan 2024 17:47:43 +0100 Subject: [PATCH 057/111] Testing and fixing closure_ingredient_task --- master_files/MesoModels.py | 43 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 421ad0f..c6d8de5 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1694,6 +1694,8 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): metric[0,0] = -1. metric[1,1] = metric[2,2] = 1. + spatial_dims = 2 + # Computing the Favre density and velocity n_t = np.sqrt(-Base.Mink_dot(BC, BC)) u_t = np.multiply(1./ n_t, BC) @@ -1709,7 +1711,7 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): # Additional quantities needed for residuals Pi_res = s - p_t - s_ab_tracefree = s_ab - np.multiply(s/2., metric + np.einsum('i,j->ij', u_t, u_t)) + s_ab_tracefree = s_ab - np.multiply(s/spatial_dims, metric + np.einsum('i,j->ij', u_t, u_t)) T_t = p_t/n_t EOS_res = p_filt - p_t @@ -1967,25 +1969,42 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): metric[1,1] = metric[2,2] = 1 # CLOSURE INGREDIENTS: FAVRE OBS DERIVATIVE DECOMPOSITION - WORKING WITH (2,0) + # The decomposition below is exact if certain algebraic constraints are satisfied, which they won't be + # due to numerical errors. Nonetheless, it appears the only quantity you need to correct is the acceleration u_t_cov = np.einsum('ij,j->i', metric, u_t) nabla_u = np.einsum('ij,jk->ik', metric, nabla_u) #this is a rank (2,0) tensor acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector - # The following two quantities should be identically zero, but they won't be due to numerical errors. - # Computing them and removing to project velocity gradients - unit_norm_violation = np.einsum('ij,j', nabla_u, u_t_cov) #vector acc_orthogonality_violation = np.einsum('i,i->', u_t_cov, acc_t) - Daub = nabla_u + np.einsum('i,j->ij', u_t, acc_t) + np.einsum('i,j->ij', unit_norm_violation, u_t) +\ - np.multiply(acc_orthogonality_violation, np.einsum('i,j->ij', u_t, u_t)) #Should be a (2,0) tensor - h_ab = metric + np.einsum('i,j->ij', u_t, u_t) - exp_t = np.einsum('ii->',Daub) - shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( 1/spatial_dims * exp_t, h_ab) - vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + acc_t = acc_t + acc_orthogonality_violation * u_t - # CLOSURE INGREDIENTS: HEAT FLUX projector = np.zeros((3,3)) np.fill_diagonal(projector, 1) - projector += np.einsum('i,j->ij',u_t_cov, u_t) + projector += np.einsum('i,j->ij',u_t, u_t_cov) + h_ab = metric + np.einsum('i,j->ij', u_t, u_t) #Rank (2,0) tensor + + Daub = np.einsum('ij,kl,jl->ik', projector, projector, nabla_u) + exp_t = np.einsum('ij,ji->', Daub, metric) + shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( exp_t/ spatial_dims, h_ab) + vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + + # The following lines should be used in case the deviations from the algebraic constraints being zero + # become too large + # Daub = np.einsum('ij,kl,jl->ik', projector, projector, nabla_u) + # orthogonality_violation_1 = np.einsum('i,ij->j', u_t_cov, Daub) + # orthogonality_violation_2 = np.einsum('ij,j->i', Daub, u_t_cov) + # orthogonality_violation_3 = np.einsum('i,j,ij->', u_t_cov, u_t_cov, Daub) + # Daub = Daub + np.einsum('i,j->ij', u_t, orthogonality_violation_1) + np.einsum('i,j->ij', orthogonality_violation_2, u_t) + \ + # np.multiply(orthogonality_violation_3, np.einsum('i,j->ij', u_t, u_t) ) + # exp_t = np.einsum('ij,ji->', Daub, metric) + # shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( exp_t/ spatial_dims, h_ab) + # vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) + + + # CLOSURE INGREDIENTS: HEAT FLUX + projector = np.zeros((3,3)) + np.fill_diagonal(projector, 1) + projector += np.einsum('i,j->ij',u_t_cov, u_t) #This is different from the projector above: look at indices! DaT = np.einsum('ij,j->i', projector, nabla_T) Theta_tilde = DaT + np.multiply(T_t, np.einsum('ij,j->i', metric, acc_t)) From c728f32533b98019e2406bbe25a6340065373aee Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 15 Jan 2024 17:54:23 +0100 Subject: [PATCH 058/111] Cleaning PoP repo --- Proof_of_principle/calibrating_meso.py | 62 --------- Proof_of_principle/config_calibrating.txt | 14 --- .../visualizing_decomposition.py | 118 ------------------ 3 files changed, 194 deletions(-) delete mode 100644 Proof_of_principle/calibrating_meso.py delete mode 100644 Proof_of_principle/config_calibrating.txt delete mode 100644 Proof_of_principle/visualizing_decomposition.py diff --git a/Proof_of_principle/calibrating_meso.py b/Proof_of_principle/calibrating_meso.py deleted file mode 100644 index 9dda819..0000000 --- a/Proof_of_principle/calibrating_meso.py +++ /dev/null @@ -1,62 +0,0 @@ -import sys -sys.path.append('../master_files/') -import pickle -import time -import configparser -import json - -from FileReaders import * -from MicroModels import * -from Filters import * -from MesoModels import * - -if __name__ == '__main__': - - # READING SIMULATION SETTINGS FROM CONFIG FILE - if len(sys.argv) == 1: - print(f"You must pass the configuration file for the simulations.") - raise Exception() - - config = configparser.ConfigParser() - config.read(sys.argv[1]) - - # LOADING THE FILTERED MESO MODEL - directory = config['Directories']['pickled_files_dir'] - meso_setup_filename = config['Filenames']['meso_setup_pickle_name'] - MesoModelLoadFile = directory + meso_setup_filename - with open(MesoModelLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) - - - Nt = meso_model.domain_vars['Nt'] - Nx = meso_model.domain_vars['Nx'] - Ny = meso_model.domain_vars['Ny'] - - num_points = Nt * Nx * Ny - print(f'Number of meso gridpoints: {num_points}') - - # DECOMPOSING AND COMPUTING CLOSURE INGREDIENTS - n_cpus = int(config['Meso_model_settings']['n_cpus']) - - start_time = time.perf_counter() - meso_model.decompose_structures_parallel(n_cpus) - time_taken = time.perf_counter() - start_time - print('Finished decomposing meso structures in parallel, time taken: {}'.format(time_taken)) - - start_time = time.perf_counter() - meso_model.calculate_derivatives() - time_taken = time.perf_counter() - start_time - print('Finished computing derivatives (serial), time taken: {}'.format(time_taken)) - - start_time = time.perf_counter() - meso_model.closure_ingredients_parallel(n_cpus) - time_taken = time.perf_counter() - start_time - print('Finished computing the closure ingredients in parallel, time taken: {}'.format(time_taken)) - - # NOW SAVING - filename = str(config['Filenames']['meso_decomposed_pickle_name']) - MesoModelDumpFile = directory + filename - with open(MesoModelDumpFile, 'wb') as filehandle: - pickle.dump(meso_model, filehandle) - - diff --git a/Proof_of_principle/config_calibrating.txt b/Proof_of_principle/config_calibrating.txt deleted file mode 100644 index 0582044..0000000 --- a/Proof_of_principle/config_calibrating.txt +++ /dev/null @@ -1,14 +0,0 @@ -[Directories] -pickled_files_dir = ./tests -figures_dir = ./tests/figures - - -[Filenames] -meso_setup_pickle_name = /rHD2d_setup_small_ranges_FW_8dx.pickle -meso_decomposed_pickle_name = /rHD2d_decomposed_small_ranges_FW_8dx.pickle - -[Meso_model_settings] -n_cpus = 40 - -[Plot_settings] -plot_ranges = {"x_range": [0.2, 0.6], "y_range": [0.1, 0.4]} \ No newline at end of file diff --git a/Proof_of_principle/visualizing_decomposition.py b/Proof_of_principle/visualizing_decomposition.py deleted file mode 100644 index 8f3d83c..0000000 --- a/Proof_of_principle/visualizing_decomposition.py +++ /dev/null @@ -1,118 +0,0 @@ -import sys -sys.path.append('../master_files/') -import pickle -import time -import configparser -import json - -from FileReaders import * -from MicroModels import * -from Filters import * -from MesoModels import * -from Visualization import * - -if __name__ == '__main__': - - - - # READING SIMULATION SETTINGS FROM CONFIG FILE - if len(sys.argv) == 1: - print(f"You must pass the configuration file for the simulations.") - raise Exception() - - config = configparser.ConfigParser() - config.read(sys.argv[1]) - - # LOADING THE DECOMPOSED MESO MODEL - directory = config['Directories']['pickled_files_dir'] - meso_decomposed_filename = config['Filenames']['meso_decomposed_pickle_name'] - MesoModelLoadFile = directory + meso_decomposed_filename - with open(MesoModelLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) - - - ########################################################## - # CODE FOR TESTING THE ALGEBRAIC DECOMPOSITION OF SET - ########################################################## - - # WORKING OUT THE ALGEBRAIC CONSTRAINTS - heat_flux = meso_model.meso_vars['q_res'] - aniso_stresses = meso_model.meso_vars['pi_res'] - favre_vel = meso_model.meso_vars['u_tilde'] - - Nt = meso_model.domain_vars['Nt'] - Nx = meso_model.domain_vars['Nx'] - Ny = meso_model.domain_vars['Ny'] - - trace_free_constr = np.zeros((Nt, Nx, Ny)) - stress_ort_1st_constr = np.zeros((Nt, Nx, Ny)) - stress_ort_2ns_constr = np.zeros((Nt, Nx, Ny)) - heat_flux_ort_constr = np.zeros((Nt, Nx, Ny)) - - metric = meso_model.metric - - for h in range(Nt): - for i in range(Nx): - for j in range(Ny): - favre_vel_covector = np.einsum('ij,j', metric, favre_vel[h,i,j]) - - trace_free_constr[h,i,j] = np.einsum('ij,ji->', aniso_stresses[h,i,j], metric) - stress_ort_1st_constr[h,i,j] = np.einsum('i,ij->', favre_vel_covector, aniso_stresses[h,i,j]) - stress_ort_2ns_constr[h,i,j] = np.einsum('ij,j->', aniso_stresses[h,i,j], favre_vel_covector) - heat_flux_ort_constr[h,i,j] = np.einsum('i,i->', favre_vel_covector, heat_flux[h,i,j]) - - # Now choosing one particular slice: it should not matter which - slice_num = 0 - trace_free_check = trace_free_constr[slice_num,:,:] - stress_ort_1st_check = stress_ort_1st_constr[slice_num,:,:] - stress_ort_2nd_check = stress_ort_2ns_constr[slice_num,:,:] - heat_flux_ort_check = stress_ort_1st_constr[slice_num,:,:] - - # NOW PLOTTING ALL THESE QUANTITIES - fig, axes = plt.subplots(2,2) - - im = axes[0,0].imshow(trace_free_check) - divider = make_axes_locatable(axes[0,0]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - - im = axes[0,1].imshow(stress_ort_1st_check) - divider = make_axes_locatable(axes[0,1]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - - im = axes[1,0].imshow(stress_ort_2nd_check) - divider = make_axes_locatable(axes[1,0]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - - im = axes[1,1].imshow(heat_flux_ort_check) - divider = make_axes_locatable(axes[1,1]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') - - # Setting titles and labelling axes - axes[0,0].set_title(r'$\pi^a_a$') - axes[0,1].set_title(r'$u_a \pi^{ab}$') - axes[1,0].set_title(r'$\pi^{ab}u_{b}$') - axes[1,1].set_title(r'$q^a u_a$') - fig.suptitle('Checking algebraic constraints in decomposing filtered SET') - fig.tight_layout() - - for i in range(2): - for j in range(2): - axes[i,j].set_xlabel(r'$y$') - axes[i,j].set_ylabel(r'$x$') - - - saving_directory = config['Directories']['figures_dir'] - figurename = '/SET_algebraic_constraints.pdf' - plt.savefig(saving_directory + figurename, format = 'pdf') - - - ########################################################## - # CODE FOR TESTING THE ALGEBRAIC DECOMPOSITION OF SET - ########################################################## - - # shear = meso_model.domain_vars['shear_tilde'] - # acceleration = meso_model.domain_vars['acc_tilde'] \ No newline at end of file From c7c558b382cc4fab398e9038c94faa4c7d559f50 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 15 Jan 2024 18:00:57 +0100 Subject: [PATCH 059/111] Testing and fixing closure_igredients_task --- master_files/MesoModels.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index c6d8de5..8f2ff13 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1970,7 +1970,8 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): # CLOSURE INGREDIENTS: FAVRE OBS DERIVATIVE DECOMPOSITION - WORKING WITH (2,0) # The decomposition below is exact if certain algebraic constraints are satisfied, which they won't be - # due to numerical errors. Nonetheless, it appears the only quantity you need to correct is the acceleration + # due to numerical errors. + # However, it appears the only quantity to be corrected is the acceleration u_t_cov = np.einsum('ij,j->i', metric, u_t) nabla_u = np.einsum('ij,jk->ik', metric, nabla_u) #this is a rank (2,0) tensor acc_t = np.einsum('i,ij', u_t_cov, nabla_u) # vector @@ -1988,8 +1989,7 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): shear_t = np.multiply(1/2., Daub + np.einsum('ij->ji', Daub)) - np.multiply( exp_t/ spatial_dims, h_ab) vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) - # The following lines should be used in case the deviations from the algebraic constraints being zero - # become too large + # #The following should be used if the violation of the algebraic constraints become too large # Daub = np.einsum('ij,kl,jl->ik', projector, projector, nabla_u) # orthogonality_violation_1 = np.einsum('i,ij->j', u_t_cov, Daub) # orthogonality_violation_2 = np.einsum('ij,j->i', Daub, u_t_cov) From d0431a00d57868c5e2c4d61731ff541e38a36048 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 15 Jan 2024 18:05:58 +0100 Subject: [PATCH 060/111] Update to PoP mainfile: adding tested routines --- Proof_of_principle/config_meso.txt | 2 +- Proof_of_principle/pickling_meso_setup.py | 22 +++++++++++++++++++++- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index 9a2d5ee..9292f2b 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -8,4 +8,4 @@ filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 8.0 } n_cpus = 40 [Pickled_filenames] -meso_pickled = /rHD2d_setup_small_ranges \ No newline at end of file +meso_pickled = /rHD2d_decomposed \ No newline at end of file diff --git a/Proof_of_principle/pickling_meso_setup.py b/Proof_of_principle/pickling_meso_setup.py index f720a3e..5c3f7e4 100644 --- a/Proof_of_principle/pickling_meso_setup.py +++ b/Proof_of_principle/pickling_meso_setup.py @@ -55,7 +55,7 @@ num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] print('Number of points: {}'.format(num_points)) - # FINDING THE OBSERVERS AND FILTERING, THEN PICKLE SAVE + # FINDING THE OBSERVERS AND FILTERING n_cpus = int(config['Meso_model_settings']['n_cpus']) start_time = time.perf_counter() @@ -68,6 +68,26 @@ time_taken = time.perf_counter() - start_time print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) + + # DECOMPOSING AND CALCULATING THE CLOSURE INGREDIENTS + start_time = time.perf_counter() + meso_model.decompose_structures_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Finished decomposing meso structures in parallel, time taken: {}'.format(time_taken)) + + + start_time = time.perf_counter() + meso_model.calculate_derivatives() + time_taken = time.perf_counter() - start_time + print('Finished computing derivatives (serial), time taken: {}'.format(time_taken)) + + start_time = time.perf_counter() + meso_model.closure_ingredients_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Finished computing the closure ingredients in parallel, time taken: {}'.format(time_taken)) + + + # PICKLING THE CLASS INSTANCE FOR FUTURE USE pickled_directory = config['Directories']['pickled_files_dir'] filename = config['Pickled_filenames']['meso_pickled'] + "_FW_" +str(int(filter_width_ratio)) + "dx" + ".pickle" MesoModelPickleDumpFile = pickled_directory + filename From 1e8a916a0ef3068ea6019a625b5f02222a880d97 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 15 Jan 2024 18:25:53 +0100 Subject: [PATCH 061/111] Update to PoP mainfile: adding tested routines --- Proof_of_principle/config_meso.txt | 10 +++++----- .../{pickling_meso_setup.py => pickling_meso.py} | 0 2 files changed, 5 insertions(+), 5 deletions(-) rename Proof_of_principle/{pickling_meso_setup.py => pickling_meso.py} (100%) diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index 9292f2b..a569c04 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -1,11 +1,11 @@ [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_1.0 -pickled_files_dir = ./tests +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/1600X1600/METHOD_output/ET_3.0 +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/1600X1600/pickled_files [Meso_model_settings] -meso_grid = {"x_range": [0.2, 0.6], "y_range": [0.1, 0.4]} -filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 8.0 } +meso_grid = {"x_range": [0.02, 0.98], "y_range": [0.2, 0.98]} +filtering_options = {"box_len_ratio": 8.0, "filter_width_ratio": 8.0 } n_cpus = 40 [Pickled_filenames] -meso_pickled = /rHD2d_decomposed \ No newline at end of file +meso_pickled = /rHD2d_decomposed_ET_3.0 \ No newline at end of file diff --git a/Proof_of_principle/pickling_meso_setup.py b/Proof_of_principle/pickling_meso.py similarity index 100% rename from Proof_of_principle/pickling_meso_setup.py rename to Proof_of_principle/pickling_meso.py From a40b14dab5f59fb7c6c5ba9949d1c5f7829f81ae Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 15 Jan 2024 18:35:26 +0100 Subject: [PATCH 062/111] Removing unneeded files from master --- master_files/py_submit_parallel.slurm | 33 --------------------------- 1 file changed, 33 deletions(-) delete mode 100644 master_files/py_submit_parallel.slurm diff --git a/master_files/py_submit_parallel.slurm b/master_files/py_submit_parallel.slurm deleted file mode 100644 index 8fd07a4..0000000 --- a/master_files/py_submit_parallel.slurm +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#################################### -# JOB INFO: single node allocating max core available -#################################### - -#SBATCH --output=outputs/slurm-%A.out -#SBATCH --nodes=1 -#SBATCH --ntasks=1 #number of MPI processes -#SBATCH --cpus-per-task=40 #number of threads per MPI process -#SBATCH --time=00:30:00 - -#SBATCH --mail-type=begin -#SBATCH --mail-type=fail -#SBATCH --mail-type=end -#SBATCH --mail-user=celora@ice.csic.es - - -#################################### -# LOADING THE CONDA ENVIRONMENT -#################################### - -module load conda/py3-latest -# I get warnings to use 'conda deactivate' instead but that doesn't work -source deactivate -conda activate myenv - - -#################################### -# LAUNCHING THE JOBS -#################################### - -# python3 -u Filters.py -python3 -u MesoModels.py From f6f02e6c176b4e12c55eda526cded44c3b90dab9 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 22 Jan 2024 17:35:01 +0100 Subject: [PATCH 063/111] Update: plot_vars_models_comparison now uses automatically raw data with exception of difference plots (relative or absolute) --- master_files/Visualization.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 348cc9d..11f0d05 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -92,7 +92,7 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me compatible = interp_dims != None and len(interp_dims) ==2 if not compatible: print('Error: when using (linearly spaced) interpolated data, you must' +\ - 'specify # points in each direction! Exiting.') + 'specify # points in each spatial direction! Exiting.') return None nx, ny = interp_dims[:] @@ -220,9 +220,10 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int components_indices=None, diff_plot=False, rel_diff=False): """ Plot variables from a number of models. If 2 models are given, a third - plot of the difference can be added too. - If 'raw_data' is chosen as the method, will check to see if the data points in the model - lie at the same coordinates, which they must. + plot of the difference (relative or absolute) can be added too. + The method refers to the difference plot(s): for models with different grid spacings, data must + be extracted via interpolation, so setting method = 'interpolate'. + In any other case, should set method = 'raw_data'. Parameters ---------- @@ -296,8 +297,8 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int fig, axes = plt.subplots(n_rows, n_cols, squeeze=False, figsize=figsize) for j in range(len(models)): - for i in range(n_rows): - data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, interp_dims, method, components_indices[j][i]) + for i in range(n_rows): + data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, None, 'raw_data', components_indices[j][i]) im=axes[i,j].imshow(data_to_plot, extent=extent) divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) From 2edf79f4676f3fdc73dec3d11c3fff82e99b920e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 1 Feb 2024 16:36:16 +0100 Subject: [PATCH 064/111] Implementing a user-defined symlog norm --- master_files/Visualization.py | 118 +++++++--- master_files/system/BaseFunctionality.py | 274 ++++++++++++++++++++++- 2 files changed, 345 insertions(+), 47 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 11f0d05..fbee561 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -36,7 +36,7 @@ def __init__(self, screen_size = [11.97, 8.36]): self.screen_size = np.array(screen_size) - def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, method= 'raw_data', component_indices=()): + def get_var_data(self, model, var_str, t, x_range, y_range, component_indices=(), method= 'raw_data', interp_dims=None): """ Retrieves the required data from model to plot a variable defined by var_str over coordinates t, x_range, y_range, either from the model's @@ -53,13 +53,13 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me defines range of x coordinates within foliation. y_range : list of 2 floats: y_start and y_end defines range of y coordinates within foliation. - interp_dims : tuple of integers - defines the number of points to interpolate at in x and y directions. - method : str - currently either raw_data or interpolate. component_indices : tuple the indices of the component to pick out if the variable is a vector/tensor. - + method : str + currently either raw_data or interpolate. + interp_dims : tuple of integers + defines the number of points to interpolate at in x and y directions. + Returns ------- data_to_plot : numpy array of floats @@ -136,7 +136,8 @@ def get_var_data(self, model, var_str, t, x_range, y_range, interp_dims=None, me print(f'{var_str} is not a plottable variable of the model!') return None - def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', components_indices=None): #[()]): + def plot_vars(self, model, var_strs, t, x_range, y_range, components_indices=None, method='raw_data', interp_dims=None, + norms=None, cmaps=None): """ Plot variable(s) from model, defined by var_strs, over coordinates t, x_range, y_range. Either from the model's raw data or by interpolating @@ -153,16 +154,28 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth defines range of x coordinates within foliation. y_range : list of 2 floats: y_start and y_end defines range of y coordinates within foliation. - interp_dims : tuple of integers - defines the number of points to interpolate at in x and y directions. - method : str - currently either raw_data or . components_indices : list of tuple(s) the indices of the components to pick out if the variables are vectors/tensors. Can be omitted if all variables are scalars, otherwise must be a list of tuples matching the length of var_strs that corresponds with each variable in the list. + method : str + currently either raw_data or interpolate + interp_dims : tuple of integers + defines the number of points to interpolate at in x and y directions. + norms = list of strs + each entry of the list is passed as option to imshow as norm=str + when plotting the corresponding var + + valid choices include all the standard norms like log or symlog, + and 'mysymlog' which is implemented in BaseFunctionality + cmaps = list of strs + each entry of the list is passed to imshow as cmap=cmaps[i] + when plotting the corresponding var + + valid choices are all the std ones + Output ------- Plots the (2D) data using imshow. Note that the plotting data's coordinates @@ -172,6 +185,19 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth """ n_plots = len(var_strs) + + if norms == None: + norms = [None for _ in range(len(var_strs))] + elif len(var_strs)!= len(norms): + print('The norms provided are not the same number as the variables: setting these to auto') + norms = [None for _ in range(len(var_strs))] + + if cmaps == None: + cmaps = [None for _ in range(len(var_strs))] + elif len(var_strs)!= len(cmaps): + print('The norms provided are not the same number as the variables: setting these to auto') + cmaps = [None for _ in range(len(var_strs))] + n_rows, n_cols = self.plot_vars_subplots_dims[n_plots] # Block to determine adaptively the figsize. @@ -196,19 +222,33 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth print('No list of components indices passed: setting this to an empty list.') components_indices = [ () for _ in range(len(var_strs))] - for var_str, component_indices, ax in zip(var_strs, components_indices, axes): + for i, (var_str, component_indices, ax) in enumerate(zip(var_strs, components_indices, axes)): # data_to_plot, points = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] - data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, interp_dims, method, component_indices) - im = ax.imshow(data_to_plot, extent=extent) - divider = make_axes_locatable(ax) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') + data_to_plot, extent = self.get_var_data(model, var_str, t, x_range, y_range, component_indices, method, interp_dims) + + if norms[i] == 'mysymlog': + ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) + data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) + mynorm = MyThreeNodesNorm(nodes) + im = ax.imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[i]) + divider = make_axes_locatable(ax) + cax = divider.append_axes('right', size='5%', pad=0.05) + cbar = fig.colorbar(im, cax=cax, orientation='vertical') + cbar.set_ticks(ticks) + cbar.ax.set_yticklabels(labels) + + elif norms[i] != 'mysymlog': + im = ax.imshow(data_to_plot, extent=extent, norm=norms[i], cmap=cmaps[i]) + divider = make_axes_locatable(ax) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + title = var_str if component_indices != (): title = title + ", {}-component".format(component_indices) ax.set_title(title) - ax.set_xlabel(r'$y$') # WHY? + ax.set_xlabel(r'$y$') ax.set_ylabel(r'$x$') fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), t), fontsize = 12) @@ -216,8 +256,8 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, interp_dims=None, meth # plt.show() return fig - def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, interp_dims=None, method='raw_data', \ - components_indices=None, diff_plot=False, rel_diff=False): + def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, components_indices=None, method='raw_data',\ + interp_dims=None, diff_plot=False, rel_diff=False, norms=None): """ Plot variables from a number of models. If 2 models are given, a third plot of the difference (relative or absolute) can be added too. @@ -236,17 +276,21 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int defines range of x coordinates within foliation. y_range : list of 2 floats: y_start and y_end defines range of y coordinates within foliation. - interp_dims : tuple of integers - defines the number of points to interpolate at in x and y directions. - method : str - currently either raw_data or interpolate. component_indices : list of list of tuples each tuple identifies the indices of the component to pick out if the variable is a vector/tensor. + method : str + currently either raw_data or interpolate. + interp_dims : tuple of integers + defines the number of points to interpolate at in x and y directions. diff_plot: bool Whether to add a column to show difference between models rel_diff: bool Whether to plot the absolute or relative difference between models + norms = list of strs + each entry of the list is passed as option to imshow as norm=str + the list should be as long as the number of vars passed + useful options are log or symlog Output ------- @@ -264,6 +308,12 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int if len(var_strs[i]) != num_vars_1st_model: print("The number of variables per model must be the same. Exiting.") return None + + if len(var_strs)!= len(norms): + print('The norms provided are not the same number as the variables: setting these to auto') + norms = [None for _ in range(len(var_strs))] + elif norms == None: + norms = [None for _ in range(len(var_strs))] n_cols = len(models) if diff_plot: @@ -289,17 +339,13 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int for i in range(len(models)): components_indices.append(empty_tuple_components) - # Block to determine adaptively the figsize. - # figsize = np.array([1,2/3.]) * self.screen_size figsize = self.screen_size - # gridspec_dict = {'wspace' : 0.3, 'hspace': 0.3} - # fig, axes = plt.subplots(n_rows, n_cols, squeeze=False, gridspec_kw=gridspec_dict, figsize=figsize) fig, axes = plt.subplots(n_rows, n_cols, squeeze=False, figsize=figsize) for j in range(len(models)): for i in range(n_rows): - data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, None, 'raw_data', components_indices[j][i]) - im=axes[i,j].imshow(data_to_plot, extent=extent) + data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, components_indices[j][i], 'raw_data', None) + im=axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[i]) divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -314,13 +360,13 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int if diff_plot and len(models)==2: try: for i in range(len(var_strs[0])): - data1, extent1 = self.get_var_data(models[0], var_strs[0][i], t, x_range, y_range, interp_dims, method, components_indices[0][i]) - data2, extent2 = self.get_var_data(models[1], var_strs[1][i], t, x_range, y_range, interp_dims, method, components_indices[1][i]) + data1, extent1 = self.get_var_data(models[0], var_strs[0][i], t, x_range, y_range, components_indices[0][i], method, interp_dims) + data2, extent2 = self.get_var_data(models[1], var_strs[1][i], t, x_range, y_range, components_indices[1][i], method, interp_dims) if extent1 != extent2: print("Cannot plot the difference between the vars: data not aligned.") continue data_to_plot = data1 - data2 - im = axes[i,2].imshow(data_to_plot, extent=extent1) + im = axes[i,2].imshow(data_to_plot, extent=extent1, norm=norms[i]) divider = make_axes_locatable(axes[i,2]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -335,8 +381,8 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int if rel_diff and len(models)==2: try: for i in range(len(var_strs[0])): - data1, extent1 = self.get_var_data(models[0], var_strs[0][i], t, x_range, y_range, interp_dims, method, components_indices[0][i]) - data2, extent2 = self.get_var_data(models[1], var_strs[1][i], t, x_range, y_range, interp_dims, method, components_indices[1][i]) + data1, extent1 = self.get_var_data(models[0], var_strs[0][i], t, x_range, y_range, components_indices[0][i], method, interp_dims) + data2, extent2 = self.get_var_data(models[1], var_strs[1][i], t, x_range, y_range, components_indices[1][i], method, interp_dims) if extent1 != extent2: print("Cannot plot the difference between the vars: data not aligned.") continue @@ -346,7 +392,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, int column = 3 else: column = 2 - im = axes[i,column].imshow(data_to_plot, extent=extent1) + im = axes[i,column].imshow(data_to_plot, extent=extent1, norm=norms[i]) divider = make_axes_locatable(axes[i,column]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') diff --git a/master_files/system/BaseFunctionality.py b/master_files/system/BaseFunctionality.py index c410d74..aaea44b 100644 --- a/master_files/system/BaseFunctionality.py +++ b/master_files/system/BaseFunctionality.py @@ -5,22 +5,17 @@ @author: mjh1n20 """ -from multiprocessing import Process, Pool -import os +# from multiprocessing import Process, Pool import numpy as np -#import matplotlib.pyplot as plt -import pickle -from timeit import default_timer as timer -import h5py -from scipy.interpolate import interpn -from scipy.optimize import root, minimize -#from mpl_toolkits.mplot3d import Axes3D -from scipy.integrate import solve_ivp, quad +# from timeit import default_timer as timer import cProfile, pstats, io import math +import matplotlib as mpl +import matplotlib.pyplot as plt +from mpl_toolkits.axes_grid1 import make_axes_locatable +import random class Base(object): - @staticmethod def Mink_dot(vec1, vec2): """ @@ -147,3 +142,260 @@ def inner(*args, **kwargs): print(s.getvalue()) return retval return inner + +class MySymLogPlotting(object): + + @staticmethod + def symlog_num(num): + """ + Return the symlog of a number + + Parameters: + ----------- + num: float + + Returns: + -------- + The symlog of the input num (separate copy) + """ + result = np.sign(num) * np.log10(np.abs(num)+1) + return result + + @staticmethod + def inverse_symlog_num(num): + if num > 0: + return 1 + 10 ** num + elif num < 0: + return 1- 10 ** (- num) + else: + return 0 + + @staticmethod + def symlog_var(var): + """ + Return the symlog of an array. + + Parameters: + ----------- + var: np.array of any shape + + Returns: + -------- + The symlog of the input var (separate copy) + """ + count_zeros=0 + temp = np.empty_like(var) + for index in np.ndindex(var.shape): + if var[index] == 0: + count_zeros +=1 + else: + value = var[index] + temp[index] = np.sign(value) * np.log10(np.abs(value)+1) + if count_zeros >= 1: + print('Careful: there are {} zeros in the data'.format(count_zeros)) + return temp + + @staticmethod + def get_mysymlog_var_ticks(var): + """ + Method to automatize the computation of the ticks and nodes for a variable. + Nodes are then to be used within the class MyThreeNodesNorm. + Ticks and labels are for the colorbar of the plot of input 'var'. + + Parameters: + ----------- + var: np.array + This HAS TO take both positive and negative values + + Returns: + -------- + ticks: list + list of tick points to be used by the colorbar + + ticks_labels: list + list of corresponding labels for the colorbar + + nodes: list of len=5 + the extrame and the three central nodes to be used by MyThreeNodesNorm + + Notes: + ------ + The ticks/nodes and labels are computed like this: start from negative values, identify min + and max values of the negative part of input 'var' to identify relevant ticks and nodes. + Then add a zero (tick and node) and proceed to the positive values. + """ + ticks = [] + ticks_labels = [] + nodes = [] + + pos_var = np.where(var > 0, var, 0).flatten() + pos_var = pos_var[pos_var!=0] + + neg_var = np.where(var < 0, var, 0).flatten() + neg_var = neg_var[neg_var!=0] + + pos_var_small = np.where(pos_var < 1, pos_var, 0) + pos_var_small = pos_var_small[pos_var_small!=0] + + pos_var_large = np.where(pos_var >= 1, pos_var, 0) + pos_var_large = pos_var_large[pos_var_large!=0] + + neg_var_small = np.where(neg_var > -1 , neg_var, 0) + neg_var_small = neg_var_small[neg_var_small!=0] + + neg_var_large = np.where(neg_var <= -1, neg_var, 0) + neg_var_large = neg_var_large[neg_var_large!=0] + + # Working out nodes, ticks and ticks_labels for the negative range + if len(neg_var_large) >0: + # print('There are negative large values', flush=True) + vmin = np.amin(neg_var_large) + new_nodes = [MySymLogPlotting.symlog_num(vmin)] + new_ticks = new_nodes + new_ticks_labels = [r'$-10^{%d}$'%(int(np.log10(-vmin)))] + + nodes += new_nodes + ticks += new_ticks + ticks_labels += new_ticks_labels + + + if len(neg_var_small) == 0: + print('Actually: only negative large values', flush=True) + vmax = np.amax(neg_var_large) + new_nodes = [MySymLogPlotting.symlog_num(vmax)] + new_ticks = new_nodes + new_ticks_labels = [r'$-10^{%d}$'%(int(np.log10(-vmax)))] + + nodes += new_nodes + ticks += new_ticks + ticks_labels += new_ticks_labels + + else: + # print('And also negative small values', flush=True) + vmin = np.amin(neg_var_small) + vmax = np.amax(neg_var_small) + + new_nodes = [MySymLogPlotting.symlog_num(vmax)] + # new_ticks = [symlog_num(vmin), symlog_num(vmax)] + new_ticks = [MySymLogPlotting.symlog_num(vmax)] + # new_ticks_labels = [r'$-10^{%d}$'%(int(d)) for d in np.log10([-vmin,-vmax])] + new_ticks_labels = [r'$-10^{%d}$'%(int(d)) for d in np.log10([-vmax])] + + ticks += new_ticks + ticks_labels += new_ticks_labels + nodes += new_nodes + + else: # len(neg_var_large)==0: + # print('Only negative small values', flush=True) + vmin = np.amin(neg_var_small) + vmax = np.amax(neg_var_small) + + new_nodes = [MySymLogPlotting.symlog_num(vmin), MySymLogPlotting.symlog_num(vmax)] + new_ticks = new_nodes + new_ticks_labels = [r'$-10^{%d}$'%(int(d)) for d in np.log10([-vmin,-vmax])] + + ticks += new_ticks + ticks_labels += new_ticks_labels + nodes += new_nodes + + + nodes += [0.] + ticks += [0.] + ticks_labels += ['0'] + + # Working out the remaining nodes, ticks and ticks_labels for the positive range + + if len(pos_var_large)==0: + # print('Only positive small values', flush=True) + vmin = np.amin(pos_var_small) + vmax = np.amax(pos_var_small) + + new_nodes = [MySymLogPlotting.symlog_num(vmin), MySymLogPlotting.symlog_num(vmax)] + new_ticks = new_nodes + new_ticks_labels = [r'$10^{%d}$'%(int(d)) for d in np.log10([vmin,vmax])] + + ticks += new_ticks + ticks_labels += new_ticks_labels + nodes += new_nodes + + else: # len(pos_var_large) > 0: + # print('There are positive large values', flush=True) + + if len(pos_var_small) >0: + # print('And also positive small values', flush=True) + vmin = np.amin(pos_var_small) + vmax = np.amax(pos_var_small) + + # new_nodes = [symlog_num(vmax)] + new_nodes = [MySymLogPlotting.symlog_num(vmin)] + # new_ticks = [symlog_num(vmin), symlog_num(vmax)] + new_ticks = [MySymLogPlotting.symlog_num(vmin)] + # new_ticks_labels = [r'$10^{%d}$'%(int(d)) for d in np.log10([vmin,vmax])] + new_ticks_labels = [r'$10^{%d}$'%(int(d)) for d in np.log10([vmin])] + + ticks += new_ticks + ticks_labels += new_ticks_labels + nodes += new_nodes + + vmax = np.amax(pos_var_large) + new_nodes = [MySymLogPlotting.symlog_num(vmax)] + new_ticks = new_nodes + new_ticks_labels = [r'$10^{%d}$'%(int(np.log10(vmax)))] + + ticks += new_ticks + ticks_labels += new_ticks_labels + nodes += new_nodes + + + else: # len(pos_var_small) ==0: + vmin = np.amin(pos_var_large) + vmax = np.amax(pos_var_large) + + new_nodes = [MySymLogPlotting.symlog_num(vmin), MySymLogPlotting.symlog_num(vmax)] + new_ticks = new_nodes + new_ticks_labels = [r'$10^{%d}$'%(int(d)) for d in np.log10([vmin,vmax])] + + ticks += new_ticks + ticks_labels += new_ticks_labels + nodes += new_nodes + + return ticks, ticks_labels, nodes + + +class MyThreeNodesNorm(mpl.colors.Normalize): + """ + Sub-classing colors.Normalize: the norm has three inner nodes plus the extrema. + Within each segment (delimited by a node or extrema) you have linear interpolation. + + Should be used when plotting quantities that are both positive and negative, and you + want to highlight 1) where a critical value (middle_node) is 2) the closest values some + variable takes to its left and right + """ + def __init__(self, nodes, clip=False): + """ + Parameters: + + nodes: array of five numbers in strictly ascending order (the nodes) + """ + if len(nodes)!=5: + raise ValueError('The class MyThreeNodesNorm requires 5 nodes: the extrema and the three central') + + for i in range(len(nodes)-1): + if nodes[i+1] <=nodes[i]: + raise ValueError('nodes must be in monotonically ascending order!') + + super().__init__(nodes[0], nodes[4], clip) + self.first_node = nodes[1] + self.central_node = nodes[2] + self.third_node = nodes[3] + + def __call__(self, value, clip=None): + x = [self.vmin, self.first_node, self.central_node, self.third_node, self.vmax] + y = [0, 0.4, 0.5, 0.6, 1.] + return np.ma.masked_array(np.interp(value, x, y, + left=-np.inf, right=np.inf)) + + def inverse(self, value): + y = [self.vmin, self.first_node, self.central_node, self.third_node, self.vmax] + x = [0, 0.4, 0.5, 0.6, 1.] + return np.interp(value, x, y, left=-np.inf, right=np.inf) From fcab681a23156c45fd49d1fe836da5c223e51dd0 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 1 Feb 2024 16:37:26 +0100 Subject: [PATCH 065/111] Improving the parallelization --- master_files/Analysis.py | 17 ++++- master_files/FileReaders.py | 14 +++- master_files/Filters.py | 145 +++++++++++++++++++++++------------- master_files/MesoModels.py | 131 +++++++++++++++++++------------- 4 files changed, 198 insertions(+), 109 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index ba60c29..8df3176 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -9,7 +9,7 @@ import matplotlib.pyplot as plt import numpy as np -# import pandas as pd +import pandas as pd import h5py import pickle import seaborn as sns @@ -298,10 +298,12 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod print('Trimming dataset for correlation plot') X = self.trim_data(x, ranges, model_points) Y = self.trim_data(y, ranges, model_points) - + print('Finished trimming data') + else: + X, Y = x, y X=X.flatten() Y=Y.flatten() - + print('Data flattened') with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') @@ -311,6 +313,7 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod sns.histplot(y=Y, ax=g.ax_marg_y, kde=True) g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) g.fig.tight_layout() + print('Figure produced inside Coeff Analysis') return g def visualize_many_correlations(self, data, labels, ranges=None, model_points=None): @@ -356,13 +359,17 @@ def visualize_many_correlations(self, data, labels, ranges=None, model_points=No print('Trimming dataset for correlation plot') for i in range(len(data)): data[i]=self.trim_data(data[i], ranges, model_points) - + print('Finished trimming data') + Data = [] for i in range(len(data)): Data.append(data[i].flatten()) + print('Data_flattened') + Data=np.column_stack(Data) Data_df = pd.DataFrame(Data, columns=labels) + print('Flattened data converted to a dataframe') with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') @@ -371,6 +378,8 @@ def visualize_many_correlations(self, data, labels, ranges=None, model_points=No g.map_lower(sns.kdeplot) g.map_diag(sns.histplot, kde=True) g.fig.tight_layout() + + print('Finished producing figure inside CoefficientsAnalysis') return g def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_wanted = 1.): diff --git a/master_files/FileReaders.py b/master_files/FileReaders.py index 47c761b..bbdae5d 100644 --- a/master_files/FileReaders.py +++ b/master_files/FileReaders.py @@ -11,7 +11,7 @@ class METHOD_HDF5(object): - def __init__(self, directory): + def __init__(self, directory, fewer_snaps=False, smaller_list=None): """ Set up the list of files (from hdf5) and dictionary with dataset names in the hdf5 file. @@ -20,8 +20,20 @@ def __init__(self, directory): ---------- directory: string the filenames in the directory have to be incremental (sorted is used) + + fewer_snaps: bool + set to true if you want micromodel to store fewer snapshots than can be found + in directory + + smaller_list: list + indices to be retained of list orderd via sorted(glob.glob(directory+str('*.hdf5'))) """ hdf5_filenames = sorted( glob.glob(directory+str('*.hdf5'))) + if fewer_snaps: + if smaller_list: + temp = [hdf5_filenames[i] for i in smaller_list] + hdf5_filenames = temp + self.hdf5_files = [] for filename in hdf5_filenames: self.hdf5_files.append(h5py.File(filename,'r')) diff --git a/master_files/Filters.py b/master_files/Filters.py index 37ea187..75924b0 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -11,6 +11,7 @@ import scipy.integrate as integrate import multiprocessing as mp from scipy.optimize import minimize, root +from scipy.interpolate import interpn from itertools import product from system.BaseFunctionality import * @@ -835,7 +836,7 @@ def get_tetrad_from_vels(spatial_vels): return tetrad @staticmethod - def initializer(spatial_dims, L): + def initializer(L, grid, BC): """ Initializer for processes in pool. Build the adapted coordinates and weights: this is independent of specific point @@ -853,17 +854,28 @@ def initializer(spatial_dims, L): """ global adapt_coords global totws + global micro_spatial_dims + global micro_grid + global micro_BC + global box_len + + micro_grid = grid + micro_spatial_dims = len(grid)-1 + micro_BC = BC + box_len = L + # print(f'Baryon current passed to initializer: {micro_BC}') + ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] ws1d = [8./9. , 5./9. , 5./9.] xs = [] ws = [] - for _ in range(spatial_dims+1): + for _ in range(micro_spatial_dims+1): xs.append(ps1d) ws.append(ws1d) adapt_coords = [] for element in product(*xs): - adapt_coords.append(np.multiply(L/2, np.array(element) )) + adapt_coords.append(np.multiply(box_len/2, np.array(element))) totws = [] for element in product(*ws): @@ -873,7 +885,8 @@ def initializer(spatial_dims, L): totws.append(temp) # print('Initialized process in the pool', flush=True) - def find_observer_Gauss(self, point_pos): + @staticmethod + def find_observer_Gauss(point, pos_in_list_points): """ CPU-bound task to be run in parralel. The routine combines what has been split into many in the serial @@ -900,30 +913,40 @@ def find_observer_Gauss(self, point_pos): # Declaring global vars set up by initializer global adapt_coords global totws + global micro_spatial_dims + global micro_grid + global micro_BC + global box_len # Building the initial guess - point = point_pos[0] - pos_in_list_points = point_pos[1] + # point = point_pos[0] + # pos_in_list_points = point_pos[1] guess = [] - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) - for i in range(1, len(U)): - guess.append(U[i] / U[0]) - + BC_point = interpn(micro_grid, micro_BC, point)[0] + n_point = np.sqrt(-Base.Mink_dot(BC_point, BC_point)) + U_point = np.multiply( 1 / n_point , BC_point) + # U_point = np.multiply( 1 / self.micro_model.get_interpol_var('n', point), self.micro_model.get_interpol_var('BC', point) + for i in range(1, len(U_point)): + guess.append(U_point[i] / U_point[0]) + + # Routine to compute drift residual via Gauss-Legendre quadrature. - def residual_gauss(spatial_vels, point, micro_model): - spatial_dims = micro_model.get_spatial_dims() + def residual_gauss(spatial_vels, point): #, micro_model): + # spatial_dims = micro_model.get_spatial_dims() tetrad = FindObs_root_parallel.get_tetrad_from_vels(spatial_vels) coords = [] for coord in adapt_coords: temp = np.array(point) - for i in range(spatial_dims+1): + for i in range(micro_spatial_dims+1): temp += np.multiply(coord[i], tetrad[i]) coords.append(temp) - integral = np.zeros(1 + spatial_dims) + integral = np.zeros(1 + micro_spatial_dims) for i, coord in enumerate(coords): - integral += np.multiply(totws[i], micro_model.get_interpol_var('BC', coord)) - integral *= (self.L /2 ) ** (spatial_dims+1) + BC_coord = interpn(micro_grid, micro_BC, coord)[0] + # integral += np.multiply(totws[i], micro_model.get_interpol_var('BC', coord)) + integral += np.multiply(totws[i], BC_coord) + integral *= (box_len /2) ** (micro_spatial_dims+1) drifts = [] for i in range(len(tetrad)-1): @@ -931,7 +954,7 @@ def residual_gauss(spatial_vels, point, micro_model): return drifts # observer: root of the residual gauss routine - sol = root(residual_gauss, x0 = guess, args = (point, self.micro_model)) + sol = root(residual_gauss, x0 = guess, args = (point)) #, self.micro_model)) if sol.success: observer = Base.get_rel_vel(sol.x) avg_error = np.sum(sol.fun[1]) @@ -972,16 +995,18 @@ def find_observers_parallel(self, points, n_cpus): success_pos = [] failed_pos = [] - spatial_dims = self.micro_model.get_spatial_dims() + # spatial_dims = self.micro_model.get_spatial_dims() L = self.L - args_for_pool = [ [points[i], i] for i in range(len(points))] + BC = self.micro_model.vars['BC'] + grid = self.micro_model.domain_vars['points'] + args_for_pool = [ (points[i], i) for i in range(len(points))] init = FindObs_root_parallel.initializer - initargs = (spatial_dims, L) + # initargs = (spatial_dims, L) + initargs = (L, grid, BC) with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: - # with mp.Pool() as pool: - print('Finding observers in parallel with {} processes\n'.format(n_cpus), flush=True) - for result in pool.map(self.find_observer_Gauss, args_for_pool): + print('Finding observers in parallel with {} processes\n'.format(pool._processes), flush=True) + for result in pool.starmap(self.find_observer_Gauss, args_for_pool): if (result[0] == True): success_pos.append(result[1]) observers.append(result[2]) @@ -1326,7 +1351,7 @@ def complete_U_tetrad(U): return triad @staticmethod - def initializer(spatial_dims, filter_width): + def initializer(spatial_dims, filter_width, grid, var): """ Initializer for processes in pool. Build the adapted coordinates and weights: this is independent of specific point @@ -1336,6 +1361,14 @@ def initializer(spatial_dims, filter_width): """ global abstract_coords global totws + global micro_spatial_dims + global micro_var + global micro_grid + + + micro_spatial_dims = spatial_dims + micro_grid = grid + micro_var = var ps1d = [0, + np.sqrt(3/5), - np.sqrt(3/5)] ws1d = [8./9. , 5./9. , 5./9.] @@ -1357,7 +1390,8 @@ def initializer(spatial_dims, filter_width): temp *= w totws.append(temp) - def filter_vars_point_gauss(self, packed_args): + @staticmethod + def filter_var_point_gauss(point, observer, pos_in_list_points): """ CPU-bound task to be run in parralel. The routine combines what has been split into many in the serial @@ -1390,38 +1424,42 @@ def filter_vars_point_gauss(self, packed_args): # declaring global vars set up by initializer global abstract_coords global totws + global micro_spatial_dims + global micro_var + global micro_grid # Unpacking arguments passed by pool.map - unpacked_args= packed_args[0] - pos = packed_args[1] - point = unpacked_args[0] - observer = unpacked_args[1] - vars_strs = unpacked_args[2] + # unpacked_args= packed_args[0] + # pos = packed_args[1] + # point = unpacked_args[0] + # observer = unpacked_args[1] + # vars_strs = unpacked_args[2] # From abstract to "real" coordinates vecs = box_filter_parallel.complete_U_tetrad(observer) sample_points = [] for coord in abstract_coords: temp = np.array(point) - for i in range(self.spatial_dims): + for i in range(micro_spatial_dims): temp += np.multiply(coord[i], vecs[i]) sample_points.append(temp) - # Filtering the vars - filtered_vars = [] - for var in vars_strs: - filtered_var = np.zeros(self.micro_model.get_var_gridpoint(var,0,0,0).shape) - for i, sample in enumerate(sample_points): - filtered_var += totws[i] * self.micro_model.get_interpol_var(var, sample) - filtered_var = np.multiply(filtered_var, 1 / (2**self.spatial_dims)) - filtered_vars.append(filtered_var) + # Filtering the var + # filtered_vars = [] + # for var in vars_strs: + # filtered_var = np.zeros(self.micro_model.get_var_gridpoint(var,0,0,0).shape) + filtered_var = np.zeros(micro_var[tuple([ 0 for _ in range(len(micro_grid))])].shape) + for i, sample in enumerate(sample_points): + filtered_var += totws[i] * interpn(micro_grid, micro_var, sample)[0] + filtered_var = np.multiply(filtered_var, 1 / (2**micro_spatial_dims)) + # filtered_vars.append(filtered_var) # Different variables are returned as list (ordered as var_strs) # Safer to return a dictionary: check redux in performance though. - return pos, filtered_vars + return pos_in_list_points, filtered_var - def filter_vars_parallel(self, list_packed_args, n_cpus): + def filter_var_parallel(self, points_observers, var, n_cpus): """ Method to run filter_vars_point_gauss in parallel given a list of points, observers and vars. @@ -1436,7 +1474,7 @@ def filter_vars_parallel(self, list_packed_args, n_cpus): observer: nd.array [vars_to_be_filtered]: list of strs - each item must much a var in micromodel + each item must match a var in micromodel n_cpus: int number of processes @@ -1453,19 +1491,22 @@ def filter_vars_parallel(self, list_packed_args, n_cpus): explicitely in the tests below, or decided at the MesoModel level. """ position_in_list = [] - filtered_vars = [] - args_for_pool = [ [list_packed_args[i], i] for i in range(len(list_packed_args))] + filtered_var = [] + args_for_pool = [ tuple([*points_observers[i], i]) for i in range(len(points_observers))] + # print(args_for_pool[0]) init = box_filter_parallel.initializer - initargs=(self.spatial_dims, self.filter_width) + micro_grid = self.micro_model.domain_vars['points'] + micro_var = self.micro_model.vars[var] + initargs=(self.spatial_dims, self.filter_width, micro_grid, micro_var) with mp.Pool(initializer=init, initargs=initargs, processes=n_cpus) as pool: - print('Filtering in parallel with {} processes\n'.format(n_cpus), flush=True) - for result in pool.map(self.filter_vars_point_gauss, args_for_pool): + print('Filtering {} in parallel with {} processes\n'.format(var, pool._processes), flush=True) + for result in pool.starmap(self.filter_var_point_gauss, args_for_pool): position_in_list.append(result[0]) - filtered_vars.append(result[1]) + filtered_var.append(result[1]) - return position_in_list, filtered_vars + return position_in_list, filtered_var if __name__ == '__main__': @@ -1562,11 +1603,13 @@ def filter_vars_parallel(self, list_packed_args, n_cpus): # Preparing args for parallel filter routine args_list = [] for elem in zip(points, observers): - args_list.append([*elem, vars]) + # args_list.append([*elem, vars]) + args_list.append(tuple(elem)) start_time = time.perf_counter() parallel_filter = box_filter_parallel(micro_model, 0.003) - parallel_filter.filter_vars_parallel(args_list, n_cpus) + for var in vars: + parallel_filter.filter_var_parallel(args_list, var, n_cpus) parallel_time = time.perf_counter() - start_time print('Finished filtering in parallel, time-taken: {}'.format(parallel_time)) # print('Speed-up factor: {}\n'.format(serial_time/parallel_time)) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 8f2ff13..ce5f868 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -497,8 +497,10 @@ def get_interpol_var(self, var, point): return interpn(self.domain_vars['Points'], self.meso_vars[var], point, method = self.interp_method)[0] elif var in self.deriv_vars: return interpn(self.domain_vars['Points'], self.deriv_vars[var], point, method = self.interp_method)[0] + elif var in self.filter_vars: + return interpn(self.domain_vars['Points'], self.filter_vars[var], point, method = self.interp_method)[0] else: - print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) + print('{} does not belong to either meso_structures/meso_vars/deriv_vars/filter_vars. Check! (Method: get_interpol_var(var, point))'.format(var)) @multimethod def get_var_gridpoint(self, var: str, h: object, i: object, j: object): @@ -528,8 +530,10 @@ def get_var_gridpoint(self, var: str, h: object, i: object, j: object): return self.meso_vars[var][h,i,j] elif var in self.deriv_vars: return self.deriv_vars[var][h,i,j] + elif var in self.filter_vars: + return self.filter_vars[var][h,i,j] else: - print("{} is not a variable of resMHD_2D!".format(var)) + print('{} does not belong to either meso_structures/meso_vars/deriv_vars/filter_vars. Check! (Method: get_var_gridpoint(h,i,j))'.format(var)) return None @multimethod @@ -561,8 +565,10 @@ def get_var_gridpoint(self, var: str, point: object): return self.meso_vars[var][tuple(indices)] elif var in self.deriv_vars: return self.deriv_vars[var][tuple(indices)] + elif var in self.filter_vars: + return self.filter_vars[var][tuple(indices)] else: - print("{} is not a variable of resMHD_2D!".format(var)) + print('{} does not belong to either meso_structures/meso_vars/deriv_vars/filter_vars. Check! (Method: get_var_gridpoint(point))'.format(var)) return None def get_model_name(self): @@ -1227,8 +1233,10 @@ def get_interpol_var(self, var, point): return interpn(self.domain_vars['Points'], self.meso_vars[var], point, method = self.interp_method)[0] elif var in self.deriv_vars: return interpn(self.domain_vars['Points'], self.deriv_vars[var], point, method = self.interp_method)[0] + elif var in self.filter_vars: + return interpn(self.domain_vars['Points'], self.filter_vars[var], point, method=self.interp_method)[0] else: - print('{} does not belong to either meso_structures/meso_vars or deriv_vars. Check!'.format(var)) + print('Cannot interpolate value of {} using data fromfilter_vars meso_structures/meso_varsderiv_vars/filter_vars. Check!'.format(var)) @multimethod def get_var_gridpoint(self, var: str, h: object, i: object, j: object): @@ -1261,7 +1269,7 @@ def get_var_gridpoint(self, var: str, h: object, i: object, j: object): elif var in self.filter_vars: return self.filter_vars[var][h,i,j] else: - print("{} is not a variable of resHD_2D!".format(var)) + print(f'Cannot get value of {var} at h,i,j from data in meso_vars/meso_structures/deriv_vars/filter_vars') return None @multimethod @@ -1294,9 +1302,9 @@ def get_var_gridpoint(self, var: str, point: object): elif var in self.deriv_vars: return self.deriv_vars[var][tuple(indices)] elif var in self.filter_vars: - return self.filter_vars[var][h,i,j] + return self.filter_vars[var][tuple(indices)] else: - print("{} is not a variable of resHD_2D!".format(var)) + print(f'Cannot get value of {var} at h,i,j from data in meso_vars/meso_structures/deriv_vars/filter_vars') return None def get_model_name(self): @@ -1305,11 +1313,11 @@ def get_model_name(self): def set_find_obs_method(self, find_obs): self.find_obs = find_obs - def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): + def setup_meso_grid(self, patch_bdrs, coarse_factor = 1, coarse_time = False): """ Builds the meso_model grid using the micro_model grid points contained in the input patch (defined via 'patch_bdrs'). The method allows for coarse graining - the grid in the spatial directions ONLY. + the grid (both in space dirs only or also in time) Then store the info about the meso grid and set up arrays of definite rank and size for the quantities needed later. @@ -1320,6 +1328,8 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): coarse_factor: integer + coarse_time: boolean + If true, coarsening is also applied to the time direction. Notes: ------ If the patch_bdrs are larger than micro_grid, the method will not set-up the meso_grid, @@ -1346,6 +1356,10 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): # Set meso_grid spacings self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] + if coarse_time: + self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] * coarse_factor + else: + self.domain_vars['Dt'] = self.micro_model.domain_vars['dt'] self.domain_vars['Dx'] = self.micro_model.domain_vars['dx'] * coarse_factor self.domain_vars['Dy'] = self.micro_model.domain_vars['dy'] * coarse_factor @@ -1354,7 +1368,10 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): while h <= idx_maxs[0]: t = self.micro_model.domain_vars['t'][h] self.domain_vars['T'].append(t) - h += 1 + if coarse_time: + h += coarse_factor + else: + h += 1 while i <= idx_maxs[1]: x = self.micro_model.domain_vars['x'][i] self.domain_vars['X'].append(x) @@ -1399,24 +1416,10 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1): for j in range(Ny): self.filter_vars['U_success'].update({(h,i,j): False}) - - # Setup arrays for derivatives of the model. - # THOMAS'S WAY (SAVE THEM AS TENSORS) self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) self.deriv_vars['D_T_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) - - # MARCUS'S WAY - # for str in self.non_local_vars_strs: - # if str in self.meso_vars: - # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_vars[str])}) - # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_vars[str])}) - # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_vars[str])}) - # if str in self.meso_structures: - # self.deriv_vars.update({'dt'+str:np.zeros_like(self.meso_structures[str])}) - # self.deriv_vars.update({'dx'+str:np.zeros_like(self.meso_structures[str])}) - # self.deriv_vars.update({'dy'+str:np.zeros_like(self.meso_structures[str])}) - + def find_observers(self): """ Method to compute filtering observers at grid points built with setup_meso_grid. @@ -1563,22 +1566,30 @@ def filter_micro_vars_parallel(self, n_cpus): return None vars = ['BC', 'SET', 'p'] - args_for_filtering_parallel = [] + points_observers = [] for i in range(len(points)): - args_for_filtering_parallel.append([points[i], observers[i], vars]) + # args_for_filtering_parallel.append([points[i], observers[i], vars]) + points_observers.append([points[i], observers[i]]) - positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel, n_cpus) + # positions, filtered_vars = self.filter.filter_vars_parallel(args_for_filtering_parallel, n_cpus) + filtered_vars = dict.fromkeys(vars) + for var in vars: + positions, filtered_vars[var] = self.filter.filter_var_parallel(points_observers, var, n_cpus) + for i in range(len(positions)): point_indxs_meso_grid = indices_meso_grid[positions[i]] - self.meso_structures['BC'][point_indxs_meso_grid] = filtered_vars[i][0] - self.meso_structures['SET'][point_indxs_meso_grid] = filtered_vars[i][1] - self.meso_vars['p_filt'][point_indxs_meso_grid] = filtered_vars[i][2] + self.meso_structures['BC'][point_indxs_meso_grid] = filtered_vars['BC'][i] + self.meso_structures['SET'][point_indxs_meso_grid] = filtered_vars['SET'][i] + self.meso_vars['p_filt'][point_indxs_meso_grid] = filtered_vars['p'][i] + # self.meso_structures['BC'][point_indxs_meso_grid] = filtered_vars[i][0] + # self.meso_structures['SET'][point_indxs_meso_grid] = filtered_vars[i][1] + # self.meso_vars['p_filt'][point_indxs_meso_grid] = filtered_vars[i][2] def p_from_EOS(self, eps, n): """ Compute pressure from Gamma-law EoS """ - return (self.coefficients['Gamma']-1)*eps*n + return (self.coefficients['Gamma']-1)*(eps-n) def decompose_structures_gridpoint(self, h, i, j): """ @@ -1710,12 +1721,14 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) # Additional quantities needed for residuals - Pi_res = s - p_t + # Pi_res = s - p_t + Pi_res = s - p_filt s_ab_tracefree = s_ab - np.multiply(s/spatial_dims, metric + np.einsum('i,j->ij', u_t, u_t)) T_t = p_t/n_t - EOS_res = p_filt - p_t + # EOS_res = p_filt - p_t - return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, EOS_res, T_t], [h,i,j] + # return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, T_t, EOS_res], [h,i,j] #Uncomment if EoS residual is modelled + return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, T_t], [h,i,j] def decompose_structures_parallel(self, n_cpus): """ @@ -1738,7 +1751,7 @@ def decompose_structures_parallel(self, n_cpus): args_for_pool.append((BC, SET, p_filt, h,i,j)) with mp.Pool(processes=n_cpus) as pool: - print('Running with {} processes\n'.format(n_cpus), flush=True) + print('Running with {} processes\n'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.decompose_structures_task, args_for_pool): h,i,j = result[1] self.meso_vars['n_tilde'][h,i,j] = result[0][0] @@ -1748,8 +1761,8 @@ def decompose_structures_parallel(self, n_cpus): self.meso_vars['pi_res'][h,i,j,:,:] = result[0][4] self.meso_vars['p_tilde'][h,i,j] = result[0][5] self.meso_vars['Pi_res'][h,i,j] = result[0][6] - self.meso_vars['eos_res'][h,i,j] = result[0][7] - self.meso_vars['T_tilde'][h,i,j] = result[0][8] + self.meso_vars['T_tilde'][h,i,j] = result[0][7] + # self.meso_vars['eos_res'][h,i,j] = result[0][8] # Uncomment if EOS_res is modelled def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): """ @@ -2040,7 +2053,7 @@ def closure_ingredients_parallel(self, n_cpus): args_for_pool.append((u_t, nabla_u, T_t, nabla_T, h, i, j)) with mp.Pool(processes=n_cpus) as pool: - print('Running with {} processes\n'.format(n_cpus), flush=True) + print('Running with {} processes'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.closure_ingredients_task, args_for_pool): keys, values, grid_idxs = result # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True @@ -2165,7 +2178,7 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): # CALCULATING SHEAR VISCOUS COEFF pi_res_sq = np.einsum('ij,kl,ik,jl->', pi_res, pi_res, metric, metric) shear_sq = np.einsum('ij,kl,ik,jl->', shear, shear, metric, metric) - eta = pi_res_sq/shear_sq + eta = np.sqrt(pi_res_sq/shear_sq) # Compute sign of eta by looking at the PA of pi_res with higher eigenvalue. # Change shear to the eigenbasis of pi_res (using that the matrix is unitary as pi_res is sym) pi_eig, pi_eigv = np.linalg.eigh(pi_res) @@ -2176,6 +2189,12 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): coefficients_names.append('eta') coefficients.append(eta) + # this is just to check - to be removed + coefficients_names.append('shear_sq') + coefficients.append(shear_sq) + coefficients_names.append('pi_res_sq') + coefficients.append(pi_res_sq) + # CALCULATING THE HEAT CONDUCTIVITIY q_res_sq = np.einsum('i,ij,j', q_res, metric, q_res) Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) @@ -2184,10 +2203,15 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): kappa = sign * np.sqrt(q_res_sq / Theta_sq) coefficients_names.append('kappa') coefficients.append(kappa) + + # this is just to check - to be removed + coefficients_names.append('q_res_sq') + coefficients.append(q_res_sq) + coefficients_names.append('Theta_sq') + coefficients.append(Theta_sq) return coefficients_names, coefficients, [h,i,j] - # this is to be modified def EL_style_closure_parallel(self, n_cpus): """ Routine to launch EL_style_closure_task in parallel across multiple gridpoints @@ -2226,6 +2250,7 @@ def EL_style_closure_parallel(self, n_cpus): with mp.Pool(processes=n_cpus) as pool: + print('Computing dissipative coefficients with {} processes'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.EL_style_closure_task, args_for_pool): keys, values, grid_idxs = result # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True @@ -2410,13 +2435,13 @@ def EL_style_closure_regression(self, CoefficientAnalysis): micro_model.setup_structures() t_range = [1.502, 1.504] - x_range = [0.05, 0.55] - y_range = [0.05, 0.55] + x_range = [0.05, 0.06] + y_range = [0.05, 0.06] # set up meso model and grid find_obs = FindObs_root_parallel(micro_model, 0.001) filter = box_filter_parallel(micro_model, 0.003) - meso_model = resHD2D(micro_model, find_obs, filter, local_or_slurm=True) + meso_model = resHD2D(micro_model, find_obs, filter) meso_model.setup_meso_grid([t_range, x_range, y_range]) num_points = meso_model.domain_vars['Nt'] * meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] @@ -2424,25 +2449,25 @@ def EL_style_closure_regression(self, CoefficientAnalysis): # fin obs - parallel start_time = time.perf_counter() - meso_model.find_observers_parallel() + meso_model.find_observers_parallel(os.cpu_count()) parallel_time = time.perf_counter() - start_time print('Finished finding observers in parallel, execution time: {}\n'.format(parallel_time)) # now filtering in parallel start_time = time.perf_counter() - meso_model.filter_micro_vars_parallel() + meso_model.filter_micro_vars_parallel(os.cpu_count()) parallel_time = time.perf_counter() - start_time print('Finished filtering in parallel, time taken {}\n'.format(parallel_time)) - # now testing serial vs parallel decomposition - start_time = time.perf_counter() - meso_model.decompose_structures() - serial_time = time.perf_counter() - start_time - print('Finished decomposing in serial, time taken {}\n'.format(serial_time)) + # # now testing serial vs parallel decomposition + # start_time = time.perf_counter() + # meso_model.decompose_structures() + # serial_time = time.perf_counter() - start_time + # print('Finished decomposing in serial, time taken {}\n'.format(serial_time)) start_time = time.perf_counter() - meso_model.decompose_structures_parallel() + meso_model.decompose_structures_parallel(os.cpu_count()) parallel_time = time.perf_counter() - start_time print('Finished decomposing in parallel, time taken {}\n'.format(parallel_time)) - print('Speed-up factor: {}'.format(serial_time/parallel_time)) + # print('Speed-up factor: {}'.format(serial_time/parallel_time)) From 0467c5c82535c42d4927c8bc9d4293f4439c72ea Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 1 Feb 2024 16:38:56 +0100 Subject: [PATCH 066/111] Updates to PoP routines --- Proof_of_principle/config_meso.txt | 26 ++++- Proof_of_principle/config_visualizing.txt | 36 ++++-- Proof_of_principle/pickling_meso.py | 123 +++++++++++++-------- Proof_of_principle/visualizing_meso.py | 127 +++++++++++++++++++--- Proof_of_principle/visualizing_micro.py | 17 ++- Proof_of_principle/visualizing_obs.py | 39 +++++-- 6 files changed, 279 insertions(+), 89 deletions(-) diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index a569c04..12255ee 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -1,11 +1,27 @@ +# CONFIGURATION FILE FOR SCRIPTS PICKLING_MESO.PY +################################################# + [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/1600X1600/METHOD_output/ET_3.0 -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/1600X1600/pickled_files + +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/METHOD_output/ET_8.0 + +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/pickled_files + +[Micro_model_settings] +#snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. +#the options is activated using the first key, while the list refers to the index of the elements of +#ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) + +snapshots_opts = {"fewer_snaps_required": true, "smaller_list": [5,6,7,8,9,10,11,12,13,14,15]} [Meso_model_settings] -meso_grid = {"x_range": [0.02, 0.98], "y_range": [0.2, 0.98]} -filtering_options = {"box_len_ratio": 8.0, "filter_width_ratio": 8.0 } + +meso_grid = {"x_range": [0.02, 0.98], "y_range": [0.02, 0.98], "num_T_slices": 3, "coarse_grain_factor": 4, "coarse_grain_time": true} + +filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 4.0} + n_cpus = 40 [Pickled_filenames] -meso_pickled = /rHD2d_decomposed_ET_3.0 \ No newline at end of file + +meso_pickled = /rHD2d_ET8_cg=fw=bl=4dx_dt.pickle \ No newline at end of file diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt index 66738db..178330c 100644 --- a/Proof_of_principle/config_visualizing.txt +++ b/Proof_of_principle/config_visualizing.txt @@ -1,15 +1,31 @@ +# CONFIGURATION FILE FOR SCRIPTS VISUALIZING_*.PY +################################################# + [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_1.0 -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 -figures_dir = ./tests -[Filenames] -meso_pickled_filename = /resHD_2D_ET_1.0_FW_8dx.pickle -micro_pickled_filename = /HD_2D_ET_1.0_micro.pickle +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/METHOD_output/ET_8.0 + +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/pickled_files + +figures_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/figures/ET8/cg=bl=fw_dt/cg4 + +[Filenames] + +meso_pickled_filename = /rHD2d_ET8_cg=fw=bl=4dx_dt.pickle + +[Models_settings] +#snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. +#relevant only for visualizing_micro.py + +snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [6,7,8,9,10,11,12,13,14]} + +mesogrid_T_slices_num = 3 [Plot_settings] -plot_ranges = {"x_range": [0.2, 0.5], "y_range": [0.2, 0.5]} +# method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids +# interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid + +plot_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} + +diff_plot_settings = {"method": "raw_data", "interp_dims": [700, 700]} -[Figures_name_param] -filter_width = 8.0 -box_length = 2.0 diff --git a/Proof_of_principle/pickling_meso.py b/Proof_of_principle/pickling_meso.py index 5c3f7e4..904f0e2 100644 --- a/Proof_of_principle/pickling_meso.py +++ b/Proof_of_principle/pickling_meso.py @@ -21,52 +21,80 @@ config = configparser.ConfigParser() config.read(sys.argv[1]) - hdf5_directory = config['Directories']['hdf5_dir'] - filenames = hdf5_directory - FileReader = METHOD_HDF5(filenames) - num_snaps = FileReader.num_files - micro_model = IdealHD_2D() - FileReader.read_in_data_HDF5_missing_xy(micro_model) - micro_model.setup_structures() - print('Finished reading micro data from hdf5, structures also set up.') - - # SETTING UP THE MESO MODEL - CPU_start_time = time.perf_counter() - central_slice_num = int(num_snaps/2) - t_range = [micro_model.domain_vars['t'][central_slice_num-1], micro_model.domain_vars['t'][central_slice_num+1]] - - meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) - filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) - - x_range = meso_grid['x_range'] - y_range = meso_grid['y_range'] - box_len_ratio = float(filtering_options['box_len_ratio']) - filter_width_ratio = filtering_options['filter_width_ratio'] - - box_len = box_len_ratio * micro_model.domain_vars['dx'] - width = filter_width_ratio * micro_model.domain_vars['dx'] - find_obs = FindObs_root_parallel(micro_model, box_len) - filter = box_filter_parallel(micro_model, width) - meso_model = resHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([t_range, x_range, y_range]) - - time_taken = time.perf_counter() - CPU_start_time - print('Grid is set up, time taken: {}'.format(time_taken)) - num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] - print('Number of points: {}'.format(num_points)) - - # FINDING THE OBSERVERS AND FILTERING + # start_time = time.perf_counter() + # hdf5_directory = config['Directories']['hdf5_dir'] + + # print(f'Starting job with data from {hdf5_directory}') + # print('==============================================\n') + # filenames = hdf5_directory + # snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) + # fewer_snaps_required = snapshots_opts['fewer_snaps_required'] + # smaller_list = snapshots_opts['smaller_list'] + # FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) + # num_snaps = FileReader.num_files + # micro_model = IdealHD_2D() + # FileReader.read_in_data_HDF5_missing_xy(micro_model) + # micro_model.setup_structures() + # time_taken = time.perf_counter() - start_time + # print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}') + + # # SETTING UP THE MESO MODEL + # start_time = time.perf_counter() + # meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) + # filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) + + # coarse_factor = meso_grid['coarse_grain_factor'] + # coarse_time = meso_grid['coarse_grain_time'] + # central_slice_num = int(num_snaps/2) + # num_T_slices = meso_grid['num_T_slices'] + # furthest_slice_number = (num_T_slices-1)/2 + # if coarse_time: + # furthest_slice_number = int(coarse_factor * furthest_slice_number) + # t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] + # x_range = meso_grid['x_range'] + # y_range = meso_grid['y_range'] + + # box_len_ratio = float(filtering_options['box_len_ratio']) + # filter_width_ratio = filtering_options['filter_width_ratio'] + + # box_len = box_len_ratio * micro_model.domain_vars['dx'] + # width = filter_width_ratio * micro_model.domain_vars['dx'] + # find_obs = FindObs_root_parallel(micro_model, box_len) + # filter = box_filter_parallel(micro_model, width) + # meso_model = resHD2D(micro_model, find_obs, filter) + # meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) + + # time_taken = time.perf_counter() - start_time + # print('Grid is set up, time taken: {}'.format(time_taken)) + # num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] + # print('Number of points: {}'.format(num_points)) + + # # FINDING THE OBSERVERS AND FILTERING + # n_cpus = int(config['Meso_model_settings']['n_cpus']) + + # start_time = time.perf_counter() + # meso_model.find_observers_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Observers found in parallel: time taken= {}'.format(time_taken)) + + # start_time = time.perf_counter() + # meso_model.filter_micro_vars_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) + + + # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename n_cpus = int(config['Meso_model_settings']['n_cpus']) - start_time = time.perf_counter() - meso_model.find_observers_parallel(n_cpus) - time_taken = time.perf_counter() - start_time - print('Observers found in parallel: time taken= {}'.format(time_taken)) - - start_time = time.perf_counter() - meso_model.filter_micro_vars_parallel(n_cpus) - time_taken = time.perf_counter() - start_time - print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) + print('=========================================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('=========================================================================\n\n') + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) # DECOMPOSING AND CALCULATING THE CLOSURE INGREDIENTS @@ -86,10 +114,15 @@ time_taken = time.perf_counter() - start_time print('Finished computing the closure ingredients in parallel, time taken: {}'.format(time_taken)) + start_time = time.perf_counter() + meso_model.EL_style_closure_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Finished computing the EL_style closure in parallel, time taken: {}'.format(time_taken)) + # PICKLING THE CLASS INSTANCE FOR FUTURE USE pickled_directory = config['Directories']['pickled_files_dir'] - filename = config['Pickled_filenames']['meso_pickled'] + "_FW_" +str(int(filter_width_ratio)) + "dx" + ".pickle" + filename = config['Pickled_filenames']['meso_pickled'] MesoModelPickleDumpFile = pickled_directory + filename with open(MesoModelPickleDumpFile, 'wb') as filehandle: pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index 8016c9a..4139420 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -4,18 +4,20 @@ import pickle import configparser import json +import time from FileReaders import * from MicroModels import * from MesoModels import * from Visualization import * +from Analysis import * if __name__ == '__main__': - ############################################################### - # PRODUCE PLOTS TO COMPARE MICRO AND MESO MODELS - # FOR EACH PLOT THE DIFFERENCE (ABSOLUTE AND RELATIVE). - ############################################################### + # ############################################################## + # #PLOT KEY QUANTITIES TO COMPARE MICRO AND MESO MODELS + # #AND THOSE RELEVANT TO CALIBRATE CLOSURE + # ############################################################## # READING SIMULATION SETTINGS FROM CONFIG FILE if len(sys.argv) == 1: @@ -28,47 +30,140 @@ # LOADING MESO AND MICRO MODELS pickle_directory = config['Directories']['pickled_files_dir'] meso_pickled_filename = config['Filenames']['meso_pickled_filename'] - MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('=========================================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('=========================================================================\n\n') + with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - micro_model = meso_model.micro_model + print('Finished reading pickled data') + + # # If you need to recompute, e.g. the closure coefficients + # n_cpus = int(config['Meso_model_settings']['n_cpus']) + # meso_model.EL_style_closure_parallel(n_cpus) + # print('Finished re-decomposing the meso_model! Now re-saving it') + # with open(MesoModelLoadFile, 'wb') as filehandle: + # pickle.dump(meso_model, filehandle) + # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') + # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE num_snaps = micro_model.domain_vars['nt'] central_slice_num = int(num_snaps/2.) time_micro = micro_model.domain_vars['t'][central_slice_num] - time_meso = meso_model.domain_vars['T'][1] #Change this if meso_grid is not set up using three central micro_slices + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] if time_meso != time_micro: print("Slices of meso and micro model do not coincide. Careful!") else: print("Comparing data at same time-slice, hurray!") - # PLOT SETTINGS + # # PLOT SETTINGS plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) x_range = plot_ranges['x_range'] y_range = plot_ranges['y_range'] saving_directory = config['Directories']['figures_dir'] visualizer = Plotter_2D([11.97, 8.36]) + diff_plot_settings =json.loads(config['Plot_settings']['diff_plot_settings']) + diff_method = diff_plot_settings['method'] + interp_dims = diff_plot_settings['interp_dims'] - # FINALLY PLOTTING + # PLOTTING MICRO VS FILTERED BC AND SET + # ##################################### + start_time = time.perf_counter() vars = [['BC', 'SET'], ['BC', 'SET']] models = [micro_model, meso_model] components = [[(0,), (0,0)], [(0,), (0,0)]] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, diff_plot=True, rel_diff=True) + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, + interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=False) fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filter_width = str(config['Figures_name_param']['filter_width']) - filename = "/ET_"+ time_for_filename+ "_fw_" + filter_width + "_scaling.pdf" + filename = "/microVSmeso_scaling.pdf" plt.savefig(saving_directory + filename, format = 'pdf') vars = [['BC', 'SET'], ['BC', 'SET',]] models = [micro_model, meso_model] components = [[(2,), (0,2)], [(2,), (0,2)]] - fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, diff_plot=True, rel_diff=True) + fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, + interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=False) + fig.tight_layout() + filename = "/microVSmeso_notscaling.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + + time_taken = time.perf_counter() - start_time + print(f'Finished plotting model comparison: time taken (X2) ={time_taken}') + + # # PLOTTING THE DECOMPOSED SET + # ############################# + vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] + components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/DecomposedSET_1.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + + vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] + components = [(0,), (1,), (2,), (), (), ()] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/DecomposedSET_2.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting decomposition of SET') + + # # PLOTTING THE DERIVATIVES OF FAVRE VEL AND TEMPERATURE + ######################################################### + favre_vel_components = [0,1,2] + for i in range(len(favre_vel_components)): + components = [tuple([favre_vel_components[i]])] + for j in range(3): + components.append(tuple([j,favre_vel_components[i]])) + vars_strs = ['u_tilde', 'D_u_tilde', 'D_u_tilde', 'D_u_tilde'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/D_favre_comp={}.pdf".format(favre_vel_components[i]) + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting derivatives of favre velocity', flush=True) + + vars_strs = ['T_tilde', 'D_T_tilde', 'D_T_tilde', 'D_T_tilde'] + components = [(), (0,), (1,), (2,)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) fig.tight_layout() time_for_filename = str(round(time_meso,2)) - filter_width = str(config['Figures_name_param']['filter_width']) - filename = "/ET_"+ time_for_filename+ "_fw_" + filter_width + "_not_scaling.pdf" + filename = "/D_T.pdf" plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting temperature derivatives', flush=True) + + # # PLOTTING SHEAR, ACCELERATION, EXPANSION AND TEMPERATURE DERIVATIVES + ######################################################################## + vars_strs = ['shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde'] + components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Shear_comps.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting shear', flush=True) + + vars_strs = ['acc_tilde', 'acc_tilde', 'acc_tilde', 'Theta_tilde', 'Theta_tilde', 'Theta_tilde'] + components = [(0,), (1,), (2,), (0,), (1,), (2,)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Acc+Tderivs.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting DaT', flush=True) + + vars_strs = ['exp_tilde'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Expansion.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting expansion', flush=True) + + diff --git a/Proof_of_principle/visualizing_micro.py b/Proof_of_principle/visualizing_micro.py index dd90797..a3316b4 100644 --- a/Proof_of_principle/visualizing_micro.py +++ b/Proof_of_principle/visualizing_micro.py @@ -25,7 +25,13 @@ if micro_from_hdf5: hdf5_directory = config['Directories']['hdf5_dir'] filenames = hdf5_directory - FileReader = METHOD_HDF5(filenames) + print('=========================================================================') + print(f'Starting job on data from {hdf5_directory}') + print('=========================================================================\n\n') + snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) + fewer_snaps_required = snapshots_opts['fewer_snaps_required'] + smaller_list = snapshots_opts['smaller_list'] + FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) num_snaps = FileReader.num_files micro_model = IdealHD_2D() FileReader.read_in_data_HDF5_missing_xy(micro_model) @@ -36,6 +42,9 @@ pickle_directory = config['Directories']['pickled_files_dir'] micro_pickled_filename = config['Filenames']['micro_pickled_filename'] MicroModelLoadFile = pickle_directory + micro_pickled_filename + print('=========================================================================') + print(f'Starting job on data from {MicroModelLoadFile}') + print('=========================================================================\n\n') with open(MicroModelLoadFile, 'rb') as filehandle: micro_model = pickle.load(filehandle) @@ -58,7 +67,7 @@ fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components) fig.tight_layout() time_for_filename = str(round(plot_time,2)) - filename = "/micro_ET_" + time_for_filename + "_BC.pdf" + filename = "/micro_T_" + time_for_filename + "_BC.pdf" plt.savefig(saving_directory + filename, format = "pdf") # Plotting the stress energy tensor @@ -67,7 +76,7 @@ fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components) fig.tight_layout() time_for_filename = str(round(plot_time,2)) - filename = "/micro_ET_" + time_for_filename + "_SET.pdf" + filename = "/micro_T_" + time_for_filename + "_SET.pdf" plt.savefig(saving_directory + filename, format = "pdf") # plotting primitive quantities @@ -75,6 +84,6 @@ fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range) fig.tight_layout() time_for_filename = str(round(plot_time,2)) - filename = "/micro_ET_" + time_for_filename + "_prims.pdf" + filename = "/micro_T_" + time_for_filename + "_prims.pdf" plt.savefig(saving_directory + filename, format = "pdf") diff --git a/Proof_of_principle/visualizing_obs.py b/Proof_of_principle/visualizing_obs.py index c13a999..d9d554d 100644 --- a/Proof_of_principle/visualizing_obs.py +++ b/Proof_of_principle/visualizing_obs.py @@ -25,18 +25,24 @@ # LOADING MESO AND MICRO MODELS pickle_directory = config['Directories']['pickled_files_dir'] meso_pickled_filename = config['Filenames']['meso_pickled_filename'] - MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('=========================================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('=========================================================================\n\n') + with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - micro_model = meso_model.micro_model + print(f'Finished reading data from {MesoModelLoadFile}') + # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE num_snaps = micro_model.domain_vars['nt'] central_slice_num = int(num_snaps/2.) time_micro = micro_model.domain_vars['t'][central_slice_num] - time_meso = meso_model.domain_vars['T'][1] #Change this if meso_grid is not set up using three central micro_slices + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] if time_meso != time_micro: print("Slices of meso and micro model do not coincide. Careful!") else: @@ -44,19 +50,34 @@ # PLOT SETTINGS + saving_directory = config['Directories']['figures_dir'] plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) x_range = plot_ranges['x_range'] y_range = plot_ranges['y_range'] - saving_directory = config['Directories']['figures_dir'] + diff_plot_settings = json.loads(config['Plot_settings']['diff_plot_settings']) + diff_method = diff_plot_settings['method'] + interp_dims = diff_plot_settings['interp_dims'] visualizer = Plotter_2D([11.97, 8.36]) # FINALLY, PLOTTING models = [micro_model, meso_model] vars = [['bar_vel', 'bar_vel', 'bar_vel'],['U', 'U', 'U']] components_indices= [[(0,),(1,),(2,)], [(0,), (1,), (2,)]] - fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices=components_indices, diff_plot=True, rel_diff=True) + fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices=components_indices, method=diff_method, + interp_dims=interp_dims, diff_plot=False, rel_diff=False) fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - box_len = str(config['Figures_name_param']['box_length']) - filename="/Meso_obs_" + box_len + "dx_ET_"+ time_for_filename +".pdf" - plt.savefig(saving_directory + filename, format="pdf") \ No newline at end of file + filename="/ObsVSmicro.pdf" + plt.savefig(saving_directory + filename, format="pdf") + print('Finished plotting the observers') + + + models = [micro_model, meso_model] + vars = [['bar_vel', 'bar_vel', 'bar_vel'],['u_tilde', 'u_tilde', 'u_tilde']] + components_indices= [[(0,),(1,),(2,)], [(0,), (1,), (2,)]] + fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices=components_indices, method=diff_method, + interp_dims=interp_dims, diff_plot=False, rel_diff=False) + fig.tight_layout() + filename="/favreVSmicro.pdf" + plt.savefig(saving_directory + filename, format="pdf") + print('Finished plotting the favre velocities') + From 556f82875462c63cf13209aeb03e31f8d94d3ed6 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 1 Feb 2024 16:39:35 +0100 Subject: [PATCH 067/111] Adding routine to visualize coefficients to PoP --- .../visualizing_correlations.py | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 Proof_of_principle/visualizing_correlations.py diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py new file mode 100644 index 0000000..0a5f386 --- /dev/null +++ b/Proof_of_principle/visualizing_correlations.py @@ -0,0 +1,93 @@ +import sys +# import os +sys.path.append('../master_files/') +import pickle +import configparser +import json +import time +import math + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ############################################################## + # #PLOT KEY QUANTITIES RELEVANT TO CALIBRATE CLOSURE + # ############################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + # #UNCOMMENT IF YOU NEED TO RE-COMPUTE, SAY, THE CLOSURE COEFFICIENTS + # ######################################## + # n_cpus = int(config['Meso_model_settings']['n_cpus']) + # meso_model.EL_style_closure_parallel(n_cpus) + # print('Finished re-decomposing the meso_model! Now re-saving it') + # with open(MesoModelLoadFile, 'wb') as filehandle: + # pickle.dump(meso_model, filehandle) + # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') + + + plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) + x_range = plot_ranges['x_range'] + y_range = plot_ranges['y_range'] + # extent = [y_range[0], y_range[-1], x_range[0], x_range[-1]] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + saving_directory = config['Directories']['figures_dir'] + + visualizer = Plotter_2D() + + vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] + norms = ['mysymlog', 'mysymlog', 'log'] + cmaps = ['seismic', 'seismic', None] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Bulk_viscosity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished bulk viscosity') + + vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] + norms = ['mysymlog', 'log', 'log'] + cmaps = ['seismic', None, None] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Shear_viscosity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished shear viscosity') + + vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] + norms = ['mysymlog', 'log', 'log'] + cmaps = ['seismic', None, None] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Heat_conductivity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished heat conductivity') + + + + \ No newline at end of file From c4a755004aa5df45a8c494ed8b81e7d7e61196c5 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 8 Feb 2024 17:54:11 +0100 Subject: [PATCH 068/111] Updates to PoP routines --- Proof_of_principle/config_meso.txt | 12 +- Proof_of_principle/config_visualizing.txt | 10 +- Proof_of_principle/pickling_meso.py | 138 +++++++++++----------- Proof_of_principle/visualizing_meso.py | 8 +- Proof_of_principle/visualizing_micro.py | 4 +- 5 files changed, 88 insertions(+), 84 deletions(-) diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index 12255ee..3b62819 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -1,22 +1,22 @@ -# CONFIGURATION FILE FOR SCRIPTS PICKLING_MESO.PY +# CONFIGURATION FILE FOR SCRIPT PICKLING_MESO.PY ################################################# [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/METHOD_output/ET_8.0 +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_2.0 -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/pickled_files +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 [Micro_model_settings] #snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. #the options is activated using the first key, while the list refers to the index of the elements of #ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) -snapshots_opts = {"fewer_snaps_required": true, "smaller_list": [5,6,7,8,9,10,11,12,13,14,15]} +snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [3,4,5,6,7,8,9,10,11,12,13,14,15,16,17]} [Meso_model_settings] -meso_grid = {"x_range": [0.02, 0.98], "y_range": [0.02, 0.98], "num_T_slices": 3, "coarse_grain_factor": 4, "coarse_grain_time": true} +meso_grid = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97], "num_T_slices": 3, "coarse_grain_factor": 4, "coarse_grain_time": true} filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 4.0} @@ -24,4 +24,4 @@ n_cpus = 40 [Pickled_filenames] -meso_pickled = /rHD2d_ET8_cg=fw=bl=4dx_dt.pickle \ No newline at end of file +meso_pickled = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle \ No newline at end of file diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt index 178330c..dee07b3 100644 --- a/Proof_of_principle/config_visualizing.txt +++ b/Proof_of_principle/config_visualizing.txt @@ -3,21 +3,21 @@ [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/METHOD_output/ET_8.0 +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_2.0 -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/pickled_files +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 -figures_dir = /scratch/tc2m23/KHIRandom/hydro/ET_3_5_8_20dx/800X800/figures/ET8/cg=bl=fw_dt/cg4 +figures_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 [Filenames] -meso_pickled_filename = /rHD2d_ET8_cg=fw=bl=4dx_dt.pickle +meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle [Models_settings] #snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. #relevant only for visualizing_micro.py -snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [6,7,8,9,10,11,12,13,14]} +snapshots_opts = {"fewer_snaps_required": true, "smaller_list": [6,7,8,9,10,11,12,13,14]} mesogrid_T_slices_num = 3 diff --git a/Proof_of_principle/pickling_meso.py b/Proof_of_principle/pickling_meso.py index 904f0e2..7471e34 100644 --- a/Proof_of_principle/pickling_meso.py +++ b/Proof_of_principle/pickling_meso.py @@ -21,80 +21,80 @@ config = configparser.ConfigParser() config.read(sys.argv[1]) - # start_time = time.perf_counter() - # hdf5_directory = config['Directories']['hdf5_dir'] - - # print(f'Starting job with data from {hdf5_directory}') - # print('==============================================\n') - # filenames = hdf5_directory - # snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) - # fewer_snaps_required = snapshots_opts['fewer_snaps_required'] - # smaller_list = snapshots_opts['smaller_list'] - # FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) - # num_snaps = FileReader.num_files - # micro_model = IdealHD_2D() - # FileReader.read_in_data_HDF5_missing_xy(micro_model) - # micro_model.setup_structures() - # time_taken = time.perf_counter() - start_time - # print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}') - - # # SETTING UP THE MESO MODEL - # start_time = time.perf_counter() - # meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) - # filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) - - # coarse_factor = meso_grid['coarse_grain_factor'] - # coarse_time = meso_grid['coarse_grain_time'] - # central_slice_num = int(num_snaps/2) - # num_T_slices = meso_grid['num_T_slices'] - # furthest_slice_number = (num_T_slices-1)/2 - # if coarse_time: - # furthest_slice_number = int(coarse_factor * furthest_slice_number) - # t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] - # x_range = meso_grid['x_range'] - # y_range = meso_grid['y_range'] - - # box_len_ratio = float(filtering_options['box_len_ratio']) - # filter_width_ratio = filtering_options['filter_width_ratio'] - - # box_len = box_len_ratio * micro_model.domain_vars['dx'] - # width = filter_width_ratio * micro_model.domain_vars['dx'] - # find_obs = FindObs_root_parallel(micro_model, box_len) - # filter = box_filter_parallel(micro_model, width) - # meso_model = resHD2D(micro_model, find_obs, filter) - # meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) - - # time_taken = time.perf_counter() - start_time - # print('Grid is set up, time taken: {}'.format(time_taken)) - # num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] - # print('Number of points: {}'.format(num_points)) - - # # FINDING THE OBSERVERS AND FILTERING - # n_cpus = int(config['Meso_model_settings']['n_cpus']) - - # start_time = time.perf_counter() - # meso_model.find_observers_parallel(n_cpus) - # time_taken = time.perf_counter() - start_time - # print('Observers found in parallel: time taken= {}'.format(time_taken)) + start_time = time.perf_counter() + hdf5_directory = config['Directories']['hdf5_dir'] + + print(f'Starting job with data from {hdf5_directory}') + print('==============================================\n') + filenames = hdf5_directory + snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) + fewer_snaps_required = snapshots_opts['fewer_snaps_required'] + smaller_list = snapshots_opts['smaller_list'] + FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) + num_snaps = FileReader.num_files + micro_model = IdealHD_2D() + FileReader.read_in_data_HDF5_missing_xy(micro_model) + micro_model.setup_structures() + time_taken = time.perf_counter() - start_time + print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}') - # start_time = time.perf_counter() - # meso_model.filter_micro_vars_parallel(n_cpus) - # time_taken = time.perf_counter() - start_time - # print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) + # SETTING UP THE MESO MODEL + start_time = time.perf_counter() + meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) + filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) + + coarse_factor = meso_grid['coarse_grain_factor'] + coarse_time = meso_grid['coarse_grain_time'] + central_slice_num = int(num_snaps/2) + num_T_slices = meso_grid['num_T_slices'] + furthest_slice_number = (num_T_slices-1)/2 + if coarse_time: + furthest_slice_number = int(coarse_factor * furthest_slice_number) + t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] + x_range = meso_grid['x_range'] + y_range = meso_grid['y_range'] + + box_len_ratio = float(filtering_options['box_len_ratio']) + filter_width_ratio = filtering_options['filter_width_ratio'] + + box_len = box_len_ratio * micro_model.domain_vars['dx'] + width = filter_width_ratio * micro_model.domain_vars['dx'] + find_obs = FindObs_root_parallel(micro_model, box_len) + filter = box_filter_parallel(micro_model, width) + meso_model = resHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) + time_taken = time.perf_counter() - start_time + print('Grid is set up, time taken: {}'.format(time_taken)) + num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] + print('Number of points: {}'.format(num_points)) - # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: - # LOADING MESO MODEL - pickle_directory = config['Directories']['pickled_files_dir'] - meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] - MesoModelLoadFile = pickle_directory + meso_pickled_filename + # FINDING THE OBSERVERS AND FILTERING n_cpus = int(config['Meso_model_settings']['n_cpus']) - print('=========================================================================') - print(f'Starting job on data from {MesoModelLoadFile}') - print('=========================================================================\n\n') - with open(MesoModelLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) + start_time = time.perf_counter() + meso_model.find_observers_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Observers found in parallel: time taken= {}'.format(time_taken)) + + start_time = time.perf_counter() + meso_model.filter_micro_vars_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) + + + # # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: + # # LOADING MESO MODEL + # pickle_directory = config['Directories']['pickled_files_dir'] + # meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] + # MesoModelLoadFile = pickle_directory + meso_pickled_filename + # n_cpus = int(config['Meso_model_settings']['n_cpus']) + + # print('=========================================================================') + # print(f'Starting job on data from {MesoModelLoadFile}') + # print('=========================================================================\n\n') + # with open(MesoModelLoadFile, 'rb') as filehandle: + # meso_model = pickle.load(filehandle) # DECOMPOSING AND CALCULATING THE CLOSURE INGREDIENTS diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index 4139420..a6d1ac3 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -98,16 +98,20 @@ # # PLOTTING THE DECOMPOSED SET # ############################# vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] + norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] + cmaps = ['seismic', 'seismic', 'seismic', 'seismic', 'seismic', 'seismic'] components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) filename = "/DecomposedSET_1.pdf" plt.savefig(saving_directory + filename, format = 'pdf') vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] + norms = ['mysymlog', 'mysymlog', 'mysymlog', None, None, None] + cmaps = ['seismic', 'seismic', 'seismic', None, None, None] components = [(0,), (1,), (2,), (), (), ()] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) filename = "/DecomposedSET_2.pdf" diff --git a/Proof_of_principle/visualizing_micro.py b/Proof_of_principle/visualizing_micro.py index a3316b4..21454e6 100644 --- a/Proof_of_principle/visualizing_micro.py +++ b/Proof_of_principle/visualizing_micro.py @@ -28,10 +28,10 @@ print('=========================================================================') print(f'Starting job on data from {hdf5_directory}') print('=========================================================================\n\n') - snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) + snapshots_opts = json.loads(config['Models_settings']['snapshots_opts']) fewer_snaps_required = snapshots_opts['fewer_snaps_required'] smaller_list = snapshots_opts['smaller_list'] - FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) + FileReader = METHOD_HDF5(filenames, fewer_snaps_required, smaller_list) num_snaps = FileReader.num_files micro_model = IdealHD_2D() FileReader.read_in_data_HDF5_missing_xy(micro_model) From 3154ecd07a416afd81efaaaafa4c5bed00089ed6 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 8 Feb 2024 17:56:14 +0100 Subject: [PATCH 069/111] Adding dictionary for plots etc --- .../visualizing_correlations.py | 93 ------------------- master_files/MesoModels.py | 42 ++++++++- master_files/MicroModels.py | 12 +++ master_files/Visualization.py | 31 +++++-- 4 files changed, 76 insertions(+), 102 deletions(-) delete mode 100644 Proof_of_principle/visualizing_correlations.py diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py deleted file mode 100644 index 0a5f386..0000000 --- a/Proof_of_principle/visualizing_correlations.py +++ /dev/null @@ -1,93 +0,0 @@ -import sys -# import os -sys.path.append('../master_files/') -import pickle -import configparser -import json -import time -import math - -from FileReaders import * -from MicroModels import * -from MesoModels import * -from Visualization import * -from Analysis import * - -if __name__ == '__main__': - - # ############################################################## - # #PLOT KEY QUANTITIES RELEVANT TO CALIBRATE CLOSURE - # ############################################################## - - # READING SIMULATION SETTINGS FROM CONFIG FILE - if len(sys.argv) == 1: - print(f"You must pass the configuration file for the simulations.") - raise Exception() - - config = configparser.ConfigParser() - config.read(sys.argv[1]) - - # LOADING MESO AND MICRO MODELS - pickle_directory = config['Directories']['pickled_files_dir'] - meso_pickled_filename = config['Filenames']['meso_pickled_filename'] - MesoModelLoadFile = pickle_directory + meso_pickled_filename - - print(f'Starting job on data from {MesoModelLoadFile}') - print('================================================\n\n') - - with open(MesoModelLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) - - # #UNCOMMENT IF YOU NEED TO RE-COMPUTE, SAY, THE CLOSURE COEFFICIENTS - # ######################################## - # n_cpus = int(config['Meso_model_settings']['n_cpus']) - # meso_model.EL_style_closure_parallel(n_cpus) - # print('Finished re-decomposing the meso_model! Now re-saving it') - # with open(MesoModelLoadFile, 'wb') as filehandle: - # pickle.dump(meso_model, filehandle) - # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') - - - plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) - x_range = plot_ranges['x_range'] - y_range = plot_ranges['y_range'] - # extent = [y_range[0], y_range[-1], x_range[0], x_range[-1]] - num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) - time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] - saving_directory = config['Directories']['figures_dir'] - - visualizer = Plotter_2D() - - vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] - norms = ['mysymlog', 'mysymlog', 'log'] - cmaps = ['seismic', 'seismic', None] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Bulk_viscosity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished bulk viscosity') - - vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] - norms = ['mysymlog', 'log', 'log'] - cmaps = ['seismic', None, None] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Shear_viscosity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished shear viscosity') - - vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] - norms = ['mysymlog', 'log', 'log'] - cmaps = ['seismic', None, None] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Heat_conductivity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished heat conductivity') - - - - \ No newline at end of file diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index ce5f868..bf4bf7e 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1147,7 +1147,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): for var in self.meso_structures: self.meso_structures[var] = [] - self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'sigma_tilde', 'T_tilde'] + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] self.meso_vectors_strs = ['u_tilde', 'q_res'] self.meso_r2tensors_strs = ['pi_res'] self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs @@ -1191,6 +1191,44 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): if not compatible: print("Meso and Micro models are incompatible:"+error) + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] + self.meso_vectors_strs = ['u_tilde', 'q_res'] + self.meso_r2tensors_strs = ['pi_res'] + self.coefficient_strs = ["Gamma"] + self.labels_var_dict = {'SET' : r'$$', + 'BC' : r'$$', + 'eps_tilde' : r'$\tilde{\varepsilon}$', + 'n_tilde' : r'$\tilde{n}$', + 'p_tilde' : r'$\tilde{p}$', + 'p_filt' : r'$

$', + 'eos_res' : r'$M$', + 'Pi_res' : r'$\tilde{\Pi}$', + 'T_tilde' : r'$\tilde{T}$', + 'u_tilde' : r'$\tilde{u}^a$', + 'D_u_tilde' : r'$\nabla_{a}\tilde{u}^b$', + 'D_T_tilde' : r'$\nabla_{a}\tilde{T}$', + 'q_res' : r'$\tilde{q}^a$', + 'pi_res' : r'$\tilde{\pi}^{ab}$', + 'Gamma' : r'$\Gamma$', + 'shear_tilde' : r'$\tilde{\sigma}^{ab}$', + 'acc_tilde' : r'$\tilde{a}^a$', + 'exp_tilde' : r'$\tilde{\theta}$', + 'Theta_tilde' : r'$\tilde{\Theta}^a$', + 'eta' : r'$\eta$', + 'zeta' : r'$\zeta$', + 'kappa' : r'$\kappa$', + 'Pi_res_sq' : r'$\tilde{\Pi}^2$', + 'pi_res_sq' : r'$\tilde{\pi}_{ab}\tilde{\pi}^{ab}$', + 'shear_sq' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}$', + 'Theta_sq': r'$\tilde{\Theta}_a\tilde{\Theta}^a$', + 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$'} + + def upgrade_labels_dict(self, entry_dict): + """ + pretty self-explanatory + """ + self.labels_var_dict.update(entry_dict) + def set_find_obs(self, find_obs): self.find_obs = find_obs @@ -1672,7 +1710,7 @@ def p_Gamma_law(eps, n, Gamma): Gamma: float Gamma factor of the Gamma law """ - return (Gamma-1)* eps*n + return (Gamma-1)* (eps-n) @staticmethod def decompose_structures_task(BC, SET , p_filt, h, i, j): diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index 0e28a23..166fe4e 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -554,6 +554,18 @@ def __init__(self, interp_method = "linear"): for str in self.structures_strs: self.structures[str] = [] + self.labels_var_dict = {'BC' : r'$n^{a}$', + 'SET' : r'$T^{ab}$', + 'bar_vel' : r'$u^a$', + 'vx' : r'$v_x$', + 'vy' : r'$v_y$'} + + def upgrade_labels_dict(self, entry_dict): + """ + pretty self-explanatory + """ + self.labels_var_dict.update(entry_dict) + def get_spatial_dims(self): return self.spatial_dims diff --git a/master_files/Visualization.py b/master_files/Visualization.py index fbee561..636c25f 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -35,6 +35,10 @@ def __init__(self, screen_size = [11.97, 8.36]): self.screen_size = np.array(screen_size) + + # Change to latex font + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") def get_var_data(self, model, var_str, t, x_range, y_range, component_indices=(), method= 'raw_data', interp_dims=None): """ @@ -245,13 +249,17 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, components_indices=Non fig.colorbar(im, cax=cax, orientation='vertical') title = var_str + if hasattr(model, 'labels_var_dict'): + if title in model.labels_var_dict.keys(): + title = model.labels_var_dict[title] if component_indices != (): title = title + ", {}-component".format(component_indices) ax.set_title(title) ax.set_xlabel(r'$y$') ax.set_ylabel(r'$x$') - fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), t), fontsize = 12) + time_for_filename = str(round(t,2)) + fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), time_for_filename), fontsize = 12) fig.tight_layout() # plt.show() return fig @@ -309,12 +317,12 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com print("The number of variables per model must be the same. Exiting.") return None - if len(var_strs)!= len(norms): + if not norms: + norms = [None for _ in range(len(var_strs[0]))] + elif len(var_strs)!= len(norms): print('The norms provided are not the same number as the variables: setting these to auto') - norms = [None for _ in range(len(var_strs))] - elif norms == None: - norms = [None for _ in range(len(var_strs))] - + norms = [None for _ in range(len(var_strs[0]))] + n_cols = len(models) if diff_plot: if len(models)!=2: @@ -349,7 +357,16 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') - title = models[j].get_model_name() + "\n"+var_strs[j][i] + # title = models[j].get_model_name() + "\n"+var_strs[j][i] + title = models[j].get_model_name() + '\n' + if hasattr(models[j], 'labels_var_dict'): + if var_strs[j][i] in models[j].labels_var_dict.keys(): + title += models[j].labels_var_dict[var_strs[j][i]] + else: + title += var_strs[j][i] + else: + title += var_strs[j][i] + if components_indices[j][i] != (): title += " {}-component".format(components_indices[j][i]) axes[i,j].set_title(title) From 625c2e72d72a2072b6cba6ad1d2bb5141ab1773f Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 8 Feb 2024 17:56:51 +0100 Subject: [PATCH 070/111] Work on regression routine + visualize correlations --- master_files/Analysis.py | 179 ++++++++++++++++++++++++++++++--------- 1 file changed, 139 insertions(+), 40 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 8df3176..e375ef8 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -30,7 +30,7 @@ class CoefficientsAnalysis(object): """ Class containing a number of methods for performing statistical analysis on gridded data. - Methods include: regression, visualizing correlations, PCAs + Methods include: regression, visualizing correlations, PCA routines """ def __init__(self): #, visualizer, spatial_dims): """ @@ -41,9 +41,9 @@ def __init__(self): #, visualizer, spatial_dims): Also nothing... """ - # self.visualizer=visualizer # need to re-structure this... or do I - # Do you really need the visualizer here? I don't think so! - # self.spatial_dims=spatial_dims + # Change to latex font + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") def trim_data(self, data, ranges, model_points): """ @@ -83,7 +83,7 @@ def trim_data(self, data, ranges, model_points): return data - def scalar_regression(self, y, X, ranges = None, model_points = None, weights=None, add_intercept=False): + def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, add_intercept=False): """ Routine to perform ordinary or weighted (multivariate) regression on some gridded data. @@ -103,14 +103,13 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No weights: ndarray of gridded weights add_intercept: bool - whether to extemd the dataset to account for a constant offset in the + whether to extend the dataset to account for a constant offset in the regression model. Returns: -------- - list of fitted parameters - list of std errors associated with the fitted parameters, - (i.e. parameter * std error of the corresponding regressor.) + list of (fitted parameters, std errors) + (std errors = parameter * std error of the corresponding regressor.) Notes: ------ @@ -122,14 +121,15 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No Intercept must be added manually: for the future, add this via true/false block. """ - # CHECKING ALIGNMENT OF PASSED DATA + + # CHECKING ALIGNMENT OF PASSED DATA dep_shape = np.shape(y) n_reg = len(X) for i in range(n_reg): if np.shape(X[i]) != dep_shape: print(f'The {i}-th regressor data is not aligned with dependent data, removing {i}-th regressor data.') X.remove(X[i]) - + if len(X)==0: print('None of the passed regressors is compatible with dep data. Exiting.') return None @@ -138,33 +138,38 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No if np.shape(y) != np.shape(weights): print(f'The weights passed are not not aligned with data, setting these to 1') weights=np.ones(y.shape) - - # TRIMMING THE DATA TO WITHIN RANGES + + + # TRIMMING THE DATA TO WITHIN RANGES + ADDING COLUMN FOR INTERCEPT (if necessary) if ranges != None and model_points != None: print('Trimming dataset for regression') Y = self.trim_data(y, ranges, model_points) - if weights: - Weights = self.trim_data(weights, ranges, model_points) XX = [] for x in X: XX.append(self.trim_data(x, ranges, model_points)) + if weights: + Weights = self.trim_data(weights, ranges, model_points) + else: + XX , Y = X , y + if weights: + Weights = weights # FLATTENING + FITTING Y = Y.flatten() if weights: Weights = Weights.flatten() - for i in range(len(X)): - XX[i]=XX[i].flatten() - if add_intercept: const = np.ones(Y.shape) - XX.insert(0,const) - n_reg = len(XX) - n_data = len(Y) - XX= np.reshape(XX, (n_data, n_reg)) + # print(const) + XX.insert(0, const) + n_reg = n_reg + 1 + + for i in range(len(XX)): + XX[i]=XX[i].flatten() + XX = np.einsum('ij->ji', XX) - # VERSION USING STATSMODELS + # # VERSION USING STATSMODELS # if weights: # model=sm.WLS(y, Xfl, W) # result= model.fit() @@ -177,9 +182,15 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No # VERSION USING SKLEARN: model=LinearRegression(fit_intercept=False) if weights: - model.fit(XX,Y,sample_weight=weights) + print('Fitting with weights') + model.fit(XX, Y, sample_weight=Weights) else: + print('Fitting without weights') model.fit(XX,Y) + regress_coeff = list(model.coef_) + + # Computing std errors on the regressed coefficients + n_data = len(Y) Y_hat = model.predict(XX) res = Y - Y_hat res_sum_of_sq = np.dot(res,res) @@ -187,8 +198,7 @@ def scalar_regression(self, y, X, ranges = None, model_points = None, weights=No var_beta = np.linalg.inv(np.einsum('ji,jl->il',XX,XX)) * sigma_res_sq bse = [var_beta[i,i]**0.5 for i in range(n_reg)] - result = [(model.coef_[i], bse[i]) for i in range(n_reg)] - return result + return regress_coeff, bse def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_points=None, components=None, weights=None, add_intercept=False): """ @@ -266,7 +276,8 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) return results - def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None): + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None, \ + hue_array=None, style_array=None, legend_dict=None, palette=None, markers=None): """ Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. Possibility to cut data to lie within ranges. @@ -298,22 +309,39 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod print('Trimming dataset for correlation plot') X = self.trim_data(x, ranges, model_points) Y = self.trim_data(y, ranges, model_points) + if hue_array is not None: + hue_array = self.trim_data(hue_array, ranges, model_points) + if style_array is not None: + style_array = self.trim_data(style_array, ranges, model_points) print('Finished trimming data') else: X, Y = x, y + X=X.flatten() Y=Y.flatten() + if hue_array is not None: + hue_array = hue_array.flatten() + if style_array is not None: + style_array = style_array.flatten() print('Data flattened') + with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') g=sns.JointGrid() - sns.scatterplot(x=X, y=Y, ax=g.ax_joint) - sns.kdeplot(x=X, ax=g.ax_marg_x) - sns.histplot(y=Y, ax=g.ax_marg_y, kde=True) + scatter = sns.scatterplot(x=X, y=Y, ax=g.ax_joint, s=4, c='red', hue=hue_array, style=style_array, + palette=palette, markers=markers) + sns.histplot(x=X, ax=g.ax_marg_x, kde=True, color= 'red') + sns.histplot(y=Y, ax=g.ax_marg_y, kde=True, color= 'red') g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) g.fig.tight_layout() - print('Figure produced inside Coeff Analysis') + + if legend_dict is not None: + handles, labels = scatter.get_legend_handles_labels() + new_labels = [legend_dict[label] for label in labels] + scatter.legend(handles, new_labels) + + # print('Figure produced inside Coeff Analysis') return g def visualize_many_correlations(self, data, labels, ranges=None, model_points=None): @@ -471,16 +499,78 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w return comp_decomp, g - def PCA_find_regressors(): + def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, model_points=None, pcs_num=1): """ - Idea pass both the residuals and a large list of quantities, identify which among these are - correlated with the residual. Do this by checking the scores of the residual (the first var - of the dataset, say) on the various principal components. - Then return the weights. This will then be used in regression (possibyly after checking they're - not correlated). + Idea: pass both the residuals/coefficient and a list of quantities. Identify which among these are + correlated with the residual/coefficient, by checking the scores of this on the + principal components. Then return the linear combination of the (highest) components with respect + to the explanatory vars. """ - pass - + # CHECKING AND PREPROCESSING THE DATA + dep_shape = dependent_var.shape + n_expl_vars = len(explanatory_vars) + Expl_vars = [] + for i in range(0, n_expl_vars): + if explanatory_vars[i].shape != dep_shape: + print(f'The {i}-th feature passed is not aligned with the first, removing {i}-th feature.') + else: + Expl_vars.append(explanatory_vars[i]) + + if len(Expl_vars)==0: + print('No two vars are compatible. Exiting.') + return None + + if ranges != None and model_points != None: + print('Trimming dataset for PCA analysis.') + Dep_var = self.trim_data(dependent_var, ranges, model_points) + for i in range(len(explanatory_vars)): + Expl_vars[i] = self.trim_data(Expl_vars[i], ranges, model_points) + + Dep_var = Dep_var.flatten() + for i in range(len(Expl_vars)): + Expl_vars[i] = Expl_vars[i].flatten() + + #STANDARDIZING DATA TO ZERO MEAN AND UNIT VARIANCE + Data = [] + for i in range(len(Expl_vars)): + x = Expl_vars[i] + mean = np.mean(x) + var = np.var(x) + y = np.array([x[j]-mean for j in range(len(x))]) + Data.append(y/var) + # Data.append(y) + mean = np.mean(Dep_var) + var = np.var(Dep_var) + y = (Dep_var - mean) + y=y/var + Data = np.column_stack(tuple([y] + [Data[i] for i in range(len(Expl_vars))])) + + #IDENTIFYING THE COMPONENTS WITH HIGHEST SCOREs ON THE DEPENDENT VAR + pca_model = PCA().fit(Data) + var2comp = pca_model.components_ + comp2var = np.einsum('ij->ji', var2comp) + + print('components_: \n{}\n'.format(var2comp)) + scores_of_dep_var = var2comp[:,0] + print(f'Scores of dependent var on the principal components: \n{scores_of_dep_var}\n') + # sorted_scores_indices = np.argsort(scores_of_dep_var) + sorted_scores_indices = np.argsort(np.abs(scores_of_dep_var)) + pos_of_highest_pcs = sorted_scores_indices[-pcs_num:] #identifying the index of the ones with highest score + # this is ordering the scores, but you should care about their magnitude, no? + + highest_pcs_decomp = [] + corresponding_scores = [] + tot_explained_var = 0 + for i in range(pcs_num-1, -1, -1): + highest_pcs_decomp.append(comp2var[:,pos_of_highest_pcs[i]]) + corresponding_scores.append(scores_of_dep_var[pos_of_highest_pcs[i]]) + tot_explained_var += pca_model.explained_variance_ratio_[pos_of_highest_pcs[i]] + + print(f'Total explained variance: {tot_explained_var}\n') + + + return highest_pcs_decomp, corresponding_scores + # Not sure about these two methods. def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ interp_dims, method, y_component_indices, x_component_indices): @@ -509,4 +599,13 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met if __name__ == '__main__': - pass + + x0 = np.arange(100).reshape((10,10)) + x1 = np.sin(np.arange(100).reshape((10,10))) + X = [x0, x1] + y = 1 + 2 * x0 + 3* x1 + random.randint(20,30) + + statistical_tool = CoefficientsAnalysis() + result, errors = statistical_tool.scalar_regression(y,X, add_intercept=True) + print('Coefficients: {}'.format(result)) + print('Errors: {}\n'.format(errors)) From e816e6a1e18072f2646d03a66365c024ea94bd82 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 8 Feb 2024 17:58:27 +0100 Subject: [PATCH 071/111] Tracking routines to visualize and regress over residuals --- Proof_of_principle/regressing_residual.py | 116 ++++++++++++++++ Proof_of_principle/visualizing_residuals.py | 140 ++++++++++++++++++++ 2 files changed, 256 insertions(+) create mode 100644 Proof_of_principle/regressing_residual.py create mode 100644 Proof_of_principle/visualizing_residuals.py diff --git a/Proof_of_principle/regressing_residual.py b/Proof_of_principle/regressing_residual.py new file mode 100644 index 0000000..52f786b --- /dev/null +++ b/Proof_of_principle/regressing_residual.py @@ -0,0 +1,116 @@ +import sys +# import os +sys.path.append('../master_files/') +import pickle +import configparser +import json +import time +import math + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################## + # # RUN THE REGRESSION ROUTINE GIVEN DEPENDENT DATA AND EXPLANATORY + # # VARS. TAKE THE LOG SO TO LOOK FOR REGRESSION IN LOGSPACE. + # # TO THINK: HOW TO DEAL WITH POSITIVE AND NEGATIVE QUANTITIES? + # ################################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + # Which data you want to run the routine on? + dep_var_str = config['Regression_settings']['dependent_var'] + regressors_strs = json.loads(config['Regression_settings']['regressors']) + add_intercept = not not int(config['Regression_settings']['add_intercept']) + regressors = [] + for i in range(len(regressors_strs)): + temp = meso_model.meso_vars[regressors_strs[i]] + regressors.append(np.log10(temp)) + dep_var = np.log10(meso_model.meso_vars[dep_var_str]) + print(f'Dependent var: {dep_var_str}, Explanatory vars: {regressors_strs}\n') + + # Which ranges? (later: add tools to randomly extract from within ranges?) + regression_ranges = json.loads(config['Regression_settings']['regression_ranges']) + x_range = regression_ranges['x_range'] + y_range = regression_ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + # Performing the regression + statistical_tool = CoefficientsAnalysis() + model_points = meso_model.domain_vars['Points'] + coeffs, std_errors = statistical_tool.scalar_regression(dep_var, regressors, ranges=ranges, model_points=model_points, + add_intercept=add_intercept) + print('regression coeffs: {}\n'.format(coeffs)) + print('Corresponding std errors: {}\n'.format(std_errors)) + + # Bulding the regressed model for scatterplot + regressors_strs = [None] + regressors_strs + regressors = [np.ones(regressors[0].shape)] + regressors + if not add_intercept: + coeffs = [0] + coeffs + + dep_var_model = np.zeros(dep_var.shape) + for i in range(len(regressors)): + dep_var_model += np.multiply(coeffs[i], regressors[i]) + + # Plotting dependent var VS its model: to visually check if the regression is decent + correlation_ranges = json.loads(config['Figure_settings']['correlation_ranges']) + x_range = correlation_ranges['x_range'] + y_range = correlation_ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + ylabel = 'log({})'.format(dep_var_str) + if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): + ylabel = r"$\log($" + meso_model.labels_var_dict[dep_var_str] + ")" + + xlabel = "" + if coeffs[0] != 0: + sign, val = int(np.sign(coeffs[0])), str(round(np.abs(coeffs[0]),3)) + xlabel = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + + for i in range(1, len(coeffs)): + sign, val = int(np.sign(coeffs[i])), str(round(np.abs(coeffs[i]),3)) + xlabel += r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + text = r"$\log($" + if hasattr(meso_model, 'labels_var_dict') and regressors_strs[i] in meso_model.labels_var_dict.keys(): + text += meso_model.labels_var_dict[regressors_strs[i]] + ")" + else: + text += regressors_strs[i] + ")" + xlabel +=text + + statistical_tool.visualize_correlation(dep_var_model, dep_var , xlabel, ylabel, ranges, model_points) + saving_directory = config['Directories']['figures_dir'] + filename = '/{}_vs_{}'.format(dep_var_str, json.loads(config['Regression_settings']['regressors'])) + if add_intercept: + filename += "_intercept" + filename += ".pdf" + plt.savefig(saving_directory + filename, format='pdf') + print(f'Finished regression and scatter plot for {dep_var_str}, saved as {filename}\n\n') + + diff --git a/Proof_of_principle/visualizing_residuals.py b/Proof_of_principle/visualizing_residuals.py new file mode 100644 index 0000000..f324bd6 --- /dev/null +++ b/Proof_of_principle/visualizing_residuals.py @@ -0,0 +1,140 @@ +import sys +# import os +sys.path.append('../master_files/') +import pickle +import configparser +import json +import time +import math + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################## + # #CORRELATION PLOTS: RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS + # ################################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + # #UNCOMMENT IF YOU NEED TO RE-COMPUTE, SAY, THE CLOSURE COEFFICIENTS + # ######################################## + # n_cpus = int(config['Meso_model_settings']['n_cpus']) + # meso_model.EL_style_closure_parallel(n_cpus) + # print('Finished re-decomposing the meso_model! Now re-saving it') + # with open(MesoModelLoadFile, 'wb') as filehandle: + # pickle.dump(meso_model, filehandle) + # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') + + + + statistical_tool = CoefficientsAnalysis() + correlation_ranges = json.loads(config['Figure_settings']['correlation_ranges']) + x_range = correlation_ranges['x_range'] + y_range = correlation_ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + print('Producing correlation plot for pi_res') + x = np.log10(meso_model.meso_vars['shear_sq']) + y = np.log10(meso_model.meso_vars['pi_res_sq']) + xlabel = r'$\log(\tilde{\sigma}_{ab}\tilde{\sigma}^{ab})$' + ylabel = r'$\log(\tilde{\pi}_{ab}\tilde{\pi}^{ab})$' + model_points = meso_model.domain_vars['Points'] + g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel, ranges=ranges, model_points=model_points) + saving_directory = config['Directories']['figures_dir'] + filename = '/pi_aniso_correlation.pdf' + plt.savefig(saving_directory + filename, format='pdf') + print(f'Finished correlation plot for pi_res, saved as {filename}\n\n') + + + # print('Producing correlation plot for q_res') + # x = np.log10(meso_model.meso_vars['q_res_sq']) + # y = np.log10(meso_model.meso_vars['Theta_sq']) + # xlabel = r'$\log(\tilde{q}_a\tilde{q}^a)$' + # ylabel = r'$\log(\tilde{\Theta}_a \tilde{\Theta}^a)$' + # model_points = meso_model.domain_vars['Points'] + # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel, ranges=ranges, model_points=model_points) + # saving_directory = config['Directories']['figures_dir'] + # filename = '/q_res_correlation.pdf' + # plt.savefig(saving_directory + filename, format='pdf') + # print(f'Finished correlation plot for q_res, saved as {filename}\n\n') + + + # print('Producing correlation plot for Pi_res') + # x= np.log10(np.power(meso_model.meso_vars['Pi_res'], 2) ) + # y= np.log10(np.power(meso_model.meso_vars['exp_tilde'], 2)) + # xlabel = r'$\log(\tilde{\Pi}^2)$' + # ylabel = r'$\log(\tilde{\theta}^2)$' + # model_points = meso_model.domain_vars['Points'] + # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel, ranges=ranges, model_points=model_points) + # saving_directory = config['Directories']['figures_dir'] + # filename = '/Pi_res_correlation.pdf' + # plt.savefig(saving_directory + filename, format='pdf') + # print(f'Finished correlation plot for Pi_res, saved as {filename}\n\n') + + + # ############################################################## + # #PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS + # ############################################################## + + # plot_ranges = json.loads(config['Figure_settings']['plot_ranges']) + # x_range = plot_ranges['x_range'] + # y_range = plot_ranges['y_range'] + # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + # saving_directory = config['Directories']['figures_dir'] + + # visualizer = Plotter_2D() + + # vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] + # norms = ['mysymlog', 'mysymlog', 'log'] + # cmaps = ['seismic', 'seismic', None] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Bulk_viscosity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished bulk viscosity') + + # vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] + # norms = ['mysymlog', 'log', 'log'] + # cmaps = ['seismic', None, None] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Shear_viscosity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished shear viscosity') + + # vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] + # norms = ['mysymlog', 'log', 'log'] + # cmaps = ['seismic', None, None] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Heat_conductivity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished heat conductivity') \ No newline at end of file From 178aa96809cd773423bc27864115513ad5b96c67 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 8 Feb 2024 18:00:29 +0100 Subject: [PATCH 072/111] Tracking routines to visualize and regress over residuals --- Proof_of_principle/config_calibration.txt | 43 +++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 Proof_of_principle/config_calibration.txt diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt new file mode 100644 index 0000000..ddfdf8f --- /dev/null +++ b/Proof_of_principle/config_calibration.txt @@ -0,0 +1,43 @@ +# CONFIGURATION FILE FOR SCRIPTS: +########################################################## + +[Directories] + +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 + +figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/regression +#/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 + +[Filenames] + +meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle + +[Models_settings] + +mesogrid_T_slices_num = 3 + +[Figure_settings] + +plot_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} + +correlation_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} + +regression_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} + +[PCA_settings] + +dependent_var = Pi_res +explanatory_vars = ["T_tilde", "n_tilde", "p_tilde", "eps_tilde"] +pcs_num = 1 + +# The following will be used if the quantity to be plotted is not a scalar +dep_var_dict = {"shear_tilde": [[1, 2], [0, 1]]} +explanatory_vars_dict = {"shear_tilde": [[1, 2], [0, 1]]} + +[Regression_settings] + +dependent_var = Pi_res +regressors = ["T_tilde", "n_tilde"] +add_intercept = 0 + +regression_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} \ No newline at end of file From ae2826f5bdf40efe465804bca08ebcab74de8990 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 9 Feb 2024 20:10:15 +0100 Subject: [PATCH 073/111] Adding methods to preprocess data --- master_files/Analysis.py | 209 +++++++++++++++++++++++++++++++++------ 1 file changed, 181 insertions(+), 28 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index e375ef8..f973280 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -9,6 +9,7 @@ import matplotlib.pyplot as plt import numpy as np +import numpy.ma as ma import pandas as pd import h5py import pickle @@ -79,9 +80,115 @@ def trim_data(self, data, ranges, model_points): IdxsToRemove.append([ j for j in range(num_points[i]) if j < start_indices[i] or j > end_indices[i]]) for i in range(len(ranges)): - data = np.delete(data, IdxsToRemove[i], axis=i) + newdata = np.delete(data, IdxsToRemove[i], axis=i) - return data + return newdata + + def get_pos_or_neg_mask(self, pos_or_neg, array): + """ + Function that return the mask that would be applied to an array in order to select + positive or negative values + + Parameters: + ----------- + pos_or_neg: can be int (0 or 1) or bool True or False + + array: np.array + + Notes: + ------ + To be combined with others before applying all together + """ + if pos_or_neg: + mask = ma.masked_where(array<0, array, copy=True).mask + else: + mask = ma.masked_where(array>0, array, copy=True).mask + + return mask + + def preprocess_data(self, list_of_arrays, preprocess_data, ranges=None, model_points=None): + """ + Takes input list of arrays with dictionary on how to preprocess_data and info about + trimming of arrays to selected range within grid + + Parameters: + ---------- + list_of_arrays: + list with the combined data to be processed + + preprocess_data: dictionary + each value of the dictionary must be a list long as the arrays passed (checked for) + Expected keys are: + + 'pos_or_neg' is 1 (0) if you want to select positive (negative) values + + 'log_or_not' is 1 if you want to take the logarithm of the data + + ranges: list of lists of 2 floats --> this is passed to trim_data + the min and max in each direction + + model_points: list of list of floats --> this is passed to trim_data + the gridpoints of the array + """ + + num_arrays = len(list_of_arrays) + condition = False + for key in preprocess_data: + if len(preprocess_data[key]) != num_arrays: + condition=True + if condition: + print('Preprocess_data dictionary is not compatible with list_of_arrays') + + # Trimming data to lie within range + if ranges != None and model_points != None: + new_list = [] + for i in range(len(list_of_arrays)): + new_list.append(self.trim_data(list_of_arrays[i], ranges=ranges, model_points=model_points)) + + # Combining the masks of the different arrays + masks = [] + for i in range(len(list_of_arrays)): + pos_or_neg = preprocess_data['pos_or_neg'][i] + temp = self.get_pos_or_neg_mask(pos_or_neg, list_of_arrays[i]) + masks.append(temp) + + tot_mask = masks[0] + for i in range(1,len(masks)): + tot_mask = np.logical_or(tot_mask, masks[i]) + + # Masking the combined dataset with combined mask and taking log + processed_list = [] + for i in range(len(list_of_arrays)): + temp = ma.masked_array(list_of_arrays[i], tot_mask) + processed_list.append(temp.compressed()) + #when array is compressed, this is automatically flattened! + + for i in range(len(processed_list)): + if preprocess_data['log_or_not'][i]: + processed_list[i] = np.log10(np.abs(processed_list[i])) + + return processed_list + + def extract_randomly(self, array, num_extractions): + """ + Extract randomly from data. + + Parameters: + ----------- + data: np.array (flattened or not) + num_extractions: number of values to be extracted + """ + new_array = array.copy() + if new_array.shape != (1,): + print('Array is not flat, flattening it.') + new_array = new_array.flatten() + + max_idx = len(new_array) - 1 + extracted_vals = [] + for i in range(num_extractions): + temp = new_array[random.randint(0,max_idx)] + extracted_vals.append(temp) + return np.array(extracted_vals) def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, add_intercept=False): """ @@ -139,6 +246,13 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, print(f'The weights passed are not not aligned with data, setting these to 1') weights=np.ones(y.shape) + # Here: + # if pre_process_data != None: + # aggregate Y and X into data + # preprocess data + # return newY,newX + + # elif ranges and model points do as below (including flattening which is otherwise taken care of by process data) # TRIMMING THE DATA TO WITHIN RANGES + ADDING COLUMN FOR INTERCEPT (if necessary) if ranges != None and model_points != None: @@ -393,21 +507,18 @@ def visualize_many_correlations(self, data, labels, ranges=None, model_points=No for i in range(len(data)): Data.append(data[i].flatten()) - print('Data_flattened') - Data=np.column_stack(Data) Data_df = pd.DataFrame(Data, columns=labels) - print('Flattened data converted to a dataframe') + with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') g=sns.PairGrid(Data_df) - g.map_upper(sns.scatterplot) - g.map_lower(sns.kdeplot) - g.map_diag(sns.histplot, kde=True) + g.map_upper(sns.scatterplot, s=4, c='red') + g.map_lower(sns.kdeplot, color='red') + g.map_diag(sns.histplot, kde=True, color='red') g.fig.tight_layout() - print('Finished producing figure inside CoefficientsAnalysis') return g def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_wanted = 1.): @@ -445,33 +556,34 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w # CHECKING AND PREPROCESSING THE DATA ref_shape = data[0].shape n_vars = len(data) + Data =[data[0]] for i in range(1, n_vars): if data[i].shape != ref_shape: print(f'The {i}-th feature passed is not aligned with the first, removing {i}-th feature.') - data.remove(data[i]) + else: + Data.append(data[i]) - if len(data)==0: + if len(Data)==0: print('No two vars are compatible. Exiting.') return None if ranges != None and model_points != None: print('Trimming dataset for PCA analysis.') - for i in range(len(data)): - data[i] = self.trim_data(data[i], ranges, model_points) + for i in range(len(Data)): + Data[i] = self.trim_data(Data[i], ranges, model_points) - for i in range(n_vars): - data[i] = data[i].flatten() + for i in range(len(Data)): + Data[i] = Data[i].flatten() #STANDARDIZING DATA TO ZERO MEAN AND UNIT VARIANCE - Data = [] - for i in range(len(data)): - x = data[i] + for i in range(len(Data)): + x = Data[i] mean = np.mean(x) var = np.var(x) y = np.array([x[j]-mean for j in range(len(x))]) - Data.append(y/var) - # Data.append(y) - Data = np.column_stack(tuple([Data[i] for i in range(n_vars)])) + Data[i] = y/var + # Data[i] = y + Data = np.column_stack(tuple([Data[i] for i in range(len(Data))])) # HOW MANY PRINCIPAL COMPONENTS HAVE TO BE RETAINED? pca_model = PCA().fit(Data) @@ -600,12 +712,53 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met if __name__ == '__main__': - x0 = np.arange(100).reshape((10,10)) - x1 = np.sin(np.arange(100).reshape((10,10))) - X = [x0, x1] - y = 1 + 2 * x0 + 3* x1 + random.randint(20,30) + # #TESTING REGRESSION + # x0 = np.arange(100).reshape((10,10)) + # x1 = np.sin(np.arange(100).reshape((10,10))) + # X = [x0, x1] + # y = 1 + 2 * x0 + 3* x1 + random.randint(20,30) + + # statistical_tool = CoefficientsAnalysis() + # result, errors = statistical_tool.scalar_regression(y,X, add_intercept=True) + # print('Coefficients: {}'.format(result)) + # print('Errors: {}\n'.format(errors)) + + # #TESTING PRE-PROCESS DATA + x = np.zeros((100,100)) + for idx in np.ndindex(x.shape): + signum = [-1,1][random.randint(0,1)] + x[idx] = signum * random.randint(0,100) + + y = np.zeros((100,100)) + for idx in np.ndindex(x.shape): + signum = [-1,1][random.randint(0,1)] + if signum==1: + exp = random.randint(-5,5) + elif signum ==-1: + exp = random.randint(-3,3) + y[idx] = signum * (10 ** exp) + print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) + print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + + + + preprocess_data = {'pos_or_neg' : [1,0], + 'log_or_not' : [0,1]} + statistical_tool = CoefficientsAnalysis() - result, errors = statistical_tool.scalar_regression(y,X, add_intercept=True) - print('Coefficients: {}'.format(result)) - print('Errors: {}\n'.format(errors)) + data = [x,y] + + + + + x, y = statistical_tool.preprocess_data(data, preprocess_data) + + print('Processing data....\n') + print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) + print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + + + print('Extracting random vals from x...\n') + extracted = statistical_tool.extract_randomly(x, 10) + print(extracted) From 470e9c8dd064f4822c99c047fc7b3617597448d1 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 9 Feb 2024 20:19:14 +0100 Subject: [PATCH 074/111] Adding methods to preprocess data --- master_files/Analysis.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index f973280..7fdac84 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -106,7 +106,7 @@ def get_pos_or_neg_mask(self, pos_or_neg, array): return mask - def preprocess_data(self, list_of_arrays, preprocess_data, ranges=None, model_points=None): + def preprocess_data(self, list_of_arrays, preprocess_data):#, ranges=None, model_points=None): """ Takes input list of arrays with dictionary on how to preprocess_data and info about trimming of arrays to selected range within grid @@ -124,11 +124,12 @@ def preprocess_data(self, list_of_arrays, preprocess_data, ranges=None, model_po 'log_or_not' is 1 if you want to take the logarithm of the data - ranges: list of lists of 2 floats --> this is passed to trim_data - the min and max in each direction - model_points: list of list of floats --> this is passed to trim_data - the gridpoints of the array + # ranges: list of lists of 2 floats --> this is passed to trim_data + # the min and max in each direction + + # model_points: list of list of floats --> this is passed to trim_data + # the gridpoints of the array """ num_arrays = len(list_of_arrays) @@ -139,11 +140,11 @@ def preprocess_data(self, list_of_arrays, preprocess_data, ranges=None, model_po if condition: print('Preprocess_data dictionary is not compatible with list_of_arrays') - # Trimming data to lie within range - if ranges != None and model_points != None: - new_list = [] - for i in range(len(list_of_arrays)): - new_list.append(self.trim_data(list_of_arrays[i], ranges=ranges, model_points=model_points)) + # # Trimming data to lie within range: you probably don't want this to habben here actually + # if ranges != None and model_points != None: + # new_list = [] + # for i in range(len(list_of_arrays)): + # new_list.append(self.trim_data(list_of_arrays[i], ranges=ranges, model_points=model_points)) # Combining the masks of the different arrays masks = [] @@ -246,13 +247,6 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, print(f'The weights passed are not not aligned with data, setting these to 1') weights=np.ones(y.shape) - # Here: - # if pre_process_data != None: - # aggregate Y and X into data - # preprocess data - # return newY,newX - - # elif ranges and model points do as below (including flattening which is otherwise taken care of by process data) # TRIMMING THE DATA TO WITHIN RANGES + ADDING COLUMN FOR INTERCEPT (if necessary) if ranges != None and model_points != None: @@ -268,6 +262,9 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, if weights: Weights = weights + # Trim data first, then pre-process data + # if preprocess_data then do it. + # FLATTENING + FITTING Y = Y.flatten() if weights: @@ -750,8 +747,6 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met data = [x,y] - - x, y = statistical_tool.preprocess_data(data, preprocess_data) print('Processing data....\n') From 0194efa61eef2557b03d06e4e5bfe7134786f972 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 16 Feb 2024 16:25:27 +0100 Subject: [PATCH 075/111] Preprocess inside regression etcetera --- master_files/Analysis.py | 194 ++++++++++++++++++++++++---------- master_files/Visualization.py | 52 +++++---- 2 files changed, 171 insertions(+), 75 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 7fdac84..94f29cd 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -106,7 +106,7 @@ def get_pos_or_neg_mask(self, pos_or_neg, array): return mask - def preprocess_data(self, list_of_arrays, preprocess_data):#, ranges=None, model_points=None): + def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): """ Takes input list of arrays with dictionary on how to preprocess_data and info about trimming of arrays to selected range within grid @@ -123,13 +123,6 @@ def preprocess_data(self, list_of_arrays, preprocess_data):#, ranges=None, model 'pos_or_neg' is 1 (0) if you want to select positive (negative) values 'log_or_not' is 1 if you want to take the logarithm of the data - - - # ranges: list of lists of 2 floats --> this is passed to trim_data - # the min and max in each direction - - # model_points: list of list of floats --> this is passed to trim_data - # the gridpoints of the array """ num_arrays = len(list_of_arrays) @@ -138,13 +131,8 @@ def preprocess_data(self, list_of_arrays, preprocess_data):#, ranges=None, model if len(preprocess_data[key]) != num_arrays: condition=True if condition: - print('Preprocess_data dictionary is not compatible with list_of_arrays') - - # # Trimming data to lie within range: you probably don't want this to habben here actually - # if ranges != None and model_points != None: - # new_list = [] - # for i in range(len(list_of_arrays)): - # new_list.append(self.trim_data(list_of_arrays[i], ranges=ranges, model_points=model_points)) + print('Preprocess_data dictionary is not compatible with list_of_arrays. Exiting') + return list_of_arrays, weights # Combining the masks of the different arrays masks = [] @@ -157,7 +145,7 @@ def preprocess_data(self, list_of_arrays, preprocess_data):#, ranges=None, model for i in range(1,len(masks)): tot_mask = np.logical_or(tot_mask, masks[i]) - # Masking the combined dataset with combined mask and taking log + # Masking the full dataset with combined mask, then taking log processed_list = [] for i in range(len(list_of_arrays)): temp = ma.masked_array(list_of_arrays[i], tot_mask) @@ -168,30 +156,55 @@ def preprocess_data(self, list_of_arrays, preprocess_data):#, ranges=None, model if preprocess_data['log_or_not'][i]: processed_list[i] = np.log10(np.abs(processed_list[i])) - return processed_list + # removing corresponding entries from weights + Weights = None + if weights != None: + Weights = np.ma.masked_array(weights, tot_mask) + return processed_list, Weights + else: + return processed_list - def extract_randomly(self, array, num_extractions): + def extract_randomly(self, data, num_extractions): """ Extract randomly from data. Parameters: ----------- - data: np.array (flattened or not) + data: list of np.arrays (flattened or not) num_extractions: number of values to be extracted - """ - new_array = array.copy() - if new_array.shape != (1,): - print('Array is not flat, flattening it.') - new_array = new_array.flatten() - max_idx = len(new_array) - 1 - extracted_vals = [] + Notes: + ------ + Data is assumed pre-processesed, that is data is a list of compatible arrays! + """ + new_data = [] + for i in range(len(data)): + new_data.append(np.array(data[i])) + if new_data[i].ndim != 1: + print(f'{i}-th array is not flat, flattening it.') + new_data[i] = new_data[i].flatten() + + # setting up the mask + mask_index = np.zeros(new_data[i].shape) + for i in range(len(mask_index)): + mask_index[i] = False + + max_idx = len(new_data[0]) - 1 + extracted_idxs = [] for i in range(num_extractions): - temp = new_array[random.randint(0,max_idx)] - extracted_vals.append(temp) - return np.array(extracted_vals) + new_extraction = random.choice(list(set(range(0,max_idx)) - set(extracted_idxs))) + extracted_idxs.append(new_extraction) + mask_index[new_extraction] = True + # need to invert the mask as it is True on the extracted indices + mask_index = np.logical_not(mask_index) + + # finally: mask, compress and return + for i in range(len(new_data)): + new_data[i] = np.ma.masked_array(new_data[i], mask_index).compressed() + + return np.array(new_data) - def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, add_intercept=False): + def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, extractions=None, preprocess_data=None, add_intercept=False): """ Routine to perform ordinary or weighted (multivariate) regression on some gridded data. @@ -248,7 +261,7 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, weights=np.ones(y.shape) - # TRIMMING THE DATA TO WITHIN RANGES + ADDING COLUMN FOR INTERCEPT (if necessary) + # TRIMMING THE DATA TO WITHIN RANGES if ranges != None and model_points != None: print('Trimming dataset for regression') Y = self.trim_data(y, ranges, model_points) @@ -262,17 +275,50 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, if weights: Weights = weights - # Trim data first, then pre-process data - # if preprocess_data then do it. + # PREPROCESSING + if preprocess_data != None: + print('preprocessing data') + data = [Y] + for i in range(len(XX)): + data.append(XX[i]) + + if weights: + processed_data = self.preprocess_data(data, preprocess_data, Weights) + else: + processed_data = self.preprocess_data(data, preprocess_data) - # FLATTENING + FITTING + Y = np.array(processed_data[0]) + for i in range(len(XX)): + XX[i] = np.array(processed_data[i+1]) + if weights: + Weights= np.array(processed_data[-1]) + + # RANDOMLY EXTRACT FROM ARRAY: DATA ACTUALLY CONSIDERED FOR REGRESSION + if extractions != None: + print('Extracting randomly from sample') + data_to_extract = [Y] + for i in range(len(XX)): + data_to_extract.append(XX[i]) + if weights: + data_to_extract.append(Weights) + extracted_data = self.extract_randomly(data_to_extract, extractions) + Y, Weights = extracted_data[0], extracted_data[-1] + for i in range(len(XX)): + XX[i] = extracted_data[i+1] + else: + extracted_data = self.extract_randomly(data_to_extract, extractions) + Y = extracted_data[0] + for i in range(len(XX)): + XX[i] = extracted_data[i+1] + + + # FLATTENING (IF NOT DONE YET IN PRE-PROCESSING) + FITTING Y = Y.flatten() if weights: Weights = Weights.flatten() if add_intercept: const = np.ones(Y.shape) - # print(const) XX.insert(0, const) n_reg = n_reg + 1 @@ -300,7 +346,7 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, model.fit(XX,Y) regress_coeff = list(model.coef_) - # Computing std errors on the regressed coefficients + # COMPUTING STD ERRORS ON REGRESSED COEFFICIENTS n_data = len(Y) Y_hat = model.predict(XX) res = Y - Y_hat @@ -387,8 +433,8 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) return results - def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None, \ - hue_array=None, style_array=None, legend_dict=None, palette=None, markers=None): + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None, hue_array=None, + style_array=None, legend_dict=None, palette=None, markers=None): """ Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. Possibility to cut data to lie within ranges. @@ -427,14 +473,13 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod print('Finished trimming data') else: X, Y = x, y - + X=X.flatten() Y=Y.flatten() if hue_array is not None: hue_array = hue_array.flatten() if style_array is not None: style_array = style_array.flatten() - print('Data flattened') with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') @@ -608,12 +653,25 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w return comp_decomp, g - def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, model_points=None, pcs_num=1): + def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, model_points=None, preprocess_data=None, + extractions=None, pcs_num=1): """ - Idea: pass both the residuals/coefficient and a list of quantities. Identify which among these are - correlated with the residual/coefficient, by checking the scores of this on the - principal components. Then return the linear combination of the (highest) components with respect - to the explanatory vars. + Idea: pass data to model and a list of explanatory variables. Identify the principal components of dataset + and find the one that has highest score on the data you want to model. This will give you the direction in + the dataset that better captures the variation in the dependent var. + + Returning the decomposition of such pc in terms of the original features, one can then visually check how + well the model is explaining the var, and use this to perform a regression analysis in terms of a reduced + set of vars. + + Parameters: + ----------- + + Returns: + -------- + + Notes: + ------ """ # CHECKING AND PREPROCESSING THE DATA dep_shape = dependent_var.shape @@ -624,17 +682,41 @@ def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, mode print(f'The {i}-th feature passed is not aligned with the first, removing {i}-th feature.') else: Expl_vars.append(explanatory_vars[i]) - if len(Expl_vars)==0: print('No two vars are compatible. Exiting.') return None + # TRIMMING DATASET TO WITHIN RANGES if ranges != None and model_points != None: print('Trimming dataset for PCA analysis.') Dep_var = self.trim_data(dependent_var, ranges, model_points) for i in range(len(explanatory_vars)): Expl_vars[i] = self.trim_data(Expl_vars[i], ranges, model_points) + # PRE-PROCESSING DATASET + if preprocess_data != None: + print('preprocessing data') + data = [Dep_var] + for i in range(len(Expl_vars)): + data.append(Expl_vars[i]) + processed_data = self.preprocess_data(data, preprocess_data) + Dep_var = np.array(processed_data[0]) + for i in range(len(Expl_vars)): + Expl_vars[i] = np.array(processed_data[i+1]) + + # RANDOMLY EXTRACT FROM ARRAY: THIS IS THE DATA ACTUALLY CONSIDERED FOR PCA + if extractions != None: + print('Extracting randomly from sample') + data_to_extract = [Dep_var] + for i in range(len(Expl_vars)): + data_to_extract.append(Expl_vars[i]) + extracted_data = self.extract_randomly(data_to_extract, extractions) + Dep_var = extracted_data[0] + for i in range(len(Expl_vars)): + Expl_vars[i] = extracted_data[i+1] + + + # FLATTENING IN CASE DATA IS NOT PRE-PROCESSED Dep_var = Dep_var.flatten() for i in range(len(Expl_vars)): Expl_vars[i] = Expl_vars[i].flatten() @@ -659,23 +741,21 @@ def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, mode var2comp = pca_model.components_ comp2var = np.einsum('ij->ji', var2comp) - print('components_: \n{}\n'.format(var2comp)) + # print('components_: \n{}\n'.format(var2comp)) scores_of_dep_var = var2comp[:,0] - print(f'Scores of dependent var on the principal components: \n{scores_of_dep_var}\n') - # sorted_scores_indices = np.argsort(scores_of_dep_var) + print(f'Scores of dependent var on principal components: \n{scores_of_dep_var}\n') sorted_scores_indices = np.argsort(np.abs(scores_of_dep_var)) - pos_of_highest_pcs = sorted_scores_indices[-pcs_num:] #identifying the index of the ones with highest score - # this is ordering the scores, but you should care about their magnitude, no? + pos_of_highest_pcs = sorted_scores_indices[-pcs_num:] highest_pcs_decomp = [] corresponding_scores = [] - tot_explained_var = 0 + # tot_explained_var = 0 for i in range(pcs_num-1, -1, -1): highest_pcs_decomp.append(comp2var[:,pos_of_highest_pcs[i]]) corresponding_scores.append(scores_of_dep_var[pos_of_highest_pcs[i]]) - tot_explained_var += pca_model.explained_variance_ratio_[pos_of_highest_pcs[i]] + # tot_explained_var += pca_model.explained_variance_ratio_[pos_of_highest_pcs[i]] - print(f'Total explained variance: {tot_explained_var}\n') + # print(f'Total explained variance: {tot_explained_var}\n') return highest_pcs_decomp, corresponding_scores @@ -754,6 +834,10 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + print('Extracting random vals from x...\n') - extracted = statistical_tool.extract_randomly(x, 10) + print(len(x)) + weights = np.ones(x.shape) + extracted, weights = statistical_tool.extract_randomly([x,y], 10) print(extracted) + diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 636c25f..601f539 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -265,7 +265,7 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, components_indices=Non return fig def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, components_indices=None, method='raw_data',\ - interp_dims=None, diff_plot=False, rel_diff=False, norms=None): + interp_dims=None, diff_plot=False, rel_diff=False, norms=None, cmaps=None): """ Plot variables from a number of models. If 2 models are given, a third plot of the difference (relative or absolute) can be added too. @@ -295,10 +295,10 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com Whether to add a column to show difference between models rel_diff: bool Whether to plot the absolute or relative difference between models - norms = list of strs - each entry of the list is passed as option to imshow as norm=str - the list should be as long as the number of vars passed - useful options are log or symlog + norms/maps = list of list of strs + these have to be compatible with the final number of rows and columns + in the plot. First index in the list runs over the columns (models and their difference), + second index runs over the rows (vars). Output ------- @@ -317,11 +317,6 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com print("The number of variables per model must be the same. Exiting.") return None - if not norms: - norms = [None for _ in range(len(var_strs[0]))] - elif len(var_strs)!= len(norms): - print('The norms provided are not the same number as the variables: setting these to auto') - norms = [None for _ in range(len(var_strs[0]))] n_cols = len(models) if diff_plot: @@ -346,6 +341,18 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com components_indices = [] for i in range(len(models)): components_indices.append(empty_tuple_components) + + inv_fig_shape = (n_cols, n_rows) + if not norms or np.array(norms).shape != inv_fig_shape: + # print('Norms provided are not compatible with figure, setting these to auto') + norms = [None for _ in range(n_rows)] + norms = [norms for _ in range(n_cols)] + + if not cmaps or np.array(cmaps).shape != inv_fig_shape: + # print('Colormaps not compatible with figure, setting these to auto') + cmaps = [None for _ in range(n_rows)] + cmaps = [cmaps for _ in range(n_cols)] + figsize = self.screen_size fig, axes = plt.subplots(n_rows, n_cols, squeeze=False, figsize=figsize) @@ -353,7 +360,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com for j in range(len(models)): for i in range(n_rows): data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, components_indices[j][i], 'raw_data', None) - im=axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[i]) + im=axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[j][i], cmap=cmaps[j][i]) divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -383,16 +390,16 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com print("Cannot plot the difference between the vars: data not aligned.") continue data_to_plot = data1 - data2 - im = axes[i,2].imshow(data_to_plot, extent=extent1, norm=norms[i]) + im = axes[i,2].imshow(data_to_plot, extent=extent1, norm=norms[2][i], cmap=cmaps[2][i]) divider = make_axes_locatable(axes[i,2]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') axes[i,2].set_title('Models difference') axes[i,2].set_xlabel(r'$y$') axes[i,2].set_ylabel(r'$x$') - except (ValueError): + except ValueError as v: print(f"Cannot plot the difference between {var_strs} in the two "+\ - "models. Likely due to the data coordinates not coinciding.") + f"models. Caught a value error: {v}") if rel_diff and len(models)==2: @@ -409,16 +416,17 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com column = 3 else: column = 2 - im = axes[i,column].imshow(data_to_plot, extent=extent1, norm=norms[i]) + + im = axes[i,column].imshow(data_to_plot, extent=extent1, norm=norms[column][i], cmap=cmaps[column][i]) divider = make_axes_locatable(axes[i,column]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') axes[i,column].set_title('Relative difference') axes[i,column].set_xlabel(r'$y$') axes[i,column].set_ylabel(r'$x$') - except (ValueError): + except ValueError as v: print(f"Cannot plot the difference between {var_strs} in the two "+\ - "models. Likely due to the data coordinates not coinciding.") + f"models. Caught a value error: {v}") models_names = [model.get_model_name() for model in models] @@ -473,11 +481,15 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com print("Finished filtering") - vars = [['BC', 'BC', 'BC', 'BC'],['BC', 'BC' ,'BC' ,'BC']] - components = [[(0,), (0,), (0,), (0,)],[(0,), (0,), (0,), (0,)]] + vars = [['BC'], ['BC']] models = [micro_model, meso_model] + components = [[(0,)],[(0,)]] + norms = [['log'], ['log'], ['symlog']] + cmaps=None + # cmaps = [['seismic'], ['seismic'], ['seismic']] # smaller_ranges = [ranges[0]+0.01, ranges[1]- 0.01] # Needed to avoid interpolation errors at boundaries # visualizer.plot_var_model_comparison(models, var, 1.502, smaller_ranges, smaller_ranges, \ # method='interpolate', interp_dims=(30,30), component_indices=component) - visualizer.plot_vars_models_comparison(models, vars, 1.502, ranges, ranges, components_indices=components, diff_plot=True, rel_diff = False) + visualizer.plot_vars_models_comparison(models, vars, 1.502, ranges, ranges, components_indices=components, diff_plot=True, rel_diff = False, + norms=norms, cmaps=cmaps) plt.show() From 749bacda406f0522eaf3e20887c1f3544c5be09c Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 16 Feb 2024 17:59:08 +0100 Subject: [PATCH 076/111] Tests and updates to PCA and regression routines and associated PoP scripts --- Proof_of_principle/config_calibration.txt | 34 ++- Proof_of_principle/hunting_correlations.py | 138 ++++++++++ Proof_of_principle/regressing_residual.py | 76 +++--- Proof_of_principle/visualizing_residuals.py | 146 ++++++----- master_files/Analysis.py | 271 +++++++++----------- 5 files changed, 403 insertions(+), 262 deletions(-) create mode 100644 Proof_of_principle/hunting_correlations.py diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index ddfdf8f..f19f792 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -5,24 +5,19 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 -figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/regression +figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/. #/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 +#/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4/fitting_eta [Filenames] - meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle [Models_settings] - mesogrid_T_slices_num = 3 -[Figure_settings] - -plot_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} - -correlation_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} +[Ranges_for_analysis] -regression_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} +ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} [PCA_settings] @@ -30,14 +25,25 @@ dependent_var = Pi_res explanatory_vars = ["T_tilde", "n_tilde", "p_tilde", "eps_tilde"] pcs_num = 1 -# The following will be used if the quantity to be plotted is not a scalar -dep_var_dict = {"shear_tilde": [[1, 2], [0, 1]]} -explanatory_vars_dict = {"shear_tilde": [[1, 2], [0, 1]]} +regressors_2_reduce = ["T_tilde", "n_tilde", "p_tilde", "eps_tilde"] +variance_wanted = 0.9 + +# Careful, only one dictionary here +preprocess_data = {"pos_or_neg": [1,1,1,1,1], "log_or_not": [1,1,1,1,1]} +#set extractions 0 if you don't want to extract randomly from sample +extractions = 10000 + [Regression_settings] dependent_var = Pi_res regressors = ["T_tilde", "n_tilde"] -add_intercept = 0 +preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} +add_intercept = 1 +#set extractions 0 if you don't want to extract randomly from sample +extractions = 10000 + + + + -regression_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} \ No newline at end of file diff --git a/Proof_of_principle/hunting_correlations.py b/Proof_of_principle/hunting_correlations.py new file mode 100644 index 0000000..fe6966e --- /dev/null +++ b/Proof_of_principle/hunting_correlations.py @@ -0,0 +1,138 @@ +import sys +# import os +sys.path.append('../master_files/') +import pickle +import configparser +import json +import time +import math + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################## + # # USING PCA TO LOOK FOR CORRELATIONS: FIND THE FIRST FEW PRINCIPAL + # # COMPONENTS IN A DATASET, AND PRINT THEIR CORRELATION PLOT WITH + # # A RESIDUAL/DISSIPATIVE COEFFICIENT + # ################################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? + dep_var_str = config['PCA_settings']['dependent_var'] + explanatory_vars_strs = json.loads(config['PCA_settings']['explanatory_vars']) + explanatory_vars = [] + for i in range(len(explanatory_vars_strs)): + temp = meso_model.meso_vars[explanatory_vars_strs[i]] + explanatory_vars.append(temp) + dep_var = meso_model.meso_vars[dep_var_str] + print(f'Dependent var: {dep_var_str}, Explanatory vars: {explanatory_vars_strs}\n') + + + # WHICH GRID-RANGES SHOULD WE CONSIDER? + PCA_ranges = json.loads(config['Ranges_for_analysis']['ranges']) + x_range = PCA_ranges['x_range'] + y_range = PCA_ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + # READING PREPROCESSING INFO FROM CONFIG FILE + preprocess_data = json.loads(config['PCA_settings']['preprocess_data']) + extractions = int(config['PCA_settings']['extractions']) + if extractions == 0: + extractions = None + + + # PRE-PROCESSING + data = [dep_var] + for i in range(len(explanatory_vars)): + data.append(explanatory_vars[i]) + statistical_tool = CoefficientsAnalysis() + model_points = meso_model.domain_vars['Points'] + new_data = statistical_tool.trim_dataset(data, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + if extractions != 0: + new_data = statistical_tool.extract_randomly(new_data, extractions) + + dep_var = new_data[0] + explanatory_vars = [] + for i in range(1,len(new_data)): + explanatory_vars.append(new_data[i]) + + # NOW DO THE PCA ANALYSIS + pcs_num = int(config['PCA_settings']['pcs_num']) + highest_pcs_decomp, scores = statistical_tool.PCA_find_regressors(dep_var, explanatory_vars, pcs_num=pcs_num) + + for i in range(len(highest_pcs_decomp)): + print(f'{i}-th highest PCs decomposition: \n{highest_pcs_decomp[i]}\nCorresponding score: {scores[i]}\n') + + # NOW, USE THE PCs DECOMPOSITION TO BUILD A LINEAR MODEL FOR DEP_VAR + # THIS IS USED FOR PRODUCING A CORRELATION PLOT TO VISUALLY GRASP HOW GOOD THE MODEL IS + # ONLY THE PC WITH HIGHEST SCORE ON DEP_VAR IS USED, THE OTHERS PROVIDE AUXILIARY INFO? + + + # BUILD THE LINEAR MODEL FROM PCA + best_pc_decomp = highest_pcs_decomp[0] + coeffs = [] + temp = 0 + for j in range(0, len(explanatory_vars)): + coeff = -best_pc_decomp[j+1] / best_pc_decomp[0] + print(f'{explanatory_vars_strs[j]}-rescaled coefficient: {coeff}\n') + coeffs.append(coeff) + temp += np.multiply(coeff, explanatory_vars[j]) + dep_var_model = temp + + # FINALLY, PLOTTING + print('\nProducing correlation plot using info from PCA') + x = dep_var_model + y = dep_var + + ylabel = dep_var_str + if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): + ylabel = meso_model.labels_var_dict[dep_var_str] + if preprocess_data['log_or_not'][0] ==1: + ylabel = r"$\log($" + ylabel + r"$)$" + + xlabel = "" + for i in range(len(explanatory_vars_strs)): + sign, val = int(np.sign(coeffs[i])), str(round(np.abs(coeffs[i]),3)) + coeff_for_label = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + text = explanatory_vars_strs[i] + if hasattr(meso_model, 'labels_var_dict') and explanatory_vars_strs[i] in meso_model.labels_var_dict.keys(): + text = meso_model.labels_var_dict[explanatory_vars_strs[i]] + if preprocess_data['log_or_not'][i+1]==1: + text = r"$\log($" + text + r"$)$" + xlabel += coeff_for_label + text + + model_points = meso_model.domain_vars['Points'] + g=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) + saving_directory = config['Directories']['figures_dir'] + filename = f'/{dep_var_str}_PCAmodel_correlation.pdf' + plt.savefig(saving_directory + filename, format='pdf') + print(f'Finished correlation plot for {dep_var_str}, saved as {filename}\n\n') + + \ No newline at end of file diff --git a/Proof_of_principle/regressing_residual.py b/Proof_of_principle/regressing_residual.py index 52f786b..d7bf6a7 100644 --- a/Proof_of_principle/regressing_residual.py +++ b/Proof_of_principle/regressing_residual.py @@ -17,8 +17,8 @@ # ################################################################## # # RUN THE REGRESSION ROUTINE GIVEN DEPENDENT DATA AND EXPLANATORY - # # VARS. TAKE THE LOG SO TO LOOK FOR REGRESSION IN LOGSPACE. - # # TO THINK: HOW TO DEAL WITH POSITIVE AND NEGATIVE QUANTITIES? + # # VARS. DATA IS PRE-PROCESSED SO TO DEAL WITH POSITIVE AND NEGATIVE + # # DATA (EXTRACTING POS OR NEG PART APPROPRIATELY) # ################################################################## # READING SIMULATION SETTINGS FROM CONFIG FILE @@ -40,71 +40,83 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - # Which data you want to run the routine on? + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['Regression_settings']['dependent_var'] + dep_var = meso_model.meso_vars[dep_var_str] regressors_strs = json.loads(config['Regression_settings']['regressors']) - add_intercept = not not int(config['Regression_settings']['add_intercept']) regressors = [] for i in range(len(regressors_strs)): temp = meso_model.meso_vars[regressors_strs[i]] - regressors.append(np.log10(temp)) - dep_var = np.log10(meso_model.meso_vars[dep_var_str]) + regressors.append(temp) print(f'Dependent var: {dep_var_str}, Explanatory vars: {regressors_strs}\n') - # Which ranges? (later: add tools to randomly extract from within ranges?) - regression_ranges = json.loads(config['Regression_settings']['regression_ranges']) + + # WHICH GRID-RANGES SHOULD WE CONSIDER? + regression_ranges = json.loads(config['Ranges_for_analysis']['ranges']) x_range = regression_ranges['x_range'] y_range = regression_ranges['y_range'] num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] - # Performing the regression - statistical_tool = CoefficientsAnalysis() + # READING PREPROCESSING INFO FROM CONFIG FILE + preprocess_data = json.loads(config['Regression_settings']['preprocess_data']) + add_intercept = not not int(config['Regression_settings']['add_intercept']) + extractions = int(config['Regression_settings']['extractions']) + if extractions == 0: + extractions = None + + # PRE-PROCESSING and REGRESSING + data = [dep_var] + for i in range(len(regressors)): + data.append(regressors[i]) + statistical_tool = CoefficientsAnalysis() model_points = meso_model.domain_vars['Points'] - coeffs, std_errors = statistical_tool.scalar_regression(dep_var, regressors, ranges=ranges, model_points=model_points, - add_intercept=add_intercept) + new_data = statistical_tool.trim_dataset(data, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + if extractions != 0: + new_data = statistical_tool.extract_randomly(new_data, extractions) + + dep_var = new_data[0] + regressors = [] + for i in range(1,len(new_data)): + regressors.append(new_data[i]) + + coeffs, std_errors = statistical_tool.scalar_regression(dep_var, regressors, add_intercept=add_intercept) print('regression coeffs: {}\n'.format(coeffs)) - print('Corresponding std errors: {}\n'.format(std_errors)) + print('Corresponding std errors: {}\n'.format(std_errors)) - # Bulding the regressed model for scatterplot + # BUILDING DATA FOR REGRESSED MODEL regressors_strs = [None] + regressors_strs regressors = [np.ones(regressors[0].shape)] + regressors if not add_intercept: coeffs = [0] + coeffs - dep_var_model = np.zeros(dep_var.shape) for i in range(len(regressors)): dep_var_model += np.multiply(coeffs[i], regressors[i]) - # Plotting dependent var VS its model: to visually check if the regression is decent - correlation_ranges = json.loads(config['Figure_settings']['correlation_ranges']) - x_range = correlation_ranges['x_range'] - y_range = correlation_ranges['y_range'] - num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) - time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] - ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] - - ylabel = 'log({})'.format(dep_var_str) + # FINALLY: PLOTTING + ylabel = dep_var_str if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): - ylabel = r"$\log($" + meso_model.labels_var_dict[dep_var_str] + ")" + ylabel = meso_model.labels_var_dict[dep_var_str] + if preprocess_data['log_or_not'][0] == 1: + ylabel = r"$\log($" + ylabel + r"$)$" xlabel = "" if coeffs[0] != 0: sign, val = int(np.sign(coeffs[0])), str(round(np.abs(coeffs[0]),3)) xlabel = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) - for i in range(1, len(coeffs)): sign, val = int(np.sign(coeffs[i])), str(round(np.abs(coeffs[i]),3)) xlabel += r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) - text = r"$\log($" + var_name = regressors_strs[i] if hasattr(meso_model, 'labels_var_dict') and regressors_strs[i] in meso_model.labels_var_dict.keys(): - text += meso_model.labels_var_dict[regressors_strs[i]] + ")" - else: - text += regressors_strs[i] + ")" - xlabel +=text + var_name = meso_model.labels_var_dict[regressors_strs[i]] + if preprocess_data['log_or_not'][i] == 1: + var_name = r"$\log($" + var_name + r"$)$" + xlabel +=var_name - statistical_tool.visualize_correlation(dep_var_model, dep_var , xlabel, ylabel, ranges, model_points) + statistical_tool.visualize_correlation(dep_var_model, dep_var , xlabel, ylabel) saving_directory = config['Directories']['figures_dir'] filename = '/{}_vs_{}'.format(dep_var_str, json.loads(config['Regression_settings']['regressors'])) if add_intercept: diff --git a/Proof_of_principle/visualizing_residuals.py b/Proof_of_principle/visualizing_residuals.py index f324bd6..48305dc 100644 --- a/Proof_of_principle/visualizing_residuals.py +++ b/Proof_of_principle/visualizing_residuals.py @@ -48,93 +48,101 @@ # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') - statistical_tool = CoefficientsAnalysis() - correlation_ranges = json.loads(config['Figure_settings']['correlation_ranges']) + correlation_ranges = json.loads(config['Ranges_for_analysis']['ranges']) x_range = correlation_ranges['x_range'] y_range = correlation_ranges['y_range'] num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + model_points = meso_model.domain_vars['Points'] print('Producing correlation plot for pi_res') x = np.log10(meso_model.meso_vars['shear_sq']) y = np.log10(meso_model.meso_vars['pi_res_sq']) + data = [x,y] + data = statistical_tool.trim_dataset(data, ranges, model_points) + x, y = data[0], data[1] xlabel = r'$\log(\tilde{\sigma}_{ab}\tilde{\sigma}^{ab})$' ylabel = r'$\log(\tilde{\pi}_{ab}\tilde{\pi}^{ab})$' - model_points = meso_model.domain_vars['Points'] - g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel, ranges=ranges, model_points=model_points) + g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) saving_directory = config['Directories']['figures_dir'] filename = '/pi_aniso_correlation.pdf' plt.savefig(saving_directory + filename, format='pdf') print(f'Finished correlation plot for pi_res, saved as {filename}\n\n') - # print('Producing correlation plot for q_res') - # x = np.log10(meso_model.meso_vars['q_res_sq']) - # y = np.log10(meso_model.meso_vars['Theta_sq']) - # xlabel = r'$\log(\tilde{q}_a\tilde{q}^a)$' - # ylabel = r'$\log(\tilde{\Theta}_a \tilde{\Theta}^a)$' - # model_points = meso_model.domain_vars['Points'] - # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel, ranges=ranges, model_points=model_points) - # saving_directory = config['Directories']['figures_dir'] - # filename = '/q_res_correlation.pdf' - # plt.savefig(saving_directory + filename, format='pdf') - # print(f'Finished correlation plot for q_res, saved as {filename}\n\n') - - - # print('Producing correlation plot for Pi_res') - # x= np.log10(np.power(meso_model.meso_vars['Pi_res'], 2) ) - # y= np.log10(np.power(meso_model.meso_vars['exp_tilde'], 2)) - # xlabel = r'$\log(\tilde{\Pi}^2)$' - # ylabel = r'$\log(\tilde{\theta}^2)$' - # model_points = meso_model.domain_vars['Points'] - # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel, ranges=ranges, model_points=model_points) - # saving_directory = config['Directories']['figures_dir'] - # filename = '/Pi_res_correlation.pdf' - # plt.savefig(saving_directory + filename, format='pdf') - # print(f'Finished correlation plot for Pi_res, saved as {filename}\n\n') + print('Producing correlation plot for q_res') + x = np.log10(meso_model.meso_vars['q_res_sq']) + y = np.log10(meso_model.meso_vars['Theta_sq']) + data = [x,y] + data = statistical_tool.trim_dataset(data, ranges, model_points) + x, y = data[0], data[1] + xlabel = r'$\log(\tilde{q}_a\tilde{q}^a)$' + ylabel = r'$\log(\tilde{\Theta}_a \tilde{\Theta}^a)$' + model_points = meso_model.domain_vars['Points'] + g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) + saving_directory = config['Directories']['figures_dir'] + filename = '/q_res_correlation.pdf' + plt.savefig(saving_directory + filename, format='pdf') + print(f'Finished correlation plot for q_res, saved as {filename}\n\n') + + + print('Producing correlation plot for Pi_res') + x= np.log10(np.power(meso_model.meso_vars['Pi_res'], 2) ) + y= np.log10(np.power(meso_model.meso_vars['exp_tilde'], 2)) + data = [x,y] + data = statistical_tool.trim_dataset(data, ranges, model_points) + x, y = data[0], data[1] + xlabel = r'$\log(\tilde{\Pi}^2)$' + ylabel = r'$\log(\tilde{\theta}^2)$' + model_points = meso_model.domain_vars['Points'] + g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) + saving_directory = config['Directories']['figures_dir'] + filename = '/Pi_res_correlation.pdf' + plt.savefig(saving_directory + filename, format='pdf') + print(f'Finished correlation plot for Pi_res, saved as {filename}\n\n') - # ############################################################## - # #PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS - # ############################################################## + ############################################################## + #PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS + ############################################################## - # plot_ranges = json.loads(config['Figure_settings']['plot_ranges']) - # x_range = plot_ranges['x_range'] - # y_range = plot_ranges['y_range'] - # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) - # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] - # saving_directory = config['Directories']['figures_dir'] - - # visualizer = Plotter_2D() - - # vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] - # norms = ['mysymlog', 'mysymlog', 'log'] - # cmaps = ['seismic', 'seismic', None] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Bulk_viscosity.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished bulk viscosity') - - # vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] - # norms = ['mysymlog', 'log', 'log'] - # cmaps = ['seismic', None, None] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Shear_viscosity.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished shear viscosity') - - # vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] - # norms = ['mysymlog', 'log', 'log'] - # cmaps = ['seismic', None, None] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Heat_conductivity.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished heat conductivity') \ No newline at end of file + plot_ranges = json.loads(config['Ranges_for_analysis']['ranges']) + x_range = plot_ranges['x_range'] + y_range = plot_ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + saving_directory = config['Directories']['figures_dir'] + + visualizer = Plotter_2D() + + vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] + norms = ['mysymlog', 'mysymlog', 'log'] + cmaps = ['seismic', 'seismic', None] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Bulk_viscosity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished bulk viscosity') + + vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] + norms = ['mysymlog', 'log', 'log'] + cmaps = ['seismic', None, None] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Shear_viscosity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished shear viscosity') + + vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] + norms = ['mysymlog', 'log', 'log'] + cmaps = ['seismic', None, None] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Heat_conductivity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished heat conductivity') \ No newline at end of file diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 94f29cd..0ad6b05 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -84,6 +84,49 @@ def trim_data(self, data, ranges, model_points): return newdata + def trim_dataset(self, list_of_data, ranges, model_points): + """ + wrapper of trim data: check the shapes of the various arrays are compatible + Then trim each of them individually. + + Parameters: + ----------- + list_of_data: list of np.arrays (gridded data) + the dataset you want to trim + + ranges: list of lists + the min and max in each direction + + model_points: list of lists + the gridpoints, len of this must be compatible with the ranges + + Returns: + -------- + list of arrays trimmed within ranges + """ + print('Trimming the dataset') + + if len(ranges) != len(model_points): + print('Ranges incompatible with model_points. Exiting') + return None + + ref_shape = list_of_data[0].shape + new_data = [list_of_data[0]] + + for i in range(1, len(list_of_data)): + if list_of_data[i].shape == ref_shape: + new_data.append(list_of_data[i]) + else: + print(f'The {i}th array in the dataset is not compatible with the first: ignoring it.') + + if len(new_data) <=1: + print('No two variables in the dataset are compatible. Exiting.') + return None + else: + for i in range(len(new_data)): + new_data[i] = self.trim_data(new_data[i], ranges, model_points) + return new_data + def get_pos_or_neg_mask(self, pos_or_neg, array): """ Function that return the mask that would be applied to an array in order to select @@ -95,6 +138,10 @@ def get_pos_or_neg_mask(self, pos_or_neg, array): array: np.array + Return: + ------- + mask: is True where values are masked! + Notes: ------ To be combined with others before applying all together @@ -108,8 +155,8 @@ def get_pos_or_neg_mask(self, pos_or_neg, array): def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): """ - Takes input list of arrays with dictionary on how to preprocess_data and info about - trimming of arrays to selected range within grid + Takes input list of arrays with dictionary on how to preprocess_data + In case data comes with weights, these can be passed to be pre-processed accordingly Parameters: ---------- @@ -123,8 +170,28 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): 'pos_or_neg' is 1 (0) if you want to select positive (negative) values 'log_or_not' is 1 if you want to take the logarithm of the data + + weights: np.array, default to None + the array with weights for each gridpoint. + + Return: + ------ + The processed data """ + print('Pre-processing dataset') + # Checking data is compatible + ref_shape = list_of_arrays[0].shape + processed_list = [list_of_arrays[0]] + for i in range(1, len(list_of_arrays)): + if list_of_arrays[i].shape == ref_shape: + processed_list.append(list_of_arrays[i]) + else: + print(f'The {i}th array in the dataset is not compatible with the first: ignoring it.') + if len(processed_list) <=1: + print('No two variables in the dataset are compatible. Exiting.') + return None + # Checking preprocess info are compatible with data num_arrays = len(list_of_arrays) condition = False for key in preprocess_data: @@ -159,30 +226,36 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): # removing corresponding entries from weights Weights = None if weights != None: - Weights = np.ma.masked_array(weights, tot_mask) + Weights = np.ma.masked_array(weights, tot_mask).compressed() return processed_list, Weights else: return processed_list - def extract_randomly(self, data, num_extractions): + def extract_randomly(self, list_of_data, num_extractions): """ Extract randomly from data. Parameters: ----------- - data: list of np.arrays (flattened or not) + list_of_data: list of np.arrays (flattened or not) + + num_extractions: number of values to be extracted - Notes: - ------ - Data is assumed pre-processesed, that is data is a list of compatible arrays! + Returns: + -------- + list of arrays with values randomly extracted from input ones """ - new_data = [] - for i in range(len(data)): - new_data.append(np.array(data[i])) - if new_data[i].ndim != 1: - print(f'{i}-th array is not flat, flattening it.') - new_data[i] = new_data[i].flatten() + print('Extracting randomly from dataset') + + # Checking data is compatible + new_data = [list_of_data[0]] + ref_shape = list_of_data[0].shape + for i in range(1,len(list_of_data)): + if list_of_data[i].shape == ref_shape: + new_data.append(list_of_data[i].flatten()) + else: + print(f'The {i}th array in the dataset is not compatible with the first: ignoring it.') # setting up the mask mask_index = np.zeros(new_data[i].shape) @@ -204,7 +277,7 @@ def extract_randomly(self, data, num_extractions): return np.array(new_data) - def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, extractions=None, preprocess_data=None, add_intercept=False): + def scalar_regression(self, y, X, weights=None, add_intercept=False): """ Routine to perform ordinary or weighted (multivariate) regression on some gridded data. @@ -216,11 +289,6 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, X: list of ndarrays of gridded data (treated as indep scalars) the "data" for the regressors - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - weights: ndarray of gridded weights add_intercept: bool @@ -234,16 +302,10 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, Notes: ------ - Gridded data and ranges are chosen at the model level, and passed here - as parameters, so that this external method will not change/access any - model quantity. - - Works in any dimensions! - - Intercept must be added manually: for the future, add this via true/false block. + Works in any dimensions """ - # CHECKING ALIGNMENT OF PASSED DATA + # CHECKING COMPATIBILITY OF PASSED DATA dep_shape = np.shape(y) n_reg = len(X) for i in range(n_reg): @@ -259,71 +321,22 @@ def scalar_regression(self, y, X, ranges=None, model_points=None, weights=None, if np.shape(y) != np.shape(weights): print(f'The weights passed are not not aligned with data, setting these to 1') weights=np.ones(y.shape) - - - # TRIMMING THE DATA TO WITHIN RANGES - if ranges != None and model_points != None: - print('Trimming dataset for regression') - Y = self.trim_data(y, ranges, model_points) - XX = [] - for x in X: - XX.append(self.trim_data(x, ranges, model_points)) - if weights: - Weights = self.trim_data(weights, ranges, model_points) - else: - XX , Y = X , y - if weights: - Weights = weights - - # PREPROCESSING - if preprocess_data != None: - print('preprocessing data') - data = [Y] - for i in range(len(XX)): - data.append(XX[i]) - - if weights: - processed_data = self.preprocess_data(data, preprocess_data, Weights) - else: - processed_data = self.preprocess_data(data, preprocess_data) - - Y = np.array(processed_data[0]) - for i in range(len(XX)): - XX[i] = np.array(processed_data[i+1]) - if weights: - Weights= np.array(processed_data[-1]) - - # RANDOMLY EXTRACT FROM ARRAY: DATA ACTUALLY CONSIDERED FOR REGRESSION - if extractions != None: - print('Extracting randomly from sample') - data_to_extract = [Y] - for i in range(len(XX)): - data_to_extract.append(XX[i]) - if weights: - data_to_extract.append(Weights) - extracted_data = self.extract_randomly(data_to_extract, extractions) - Y, Weights = extracted_data[0], extracted_data[-1] - for i in range(len(XX)): - XX[i] = extracted_data[i+1] - else: - extracted_data = self.extract_randomly(data_to_extract, extractions) - Y = extracted_data[0] - for i in range(len(XX)): - XX[i] = extracted_data[i+1] - + # FLATTENING (IF NOT DONE YET IN PRE-PROCESSING) + FITTING - Y = Y.flatten() + Y = y.flatten() if weights: Weights = Weights.flatten() - + + XX =[] if add_intercept: const = np.ones(Y.shape) - XX.insert(0, const) + # XX.insert(0, const) + XX.append(const) n_reg = n_reg + 1 - - for i in range(len(XX)): - XX[i]=XX[i].flatten() + + for i in range(len(X)): + XX.append(X[i].flatten()) XX = np.einsum('ij->ji', XX) # # VERSION USING STATSMODELS @@ -433,8 +446,7 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) return results - def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, model_points=None, hue_array=None, - style_array=None, legend_dict=None, palette=None, markers=None): + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, hue_array=None, style_array=None, legend_dict=None, palette=None, markers=None): """ Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. Possibility to cut data to lie within ranges. @@ -462,20 +474,8 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod print('Cannot check correlation: data is misaligned!') return None - if ranges != None and model_points != None: - print('Trimming dataset for correlation plot') - X = self.trim_data(x, ranges, model_points) - Y = self.trim_data(y, ranges, model_points) - if hue_array is not None: - hue_array = self.trim_data(hue_array, ranges, model_points) - if style_array is not None: - style_array = self.trim_data(style_array, ranges, model_points) - print('Finished trimming data') - else: - X, Y = x, y - - X=X.flatten() - Y=Y.flatten() + X=x.flatten() + Y=y.flatten() if hue_array is not None: hue_array = hue_array.flatten() if style_array is not None: @@ -500,7 +500,7 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, ranges=None, mod # print('Figure produced inside Coeff Analysis') return g - def visualize_many_correlations(self, data, labels, ranges=None, model_points=None): + def visualize_many_correlations(self, data, labels): """ Method that returns an instance of PairGrid of correlation plots for a list of vars. Plotted is: @@ -539,11 +539,11 @@ def visualize_many_correlations(self, data, labels, ranges=None, model_points=No if len(Idx_to_delete) >= 1: data=list(np.delete(data, Idx_to_delete, axis=0)) - if ranges != None and model_points != None: - print('Trimming dataset for correlation plot') - for i in range(len(data)): - data[i]=self.trim_data(data[i], ranges, model_points) - print('Finished trimming data') + # if ranges != None and model_points != None: + # print('Trimming dataset for correlation plot') + # for i in range(len(data)): + # data[i]=self.trim_data(data[i], ranges, model_points) + # print('Finished trimming data') Data = [] for i in range(len(data)): @@ -653,25 +653,33 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w return comp_decomp, g - def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, model_points=None, preprocess_data=None, - extractions=None, pcs_num=1): + def PCA_find_regressors(self, dependent_var, explanatory_vars, pcs_num=1): """ Idea: pass data to model and a list of explanatory variables. Identify the principal components of dataset - and find the one that has highest score on the data you want to model. This will give you the direction in - the dataset that better captures the variation in the dependent var. + and find the onew that have highest score on the data you want to model. These will give you the direction(s) in + the dataset that better capture the variation in the dependent var. - Returning the decomposition of such pc in terms of the original features, one can then visually check how + Returning the decomposition of such pc(s) in terms of the original features, one can then visually check how well the model is explaining the var, and use this to perform a regression analysis in terms of a reduced set of vars. Parameters: ----------- + dependent_var: np.array with (gridded) data + + explanatory_vars: list of np.arrays with (gridded) data + + pcs_num: integer default to 1 + the number of principal components to be looked at and whose decomposition is returned Returns: -------- + highest_pcs_decomp: list of arrays + each array contains the decomposition of the pcs in terms of original features + + corresponding_scores: list of floats + the score the one of the dependent_var on the principal components - Notes: - ------ """ # CHECKING AND PREPROCESSING THE DATA dep_shape = dependent_var.shape @@ -686,38 +694,8 @@ def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, mode print('No two vars are compatible. Exiting.') return None - # TRIMMING DATASET TO WITHIN RANGES - if ranges != None and model_points != None: - print('Trimming dataset for PCA analysis.') - Dep_var = self.trim_data(dependent_var, ranges, model_points) - for i in range(len(explanatory_vars)): - Expl_vars[i] = self.trim_data(Expl_vars[i], ranges, model_points) - - # PRE-PROCESSING DATASET - if preprocess_data != None: - print('preprocessing data') - data = [Dep_var] - for i in range(len(Expl_vars)): - data.append(Expl_vars[i]) - processed_data = self.preprocess_data(data, preprocess_data) - Dep_var = np.array(processed_data[0]) - for i in range(len(Expl_vars)): - Expl_vars[i] = np.array(processed_data[i+1]) - - # RANDOMLY EXTRACT FROM ARRAY: THIS IS THE DATA ACTUALLY CONSIDERED FOR PCA - if extractions != None: - print('Extracting randomly from sample') - data_to_extract = [Dep_var] - for i in range(len(Expl_vars)): - data_to_extract.append(Expl_vars[i]) - extracted_data = self.extract_randomly(data_to_extract, extractions) - Dep_var = extracted_data[0] - for i in range(len(Expl_vars)): - Expl_vars[i] = extracted_data[i+1] - - # FLATTENING IN CASE DATA IS NOT PRE-PROCESSED - Dep_var = Dep_var.flatten() + Dep_var = dependent_var.flatten() for i in range(len(Expl_vars)): Expl_vars[i] = Expl_vars[i].flatten() @@ -757,7 +735,6 @@ def PCA_find_regressors(self, dependent_var, explanatory_vars, ranges=None, mode # print(f'Total explained variance: {tot_explained_var}\n') - return highest_pcs_decomp, corresponding_scores # Not sure about these two methods. From 26ef0742968308839124bd8ac6e1f99b030f66fc Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 16 Feb 2024 17:59:38 +0100 Subject: [PATCH 077/111] Updates to PoP --- Proof_of_principle/config_visualizing.txt | 4 +- Proof_of_principle/visualizing_meso.py | 178 ++++++++++++---------- 2 files changed, 98 insertions(+), 84 deletions(-) diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt index dee07b3..ee177a9 100644 --- a/Proof_of_principle/config_visualizing.txt +++ b/Proof_of_principle/config_visualizing.txt @@ -25,7 +25,7 @@ mesogrid_T_slices_num = 3 # method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids # interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid -plot_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} +plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} -diff_plot_settings = {"method": "raw_data", "interp_dims": [700, 700]} +diff_plot_settings = {"method": "interpolate", "interp_dims": [700, 700]} diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index a6d1ac3..2198e5a 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -74,100 +74,114 @@ # PLOTTING MICRO VS FILTERED BC AND SET # ##################################### start_time = time.perf_counter() - vars = [['BC', 'SET'], ['BC', 'SET']] + # vars = [['BC', 'SET'], ['BC', 'SET']] + # models = [micro_model, meso_model] + # components = [[(0,), (0,0)], [(0,), (0,0)]] + vars = [['BC'],['BC']] models = [micro_model, meso_model] - components = [[(0,), (0,0)], [(0,), (0,0)]] + components= [[(0,)],[(0,)]] + norms = [[None], [None], ['symlog']] fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, - interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=False) + interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=True, norms=norms) fig.tight_layout() - filename = "/microVSmeso_scaling.pdf" + filename = "/microVSmeso_narel_new.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - vars = [['BC', 'SET'], ['BC', 'SET',]] + vars = [['BC'],['BC']] models = [micro_model, meso_model] - components = [[(2,), (0,2)], [(2,), (0,2)]] + norms = [[None], [None], ['symlog']] + components= [[(0,)],[(0,)]] fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, - interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=False) + interp_dims = interp_dims, method = diff_method, diff_plot=True, rel_diff=False, norms=norms) fig.tight_layout() - filename = "/microVSmeso_notscaling.pdf" + filename = "/microVSmeso_naabs_new.pdf" plt.savefig(saving_directory + filename, format = 'pdf') + # vars = [['BC', 'SET'], ['BC', 'SET',]] + # models = [micro_model, meso_model] + # components = [[(2,), (0,2)], [(2,), (0,2)]] + # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, + # interp_dims = interp_dims, method = diff_method, diff_plot=True, rel_diff=False) + # fig.tight_layout() + # filename = "/microVSmeso_notscaling.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + time_taken = time.perf_counter() - start_time print(f'Finished plotting model comparison: time taken (X2) ={time_taken}') - # # PLOTTING THE DECOMPOSED SET - # ############################# - vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] - norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] - cmaps = ['seismic', 'seismic', 'seismic', 'seismic', 'seismic', 'seismic'] - components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/DecomposedSET_1.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - - vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] - norms = ['mysymlog', 'mysymlog', 'mysymlog', None, None, None] - cmaps = ['seismic', 'seismic', 'seismic', None, None, None] - components = [(0,), (1,), (2,), (), (), ()] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/DecomposedSET_2.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting decomposition of SET') - - # # PLOTTING THE DERIVATIVES OF FAVRE VEL AND TEMPERATURE - ######################################################### - favre_vel_components = [0,1,2] - for i in range(len(favre_vel_components)): - components = [tuple([favre_vel_components[i]])] - for j in range(3): - components.append(tuple([j,favre_vel_components[i]])) - vars_strs = ['u_tilde', 'D_u_tilde', 'D_u_tilde', 'D_u_tilde'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/D_favre_comp={}.pdf".format(favre_vel_components[i]) - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting derivatives of favre velocity', flush=True) - - vars_strs = ['T_tilde', 'D_T_tilde', 'D_T_tilde', 'D_T_tilde'] - components = [(), (0,), (1,), (2,)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/D_T.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting temperature derivatives', flush=True) - - # # PLOTTING SHEAR, ACCELERATION, EXPANSION AND TEMPERATURE DERIVATIVES - ######################################################################## - vars_strs = ['shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde'] - components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Shear_comps.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting shear', flush=True) - - vars_strs = ['acc_tilde', 'acc_tilde', 'acc_tilde', 'Theta_tilde', 'Theta_tilde', 'Theta_tilde'] - components = [(0,), (1,), (2,), (0,), (1,), (2,)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Acc+Tderivs.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting DaT', flush=True) - - vars_strs = ['exp_tilde'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Expansion.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting expansion', flush=True) + # # # PLOTTING THE DECOMPOSED SET + # # ############################# + # vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] + # norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] + # cmaps = ['seismic', 'seismic', 'seismic', 'seismic', 'seismic', 'seismic'] + # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/DecomposedSET_1.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + + # vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] + # norms = ['mysymlog', 'mysymlog', 'mysymlog', None, None, None] + # cmaps = ['seismic', 'seismic', 'seismic', None, None, None] + # components = [(0,), (1,), (2,), (), (), ()] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/DecomposedSET_2.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished plotting decomposition of SET') + + # # # PLOTTING THE DERIVATIVES OF FAVRE VEL AND TEMPERATURE + # ######################################################### + # favre_vel_components = [0,1,2] + # for i in range(len(favre_vel_components)): + # components = [tuple([favre_vel_components[i]])] + # for j in range(3): + # components.append(tuple([j,favre_vel_components[i]])) + # vars_strs = ['u_tilde', 'D_u_tilde', 'D_u_tilde', 'D_u_tilde'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/D_favre_comp={}.pdf".format(favre_vel_components[i]) + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished plotting derivatives of favre velocity', flush=True) + + # vars_strs = ['T_tilde', 'D_T_tilde', 'D_T_tilde', 'D_T_tilde'] + # components = [(), (0,), (1,), (2,)] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/D_T.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished plotting temperature derivatives', flush=True) + + # # # PLOTTING SHEAR, ACCELERATION, EXPANSION AND TEMPERATURE DERIVATIVES + # ######################################################################## + # vars_strs = ['shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde'] + # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Shear_comps.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished plotting shear', flush=True) + + # vars_strs = ['acc_tilde', 'acc_tilde', 'acc_tilde', 'Theta_tilde', 'Theta_tilde', 'Theta_tilde'] + # components = [(0,), (1,), (2,), (0,), (1,), (2,)] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Acc+Tderivs.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished plotting DaT', flush=True) + + # vars_strs = ['exp_tilde'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Expansion.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished plotting expansion', flush=True) From 2064492238e99beb6ab01bcd56c1f42504e529e4 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 21 Feb 2024 09:32:57 +0100 Subject: [PATCH 078/111] Adding routine to compute various quantities for modelling extracted coefficients --- Proof_of_principle/pickling_meso.py | 147 ++++++++++++++------------- master_files/MesoModels.py | 151 ++++++++++++++++++++++------ 2 files changed, 195 insertions(+), 103 deletions(-) diff --git a/Proof_of_principle/pickling_meso.py b/Proof_of_principle/pickling_meso.py index 7471e34..d05bde2 100644 --- a/Proof_of_principle/pickling_meso.py +++ b/Proof_of_principle/pickling_meso.py @@ -15,7 +15,7 @@ # READING SIMULATION SETTINGS FROM CONFIG FILE if len(sys.argv) == 1: - print(f"You must pass the configuration file for the simulations.") + print(f"You must pass the configuration file for the simulations.", flush=True) raise Exception() config = configparser.ConfigParser() @@ -24,100 +24,105 @@ start_time = time.perf_counter() hdf5_directory = config['Directories']['hdf5_dir'] - print(f'Starting job with data from {hdf5_directory}') - print('==============================================\n') - filenames = hdf5_directory - snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) - fewer_snaps_required = snapshots_opts['fewer_snaps_required'] - smaller_list = snapshots_opts['smaller_list'] - FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) - num_snaps = FileReader.num_files - micro_model = IdealHD_2D() - FileReader.read_in_data_HDF5_missing_xy(micro_model) - micro_model.setup_structures() - time_taken = time.perf_counter() - start_time - print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}') + # print(f'Starting job with data from {hdf5_directory}', flush=True) + # print('==============================================\n') + # filenames = hdf5_directory + # snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) + # fewer_snaps_required = snapshots_opts['fewer_snaps_required'] + # smaller_list = snapshots_opts['smaller_list'] + # FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) + # num_snaps = FileReader.num_files + # micro_model = IdealHD_2D() + # FileReader.read_in_data_HDF5_missing_xy(micro_model) + # micro_model.setup_structures() + # time_taken = time.perf_counter() - start_time + # print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}', flush=True) + + # # SETTING UP THE MESO MODEL + # start_time = time.perf_counter() + # meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) + # filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) + + # coarse_factor = meso_grid['coarse_grain_factor'] + # coarse_time = meso_grid['coarse_grain_time'] + # central_slice_num = int(num_snaps/2) + # num_T_slices = meso_grid['num_T_slices'] + # furthest_slice_number = (num_T_slices-1)/2 + # if coarse_time: + # furthest_slice_number = int(coarse_factor * furthest_slice_number) + # t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] + # x_range = meso_grid['x_range'] + # y_range = meso_grid['y_range'] + + # box_len_ratio = float(filtering_options['box_len_ratio']) + # filter_width_ratio = filtering_options['filter_width_ratio'] + + # box_len = box_len_ratio * micro_model.domain_vars['dx'] + # width = filter_width_ratio * micro_model.domain_vars['dx'] + # find_obs = FindObs_root_parallel(micro_model, box_len) + # filter = box_filter_parallel(micro_model, width) + # meso_model = resHD2D(micro_model, find_obs, filter) + # meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) + + # time_taken = time.perf_counter() - start_time + # print('Grid is set up, time taken: {}\n'.format(time_taken), flush=True) + # num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] + # print('Number of points: {}\n'.format(num_points), flush=True) + + # # FINDING THE OBSERVERS AND FILTERING + # n_cpus = int(config['Meso_model_settings']['n_cpus']) - # SETTING UP THE MESO MODEL - start_time = time.perf_counter() - meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) - filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) - - coarse_factor = meso_grid['coarse_grain_factor'] - coarse_time = meso_grid['coarse_grain_time'] - central_slice_num = int(num_snaps/2) - num_T_slices = meso_grid['num_T_slices'] - furthest_slice_number = (num_T_slices-1)/2 - if coarse_time: - furthest_slice_number = int(coarse_factor * furthest_slice_number) - t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] - x_range = meso_grid['x_range'] - y_range = meso_grid['y_range'] - - box_len_ratio = float(filtering_options['box_len_ratio']) - filter_width_ratio = filtering_options['filter_width_ratio'] - - box_len = box_len_ratio * micro_model.domain_vars['dx'] - width = filter_width_ratio * micro_model.domain_vars['dx'] - find_obs = FindObs_root_parallel(micro_model, box_len) - filter = box_filter_parallel(micro_model, width) - meso_model = resHD2D(micro_model, find_obs, filter) - meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) + # start_time = time.perf_counter() + # meso_model.find_observers_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Observers found in parallel: time taken= {}\n'.format(time_taken), flush=True) - time_taken = time.perf_counter() - start_time - print('Grid is set up, time taken: {}'.format(time_taken)) - num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] - print('Number of points: {}'.format(num_points)) + # start_time = time.perf_counter() + # meso_model.filter_micro_vars_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken), flush=True) - # FINDING THE OBSERVERS AND FILTERING - n_cpus = int(config['Meso_model_settings']['n_cpus']) - start_time = time.perf_counter() - meso_model.find_observers_parallel(n_cpus) - time_taken = time.perf_counter() - start_time - print('Observers found in parallel: time taken= {}'.format(time_taken)) - - start_time = time.perf_counter() - meso_model.filter_micro_vars_parallel(n_cpus) - time_taken = time.perf_counter() - start_time - print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken)) - - - # # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: - # # LOADING MESO MODEL - # pickle_directory = config['Directories']['pickled_files_dir'] - # meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] - # MesoModelLoadFile = pickle_directory + meso_pickled_filename - # n_cpus = int(config['Meso_model_settings']['n_cpus']) + # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + n_cpus = int(config['Meso_model_settings']['n_cpus']) - # print('=========================================================================') - # print(f'Starting job on data from {MesoModelLoadFile}') - # print('=========================================================================\n\n') - # with open(MesoModelLoadFile, 'rb') as filehandle: - # meso_model = pickle.load(filehandle) + print('=========================================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('=========================================================================\n\n') + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) # DECOMPOSING AND CALCULATING THE CLOSURE INGREDIENTS start_time = time.perf_counter() meso_model.decompose_structures_parallel(n_cpus) time_taken = time.perf_counter() - start_time - print('Finished decomposing meso structures in parallel, time taken: {}'.format(time_taken)) + print('Finished decomposing meso structures in parallel, time taken: {}\n'.format(time_taken), flush=True) start_time = time.perf_counter() meso_model.calculate_derivatives() time_taken = time.perf_counter() - start_time - print('Finished computing derivatives (serial), time taken: {}'.format(time_taken)) + print('Finished computing derivatives (serial), time taken: {}\n'.format(time_taken), flush=True) start_time = time.perf_counter() meso_model.closure_ingredients_parallel(n_cpus) time_taken = time.perf_counter() - start_time - print('Finished computing the closure ingredients in parallel, time taken: {}'.format(time_taken)) + print('Finished computing the closure ingredients in parallel, time taken: {}\n'.format(time_taken), flush=True) start_time = time.perf_counter() meso_model.EL_style_closure_parallel(n_cpus) time_taken = time.perf_counter() - start_time - print('Finished computing the EL_style closure in parallel, time taken: {}'.format(time_taken)) + print('Finished computing the EL_style closure in parallel, time taken: {}\n'.format(time_taken), flush=True) + + start_time = time.perf_counter() + meso_model.modelling_coefficients_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Finished computing quantities to model extracted coefficients, time taken: {}\n'.format(time_taken), flush=True) # PICKLING THE CLASS INSTANCE FOR FUTURE USE diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index bf4bf7e..7e1dab5 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1221,7 +1221,10 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): 'pi_res_sq' : r'$\tilde{\pi}_{ab}\tilde{\pi}^{ab}$', 'shear_sq' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}$', 'Theta_sq': r'$\tilde{\Theta}_a\tilde{\Theta}^a$', - 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$'} + 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$', + 'det_shear': r'det$(\sigma)$', + 'vort_mod' : r'$|W|$', + 'acc_mag': r'$|a|$'} def upgrade_labels_dict(self, entry_dict): """ @@ -1742,8 +1745,7 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): metric = np.zeros((3,3)) metric[0,0] = -1. metric[1,1] = metric[2,2] = 1. - - spatial_dims = 2 + spatial_dims = 2. # Computing the Favre density and velocity n_t = np.sqrt(-Base.Mink_dot(BC, BC)) @@ -1753,20 +1755,20 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): # Computing the SET decomposition at each point eps_t = np.einsum('i,j,ik,jl,kl', u_t, u_t, metric, metric, SET) h_ab = np.einsum('ij,jk->ik', metric + np.einsum('i,j->ij', u_t, u_t), metric) # This is a rank (1,1) tensor, i.e. a real projector. - q_a = - np.einsum('ij,jk,kl,l->i', h_ab, SET, metric, u_t) + q_a = - np.einsum('ij,jk,kl,l->i', h_ab, SET, metric, u_t) s_ab = np.einsum('ij,kl,jl->ik', h_ab, h_ab, SET) - s = np.einsum('ij,ji->', s_ab, metric) - p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) + s = np.einsum('ij,ji->', s_ab, metric) + s = np.multiply(1/spatial_dims, s) + s_ab_tracefree = s_ab - np.multiply(s, metric + np.einsum('i,j->ij', u_t, u_t)) - # Additional quantities needed for residuals + p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) # Pi_res = s - p_t Pi_res = s - p_filt - s_ab_tracefree = s_ab - np.multiply(s/spatial_dims, metric + np.einsum('i,j->ij', u_t, u_t)) + EOS_res = p_filt - p_t T_t = p_t/n_t - # EOS_res = p_filt - p_t - - # return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, T_t, EOS_res], [h,i,j] #Uncomment if EoS residual is modelled - return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, T_t], [h,i,j] + + return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, EOS_res, T_t], [h,i,j] #Uncomment if EoS residual is modelled + # return [n_t, u_t, eps_t, q_a, s_ab_tracefree, p_t, Pi_res, T_t], [h,i,j] def decompose_structures_parallel(self, n_cpus): """ @@ -1789,7 +1791,7 @@ def decompose_structures_parallel(self, n_cpus): args_for_pool.append((BC, SET, p_filt, h,i,j)) with mp.Pool(processes=n_cpus) as pool: - print('Running with {} processes\n'.format(pool._processes), flush=True) + print('Decomposing structures in parallel with {} processes'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.decompose_structures_task, args_for_pool): h,i,j = result[1] self.meso_vars['n_tilde'][h,i,j] = result[0][0] @@ -1799,8 +1801,8 @@ def decompose_structures_parallel(self, n_cpus): self.meso_vars['pi_res'][h,i,j,:,:] = result[0][4] self.meso_vars['p_tilde'][h,i,j] = result[0][5] self.meso_vars['Pi_res'][h,i,j] = result[0][6] - self.meso_vars['T_tilde'][h,i,j] = result[0][7] - # self.meso_vars['eos_res'][h,i,j] = result[0][8] # Uncomment if EOS_res is modelled + self.meso_vars['eos_res'][h,i,j] = result[0][7] #If EoS is not modelled you should adjust this! + self.meso_vars['T_tilde'][h,i,j] = result[0][8] def calculate_derivatives_gridpoint(self, nonlocal_var_str, h, i, j, direction, order = 1): """ @@ -2014,7 +2016,7 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): third list contains the indices on meso-grid """ # building blocks: this task is static so no access to self - spatial_dims = 2 + spatial_dims = 2. metric = np.zeros((3,3)) metric[0,0] = -1 metric[1,1] = metric[2,2] = 1 @@ -2057,10 +2059,10 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): np.fill_diagonal(projector, 1) projector += np.einsum('i,j->ij',u_t_cov, u_t) #This is different from the projector above: look at indices! DaT = np.einsum('ij,j->i', projector, nabla_T) - Theta_tilde = DaT + np.multiply(T_t, np.einsum('ij,j->i', metric, acc_t)) + Theta_t = DaT + np.multiply(T_t, np.einsum('ij,j->i', metric, acc_t)) - closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'Theta_tilde'] - closure_vars = [shear_t, exp_t, acc_t, Theta_tilde] + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'vort_tilde','Theta_tilde'] + closure_vars = [shear_t, exp_t, acc_t, vort_t, Theta_t] return closure_vars_strs, closure_vars, [h,i,j] def closure_ingredients_parallel(self, n_cpus): @@ -2091,7 +2093,7 @@ def closure_ingredients_parallel(self, n_cpus): args_for_pool.append((u_t, nabla_u, T_t, nabla_T, h, i, j)) with mp.Pool(processes=n_cpus) as pool: - print('Running with {} processes'.format(pool._processes), flush=True) + print('Computing closure ingredients with {} processes'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.closure_ingredients_task, args_for_pool): keys, values, grid_idxs = result # if self.filter_vars['U_success'][tuple(grid_idxs)]: #this check is in practice always True @@ -2227,12 +2229,6 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): coefficients_names.append('eta') coefficients.append(eta) - # this is just to check - to be removed - coefficients_names.append('shear_sq') - coefficients.append(shear_sq) - coefficients_names.append('pi_res_sq') - coefficients.append(pi_res_sq) - # CALCULATING THE HEAT CONDUCTIVITIY q_res_sq = np.einsum('i,ij,j', q_res, metric, q_res) Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) @@ -2241,12 +2237,6 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): kappa = sign * np.sqrt(q_res_sq / Theta_sq) coefficients_names.append('kappa') coefficients.append(kappa) - - # this is just to check - to be removed - coefficients_names.append('q_res_sq') - coefficients.append(q_res_sq) - coefficients_names.append('Theta_sq') - coefficients.append(Theta_sq) return coefficients_names, coefficients, [h,i,j] @@ -2302,6 +2292,103 @@ def EL_style_closure_parallel(self, n_cpus): finally: self.meso_vars[key][tuple(grid_idxs)] = values[idx] + @staticmethod + def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, h, i, j): + """ + Task (to be used in parallel) to compute various quantities that will be used later + for modelling the extracted closure coefficients. + """ + var_names = [] + vars = [] + + metric = np.zeros((3,3)) + metric[0,0] = -1 + metric[1,1] = metric[2,2] = +1 + + # Computing various invariants of shear matrix + # det_shear = np.linalg.det(shear) + pi_eig, _ = np.linalg.eigh(shear) + det_shear = 1. + for i in range(len(pi_eig)): + if pi_eig[i] != 0.: + det_shear *= pi_eig[i] + + shear_sq = np.einsum('ij,kl,ik,jl->', shear, shear, metric, metric) + + var_names.append('det_shear') + vars.append(det_shear) + var_names.append('shear_sq') + vars.append(shear_sq) + + # Q-criterion + vort_sq = np.einsum('ij,kl,ik,jl->', vort, vort, metric, metric) + Q_criterion = shear_sq - vort_sq + var_names.append('Q') + vars.append(Q_criterion) + + # Computing the magnitude of vorticity + vort_mod = np.sqrt(vort_sq / 2. ) + var_names.append('vort_mod') + vars.append(vort_mod) + + #Computing the magnitude of the acceleration vector + acc_mag = np.sqrt(np.einsum('i,ij,j->', acc, metric, acc)) + var_names.append('acc_mag') + vars.append(acc_mag) + + #Computing squares of residuals and also of Theta_tilde + Pi_res_sq = Pi_res * Pi_res + var_names.append('Pi_res_sq') + vars.append(Pi_res_sq) + pi_res_sq = np.einsum('ij,kl,ik,jl->', pi_res, pi_res, metric, metric) + var_names.append('pi_res_sq') + vars.append(pi_res_sq) + q_res_sq = np.einsum('i,ij,j', q_res, metric, q_res) + var_names.append('q_res_sq') + vars.append(q_res_sq) + Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) + var_names.append('Theta_sq') + vars.append(Theta_sq) + + return var_names, vars, [h,i,j] + + def modelling_coefficients_parallel(self, n_cpus): + """ + Routine to parallelize the task 'Computing_calibration_vars_task' + """ + args_for_pool=[] + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + shear_t = self.meso_vars['shear_tilde'][h,i,j] + vort_t = self.meso_vars['vort_tilde'][h,i,j] + acc_t = self.meso_vars['acc_tilde'][h,i,j] + Theta_t = self.meso_vars['Theta_tilde'][h,i,j] + Pi_res = self.meso_vars['Pi_res'][h,i,j] + pi_res = self.meso_vars['pi_res'][h,i,j] + q_res = self.meso_vars['q_res'][h,i,j] + + args_for_pool.append((shear_t, vort_t, acc_t, Theta_t, Pi_res, pi_res, q_res, h, i, j)) + + with mp.Pool(processes=n_cpus) as pool: + print('Computing vars for modelling coefficients with {} processes'.format(pool._processes), flush=True) + for result in pool.starmap(resHD2D.modelling_coefficients_task, args_for_pool): + keys, values, grid_idxs = result + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key), flush=True) + shape = values[idx].shape + self.meso_vars.update({key : np.zeros(([Nt,Nx,Ny]+ list(shape)))}) + finally: + self.meso_vars[key][tuple(grid_idxs)] = values[idx] + def EL_style_closure_regression(self, CoefficientAnalysis): """ Takes in a list of correlation quantities (strings? The regressors needed are computed From 26bd1551abbb519adde7885b7a62dd745bccb3c0 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 21 Feb 2024 10:23:40 +0100 Subject: [PATCH 079/111] Work on routines and scripts to model the extracted coefficients --- Proof_of_principle/config_calibration.txt | 28 ++++-- Proof_of_principle/hunting_correlations.py | 11 ++- Proof_of_principle/reducing_regressors.py | 79 +++++++++++++++++ Proof_of_principle/regressing_residual.py | 16 ++-- .../visualizing_correlations.py | 88 +++++++++++++++++++ master_files/Analysis.py | 50 +++++------ 6 files changed, 226 insertions(+), 46 deletions(-) create mode 100644 Proof_of_principle/reducing_regressors.py create mode 100644 Proof_of_principle/visualizing_correlations.py diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index f19f792..06d0eff 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -5,7 +5,7 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 -figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/. +figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/PCA_regression #/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 #/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4/fitting_eta @@ -17,31 +17,41 @@ mesogrid_T_slices_num = 3 [Ranges_for_analysis] -ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} +ranges = {"x_range": [0.2, 0.8], "y_range": [0.2, 0.8]} + + +[Visualize_correlation] + +vars = ["T_tilde", "n_tilde"] +preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} +#set extractions 0 if you don't want to extract randomly from sample +extractions = 0 + [PCA_settings] -dependent_var = Pi_res -explanatory_vars = ["T_tilde", "n_tilde", "p_tilde", "eps_tilde"] +dependent_var = zeta +explanatory_vars = ["T_tilde", "n_tilde"] pcs_num = 1 +#["det_shear", "shear_sq", "vort_mod", "Q", "acc_mag", "T_tilde", "n_tilde"] -regressors_2_reduce = ["T_tilde", "n_tilde", "p_tilde", "eps_tilde"] +regressors_2_reduce = ["T_tilde", "n_tilde", "eps_tilde"] variance_wanted = 0.9 # Careful, only one dictionary here -preprocess_data = {"pos_or_neg": [1,1,1,1,1], "log_or_not": [1,1,1,1,1]} +preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} #set extractions 0 if you don't want to extract randomly from sample -extractions = 10000 +extractions = 0 [Regression_settings] -dependent_var = Pi_res +dependent_var = zeta regressors = ["T_tilde", "n_tilde"] preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} add_intercept = 1 #set extractions 0 if you don't want to extract randomly from sample -extractions = 10000 +extractions = 0 diff --git a/Proof_of_principle/hunting_correlations.py b/Proof_of_principle/hunting_correlations.py index fe6966e..ae7664b 100644 --- a/Proof_of_principle/hunting_correlations.py +++ b/Proof_of_principle/hunting_correlations.py @@ -34,12 +34,15 @@ meso_pickled_filename = config['Filenames']['meso_pickled_filename'] MesoModelLoadFile = pickle_directory + meso_pickled_filename + print('================================================') print(f'Starting job on data from {MesoModelLoadFile}') print('================================================\n\n') with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) + # additional_entry = {'acc_mag': r'$|a|$'} + # meso_model.upgrade_labels_dict(additional_entry) # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['PCA_settings']['dependent_var'] @@ -63,9 +66,6 @@ # READING PREPROCESSING INFO FROM CONFIG FILE preprocess_data = json.loads(config['PCA_settings']['preprocess_data']) extractions = int(config['PCA_settings']['extractions']) - if extractions == 0: - extractions = None - # PRE-PROCESSING data = [dep_var] @@ -131,7 +131,10 @@ model_points = meso_model.domain_vars['Points'] g=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) saving_directory = config['Directories']['figures_dir'] - filename = f'/{dep_var_str}_PCAmodel_correlation.pdf' + filename = f'/PCA_{dep_var_str}_vs' + for i in range(len(explanatory_vars_strs)): + filename += f'_{explanatory_vars_strs[i]}' + filename += ".pdf" plt.savefig(saving_directory + filename, format='pdf') print(f'Finished correlation plot for {dep_var_str}, saved as {filename}\n\n') diff --git a/Proof_of_principle/reducing_regressors.py b/Proof_of_principle/reducing_regressors.py new file mode 100644 index 0000000..e41478b --- /dev/null +++ b/Proof_of_principle/reducing_regressors.py @@ -0,0 +1,79 @@ +import sys +# import os +sys.path.append('../master_files/') +import pickle +import configparser +import json +import time +import math + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################### + # # USING PCA TO REDUCE A LARGE LIST OF REGRESSOR TO A SMALLER SUBSET + # ################################################################### + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + # additional_entry = {'acc_mag': r'$|a|$'} + # meso_model.upgrade_labels_dict(additional_entry) + + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON + regressors_strs = json.loads(config['PCA_settings']['regressors_2_reduce']) + regressors = [] + for i in range(len(regressors_strs)): + temp = meso_model.meso_vars[regressors_strs[i]] + regressors.append(temp) + print('Trying to reduce the following list to a smaller subset: {}\n'.format(regressors_strs)) + + # WHICH GRID-RANGES SHOULD WE CONSIDER? + ranges = json.loads(config['Ranges_for_analysis']['ranges']) + x_range = ranges['x_range'] + y_range = ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + # READING PREPROCESSING INFO FROM CONFIG FILE + preprocess_data = json.loads(config['PCA_settings']['preprocess_data']) + extractions = int(config['PCA_settings']['extractions']) + + # PRE-PROCESSING and REGRESSING + statistical_tool = CoefficientsAnalysis() + model_points = meso_model.domain_vars['Points'] + new_data = statistical_tool.trim_dataset(regressors, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + if extractions != 0: + new_data = statistical_tool.extract_randomly(new_data, extractions) + + # PERFORMING PCA TO EXTRACT PRINCIPAL COMPONENTS + var_wanted = float(config['PCA_settings']['variance_wanted']) + comp_decomp = statistical_tool.PCA_find_regressors_subset(regressors, var_wanted=var_wanted) + + print('The components decomposition are:') + for i in range(len(comp_decomp)): + print(f'The {i}-th component decomposition in terms of original vars is\n{comp_decomp[i]}\n') diff --git a/Proof_of_principle/regressing_residual.py b/Proof_of_principle/regressing_residual.py index d7bf6a7..e456868 100644 --- a/Proof_of_principle/regressing_residual.py +++ b/Proof_of_principle/regressing_residual.py @@ -19,7 +19,7 @@ # # RUN THE REGRESSION ROUTINE GIVEN DEPENDENT DATA AND EXPLANATORY # # VARS. DATA IS PRE-PROCESSED SO TO DEAL WITH POSITIVE AND NEGATIVE # # DATA (EXTRACTING POS OR NEG PART APPROPRIATELY) - # ################################################################## + # ################################################################## # READING SIMULATION SETTINGS FROM CONFIG FILE if len(sys.argv) == 1: @@ -34,12 +34,16 @@ meso_pickled_filename = config['Filenames']['meso_pickled_filename'] MesoModelLoadFile = pickle_directory + meso_pickled_filename + print('================================================') print(f'Starting job on data from {MesoModelLoadFile}') print('================================================\n\n') with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) + # additional_entry = {'acc_mag': r'$|a|$'} + # meso_model.upgrade_labels_dict(additional_entry) + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['Regression_settings']['dependent_var'] dep_var = meso_model.meso_vars[dep_var_str] @@ -63,8 +67,6 @@ preprocess_data = json.loads(config['Regression_settings']['preprocess_data']) add_intercept = not not int(config['Regression_settings']['add_intercept']) extractions = int(config['Regression_settings']['extractions']) - if extractions == 0: - extractions = None # PRE-PROCESSING and REGRESSING data = [dep_var] @@ -118,9 +120,11 @@ statistical_tool.visualize_correlation(dep_var_model, dep_var , xlabel, ylabel) saving_directory = config['Directories']['figures_dir'] - filename = '/{}_vs_{}'.format(dep_var_str, json.loads(config['Regression_settings']['regressors'])) - if add_intercept: - filename += "_intercept" + filename = f'/Regress_{dep_var_str}_vs' + for i in range(1,len(regressors_strs)): + filename += f'_{regressors_strs[i]}' + # if add_intercept: + # filename += "_intercept" filename += ".pdf" plt.savefig(saving_directory + filename, format='pdf') print(f'Finished regression and scatter plot for {dep_var_str}, saved as {filename}\n\n') diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py new file mode 100644 index 0000000..fcbf8fc --- /dev/null +++ b/Proof_of_principle/visualizing_correlations.py @@ -0,0 +1,88 @@ +import sys +# import os +sys.path.append('../master_files/') +import pickle +import configparser +import json +import time +import math + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################## + # #CORRELATION PLOTS: RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS + # ################################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? + var_strs = json.loads(config['Visualize_correlation']['vars']) + vars = [] + for i in range(len(var_strs)): + vars.append(meso_model.meso_vars[var_strs[i]]) + print(f'Producing scatter plot for the vars: {var_strs}\n') + + # WHICH GRID-RANGES SHOULD WE CONSIDER? + regression_ranges = json.loads(config['Ranges_for_analysis']['ranges']) + x_range = regression_ranges['x_range'] + y_range = regression_ranges['y_range'] + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + # READING PREPROCESSING INFO FROM CONFIG FILE, AND PREPROCESSING + preprocess_data = json.loads(config['Visualize_correlation']['preprocess_data']) + extractions = int(config['Visualize_correlation']['extractions']) + + # PRE-PROCESSING DATA + statistical_tool = CoefficientsAnalysis() + model_points = meso_model.domain_vars['Points'] + new_data = statistical_tool.trim_dataset(vars, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + if extractions != 0: + new_data = statistical_tool.extract_randomly(new_data, extractions) + vars =new_data + + # FINALLY, THE CORRELATION PLOT + labels = [] + for var_str in var_strs: + label = var_str + if hasattr(meso_model, 'labels_var_dict') and var_str in meso_model.labels_var_dict.keys(): + label = meso_model.labels_var_dict[var_str] + if preprocess_data['log_or_not'][0] == 1: + label = r"$\log($" + label + r"$)$" + labels.append(label) + if len(var_strs) ==2: + statistical_tool.visualize_correlation(vars[0], vars[1], xlabel=labels[0], ylabel=labels[1]) + else: + statistical_tool.visualize_many_correlations(vars, labels) + saving_directory = config['Directories']['figures_dir'] + filename = '/Correlation_{}'.format(var_strs) + filename += ".pdf" + plt.savefig(saving_directory + filename, format='pdf') + print(f'Finished producing correlation plot for {var_strs}, saved as {filename}\n') + diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 0ad6b05..96bb0eb 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -199,8 +199,11 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): condition=True if condition: print('Preprocess_data dictionary is not compatible with list_of_arrays. Exiting') - return list_of_arrays, weights - + if weights is not None: + return list_of_arrays, weights + else: + return list_of_arrays + # Combining the masks of the different arrays masks = [] for i in range(len(list_of_arrays)): @@ -224,7 +227,6 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): processed_list[i] = np.log10(np.abs(processed_list[i])) # removing corresponding entries from weights - Weights = None if weights != None: Weights = np.ma.masked_array(weights, tot_mask).compressed() return processed_list, Weights @@ -249,8 +251,8 @@ def extract_randomly(self, list_of_data, num_extractions): print('Extracting randomly from dataset') # Checking data is compatible - new_data = [list_of_data[0]] ref_shape = list_of_data[0].shape + new_data = [list_of_data[0].flatten()] for i in range(1,len(list_of_data)): if list_of_data[i].shape == ref_shape: new_data.append(list_of_data[i].flatten()) @@ -563,23 +565,18 @@ def visualize_many_correlations(self, data, labels): return g - def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_wanted = 1.): + def PCA_find_regressors_subset(self, data, var_wanted = 0.9): """ - Idea: pass a large list of quantities that are correlated with a residual/closure coeff. + Idea: pass list of quantities that you want to use for regression. Check if there is a smaller subset of principal components to be retained that are sufficient - to explain the enough of the observed variance in the dataset. + to explain enough of the observed variance in the dataset. - AIM: linear dimensionality reductions for regressors + Why: if regressors are highly correlated, linear regression is unstable and cannot be trusted Parameters: ----------- data: list of gridded data - ranges: list of lists of 2 floats - mins and max in each direction - - model_points: list of lists containing the gridpoints in each direction - var_wanted: float should be a number between 0. and 1. : the percetage of variance to be retained. @@ -605,15 +602,10 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w else: Data.append(data[i]) - if len(Data)==0: + if len(Data)==1: print('No two vars are compatible. Exiting.') return None - if ranges != None and model_points != None: - print('Trimming dataset for PCA analysis.') - for i in range(len(Data)): - Data[i] = self.trim_data(Data[i], ranges, model_points) - for i in range(len(Data)): Data[i] = Data[i].flatten() @@ -632,7 +624,7 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w n_comp=0 exp_var_sum = np.cumsum(pca_model.explained_variance_ratio_) var_captured = exp_var_sum[n_comp] - while var_captured < var_wanted: + while var_captured <= var_wanted: n_comp +=1 var_captured= exp_var_sum[n_comp] n_comp = n_comp+1 @@ -644,14 +636,18 @@ def PCA_find_regressors_subset(self, data, ranges=None, model_points=None, var_w # should be the inverse but var2comp is unitary, so transpose comp_decomp.append(np.einsum('ij->ji', var2comp)[:,i]) - # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEY'RE UNCORRELATED - # this block is temporary and will be removed later. - pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) - labels = [f'{i} comp' for i in range(n_comp)] - pc_for_plotting = [pc_profiles[:,i] for i in range(n_comp)] - g = self.visualize_many_correlations(pc_for_plotting, labels) + + # # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEY'RE UNCORRELATED + # # this block is temporary and will be removed later. + # pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) + # labels = [f'{i} comp' for i in range(n_comp)] + # pc_for_plotting = [pc_profiles[:,i] for i in range(n_comp)] + # g = self.visualize_many_correlations(pc_for_plotting, labels) + + # print('Built the figure\n') - return comp_decomp, g + # return comp_decomp, g + return comp_decomp def PCA_find_regressors(self, dependent_var, explanatory_vars, pcs_num=1): """ From d16e39c31d5848d609d926df5196c9f03d47e4d6 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 22 Feb 2024 16:29:50 +0100 Subject: [PATCH 080/111] Adding submit scripts --- Proof_of_principle/py_parallel.slurm | 46 +++++++++++++++++++++++ Proof_of_principle/py_serial.slurm | 55 ++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 Proof_of_principle/py_parallel.slurm create mode 100644 Proof_of_principle/py_serial.slurm diff --git a/Proof_of_principle/py_parallel.slurm b/Proof_of_principle/py_parallel.slurm new file mode 100644 index 0000000..4315025 --- /dev/null +++ b/Proof_of_principle/py_parallel.slurm @@ -0,0 +1,46 @@ +#!/bin/bash +#################################### +# JOB INFO: +# parallel, single node, max num of cores +#################################### + +#SBATCH --output=outputs/Meso-pickling-%A.out +#SBATCH --nodes=1 +#SBATCH --ntasks=40 +#SBATCH --time=02:00:00 + +# uncomment if job array +##SBATCH --array=1-6 +##SBATCH --output=outputs/slurm-%A_%a.out + +#SBATCH --mail-type=begin +#SBATCH --mail-type=fail +#SBATCH --mail-type=end +#SBATCH --mail-user=celora@ice.csic.es + + +#################################### +# LOADING THE CONDA ENVIRONMENT +#################################### + +module load conda/py3-latest +# I get warnings to use 'conda deactivate' instead but that doesn't work +source deactivate +conda activate myenv + +#################################### +# RETRIEVING PARAMETERS FOR RUNS: OLD, IGNORE +#################################### + +# ET=$(awk -v var=${SLURM_ARRAY_TASK_ID} '$1==var {print $2}' config.txt) + +#################################### +# LAUNCHING THE JOBS +#################################### + +python3 -u pickling_meso.py config_meso.txt +# python3 -u pickling_meso_setup_array.py config_meso_array.txt + + + + diff --git a/Proof_of_principle/py_serial.slurm b/Proof_of_principle/py_serial.slurm new file mode 100644 index 0000000..9cebe42 --- /dev/null +++ b/Proof_of_principle/py_serial.slurm @@ -0,0 +1,55 @@ +#!/bin/bash +#################################### +# JOB INFO: +#################################### + + +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --time=00:05:00 +#SBATCH --output=outputs/visualize_slurm-%A.out +# #SBATCH --output=outputs/calibration_slurm-%A.out +# #SBATCH --output=outputs/reducing_slurm-%A.out + +# uncomment if array: +##SBATCH --array=1-3 +##SBATCH --output=outputs/slurm-%A_%a.out + + +#SBATCH --mail-type=begin # send email when job begins +#SBATCH --mail-type=fail +#SBATCH --mail-type=end # send email when job ends +#SBATCH --mail-user=celora@ice.csic.es + + +#################################### +# LOADING THE CONDA ENVIRONMENT +#################################### + +module load conda/py3-latest +# I get warnings to use 'conda deactivate' instead but that doesn't work +source deactivate +conda activate myenv + +#################################### +# RETRIEVING THE PARAMETERS FOR THE RUNS: OLD +#################################### + +# ET=$(awk -v var=${SLURM_ARRAY_TASK_ID} '$1==var {print $2}' config.txt) + + +#################################### +# LAUNCHING THE JOBS +##################################### + +python3 -u visualizing_micro.py config_visualizing.txt +# python3 -u visualizing_obs.py config_visualizing.txt +# python3 -u visualizing_meso.py config_visualizing.txt + +# python3 -u visualizing_residuals.py config_calibration.txt + +# python3 -u visualizing_correlations.py config_calibration.txt +# python3 -u reducing_regressors.py config_calibration.txt +# python3 -u hunting_correlations.py config_calibration.txt +# python3 -u regressing_residual.py config_calibration.txt + From d1b1d12c40b61b6d3cf016880ad2737149ea243a Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 22 Feb 2024 18:46:42 +0100 Subject: [PATCH 081/111] pushing latest versions of stuff --- Proof_of_principle/config_calibration.txt | 17 +-- Proof_of_principle/config_meso.txt | 13 +- Proof_of_principle/config_visualizing.txt | 18 +-- Proof_of_principle/pickling_meso.py | 133 ++++++++++---------- Proof_of_principle/py_serial.slurm | 11 +- Proof_of_principle/visualizing_meso.py | 130 +++++++++---------- Proof_of_principle/visualizing_micro.py | 18 ++- Proof_of_principle/visualizing_residuals.py | 1 + master_files/FileReaders.py | 18 +-- master_files/Filters.py | 4 +- master_files/MesoModels.py | 13 +- master_files/MicroModels.py | 17 ++- 12 files changed, 204 insertions(+), 189 deletions(-) diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index 06d0eff..70be08d 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -3,26 +3,27 @@ [Directories] -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/PCA_regression +figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/. #/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 #/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4/fitting_eta [Filenames] -meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle +#meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle +meso_pickled_filename = /prova.pickle [Models_settings] mesogrid_T_slices_num = 3 [Ranges_for_analysis] -ranges = {"x_range": [0.2, 0.8], "y_range": [0.2, 0.8]} +ranges = {"x_range": [0.21, 0.24], "y_range": [0.21, 0.24]} [Visualize_correlation] -vars = ["T_tilde", "n_tilde"] +vars = ["T_tilde", "rho_tilde"] preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} #set extractions 0 if you don't want to extract randomly from sample extractions = 0 @@ -31,11 +32,11 @@ extractions = 0 [PCA_settings] dependent_var = zeta -explanatory_vars = ["T_tilde", "n_tilde"] +explanatory_vars = ["T_tilde", "rho_tilde"] pcs_num = 1 #["det_shear", "shear_sq", "vort_mod", "Q", "acc_mag", "T_tilde", "n_tilde"] -regressors_2_reduce = ["T_tilde", "n_tilde", "eps_tilde"] +regressors_2_reduce = ["T_tilde", "rho_tilde", "p_tilde"] variance_wanted = 0.9 # Careful, only one dictionary here @@ -47,7 +48,7 @@ extractions = 0 [Regression_settings] dependent_var = zeta -regressors = ["T_tilde", "n_tilde"] +regressors = ["T_tilde", "rho_tilde"] preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} add_intercept = 1 #set extractions 0 if you don't want to extract randomly from sample diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index 3b62819..5d9ae7f 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -3,25 +3,26 @@ [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_2.0 +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/ -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled [Micro_model_settings] #snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. #the options is activated using the first key, while the list refers to the index of the elements of #ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) -snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [3,4,5,6,7,8,9,10,11,12,13,14,15,16,17]} +snapshots_opts = {"fewer_snaps_required": true, "smaller_list": [6,7,8,9,10,11,12,13,14]} [Meso_model_settings] -meso_grid = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97], "num_T_slices": 3, "coarse_grain_factor": 4, "coarse_grain_time": true} +meso_grid = {"x_range": [0.2, 0.25], "y_range": [0.2, 0.25], "num_T_slices": 3, "coarse_grain_factor": 1, "coarse_grain_time": false} -filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 4.0} +filtering_options = {"box_len_ratio": 2.0, "filter_width_ratio": 2.0} n_cpus = 40 [Pickled_filenames] -meso_pickled = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle \ No newline at end of file +#meso_pickled = /rHD2d_nocg_fw=bl=8dx.pickle +meso_pickled = /prova \ No newline at end of file diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt index ee177a9..a423277 100644 --- a/Proof_of_principle/config_visualizing.txt +++ b/Proof_of_principle/config_visualizing.txt @@ -3,21 +3,25 @@ [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/METHOD_output/800X800/ET_2.0 +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/pickled_files/800X800 +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled + +#figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures +figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/. -figures_dir = /scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 [Filenames] -meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle +#meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle +meso_pickled_filename = /prova.pickle + [Models_settings] #snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. #relevant only for visualizing_micro.py -snapshots_opts = {"fewer_snaps_required": true, "smaller_list": [6,7,8,9,10,11,12,13,14]} +snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [9,10,11]} mesogrid_T_slices_num = 3 @@ -25,7 +29,7 @@ mesogrid_T_slices_num = 3 # method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids # interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid -plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} +plot_ranges = {"x_range": [0.2, 0.25], "y_range": [0.2, 0.25]} -diff_plot_settings = {"method": "interpolate", "interp_dims": [700, 700]} +diff_plot_settings = {"method": "raw_data", "interp_dims": [700, 700]} diff --git a/Proof_of_principle/pickling_meso.py b/Proof_of_principle/pickling_meso.py index d05bde2..94278c5 100644 --- a/Proof_of_principle/pickling_meso.py +++ b/Proof_of_principle/pickling_meso.py @@ -24,77 +24,78 @@ start_time = time.perf_counter() hdf5_directory = config['Directories']['hdf5_dir'] - # print(f'Starting job with data from {hdf5_directory}', flush=True) - # print('==============================================\n') - # filenames = hdf5_directory - # snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) - # fewer_snaps_required = snapshots_opts['fewer_snaps_required'] - # smaller_list = snapshots_opts['smaller_list'] - # FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) - # num_snaps = FileReader.num_files - # micro_model = IdealHD_2D() - # FileReader.read_in_data_HDF5_missing_xy(micro_model) - # micro_model.setup_structures() - # time_taken = time.perf_counter() - start_time - # print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}', flush=True) - - # # SETTING UP THE MESO MODEL - # start_time = time.perf_counter() - # meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) - # filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) - - # coarse_factor = meso_grid['coarse_grain_factor'] - # coarse_time = meso_grid['coarse_grain_time'] - # central_slice_num = int(num_snaps/2) - # num_T_slices = meso_grid['num_T_slices'] - # furthest_slice_number = (num_T_slices-1)/2 - # if coarse_time: - # furthest_slice_number = int(coarse_factor * furthest_slice_number) - # t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] - # x_range = meso_grid['x_range'] - # y_range = meso_grid['y_range'] - - # box_len_ratio = float(filtering_options['box_len_ratio']) - # filter_width_ratio = filtering_options['filter_width_ratio'] - - # box_len = box_len_ratio * micro_model.domain_vars['dx'] - # width = filter_width_ratio * micro_model.domain_vars['dx'] - # find_obs = FindObs_root_parallel(micro_model, box_len) - # filter = box_filter_parallel(micro_model, width) - # meso_model = resHD2D(micro_model, find_obs, filter) - # meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) - - # time_taken = time.perf_counter() - start_time - # print('Grid is set up, time taken: {}\n'.format(time_taken), flush=True) - # num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] - # print('Number of points: {}\n'.format(num_points), flush=True) - - # # FINDING THE OBSERVERS AND FILTERING - # n_cpus = int(config['Meso_model_settings']['n_cpus']) - - # start_time = time.perf_counter() - # meso_model.find_observers_parallel(n_cpus) - # time_taken = time.perf_counter() - start_time - # print('Observers found in parallel: time taken= {}\n'.format(time_taken), flush=True) + print(f'Starting job with data from {hdf5_directory}', flush=True) + print('==============================================\n') + filenames = hdf5_directory + snapshots_opts = json.loads(config['Micro_model_settings']['snapshots_opts']) + fewer_snaps_required = snapshots_opts['fewer_snaps_required'] + smaller_list = snapshots_opts['smaller_list'] + FileReader = METHOD_HDF5(filenames,fewer_snaps_required, smaller_list) + num_snaps = FileReader.num_files + micro_model = IdealHD_2D() + FileReader.read_in_data_HDF5_missing_xy(micro_model) + micro_model.setup_structures() + time_taken = time.perf_counter() - start_time + print(f'Finished reading micro data from hdf5, structures also set up. Time taken: {time_taken}', flush=True) - # start_time = time.perf_counter() - # meso_model.filter_micro_vars_parallel(n_cpus) - # time_taken = time.perf_counter() - start_time - # print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken), flush=True) + # SETTING UP THE MESO MODEL + start_time = time.perf_counter() + meso_grid = json.loads(config['Meso_model_settings']['meso_grid']) + filtering_options = json.loads(config['Meso_model_settings']['filtering_options']) + + coarse_factor = meso_grid['coarse_grain_factor'] + coarse_time = meso_grid['coarse_grain_time'] + central_slice_num = int(num_snaps/2) + num_T_slices = meso_grid['num_T_slices'] + furthest_slice_number = int((num_T_slices-1)/2) + if coarse_time: + furthest_slice_number = int(coarse_factor * furthest_slice_number) + print(furthest_slice_number) + t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] + x_range = meso_grid['x_range'] + y_range = meso_grid['y_range'] + + box_len_ratio = float(filtering_options['box_len_ratio']) + filter_width_ratio = filtering_options['filter_width_ratio'] + + box_len = box_len_ratio * micro_model.domain_vars['dx'] + width = filter_width_ratio * micro_model.domain_vars['dx'] + find_obs = FindObs_root_parallel(micro_model, box_len) + filter = box_filter_parallel(micro_model, width) + meso_model = resHD2D(micro_model, find_obs, filter) + meso_model.setup_meso_grid([t_range, x_range, y_range], coarse_factor = coarse_factor, coarse_time = coarse_time) + time_taken = time.perf_counter() - start_time + print('Grid is set up, time taken: {}\n'.format(time_taken), flush=True) + num_points = meso_model.domain_vars['Nx'] * meso_model.domain_vars['Ny'] * meso_model.domain_vars['Nt'] + print('Number of points: {}\n'.format(num_points), flush=True) - # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: - # LOADING MESO MODEL - pickle_directory = config['Directories']['pickled_files_dir'] - meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] - MesoModelLoadFile = pickle_directory + meso_pickled_filename + # FINDING THE OBSERVERS AND FILTERING n_cpus = int(config['Meso_model_settings']['n_cpus']) - print('=========================================================================') - print(f'Starting job on data from {MesoModelLoadFile}') - print('=========================================================================\n\n') - with open(MesoModelLoadFile, 'rb') as filehandle: - meso_model = pickle.load(filehandle) + start_time = time.perf_counter() + meso_model.find_observers_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Observers found in parallel: time taken= {}\n'.format(time_taken), flush=True) + + start_time = time.perf_counter() + meso_model.filter_micro_vars_parallel(n_cpus) + time_taken = time.perf_counter() - start_time + print('Parallel filtering stage ended (fw= {}): time taken= {}\n'.format(int(filter_width_ratio), time_taken), flush=True) + + + # # UNCOMMENT IF MESO_MODEL EXISTS PICKLED EXIST ALREADY AND YOU WANT TO RE-RUN SOME OF THE LATER ROUTINES: + # # LOADING MESO MODEL + # pickle_directory = config['Directories']['pickled_files_dir'] + # meso_pickled_filename = config['Pickled_filenames']['meso_pickled'] + # MesoModelLoadFile = pickle_directory + meso_pickled_filename + # n_cpus = int(config['Meso_model_settings']['n_cpus']) + + # print('=========================================================================') + # print(f'Starting job on data from {MesoModelLoadFile}') + # print('=========================================================================\n\n') + # with open(MesoModelLoadFile, 'rb') as filehandle: + # meso_model = pickle.load(filehandle) # DECOMPOSING AND CALCULATING THE CLOSURE INGREDIENTS diff --git a/Proof_of_principle/py_serial.slurm b/Proof_of_principle/py_serial.slurm index 9cebe42..bcd574a 100644 --- a/Proof_of_principle/py_serial.slurm +++ b/Proof_of_principle/py_serial.slurm @@ -42,14 +42,13 @@ conda activate myenv # LAUNCHING THE JOBS ##################################### -python3 -u visualizing_micro.py config_visualizing.txt +# python3 -u visualizing_micro.py config_visualizing.txt # python3 -u visualizing_obs.py config_visualizing.txt # python3 -u visualizing_meso.py config_visualizing.txt - # python3 -u visualizing_residuals.py config_calibration.txt -# python3 -u visualizing_correlations.py config_calibration.txt -# python3 -u reducing_regressors.py config_calibration.txt -# python3 -u hunting_correlations.py config_calibration.txt -# python3 -u regressing_residual.py config_calibration.txt +python3 -u visualizing_correlations.py config_calibration.txt +python3 -u reducing_regressors.py config_calibration.txt +python3 -u hunting_correlations.py config_calibration.txt +python3 -u regressing_residual.py config_calibration.txt diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index 2198e5a..947f3ed 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -80,7 +80,7 @@ vars = [['BC'],['BC']] models = [micro_model, meso_model] components= [[(0,)],[(0,)]] - norms = [[None], [None], ['symlog']] + norms = [[None], [None]] fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=True, norms=norms) fig.tight_layout() @@ -89,7 +89,7 @@ vars = [['BC'],['BC']] models = [micro_model, meso_model] - norms = [[None], [None], ['symlog']] + norms = [[None], [None]] components= [[(0,)],[(0,)]] fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, interp_dims = interp_dims, method = diff_method, diff_plot=True, rel_diff=False, norms=norms) @@ -111,77 +111,77 @@ # # # PLOTTING THE DECOMPOSED SET # # ############################# - # vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] - # norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] - # cmaps = ['seismic', 'seismic', 'seismic', 'seismic', 'seismic', 'seismic'] - # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/DecomposedSET_1.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') + vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] + norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] + cmaps = ['seismic', 'seismic', 'seismic', 'seismic', 'seismic', 'seismic'] + components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/DecomposedSET_1.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') - # vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] - # norms = ['mysymlog', 'mysymlog', 'mysymlog', None, None, None] - # cmaps = ['seismic', 'seismic', 'seismic', None, None, None] - # components = [(0,), (1,), (2,), (), (), ()] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/DecomposedSET_2.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting decomposition of SET') + vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] + norms = ['mysymlog', 'mysymlog', 'mysymlog', None, None, None] + cmaps = ['seismic', 'seismic', 'seismic', None, None, None] + components = [(0,), (1,), (2,), (), (), ()] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/DecomposedSET_2.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting decomposition of SET') # # # PLOTTING THE DERIVATIVES OF FAVRE VEL AND TEMPERATURE # ######################################################### - # favre_vel_components = [0,1,2] - # for i in range(len(favre_vel_components)): - # components = [tuple([favre_vel_components[i]])] - # for j in range(3): - # components.append(tuple([j,favre_vel_components[i]])) - # vars_strs = ['u_tilde', 'D_u_tilde', 'D_u_tilde', 'D_u_tilde'] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/D_favre_comp={}.pdf".format(favre_vel_components[i]) - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting derivatives of favre velocity', flush=True) - - # vars_strs = ['T_tilde', 'D_T_tilde', 'D_T_tilde', 'D_T_tilde'] - # components = [(), (0,), (1,), (2,)] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/D_T.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting temperature derivatives', flush=True) + favre_vel_components = [0,1,2] + for i in range(len(favre_vel_components)): + components = [tuple([favre_vel_components[i]])] + for j in range(3): + components.append(tuple([j,favre_vel_components[i]])) + vars_strs = ['u_tilde', 'D_u_tilde', 'D_u_tilde', 'D_u_tilde'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/D_favre_comp={}.pdf".format(favre_vel_components[i]) + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting derivatives of favre velocity', flush=True) + + vars_strs = ['T_tilde', 'D_T_tilde', 'D_T_tilde', 'D_T_tilde'] + components = [(), (0,), (1,), (2,)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/D_T.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting temperature derivatives', flush=True) # # # PLOTTING SHEAR, ACCELERATION, EXPANSION AND TEMPERATURE DERIVATIVES # ######################################################################## - # vars_strs = ['shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde'] - # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Shear_comps.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting shear', flush=True) + vars_strs = ['shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde'] + components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Shear_comps.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting shear', flush=True) - # vars_strs = ['acc_tilde', 'acc_tilde', 'acc_tilde', 'Theta_tilde', 'Theta_tilde', 'Theta_tilde'] - # components = [(0,), (1,), (2,), (0,), (1,), (2,)] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Acc+Tderivs.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting DaT', flush=True) + vars_strs = ['acc_tilde', 'acc_tilde', 'acc_tilde', 'Theta_tilde', 'Theta_tilde', 'Theta_tilde'] + components = [(0,), (1,), (2,), (0,), (1,), (2,)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Acc+Tderivs.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting DaT', flush=True) - # vars_strs = ['exp_tilde'] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Expansion.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting expansion', flush=True) + vars_strs = ['exp_tilde'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Expansion.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting expansion', flush=True) diff --git a/Proof_of_principle/visualizing_micro.py b/Proof_of_principle/visualizing_micro.py index 21454e6..7c9eb55 100644 --- a/Proof_of_principle/visualizing_micro.py +++ b/Proof_of_principle/visualizing_micro.py @@ -24,10 +24,10 @@ if micro_from_hdf5: hdf5_directory = config['Directories']['hdf5_dir'] - filenames = hdf5_directory print('=========================================================================') print(f'Starting job on data from {hdf5_directory}') print('=========================================================================\n\n') + filenames = hdf5_directory + '/' snapshots_opts = json.loads(config['Models_settings']['snapshots_opts']) fewer_snaps_required = snapshots_opts['fewer_snaps_required'] smaller_list = snapshots_opts['smaller_list'] @@ -62,9 +62,11 @@ # FINALLY, PLOTTING # Plotting the baryon current - vars = ['BC', 'BC', 'BC', 'n', 'W', 'vx'] + vars = ['BC', 'BC', 'BC', 'W', 'vx', 'vy'] + norms= ['log', 'symlog', 'symlog', 'log', 'symlog', 'symlog'] + cmaps = [None, 'seismic', 'seismic', None, 'seismic', 'seismic'] components = [(0,), (1,), (2,), (), (), ()] - fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components) + fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(plot_time,2)) filename = "/micro_T_" + time_for_filename + "_BC.pdf" @@ -73,15 +75,19 @@ # Plotting the stress energy tensor vars = ['SET', 'SET', 'SET', 'SET', 'SET', 'SET'] components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components) + norms= ['log', 'symlog', 'symlog', 'log', 'symlog', 'log'] + cmaps = [None, 'seismic', 'seismic', None, 'seismic', None] + fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(plot_time,2)) filename = "/micro_T_" + time_for_filename + "_SET.pdf" plt.savefig(saving_directory + filename, format = "pdf") # plotting primitive quantities - vars = ['W', 'vx', 'vy', 'n', 'p', 'e'] - fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range) + vars = ['W', 'vx', 'vy', 'rho', 'p', 'e'] + norms= ['log', 'symlog', 'symlog', 'log', 'log', 'log'] + cmaps = [None, 'seismic', 'seismic', None, None, None] + fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(plot_time,2)) filename = "/micro_T_" + time_for_filename + "_prims.pdf" diff --git a/Proof_of_principle/visualizing_residuals.py b/Proof_of_principle/visualizing_residuals.py index 48305dc..03ef485 100644 --- a/Proof_of_principle/visualizing_residuals.py +++ b/Proof_of_principle/visualizing_residuals.py @@ -32,6 +32,7 @@ meso_pickled_filename = config['Filenames']['meso_pickled_filename'] MesoModelLoadFile = pickle_directory + meso_pickled_filename + print('================================================') print(f'Starting job on data from {MesoModelLoadFile}') print('================================================\n\n') diff --git a/master_files/FileReaders.py b/master_files/FileReaders.py index bbdae5d..3bf657f 100644 --- a/master_files/FileReaders.py +++ b/master_files/FileReaders.py @@ -28,7 +28,7 @@ def __init__(self, directory, fewer_snaps=False, smaller_list=None): smaller_list: list indices to be retained of list orderd via sorted(glob.glob(directory+str('*.hdf5'))) """ - hdf5_filenames = sorted( glob.glob(directory+str('*.hdf5'))) + hdf5_filenames = sorted(glob.glob(directory+str('*.hdf5'))) if fewer_snaps: if smaller_list: temp = [hdf5_filenames[i] for i in smaller_list] @@ -58,10 +58,10 @@ def read_in_data(self, micro_model): self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) for prim_str in micro_model.get_prim_strs(): - if prim_str == "n": - self.translating_prims[prim_str] = "rho" - else: - self.translating_prims[prim_str] = prim_str + # if prim_str == "n": + # self.translating_prims[prim_str] = "rho" + # else: + self.translating_prims[prim_str] = prim_str for prim_var_str in micro_model.prim_vars: try: @@ -137,10 +137,10 @@ def read_in_data_HDF5_missing_xy(self, micro_model): self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) for prim_str in micro_model.get_prim_strs(): - if prim_str == "n": - self.translating_prims[prim_str] = "rho" - else: - self.translating_prims[prim_str] = prim_str + # if prim_str == "n": + # self.translating_prims[prim_str] = "rho" + # else: + self.translating_prims[prim_str] = prim_str for prim_var_str in micro_model.prim_vars: try: diff --git a/master_files/Filters.py b/master_files/Filters.py index 75924b0..9a88525 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -329,7 +329,7 @@ def find_observer(self, point, flux_str = "gauss", initial_guess = None): if initial_guess is not None and len(initial_guess) == self.spatial_dims: guess = initial_guess else: - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + U = np.multiply(1 / self.micro_model.get_interpol_var('rho', point) , self.micro_model.get_interpol_var('BC', point) ) for i in range(1, len(U)): guess.append(U[i] / U[0]) guess = np.array(guess) @@ -681,7 +681,7 @@ def find_observer(self, point, drift_str = "gauss", initial_guess= None): if initial_guess is not None and len(initial_guess) == self.spatial_dims: guess = initial_guess else: - U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) + U = np.multiply(1 / self.micro_model.get_interpol_var('rho', point) , self.micro_model.get_interpol_var('BC', point) ) for i in range(1, len(U)): guess.append(U[i] / U[0]) guess = np.array(guess) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 7e1dab5..0a1db16 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1147,7 +1147,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): for var in self.meso_structures: self.meso_structures[var] = [] - self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] + self.meso_scalars_strs = ['eps_tilde', 'rho_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] self.meso_vectors_strs = ['u_tilde', 'q_res'] self.meso_r2tensors_strs = ['pi_res'] self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs @@ -1191,14 +1191,10 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): if not compatible: print("Meso and Micro models are incompatible:"+error) - self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] - self.meso_vectors_strs = ['u_tilde', 'q_res'] - self.meso_r2tensors_strs = ['pi_res'] - self.coefficient_strs = ["Gamma"] self.labels_var_dict = {'SET' : r'$$', 'BC' : r'$$', 'eps_tilde' : r'$\tilde{\varepsilon}$', - 'n_tilde' : r'$\tilde{n}$', + 'rho_tilde' : r'$\tilde{\rho}$', 'p_tilde' : r'$\tilde{p}$', 'p_filt' : r'$

$', 'eos_res' : r'$M$', @@ -1666,7 +1662,7 @@ def decompose_structures_gridpoint(self, h, i, j): # Storing the decomposition with appropriate names. - self.meso_vars['n_tilde'][h,i,j] = n_t + self.meso_vars['rho_tilde'][h,i,j] = n_t self.meso_vars['u_tilde'][h,i,j,:] = u_t self.meso_vars['eps_tilde'][h,i,j] = eps_t self.meso_vars['q_res'][h,i,j,:] = q_a @@ -1794,7 +1790,7 @@ def decompose_structures_parallel(self, n_cpus): print('Decomposing structures in parallel with {} processes'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.decompose_structures_task, args_for_pool): h,i,j = result[1] - self.meso_vars['n_tilde'][h,i,j] = result[0][0] + self.meso_vars['rho_tilde'][h,i,j] = result[0][0] self.meso_vars['u_tilde'][h,i,j,:] = result[0][1] self.meso_vars['eps_tilde'][h,i,j] = result[0][2] self.meso_vars['q_res'][h,i,j,:] = result[0][3] @@ -2238,6 +2234,7 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): coefficients_names.append('kappa') coefficients.append(kappa) + return coefficients_names, coefficients, [h,i,j] def EL_style_closure_parallel(self, n_cpus): diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index 166fe4e..a720c04 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -537,7 +537,7 @@ def __init__(self, interp_method = "linear"): self.domain_vars[str] = [] #Dictionary for primitive var - self.prim_strs = ("vx","vy","n","p") + self.prim_strs = ("vx","vy","rho","p") self.prim_vars = dict.fromkeys(self.prim_strs) for str in self.prim_strs: self.prim_vars[str] = [] @@ -554,11 +554,16 @@ def __init__(self, interp_method = "linear"): for str in self.structures_strs: self.structures[str] = [] - self.labels_var_dict = {'BC' : r'$n^{a}$', + self.labels_var_dict = {'BC' : r'$\rho^{a}$', 'SET' : r'$T^{ab}$', 'bar_vel' : r'$u^a$', 'vx' : r'$v_x$', - 'vy' : r'$v_y$'} + 'vy' : r'$v_y$', + 'rho' : r'$\rho$', + 'W' : r'$W$', + 'e' : r'$e$', + 'h' : r'$h$', + 'p' : r'$p$'} def upgrade_labels_dict(self, entry_dict): """ @@ -710,9 +715,9 @@ def setup_structures(self): self.structures['bar_vel'][h,i,j,:] = vel_vec - self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['rho'][h,i,j], vel_vec ) - self.structures['SET'][h,i,j,:,:] = (self.prim_vars["n"][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + self.structures['SET'][h,i,j,:,:] = (self.prim_vars['rho'][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + self.prim_vars['p'][h,i,j] * self.metric @@ -775,7 +780,7 @@ def setup_structures(self): print('Structure strs: {}'.format(micro_model.get_structures_strs())) point = [1.502,0.4,0.2] # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - vars = ['BC', 'bar_vel', 'n'] + vars = ['BC', 'bar_vel', 'rho'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) From 3d5553243402b28cf315b8c299e9c091cd989f4b Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 26 Feb 2024 14:01:56 +0100 Subject: [PATCH 082/111] Fixing bug in modelling_coefficients_parallel --- Proof_of_principle/config_calibration.txt | 11 +- Proof_of_principle/config_meso.txt | 9 +- Proof_of_principle/config_visualizing.txt | 11 +- Proof_of_principle/pickling_meso.py | 3 +- Proof_of_principle/py_parallel.slurm | 2 +- Proof_of_principle/py_serial.slurm | 18 +-- Proof_of_principle/visualizing_meso.py | 92 +++++++-------- Proof_of_principle/visualizing_obs.py | 15 ++- Proof_of_principle/visualizing_residuals.py | 120 ++++++++++---------- master_files/FileReaders.py | 16 +-- master_files/Filters.py | 6 +- master_files/MesoModels.py | 26 ++--- master_files/MicroModels.py | 12 +- master_files/Visualization.py | 89 ++++++++++++--- 14 files changed, 237 insertions(+), 193 deletions(-) diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index 70be08d..718df1b 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -5,25 +5,22 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/. -#/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4 -#/scratch/tc2m23/KHIRandom/hydro/ET_1_3.5_step0.5/20dx/Figures/800X800/ET2/cg=bl=fw=dt/CG4/fitting_eta +figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1 [Filenames] -#meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle -meso_pickled_filename = /prova.pickle +meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle [Models_settings] mesogrid_T_slices_num = 3 [Ranges_for_analysis] -ranges = {"x_range": [0.21, 0.24], "y_range": [0.21, 0.24]} +ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} [Visualize_correlation] -vars = ["T_tilde", "rho_tilde"] +vars = ["eta", ] preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} #set extractions 0 if you don't want to extract randomly from sample extractions = 0 diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt index 5d9ae7f..630bacf 100644 --- a/Proof_of_principle/config_meso.txt +++ b/Proof_of_principle/config_meso.txt @@ -12,17 +12,16 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickle #the options is activated using the first key, while the list refers to the index of the elements of #ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) -snapshots_opts = {"fewer_snaps_required": true, "smaller_list": [6,7,8,9,10,11,12,13,14]} +snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [6,7,8,9,10,11,12,13,14]} [Meso_model_settings] -meso_grid = {"x_range": [0.2, 0.25], "y_range": [0.2, 0.25], "num_T_slices": 3, "coarse_grain_factor": 1, "coarse_grain_time": false} +meso_grid = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97], "num_T_slices": 3, "coarse_grain_factor": 1, "coarse_grain_time": false} -filtering_options = {"box_len_ratio": 2.0, "filter_width_ratio": 2.0} +filtering_options = {"box_len_ratio": 8.0, "filter_width_ratio": 8.0} n_cpus = 40 [Pickled_filenames] -#meso_pickled = /rHD2d_nocg_fw=bl=8dx.pickle -meso_pickled = /prova \ No newline at end of file +meso_pickled = /rHD2d_nocg_fw=bl=8dx.pickle diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt index a423277..691a82d 100644 --- a/Proof_of_principle/config_visualizing.txt +++ b/Proof_of_principle/config_visualizing.txt @@ -7,14 +7,12 @@ hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -#figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures -figures_dir = /home/tc2m23/Filtering/Proof_of_principle/tests/. +figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1 [Filenames] -#meso_pickled_filename = /rHD2d_ET2_cg=fw=bl=4dx_dt.pickle -meso_pickled_filename = /prova.pickle +meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle [Models_settings] @@ -28,8 +26,9 @@ mesogrid_T_slices_num = 3 [Plot_settings] # method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids # interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid +# Else, set method to "raw_data" to use gridded data directly -plot_ranges = {"x_range": [0.2, 0.25], "y_range": [0.2, 0.25]} +plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} -diff_plot_settings = {"method": "raw_data", "interp_dims": [700, 700]} +diff_plot_settings = {"method": "raw_data", "interp_dims": [300, 300]} diff --git a/Proof_of_principle/pickling_meso.py b/Proof_of_principle/pickling_meso.py index 94278c5..1cd36ec 100644 --- a/Proof_of_principle/pickling_meso.py +++ b/Proof_of_principle/pickling_meso.py @@ -50,7 +50,7 @@ furthest_slice_number = int((num_T_slices-1)/2) if coarse_time: furthest_slice_number = int(coarse_factor * furthest_slice_number) - print(furthest_slice_number) + t_range = [micro_model.domain_vars['t'][central_slice_num-furthest_slice_number], micro_model.domain_vars['t'][central_slice_num+furthest_slice_number]] x_range = meso_grid['x_range'] y_range = meso_grid['y_range'] @@ -120,6 +120,7 @@ time_taken = time.perf_counter() - start_time print('Finished computing the EL_style closure in parallel, time taken: {}\n'.format(time_taken), flush=True) + start_time = time.perf_counter() meso_model.modelling_coefficients_parallel(n_cpus) time_taken = time.perf_counter() - start_time diff --git a/Proof_of_principle/py_parallel.slurm b/Proof_of_principle/py_parallel.slurm index 4315025..11bfc22 100644 --- a/Proof_of_principle/py_parallel.slurm +++ b/Proof_of_principle/py_parallel.slurm @@ -7,7 +7,7 @@ #SBATCH --output=outputs/Meso-pickling-%A.out #SBATCH --nodes=1 #SBATCH --ntasks=40 -#SBATCH --time=02:00:00 +#SBATCH --time=03:00:00 # uncomment if job array ##SBATCH --array=1-6 diff --git a/Proof_of_principle/py_serial.slurm b/Proof_of_principle/py_serial.slurm index bcd574a..b7f583f 100644 --- a/Proof_of_principle/py_serial.slurm +++ b/Proof_of_principle/py_serial.slurm @@ -6,10 +6,10 @@ #SBATCH --nodes=1 #SBATCH --ntasks=1 -#SBATCH --time=00:05:00 +#SBATCH --time=02:30:00 #SBATCH --output=outputs/visualize_slurm-%A.out -# #SBATCH --output=outputs/calibration_slurm-%A.out -# #SBATCH --output=outputs/reducing_slurm-%A.out +# #SBATCH --output=outputs/vis_residuals-%A.out +# #SBATCH --output=outputs/vis_calibration-%A.out # uncomment if array: ##SBATCH --array=1-3 @@ -41,14 +41,14 @@ conda activate myenv #################################### # LAUNCHING THE JOBS ##################################### - # python3 -u visualizing_micro.py config_visualizing.txt # python3 -u visualizing_obs.py config_visualizing.txt # python3 -u visualizing_meso.py config_visualizing.txt -# python3 -u visualizing_residuals.py config_calibration.txt -python3 -u visualizing_correlations.py config_calibration.txt -python3 -u reducing_regressors.py config_calibration.txt -python3 -u hunting_correlations.py config_calibration.txt -python3 -u regressing_residual.py config_calibration.txt +python3 -u visualizing_residuals.py config_calibration.txt + +# python3 -u visualizing_correlations.py config_calibration.txt +# python3 -u reducing_regressors.py config_calibration.txt +# python3 -u hunting_correlations.py config_calibration.txt +# python3 -u regressing_residual.py config_calibration.txt diff --git a/Proof_of_principle/visualizing_meso.py b/Proof_of_principle/visualizing_meso.py index 947f3ed..6ca9679 100644 --- a/Proof_of_principle/visualizing_meso.py +++ b/Proof_of_principle/visualizing_meso.py @@ -40,15 +40,7 @@ meso_model = pickle.load(filehandle) micro_model = meso_model.micro_model - print('Finished reading pickled data') - - # # If you need to recompute, e.g. the closure coefficients - # n_cpus = int(config['Meso_model_settings']['n_cpus']) - # meso_model.EL_style_closure_parallel(n_cpus) - # print('Finished re-decomposing the meso_model! Now re-saving it') - # with open(MesoModelLoadFile, 'wb') as filehandle: - # pickle.dump(meso_model, filehandle) - # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') + print('Finished reading pickled data\n') # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE num_snaps = micro_model.domain_vars['nt'] @@ -57,9 +49,9 @@ num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] if time_meso != time_micro: - print("Slices of meso and micro model do not coincide. Careful!") + print("Slices of meso and micro model do not coincide. Careful!\n") else: - print("Comparing data at same time-slice, hurray!") + print("Comparing data at same time-slice, hurray!\n") # # PLOT SETTINGS plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) @@ -74,63 +66,51 @@ # PLOTTING MICRO VS FILTERED BC AND SET # ##################################### start_time = time.perf_counter() - # vars = [['BC', 'SET'], ['BC', 'SET']] - # models = [micro_model, meso_model] - # components = [[(0,), (0,0)], [(0,), (0,0)]] vars = [['BC'],['BC']] models = [micro_model, meso_model] components= [[(0,)],[(0,)]] - norms = [[None], [None]] + norms = [[None], [None], ['log']] fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=True, norms=norms) fig.tight_layout() - filename = "/microVSmeso_narel_new.pdf" + filename = "/Comparing_BC.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - vars = [['BC'],['BC']] + vars = [['SET'], ['SET']] models = [micro_model, meso_model] - norms = [[None], [None]] - components= [[(0,)],[(0,)]] + components = [[(0,0)], [(0,0)]] + norms = [[None], [None], ['log']] fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, - interp_dims = interp_dims, method = diff_method, diff_plot=True, rel_diff=False, norms=norms) + interp_dims = interp_dims, method = diff_method, diff_plot=False, rel_diff=True, norms=norms) fig.tight_layout() - filename = "/microVSmeso_naabs_new.pdf" + filename = "/Comparing_SET.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - # vars = [['BC', 'SET'], ['BC', 'SET',]] - # models = [micro_model, meso_model] - # components = [[(2,), (0,2)], [(2,), (0,2)]] - # fig=visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices = components, - # interp_dims = interp_dims, method = diff_method, diff_plot=True, rel_diff=False) - # fig.tight_layout() - # filename = "/microVSmeso_notscaling.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - time_taken = time.perf_counter() - start_time - print(f'Finished plotting model comparison: time taken (X2) ={time_taken}') + print(f'Finished plotting model comparison: time taken (X2) ={time_taken}\n') # # # PLOTTING THE DECOMPOSED SET # # ############################# vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] - cmaps = ['seismic', 'seismic', 'seismic', 'seismic', 'seismic', 'seismic'] + cmaps = ['seismic','seismic','seismic','seismic','seismic','seismic'] components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) - filename = "/DecomposedSET_1.pdf" + filename = "/DecomposedSET_1.pdf" plt.savefig(saving_directory + filename, format = 'pdf') vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] - norms = ['mysymlog', 'mysymlog', 'mysymlog', None, None, None] - cmaps = ['seismic', 'seismic', 'seismic', None, None, None] + norms = ['mysymlog', 'mysymlog', 'mysymlog', 'log', 'log', 'log'] + cmaps = ['seismic','seismic','seismic', 'plasma', 'plasma', 'plasma'] components = [(0,), (1,), (2,), (), (), ()] fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) filename = "/DecomposedSET_2.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting decomposition of SET') + print('Finished plotting decomposition of SET\n') # # # PLOTTING THE DERIVATIVES OF FAVRE VEL AND TEMPERATURE # ######################################################### @@ -139,49 +119,55 @@ components = [tuple([favre_vel_components[i]])] for j in range(3): components.append(tuple([j,favre_vel_components[i]])) + if i==0: + norms = ['log', 'mysymlog', 'mysymlog', 'mysymlog'] + cmaps = ['plasma', 'seismic', 'seismic', 'seismic'] + else: + norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] + cmaps = ['seismic', 'seismic', 'seismic', 'seismic'] + vars_strs = ['u_tilde', 'D_u_tilde', 'D_u_tilde', 'D_u_tilde'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/D_favre_comp={}.pdf".format(favre_vel_components[i]) + time_for_filename = str(round(time_meso,2)) + filename = "/D_favre_{}.pdf".format(favre_vel_components[i]) plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting derivatives of favre velocity', flush=True) + print('Finished plotting derivatives of favre velocity\n', flush=True) vars_strs = ['T_tilde', 'D_T_tilde', 'D_T_tilde', 'D_T_tilde'] components = [(), (0,), (1,), (2,)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + norms = ['log', 'mysymlog', 'mysymlog', 'mysymlog'] + cmaps = ['plasma', 'seismic', 'seismic', 'seismic'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) filename = "/D_T.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting temperature derivatives', flush=True) + print('Finished plotting temperature derivatives\n', flush=True) # # # PLOTTING SHEAR, ACCELERATION, EXPANSION AND TEMPERATURE DERIVATIVES # ######################################################################## vars_strs = ['shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde', 'shear_tilde'] components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + norms = ['mysymlog','mysymlog','mysymlog','mysymlog','mysymlog','mysymlog'] + cmaps = ['seismic','seismic','seismic','seismic','seismic','seismic'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) filename = "/Shear_comps.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting shear', flush=True) + print('Finished plotting shear\n', flush=True) vars_strs = ['acc_tilde', 'acc_tilde', 'acc_tilde', 'Theta_tilde', 'Theta_tilde', 'Theta_tilde'] components = [(0,), (1,), (2,), (0,), (1,), (2,)] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components) + norms = ['mysymlog','mysymlog','mysymlog','mysymlog','mysymlog','mysymlog'] + cmaps = ['seismic','seismic','seismic','seismic','seismic','seismic'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(time_meso,2)) filename = "/Acc+Tderivs.pdf" plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting DaT', flush=True) + print('Finished plotting DaT\n', flush=True) - vars_strs = ['exp_tilde'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Expansion.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished plotting expansion', flush=True) diff --git a/Proof_of_principle/visualizing_obs.py b/Proof_of_principle/visualizing_obs.py index d9d554d..f5755fb 100644 --- a/Proof_of_principle/visualizing_obs.py +++ b/Proof_of_principle/visualizing_obs.py @@ -37,7 +37,7 @@ print(f'Finished reading data from {MesoModelLoadFile}') - # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE + # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE num_snaps = micro_model.domain_vars['nt'] central_slice_num = int(num_snaps/2.) time_micro = micro_model.domain_vars['t'][central_slice_num] @@ -58,13 +58,18 @@ diff_method = diff_plot_settings['method'] interp_dims = diff_plot_settings['interp_dims'] visualizer = Plotter_2D([11.97, 8.36]) + + label_2_update = {'U' : r'$U^a$'} + meso_model.upgrade_labels_dict(label_2_update) - # FINALLY, PLOTTING + # FINALLY, PLOTTING models = [micro_model, meso_model] vars = [['bar_vel', 'bar_vel', 'bar_vel'],['U', 'U', 'U']] + norms = [['log','mysymlog','mysymlog'],['log','mysymlog','mysymlog'],['log','log','log']] + cmaps = [['plasma','seismic','seismic'], ['plasma','seismic','seismic'], ['plasma','plasma','plasma']] components_indices= [[(0,),(1,),(2,)], [(0,), (1,), (2,)]] fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices=components_indices, method=diff_method, - interp_dims=interp_dims, diff_plot=False, rel_diff=False) + interp_dims=interp_dims, diff_plot=False, rel_diff=True, norms=norms, cmaps=cmaps) fig.tight_layout() filename="/ObsVSmicro.pdf" plt.savefig(saving_directory + filename, format="pdf") @@ -74,8 +79,10 @@ models = [micro_model, meso_model] vars = [['bar_vel', 'bar_vel', 'bar_vel'],['u_tilde', 'u_tilde', 'u_tilde']] components_indices= [[(0,),(1,),(2,)], [(0,), (1,), (2,)]] + norms = [['log','mysymlog','mysymlog'],['log','mysymlog','mysymlog'],['log','log','log']] + cmaps = [['plasma','seismic','seismic'],['plasma','seismic','seismic'], ['plasma','plasma','plasma']] fig = visualizer.plot_vars_models_comparison(models, vars, time_meso, x_range, y_range, components_indices=components_indices, method=diff_method, - interp_dims=interp_dims, diff_plot=False, rel_diff=False) + interp_dims=interp_dims, diff_plot=False, rel_diff=True, norms=norms, cmaps=cmaps) fig.tight_layout() filename="/favreVSmicro.pdf" plt.savefig(saving_directory + filename, format="pdf") diff --git a/Proof_of_principle/visualizing_residuals.py b/Proof_of_principle/visualizing_residuals.py index 03ef485..4f221c7 100644 --- a/Proof_of_principle/visualizing_residuals.py +++ b/Proof_of_principle/visualizing_residuals.py @@ -39,16 +39,6 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - # #UNCOMMENT IF YOU NEED TO RE-COMPUTE, SAY, THE CLOSURE COEFFICIENTS - # ######################################## - # n_cpus = int(config['Meso_model_settings']['n_cpus']) - # meso_model.EL_style_closure_parallel(n_cpus) - # print('Finished re-decomposing the meso_model! Now re-saving it') - # with open(MesoModelLoadFile, 'wb') as filehandle: - # pickle.dump(meso_model, filehandle) - # print('Finished re-computing the dissipative coefficients and related quantities! Now re-saving it') - - statistical_tool = CoefficientsAnalysis() correlation_ranges = json.loads(config['Ranges_for_analysis']['ranges']) x_range = correlation_ranges['x_range'] @@ -57,12 +47,16 @@ time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] model_points = meso_model.domain_vars['Points'] + saving_directory = config['Directories']['figures_dir'] print('Producing correlation plot for pi_res') - x = np.log10(meso_model.meso_vars['shear_sq']) - y = np.log10(meso_model.meso_vars['pi_res_sq']) - data = [x,y] + x = meso_model.meso_vars['shear_sq'] + y = meso_model.meso_vars['pi_res_sq'] + data = [x,y] data = statistical_tool.trim_dataset(data, ranges, model_points) + preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} + data = statistical_tool.preprocess_data(data, preprocess_data) + data = statistical_tool.extract_randomly(data, 10000) x, y = data[0], data[1] xlabel = r'$\log(\tilde{\sigma}_{ab}\tilde{\sigma}^{ab})$' ylabel = r'$\log(\tilde{\pi}_{ab}\tilde{\pi}^{ab})$' @@ -74,10 +68,13 @@ print('Producing correlation plot for q_res') - x = np.log10(meso_model.meso_vars['q_res_sq']) - y = np.log10(meso_model.meso_vars['Theta_sq']) - data = [x,y] + x = meso_model.meso_vars['q_res_sq'] + y = meso_model.meso_vars['Theta_sq'] + data = [x,y] data = statistical_tool.trim_dataset(data, ranges, model_points) + preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} + data = statistical_tool.preprocess_data(data, preprocess_data) + data = statistical_tool.extract_randomly(data, 10000) x, y = data[0], data[1] xlabel = r'$\log(\tilde{q}_a\tilde{q}^a)$' ylabel = r'$\log(\tilde{\Theta}_a \tilde{\Theta}^a)$' @@ -90,10 +87,13 @@ print('Producing correlation plot for Pi_res') - x= np.log10(np.power(meso_model.meso_vars['Pi_res'], 2) ) - y= np.log10(np.power(meso_model.meso_vars['exp_tilde'], 2)) - data = [x,y] + x = np.power(meso_model.meso_vars['Pi_res'], 2) + y = np.power(meso_model.meso_vars['exp_tilde'], 2) + data = [x,y] data = statistical_tool.trim_dataset(data, ranges, model_points) + preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} + data = statistical_tool.preprocess_data(data, preprocess_data) + data = statistical_tool.extract_randomly(data, 10000) x, y = data[0], data[1] xlabel = r'$\log(\tilde{\Pi}^2)$' ylabel = r'$\log(\tilde{\theta}^2)$' @@ -105,45 +105,43 @@ print(f'Finished correlation plot for Pi_res, saved as {filename}\n\n') - ############################################################## - #PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS - ############################################################## - - plot_ranges = json.loads(config['Ranges_for_analysis']['ranges']) - x_range = plot_ranges['x_range'] - y_range = plot_ranges['y_range'] - num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) - time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] - saving_directory = config['Directories']['figures_dir'] - - visualizer = Plotter_2D() - - vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] - norms = ['mysymlog', 'mysymlog', 'log'] - cmaps = ['seismic', 'seismic', None] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Bulk_viscosity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished bulk viscosity') - - vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] - norms = ['mysymlog', 'log', 'log'] - cmaps = ['seismic', None, None] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Shear_viscosity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished shear viscosity') - - vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] - norms = ['mysymlog', 'log', 'log'] - cmaps = ['seismic', None, None] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Heat_conductivity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished heat conductivity') \ No newline at end of file + # ############################################################# + # PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS + # ############################################################# + + # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + # visualizer = Plotter_2D() + + # vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] + # norms = ['mysymlog', 'symlog', 'log'] + # cmaps = ['seismic', 'seismic', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Bulk_viscosity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished bulk viscosity') + + # vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] + # norms = ['mysymlog', 'log', 'log'] + # cmaps = ['seismic', 'plasma', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Shear_viscosity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished shear viscosity') + + # vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] + # norms = ['mysymlog', 'log', 'log'] + # cmaps = ['seismic', 'plasma', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Heat_conductivity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished heat conductivity') + + + \ No newline at end of file diff --git a/master_files/FileReaders.py b/master_files/FileReaders.py index 3bf657f..bd5dbcf 100644 --- a/master_files/FileReaders.py +++ b/master_files/FileReaders.py @@ -58,10 +58,10 @@ def read_in_data(self, micro_model): self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) for prim_str in micro_model.get_prim_strs(): - # if prim_str == "n": - # self.translating_prims[prim_str] = "rho" - # else: - self.translating_prims[prim_str] = prim_str + if prim_str == "n": + self.translating_prims[prim_str] = "rho" + else: + self.translating_prims[prim_str] = prim_str for prim_var_str in micro_model.prim_vars: try: @@ -137,10 +137,10 @@ def read_in_data_HDF5_missing_xy(self, micro_model): self.translating_prims = dict.fromkeys(micro_model.get_prim_strs()) for prim_str in micro_model.get_prim_strs(): - # if prim_str == "n": - # self.translating_prims[prim_str] = "rho" - # else: - self.translating_prims[prim_str] = prim_str + if prim_str == "n": + self.translating_prims[prim_str] = "rho" + else: + self.translating_prims[prim_str] = prim_str for prim_var_str in micro_model.prim_vars: try: diff --git a/master_files/Filters.py b/master_files/Filters.py index 9a88525..336d379 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -329,7 +329,7 @@ def find_observer(self, point, flux_str = "gauss", initial_guess = None): if initial_guess is not None and len(initial_guess) == self.spatial_dims: guess = initial_guess else: - U = np.multiply(1 / self.micro_model.get_interpol_var('rho', point) , self.micro_model.get_interpol_var('BC', point) ) + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) for i in range(1, len(U)): guess.append(U[i] / U[0]) guess = np.array(guess) @@ -681,7 +681,7 @@ def find_observer(self, point, drift_str = "gauss", initial_guess= None): if initial_guess is not None and len(initial_guess) == self.spatial_dims: guess = initial_guess else: - U = np.multiply(1 / self.micro_model.get_interpol_var('rho', point) , self.micro_model.get_interpol_var('BC', point) ) + U = np.multiply(1 / self.micro_model.get_interpol_var('n', point) , self.micro_model.get_interpol_var('BC', point) ) for i in range(1, len(U)): guess.append(U[i] / U[0]) guess = np.array(guess) @@ -998,7 +998,7 @@ def find_observers_parallel(self, points, n_cpus): # spatial_dims = self.micro_model.get_spatial_dims() L = self.L BC = self.micro_model.vars['BC'] - grid = self.micro_model.domain_vars['points'] + grid = self.micro_model.domain_vars['points'] args_for_pool = [ (points[i], i) for i in range(len(points))] init = FindObs_root_parallel.initializer # initargs = (spatial_dims, L) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 0a1db16..153e31f 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1147,7 +1147,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): for var in self.meso_structures: self.meso_structures[var] = [] - self.meso_scalars_strs = ['eps_tilde', 'rho_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] + self.meso_scalars_strs = ['eps_tilde', 'n_tilde', 'p_tilde', 'p_filt', 'eos_res', 'Pi_res', 'T_tilde'] self.meso_vectors_strs = ['u_tilde', 'q_res'] self.meso_r2tensors_strs = ['pi_res'] self.meso_vars_strs = self.meso_scalars_strs + self.meso_vectors_strs + self.meso_r2tensors_strs @@ -1194,7 +1194,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): self.labels_var_dict = {'SET' : r'$$', 'BC' : r'$$', 'eps_tilde' : r'$\tilde{\varepsilon}$', - 'rho_tilde' : r'$\tilde{\rho}$', + 'n_tilde' : r'$\tilde{n}$', 'p_tilde' : r'$\tilde{p}$', 'p_filt' : r'$

$', 'eos_res' : r'$M$', @@ -1220,7 +1220,8 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$', 'det_shear': r'det$(\sigma)$', 'vort_mod' : r'$|W|$', - 'acc_mag': r'$|a|$'} + 'acc_mag': r'$|a|$', + 'U' : r'$U^a$'} def upgrade_labels_dict(self, entry_dict): """ @@ -1662,7 +1663,7 @@ def decompose_structures_gridpoint(self, h, i, j): # Storing the decomposition with appropriate names. - self.meso_vars['rho_tilde'][h,i,j] = n_t + self.meso_vars['n_tilde'][h,i,j] = n_t self.meso_vars['u_tilde'][h,i,j,:] = u_t self.meso_vars['eps_tilde'][h,i,j] = eps_t self.meso_vars['q_res'][h,i,j,:] = q_a @@ -1790,7 +1791,7 @@ def decompose_structures_parallel(self, n_cpus): print('Decomposing structures in parallel with {} processes'.format(pool._processes), flush=True) for result in pool.starmap(resHD2D.decompose_structures_task, args_for_pool): h,i,j = result[1] - self.meso_vars['rho_tilde'][h,i,j] = result[0][0] + self.meso_vars['n_tilde'][h,i,j] = result[0][0] self.meso_vars['u_tilde'][h,i,j,:] = result[0][1] self.meso_vars['eps_tilde'][h,i,j] = result[0][2] self.meso_vars['q_res'][h,i,j,:] = result[0][3] @@ -2302,14 +2303,7 @@ def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, metric[0,0] = -1 metric[1,1] = metric[2,2] = +1 - # Computing various invariants of shear matrix - # det_shear = np.linalg.det(shear) - pi_eig, _ = np.linalg.eigh(shear) - det_shear = 1. - for i in range(len(pi_eig)): - if pi_eig[i] != 0.: - det_shear *= pi_eig[i] - + det_shear = np.linalg.det(shear) shear_sq = np.einsum('ij,kl,ik,jl->', shear, shear, metric, metric) var_names.append('det_shear') @@ -2359,7 +2353,7 @@ def modelling_coefficients_parallel(self, n_cpus): Nx = self.domain_vars['Nx'] Ny = self.domain_vars['Ny'] - for h in range(Nt): + for h in range(Nt): for i in range(Nx): for j in range(Ny): shear_t = self.meso_vars['shear_tilde'][h,i,j] @@ -2374,8 +2368,10 @@ def modelling_coefficients_parallel(self, n_cpus): with mp.Pool(processes=n_cpus) as pool: print('Computing vars for modelling coefficients with {} processes'.format(pool._processes), flush=True) + for result in pool.starmap(resHD2D.modelling_coefficients_task, args_for_pool): keys, values, grid_idxs = result + for idx, key in enumerate(keys): try: self.meso_vars[key] @@ -2385,7 +2381,7 @@ def modelling_coefficients_parallel(self, n_cpus): self.meso_vars.update({key : np.zeros(([Nt,Nx,Ny]+ list(shape)))}) finally: self.meso_vars[key][tuple(grid_idxs)] = values[idx] - + def EL_style_closure_regression(self, CoefficientAnalysis): """ Takes in a list of correlation quantities (strings? The regressors needed are computed diff --git a/master_files/MicroModels.py b/master_files/MicroModels.py index a720c04..a8555ac 100644 --- a/master_files/MicroModels.py +++ b/master_files/MicroModels.py @@ -537,7 +537,7 @@ def __init__(self, interp_method = "linear"): self.domain_vars[str] = [] #Dictionary for primitive var - self.prim_strs = ("vx","vy","rho","p") + self.prim_strs = ("vx","vy","n","p") self.prim_vars = dict.fromkeys(self.prim_strs) for str in self.prim_strs: self.prim_vars[str] = [] @@ -554,12 +554,12 @@ def __init__(self, interp_method = "linear"): for str in self.structures_strs: self.structures[str] = [] - self.labels_var_dict = {'BC' : r'$\rho^{a}$', + self.labels_var_dict = {'BC' : r'$n^{a}$', 'SET' : r'$T^{ab}$', 'bar_vel' : r'$u^a$', 'vx' : r'$v_x$', 'vy' : r'$v_y$', - 'rho' : r'$\rho$', + 'n' : r'$n$', 'W' : r'$W$', 'e' : r'$e$', 'h' : r'$h$', @@ -715,9 +715,9 @@ def setup_structures(self): self.structures['bar_vel'][h,i,j,:] = vel_vec - self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['rho'][h,i,j], vel_vec ) + self.structures['BC'][h,i,j,:] = np.multiply(self.prim_vars['n'][h,i,j], vel_vec ) - self.structures['SET'][h,i,j,:,:] = (self.prim_vars['rho'][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + self.structures['SET'][h,i,j,:,:] = (self.prim_vars['n'][h,i,j] * self.aux_vars['h'][h,i,j]) * np.outer(vel_vec, vel_vec) \ + self.prim_vars['p'][h,i,j] * self.metric @@ -780,7 +780,7 @@ def setup_structures(self): print('Structure strs: {}'.format(micro_model.get_structures_strs())) point = [1.502,0.4,0.2] # vars = ['SETfl', 'BC', 'Fab', 'SETem'] - vars = ['BC', 'bar_vel', 'rho'] + vars = ['BC', 'bar_vel', 'n'] for var in vars: res = micro_model.get_interpol_var(var, point) res2 = micro_model.get_var_gridpoint(var, point) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 601f539..b2086a8 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -344,12 +344,12 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com inv_fig_shape = (n_cols, n_rows) if not norms or np.array(norms).shape != inv_fig_shape: - # print('Norms provided are not compatible with figure, setting these to auto') + print('Norms provided are not compatible with figure, setting these to auto') norms = [None for _ in range(n_rows)] norms = [norms for _ in range(n_cols)] if not cmaps or np.array(cmaps).shape != inv_fig_shape: - # print('Colormaps not compatible with figure, setting these to auto') + print('Colormaps not compatible with figure, setting these to auto') cmaps = [None for _ in range(n_rows)] cmaps = [cmaps for _ in range(n_cols)] @@ -359,12 +359,32 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com for j in range(len(models)): for i in range(n_rows): + data_to_plot, extent = self.get_var_data(models[j], var_strs[j][i], t, x_range, y_range, components_indices[j][i], 'raw_data', None) - im=axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[j][i], cmap=cmaps[j][i]) - divider = make_axes_locatable(axes[i,j]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') + + if norms[j][i] == 'mysymlog': + ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) + data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) + mynorm = MyThreeNodesNorm(nodes) + im = axes[i,j].imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[j][i]) + divider = make_axes_locatable(axes[i,j]) + cax = divider.append_axes('right', size='5%', pad=0.05) + cbar = fig.colorbar(im, cax=cax, orientation='vertical') + cbar.set_ticks(ticks) + cbar.ax.set_yticklabels(labels) + + elif norms[j][i] != 'mysymlog': + im = axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[j][i], cmap=cmaps[j][i]) + divider = make_axes_locatable(axes[i,j]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + # im=axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[j][i], cmap=cmaps[j][i]) + # divider = make_axes_locatable(axes[i,j]) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # fig.colorbar(im, cax=cax, orientation='vertical') # title = models[j].get_model_name() + "\n"+var_strs[j][i] + title = models[j].get_model_name() + '\n' if hasattr(models[j], 'labels_var_dict'): if var_strs[j][i] in models[j].labels_var_dict.keys(): @@ -381,6 +401,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com axes[i,j].set_ylabel(r'$x$') + if diff_plot and len(models)==2: try: for i in range(len(var_strs[0])): @@ -390,10 +411,31 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com print("Cannot plot the difference between the vars: data not aligned.") continue data_to_plot = data1 - data2 - im = axes[i,2].imshow(data_to_plot, extent=extent1, norm=norms[2][i], cmap=cmaps[2][i]) - divider = make_axes_locatable(axes[i,2]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') + + if norms[2][i] == 'mysymlog': + ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) + data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) + mynorm = MyThreeNodesNorm(nodes) + im = axes[i,2].imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[2][i]) + divider = make_axes_locatable(axes[i,2]) + cax = divider.append_axes('right', size='5%', pad=0.05) + cbar = fig.colorbar(im, cax=cax, orientation='vertical') + cbar.set_ticks(ticks) + cbar.ax.set_yticklabels(labels) + + elif norms[2][i] != 'mysymlog': + im = axes[i,2].imshow(data_to_plot, extent=extent, norm=norms[2][i], cmap=cmaps[2][i]) + divider = make_axes_locatable(axes[i,2]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + + # im = axes[i,2].imshow(data_to_plot, extent=extent1, norm=norms[2][i], cmap=cmaps[2][i]) + # divider = make_axes_locatable(axes[i,2]) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # fig.colorbar(im, cax=cax, orientation='vertical') + + axes[i,2].set_title('Models difference') axes[i,2].set_xlabel(r'$y$') axes[i,2].set_ylabel(r'$x$') @@ -417,10 +459,29 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com else: column = 2 - im = axes[i,column].imshow(data_to_plot, extent=extent1, norm=norms[column][i], cmap=cmaps[column][i]) - divider = make_axes_locatable(axes[i,column]) - cax = divider.append_axes('right', size='5%', pad=0.05) - fig.colorbar(im, cax=cax, orientation='vertical') + + if norms[column][i] == 'mysymlog': + ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) + data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) + mynorm = MyThreeNodesNorm(nodes) + im = axes[i,column].imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[column][i]) + divider = make_axes_locatable(axes[i,column]) + cax = divider.append_axes('right', size='5%', pad=0.05) + cbar = fig.colorbar(im, cax=cax, orientation='vertical') + cbar.set_ticks(ticks) + cbar.ax.set_yticklabels(labels) + + elif norms[column][i] != 'mysymlog': + im = axes[i,column].imshow(data_to_plot, extent=extent, norm=norms[column][i], cmap=cmaps[column][i]) + divider = make_axes_locatable(axes[i,column]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + # im = axes[i,column].imshow(data_to_plot, extent=extent1, norm=norms[column][i], cmap=cmaps[column][i]) + # divider = make_axes_locatable(axes[i,column]) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # fig.colorbar(im, cax=cax, orientation='vertical') + axes[i,column].set_title('Relative difference') axes[i,column].set_xlabel(r'$y$') axes[i,column].set_ylabel(r'$x$') From be78650d6b0d6d128af3f4cb0126befe45e144dd Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 27 Feb 2024 13:58:01 +0100 Subject: [PATCH 083/111] Adding r-value to correlation plots + speeding up random extraction --- master_files/Analysis.py | 138 ++++++++++++++++++++++++------------- master_files/MesoModels.py | 8 ++- 2 files changed, 95 insertions(+), 51 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 96bb0eb..f5f0ea8 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -17,6 +17,7 @@ import warnings from sklearn.linear_model import LinearRegression from sklearn.decomposition import PCA +from scipy import stats # import statsmodels.api as sm from system.BaseFunctionality import * @@ -153,6 +154,22 @@ def get_pos_or_neg_mask(self, pos_or_neg, array): return mask + def restrict_data_range_mask(self, var, min=None, max=None): + """ + Takes input var and make a copy of array masking those entries that are outside the (min, max) range + Return compressed masked array. + """ + if min is None: + min = np.amin(var) + if max is None: + max = np.amax(var) + + restricted_var = ma.masked_where(var < min, var, copy = True) + restricted_var = ma.masked_where(restricted_var > max, restricted_var, copy=True) + restricted_var = restricted_var.compressed() + + return restricted_var + def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): """ Takes input list of arrays with dictionary on how to preprocess_data @@ -259,23 +276,15 @@ def extract_randomly(self, list_of_data, num_extractions): else: print(f'The {i}th array in the dataset is not compatible with the first: ignoring it.') - # setting up the mask - mask_index = np.zeros(new_data[i].shape) - for i in range(len(mask_index)): - mask_index[i] = False - - max_idx = len(new_data[0]) - 1 - extracted_idxs = [] - for i in range(num_extractions): - new_extraction = random.choice(list(set(range(0,max_idx)) - set(extracted_idxs))) - extracted_idxs.append(new_extraction) - mask_index[new_extraction] = True - # need to invert the mask as it is True on the extracted indices - mask_index = np.logical_not(mask_index) - - # finally: mask, compress and return - for i in range(len(new_data)): - new_data[i] = np.ma.masked_array(new_data[i], mask_index).compressed() + available_indices = list(np.arange(len(new_data[0]))) + selected_indices = random.sample(available_indices, num_extractions) + new_data = [] + for i in range(len(list_of_data)): + shortened_list = [] + for j in range(len(selected_indices)): + shortened_list.append(list_of_data[i][selected_indices[j]]) + shortened_array = np.array(shortened_list) + new_data.append(shortened_array) return np.array(new_data) @@ -499,7 +508,9 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, hue_array=None, new_labels = [legend_dict[label] for label in labels] scatter.legend(handles, new_labels) - # print('Figure produced inside Coeff Analysis') + r, _ = stats.pearsonr(X,Y) + g.ax_joint.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=g.ax_joint.transAxes) + return g def visualize_many_correlations(self, data, labels): @@ -554,15 +565,22 @@ def visualize_many_correlations(self, data, labels): Data=np.column_stack(Data) Data_df = pd.DataFrame(Data, columns=labels) + def corrfunc(x, y, **kws): + r, _ = stats.pearsonr(x, y) + ax = plt.gca() + ax.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=ax.transAxes) + with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') g=sns.PairGrid(Data_df) - g.map_upper(sns.scatterplot, s=4, c='red') - g.map_lower(sns.kdeplot, color='red') + g.map_lower(sns.scatterplot, s=4, c='red') + g.map_upper(sns.kdeplot, color='red') g.map_diag(sns.histplot, kde=True, color='red') g.fig.tight_layout() + g.map_lower(corrfunc) + return g def PCA_find_regressors_subset(self, data, var_wanted = 0.9): @@ -761,7 +779,7 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met if __name__ == '__main__': - + pass # #TESTING REGRESSION # x0 = np.arange(100).reshape((10,10)) # x1 = np.sin(np.arange(100).reshape((10,10))) @@ -773,44 +791,66 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # print('Coefficients: {}'.format(result)) # print('Errors: {}\n'.format(errors)) - # #TESTING PRE-PROCESS DATA - x = np.zeros((100,100)) - for idx in np.ndindex(x.shape): - signum = [-1,1][random.randint(0,1)] - x[idx] = signum * random.randint(0,100) + # # #TESTING PRE-PROCESS DATA + # x = np.zeros((100,100)) + # for idx in np.ndindex(x.shape): + # signum = [-1,1][random.randint(0,1)] + # x[idx] = signum * random.randint(0,100) - y = np.zeros((100,100)) - for idx in np.ndindex(x.shape): - signum = [-1,1][random.randint(0,1)] - if signum==1: - exp = random.randint(-5,5) - elif signum ==-1: - exp = random.randint(-3,3) - y[idx] = signum * (10 ** exp) + # y = np.zeros((100,100)) + # for idx in np.ndindex(x.shape): + # signum = [-1,1][random.randint(0,1)] + # if signum==1: + # exp = random.randint(-5,5) + # elif signum ==-1: + # exp = random.randint(-3,3) + # y[idx] = signum * (10 ** exp) - print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) - print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + # print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) + # print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) - preprocess_data = {'pos_or_neg' : [1,0], - 'log_or_not' : [0,1]} + # preprocess_data = {'pos_or_neg' : [1,0], + # 'log_or_not' : [0,1]} - statistical_tool = CoefficientsAnalysis() - data = [x,y] + # statistical_tool = CoefficientsAnalysis() + # data = [x,y] - x, y = statistical_tool.preprocess_data(data, preprocess_data) + # x, y = statistical_tool.preprocess_data(data, preprocess_data) - print('Processing data....\n') - print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) - print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + # print('Processing data....\n') + # print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) + # print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) - print('Extracting random vals from x...\n') - print(len(x)) - weights = np.ones(x.shape) - extracted, weights = statistical_tool.extract_randomly([x,y], 10) - print(extracted) + # print('Extracting random vals from x...\n') + # print(len(x)) + # weights = np.ones(x.shape) + # extracted, weights = statistical_tool.extract_randomly([x,y], 10) + # print(extracted) + + + # TESTING ADDING R-VALUE TO CORRELATION PLOTS + # a = np.zeros(500) + # for i in range(len(a)): + # err = random.randint(1,10) + # a[i] = err + # b = np.zeros((500,)) + # c = np.zeros((500,)) + # for i in range(len(b)): + # err = random.randint(10,100) + # b[i] = 3 * a[i] + err + + # for i in range(len(b)): + # err = random.randint(10,100) + # b[i] = 5 * a[i] + err + + # labels = ['a','b','c'] + # statistical_tool = CoefficientsAnalysis() + # g = statistical_tool.visualize_many_correlations([a,b,c],labels) + # g = statistical_tool.visualize_correlation(a,b) + # plt.show() \ No newline at end of file diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 153e31f..8207c32 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1219,7 +1219,8 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): 'Theta_sq': r'$\tilde{\Theta}_a\tilde{\Theta}^a$', 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$', 'det_shear': r'det$(\sigma)$', - 'vort_mod' : r'$|W|$', + 'vort_mod' : r'$|W|$', + 'vort_sq' : r'$\omega_{ab}\omega^{ab}$', 'acc_mag': r'$|a|$', 'U' : r'$U^a$'} @@ -2313,12 +2314,15 @@ def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, # Q-criterion vort_sq = np.einsum('ij,kl,ik,jl->', vort, vort, metric, metric) + var_names.append('vort_sq') + vars.append(vort_sq) + Q_criterion = shear_sq - vort_sq var_names.append('Q') vars.append(Q_criterion) # Computing the magnitude of vorticity - vort_mod = np.sqrt(vort_sq / 2. ) + vort_mod = np.sqrt(vort_sq / 2.) var_names.append('vort_mod') vars.append(vort_mod) From 3f80bce835a45ab89b33b62e3497ef345ecbf18d Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 27 Feb 2024 16:50:30 +0100 Subject: [PATCH 084/111] changes to preprocess data: now you can control which ranges not just if you wanna focus on positive or negative values --- master_files/Analysis.py | 112 ++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 55 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index f5f0ea8..4a252dc 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -130,6 +130,8 @@ def trim_dataset(self, list_of_data, ranges, model_points): def get_pos_or_neg_mask(self, pos_or_neg, array): """ + NOT USED ANYMORE!!!!!! + Function that return the mask that would be applied to an array in order to select positive or negative values @@ -164,12 +166,41 @@ def restrict_data_range_mask(self, var, min=None, max=None): if max is None: max = np.amax(var) - restricted_var = ma.masked_where(var < min, var, copy = True) - restricted_var = ma.masked_where(restricted_var > max, restricted_var, copy=True) + restricted_var = ma.masked_where(var <= min, var, copy = True) + restricted_var = ma.masked_where(restricted_var >= max, restricted_var, copy=True) restricted_var = restricted_var.compressed() return restricted_var + def get_mask_min_max(self, array, min=None, max=None): + """ + Given an input 'array' and a list of 'value_ranges' to be considered, return + mask that would have to be applied to array to mask the entries outside the given + range + + Parameters: + ----------- + min, max: floats + the ranges to be considered + + array: nd.array + + Returns: + -------- + Mask + """ + if min is None: + min = -np.inf + if max is None: + max = np.inf + + masked_array = ma.masked_where(arraymax, masked_array, copy=True) + + mask = masked_array.mask + + return mask + def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): """ Takes input list of arrays with dictionary on how to preprocess_data @@ -221,11 +252,17 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): else: return list_of_arrays + # USE THIS IF YOU WANNA REVERT TO PREVIOUS STUFF + # for i in range(len(list_of_arrays)): + # pos_or_neg = preprocess_data['pos_or_neg'][i] + # temp = self.get_pos_or_neg_mask(pos_or_neg, list_of_arrays[i]) + # masks.append(temp) + # Combining the masks of the different arrays masks = [] for i in range(len(list_of_arrays)): - pos_or_neg = preprocess_data['pos_or_neg'][i] - temp = self.get_pos_or_neg_mask(pos_or_neg, list_of_arrays[i]) + min, max = preprocess_data['value_ranges'][i] + temp = self.get_mask_min_max(list_of_arrays[i], min, max) masks.append(temp) tot_mask = masks[0] @@ -238,9 +275,9 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): temp = ma.masked_array(list_of_arrays[i], tot_mask) processed_list.append(temp.compressed()) #when array is compressed, this is automatically flattened! - + for i in range(len(processed_list)): - if preprocess_data['log_or_not'][i]: + if preprocess_data['log_abs'][i]: processed_list[i] = np.log10(np.abs(processed_list[i])) # removing corresponding entries from weights @@ -779,7 +816,7 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met if __name__ == '__main__': - pass + # pass # #TESTING REGRESSION # x0 = np.arange(100).reshape((10,10)) # x1 = np.sin(np.arange(100).reshape((10,10))) @@ -791,38 +828,26 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # print('Coefficients: {}'.format(result)) # print('Errors: {}\n'.format(errors)) - # # #TESTING PRE-PROCESS DATA - # x = np.zeros((100,100)) - # for idx in np.ndindex(x.shape): - # signum = [-1,1][random.randint(0,1)] - # x[idx] = signum * random.randint(0,100) - - # y = np.zeros((100,100)) - # for idx in np.ndindex(x.shape): - # signum = [-1,1][random.randint(0,1)] - # if signum==1: - # exp = random.randint(-5,5) - # elif signum ==-1: - # exp = random.randint(-3,3) - # y[idx] = signum * (10 ** exp) - - # print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) - # print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + # #TESTING PRE-PROCESS DATA + x = np.arange(1, 200) + y = np.arange(29, 228) + print('Min and max of x: {}, {}\n'.format(np.min(x), np.max(x))) + print('Min and max of y: {}, {}\n'.format(np.min(y), np.max(y))) + preprocess_data = {"value_ranges": [[None, None], [None, None]], + "log_abs": [1, 1]} - # preprocess_data = {'pos_or_neg' : [1,0], - # 'log_or_not' : [0,1]} - # statistical_tool = CoefficientsAnalysis() - # data = [x,y] + statistical_tool = CoefficientsAnalysis() + data = [x,y] - # x, y = statistical_tool.preprocess_data(data, preprocess_data) + x, y = statistical_tool.preprocess_data(data, preprocess_data) - # print('Processing data....\n') - # print('Max and min of x: {}, {}\n'.format(np.max(x), np.min(x))) - # print('Max and min of y: {}, {}\n'.format(np.max(y), np.min(y))) + print('Processing data....\n') + print('Min and max of x: {}, {}\n'.format(np.min(x), np.max(x))) + print('Min and max of y: {}, {}\n'.format(np.min(y), np.max(y))) @@ -831,26 +856,3 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # weights = np.ones(x.shape) # extracted, weights = statistical_tool.extract_randomly([x,y], 10) # print(extracted) - - - # TESTING ADDING R-VALUE TO CORRELATION PLOTS - # a = np.zeros(500) - # for i in range(len(a)): - # err = random.randint(1,10) - # a[i] = err - - # b = np.zeros((500,)) - # c = np.zeros((500,)) - # for i in range(len(b)): - # err = random.randint(10,100) - # b[i] = 3 * a[i] + err - - # for i in range(len(b)): - # err = random.randint(10,100) - # b[i] = 5 * a[i] + err - - # labels = ['a','b','c'] - # statistical_tool = CoefficientsAnalysis() - # g = statistical_tool.visualize_many_correlations([a,b,c],labels) - # g = statistical_tool.visualize_correlation(a,b) - # plt.show() \ No newline at end of file From f732ca2a07c311ce9bcd838e61f93f2f0d262a94 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 27 Feb 2024 16:51:35 +0100 Subject: [PATCH 085/111] Corresponding updates to PoP routines --- Proof_of_principle/config_calibration.txt | 42 +++++++---- Proof_of_principle/config_visualizing.txt | 2 +- Proof_of_principle/hunting_correlations.py | 4 +- Proof_of_principle/regressing_residual.py | 21 ++++-- .../visualizing_correlations.py | 17 +++-- Proof_of_principle/visualizing_residuals.py | 72 +++++++++---------- 6 files changed, 94 insertions(+), 64 deletions(-) diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index 718df1b..8b29c24 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -5,7 +5,7 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1 +figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8/fitting_eta/full_data [Filenames] meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle @@ -15,15 +15,21 @@ mesogrid_T_slices_num = 3 [Ranges_for_analysis] -ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} +ranges = {"x_range": [0.04, 0.96], "y_range": [0.03, 0.96]} -[Visualize_correlation] +[Visualize_correlations] + +vars = ["eta", "T_tilde", "n_tilde"] +#preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} + +# Set values to null if you do not want to restrict the data +preprocess_data = {"value_ranges": [[1e-6, null], [null, null], [null, null]], + "log_abs": [1, 1, 1]} -vars = ["eta", ] -preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} #set extractions 0 if you don't want to extract randomly from sample -extractions = 0 +extractions = 10000 + [PCA_settings] @@ -31,25 +37,35 @@ extractions = 0 dependent_var = zeta explanatory_vars = ["T_tilde", "rho_tilde"] pcs_num = 1 -#["det_shear", "shear_sq", "vort_mod", "Q", "acc_mag", "T_tilde", "n_tilde"] regressors_2_reduce = ["T_tilde", "rho_tilde", "p_tilde"] variance_wanted = 0.9 # Careful, only one dictionary here -preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} +#preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} + +# Set values to null if you do not want to restrict the data +preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null]], + "log_abs": [1, 1, 1]} + #set extractions 0 if you don't want to extract randomly from sample extractions = 0 [Regression_settings] -dependent_var = zeta -regressors = ["T_tilde", "rho_tilde"] -preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} +dependent_var = eta +regressors = ["T_tilde", "n_tilde"] add_intercept = 1 -#set extractions 0 if you don't want to extract randomly from sample -extractions = 0 + +#preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} + +# Set values to null if you do not want to restrict the data +preprocess_data = {"value_ranges": [[1e-6, null], [null, null], [null, null]], + "log_abs": [1, 1, 1]} + +#set extractions 0 if you don't want to extract randomly from sample: extraction only in the scatter plot? Not in regression? +extractions = 10000 diff --git a/Proof_of_principle/config_visualizing.txt b/Proof_of_principle/config_visualizing.txt index 691a82d..3ab53c3 100644 --- a/Proof_of_principle/config_visualizing.txt +++ b/Proof_of_principle/config_visualizing.txt @@ -7,7 +7,7 @@ hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1 +figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8 [Filenames] diff --git a/Proof_of_principle/hunting_correlations.py b/Proof_of_principle/hunting_correlations.py index ae7664b..88b1b30 100644 --- a/Proof_of_principle/hunting_correlations.py +++ b/Proof_of_principle/hunting_correlations.py @@ -114,7 +114,7 @@ ylabel = dep_var_str if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): ylabel = meso_model.labels_var_dict[dep_var_str] - if preprocess_data['log_or_not'][0] ==1: + if preprocess_data['log_abs'][0] ==1: ylabel = r"$\log($" + ylabel + r"$)$" xlabel = "" @@ -124,7 +124,7 @@ text = explanatory_vars_strs[i] if hasattr(meso_model, 'labels_var_dict') and explanatory_vars_strs[i] in meso_model.labels_var_dict.keys(): text = meso_model.labels_var_dict[explanatory_vars_strs[i]] - if preprocess_data['log_or_not'][i+1]==1: + if preprocess_data['log_abs'][i+1]==1: text = r"$\log($" + text + r"$)$" xlabel += coeff_for_label + text diff --git a/Proof_of_principle/regressing_residual.py b/Proof_of_principle/regressing_residual.py index e456868..c877c18 100644 --- a/Proof_of_principle/regressing_residual.py +++ b/Proof_of_principle/regressing_residual.py @@ -6,6 +6,7 @@ import json import time import math +from scipy import stats from FileReaders import * from MicroModels import * @@ -41,8 +42,8 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - # additional_entry = {'acc_mag': r'$|a|$'} - # meso_model.upgrade_labels_dict(additional_entry) + dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} + meso_model.upgrade_labels_dict(dict_to_add) # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['Regression_settings']['dependent_var'] @@ -76,8 +77,8 @@ model_points = meso_model.domain_vars['Points'] new_data = statistical_tool.trim_dataset(data, ranges, model_points) new_data = statistical_tool.preprocess_data(new_data, preprocess_data) - if extractions != 0: - new_data = statistical_tool.extract_randomly(new_data, extractions) + # if extractions != 0: + # new_data = statistical_tool.extract_randomly(new_data, extractions) dep_var = new_data[0] regressors = [] @@ -96,12 +97,18 @@ dep_var_model = np.zeros(dep_var.shape) for i in range(len(regressors)): dep_var_model += np.multiply(coeffs[i], regressors[i]) + + if extractions != 0: + data_for_scatter = [dep_var, dep_var_model] + new_data = statistical_tool.extract_randomly(data_for_scatter, extractions) + dep_var, dep_var_model = new_data[0], new_data[1] + # FINALLY: PLOTTING ylabel = dep_var_str if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): ylabel = meso_model.labels_var_dict[dep_var_str] - if preprocess_data['log_or_not'][0] == 1: + if preprocess_data['log_abs'][0] == 1: ylabel = r"$\log($" + ylabel + r"$)$" xlabel = "" @@ -114,7 +121,7 @@ var_name = regressors_strs[i] if hasattr(meso_model, 'labels_var_dict') and regressors_strs[i] in meso_model.labels_var_dict.keys(): var_name = meso_model.labels_var_dict[regressors_strs[i]] - if preprocess_data['log_or_not'][i] == 1: + if preprocess_data['log_abs'][i] == 1: var_name = r"$\log($" + var_name + r"$)$" xlabel +=var_name @@ -129,4 +136,4 @@ plt.savefig(saving_directory + filename, format='pdf') print(f'Finished regression and scatter plot for {dep_var_str}, saved as {filename}\n\n') - + diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py index fcbf8fc..b44af07 100644 --- a/Proof_of_principle/visualizing_correlations.py +++ b/Proof_of_principle/visualizing_correlations.py @@ -39,8 +39,11 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) + dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} + meso_model.upgrade_labels_dict(dict_to_add) + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? - var_strs = json.loads(config['Visualize_correlation']['vars']) + var_strs = json.loads(config['Visualize_correlations']['vars']) vars = [] for i in range(len(var_strs)): vars.append(meso_model.meso_vars[var_strs[i]]) @@ -55,8 +58,8 @@ ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] # READING PREPROCESSING INFO FROM CONFIG FILE, AND PREPROCESSING - preprocess_data = json.loads(config['Visualize_correlation']['preprocess_data']) - extractions = int(config['Visualize_correlation']['extractions']) + preprocess_data = json.loads(config['Visualize_correlations']['preprocess_data']) + extractions = int(config['Visualize_correlations']['extractions']) # PRE-PROCESSING DATA statistical_tool = CoefficientsAnalysis() @@ -73,15 +76,19 @@ label = var_str if hasattr(meso_model, 'labels_var_dict') and var_str in meso_model.labels_var_dict.keys(): label = meso_model.labels_var_dict[var_str] - if preprocess_data['log_or_not'][0] == 1: + if preprocess_data['log_abs'][0] == 1: label = r"$\log($" + label + r"$)$" labels.append(label) + if len(var_strs) ==2: statistical_tool.visualize_correlation(vars[0], vars[1], xlabel=labels[0], ylabel=labels[1]) else: statistical_tool.visualize_many_correlations(vars, labels) + saving_directory = config['Directories']['figures_dir'] - filename = '/Correlation_{}'.format(var_strs) + filename = '/Correlation' + for i in range(len(var_strs)): + filename += "_" + var_strs[i] filename += ".pdf" plt.savefig(saving_directory + filename, format='pdf') print(f'Finished producing correlation plot for {var_strs}, saved as {filename}\n') diff --git a/Proof_of_principle/visualizing_residuals.py b/Proof_of_principle/visualizing_residuals.py index 4f221c7..c9c211e 100644 --- a/Proof_of_principle/visualizing_residuals.py +++ b/Proof_of_principle/visualizing_residuals.py @@ -54,7 +54,7 @@ y = meso_model.meso_vars['pi_res_sq'] data = [x,y] data = statistical_tool.trim_dataset(data, ranges, model_points) - preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} + preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} data = statistical_tool.preprocess_data(data, preprocess_data) data = statistical_tool.extract_randomly(data, 10000) x, y = data[0], data[1] @@ -72,7 +72,7 @@ y = meso_model.meso_vars['Theta_sq'] data = [x,y] data = statistical_tool.trim_dataset(data, ranges, model_points) - preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} + preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} data = statistical_tool.preprocess_data(data, preprocess_data) data = statistical_tool.extract_randomly(data, 10000) x, y = data[0], data[1] @@ -91,7 +91,7 @@ y = np.power(meso_model.meso_vars['exp_tilde'], 2) data = [x,y] data = statistical_tool.trim_dataset(data, ranges, model_points) - preprocess_data = {"pos_or_neg": [1,1], "log_or_not": [1,1]} + preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} data = statistical_tool.preprocess_data(data, preprocess_data) data = statistical_tool.extract_randomly(data, 10000) x, y = data[0], data[1] @@ -109,39 +109,39 @@ # PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS # ############################################################# - # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) - # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] - # visualizer = Plotter_2D() - - # vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] - # norms = ['mysymlog', 'symlog', 'log'] - # cmaps = ['seismic', 'seismic', 'plasma'] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Bulk_viscosity.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished bulk viscosity') - - # vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] - # norms = ['mysymlog', 'log', 'log'] - # cmaps = ['seismic', 'plasma', 'plasma'] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Shear_viscosity.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished shear viscosity') - - # vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] - # norms = ['mysymlog', 'log', 'log'] - # cmaps = ['seismic', 'plasma', 'plasma'] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/Heat_conductivity.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished heat conductivity') + num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + visualizer = Plotter_2D() + + vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] + norms = ['mysymlog', 'symlog', 'log'] + cmaps = ['seismic', 'seismic', 'plasma'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Bulk_viscosity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished bulk viscosity') + + vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] + norms = ['mysymlog', 'log', 'log'] + cmaps = ['seismic', 'plasma', 'plasma'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Shear_viscosity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished shear viscosity') + + vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] + norms = ['mysymlog', 'log', 'log'] + cmaps = ['seismic', 'plasma', 'plasma'] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/Heat_conductivity.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished heat conductivity') \ No newline at end of file From 7b92de61379456ed4932df5c202fe73e3771485b Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Wed, 28 Feb 2024 14:57:11 +0100 Subject: [PATCH 086/111] Better looking correlation plots --- master_files/Analysis.py | 81 +++++++++++++++++++++++++++------------- 1 file changed, 55 insertions(+), 26 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 4a252dc..2579a52 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -533,10 +533,13 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, hue_array=None, warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') g=sns.JointGrid() - scatter = sns.scatterplot(x=X, y=Y, ax=g.ax_joint, s=4, c='red', hue=hue_array, style=style_array, - palette=palette, markers=markers) - sns.histplot(x=X, ax=g.ax_marg_x, kde=True, color= 'red') - sns.histplot(y=Y, ax=g.ax_marg_y, kde=True, color= 'red') + + sns.set_theme(style="dark") + sns.scatterplot(x=X, y=Y, s=4, color=".15", ax=g.ax_joint, hue=hue_array, style=style_array, palette=palette, markers=markers) + sns.histplot(x=X, y=Y, bins=50, ax=g.ax_joint, pthresh=.1, cmap="mako") + sns.kdeplot(x=X, y=Y, levels=5, ax=g.ax_joint, color="w", linewidths=1) + sns.histplot(x=X, ax=g.ax_marg_x, kde=True, color= 'black') + sns.histplot(y=Y, ax=g.ax_marg_y, kde=True, color= 'black') g.set_axis_labels(xlabel=xlabel, ylabel=ylabel) g.fig.tight_layout() @@ -588,12 +591,6 @@ def visualize_many_correlations(self, data, labels): if len(Idx_to_delete) >= 1: data=list(np.delete(data, Idx_to_delete, axis=0)) - - # if ranges != None and model_points != None: - # print('Trimming dataset for correlation plot') - # for i in range(len(data)): - # data[i]=self.trim_data(data[i], ranges, model_points) - # print('Finished trimming data') Data = [] for i in range(len(data)): @@ -610,10 +607,25 @@ def corrfunc(x, y, **kws): with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') - g=sns.PairGrid(Data_df) - g.map_lower(sns.scatterplot, s=4, c='red') - g.map_upper(sns.kdeplot, color='red') - g.map_diag(sns.histplot, kde=True, color='red') + g=sns.PairGrid(Data_df) #, plot_kws=dict(scatter_kws=dict(s=4))) + # g.map_lower(sns.scatterplot, s=4, c='red') + # g.map_upper(sns.kdeplot, color='red') + # g.map_diag(sns.histplot, kde=True, color='red') + + def hide_current_axis(*args, **kwds): + plt.gca().set_visible(False) + + g.map_upper(hide_current_axis) + g.map_diag(sns.histplot, kde=True, color='black') + g.map_lower(sns.scatterplot, s=4, color=".15") + g.map_lower(sns.histplot, pthresh=.1, cmap="mako")#, bins=50 + g.map_lower(sns.kdeplot, levels=5, color='w', linewidths=1) + + # sns.set_theme(style="dark") + # sns.scatterplot(x=X, y=Y, s=5, color=".15", ax=g.ax_joint, hue=hue_array, style=style_array, palette=palette, markers=markers) + # sns.histplot(x=X, y=Y, bins=50, ax=g.ax_joint, pthresh=.1, cmap="mako") + # sns.kdeplot(x=X, y=Y, levels=5, ax=g.ax_joint, color="w", linewidths=1) + g.fig.tight_layout() g.map_lower(corrfunc) @@ -829,25 +841,25 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # print('Errors: {}\n'.format(errors)) # #TESTING PRE-PROCESS DATA - x = np.arange(1, 200) - y = np.arange(29, 228) + # x = np.arange(1, 200) + # y = np.arange(29, 228) - print('Min and max of x: {}, {}\n'.format(np.min(x), np.max(x))) - print('Min and max of y: {}, {}\n'.format(np.min(y), np.max(y))) + # print('Min and max of x: {}, {}\n'.format(np.min(x), np.max(x))) + # print('Min and max of y: {}, {}\n'.format(np.min(y), np.max(y))) - preprocess_data = {"value_ranges": [[None, None], [None, None]], - "log_abs": [1, 1]} + # preprocess_data = {"value_ranges": [[None, None], [None, None]], + # "log_abs": [1, 1]} - statistical_tool = CoefficientsAnalysis() - data = [x,y] + # statistical_tool = CoefficientsAnalysis() + # data = [x,y] - x, y = statistical_tool.preprocess_data(data, preprocess_data) + # x, y = statistical_tool.preprocess_data(data, preprocess_data) - print('Processing data....\n') - print('Min and max of x: {}, {}\n'.format(np.min(x), np.max(x))) - print('Min and max of y: {}, {}\n'.format(np.min(y), np.max(y))) + # print('Processing data....\n') + # print('Min and max of x: {}, {}\n'.format(np.min(x), np.max(x))) + # print('Min and max of y: {}, {}\n'.format(np.min(y), np.max(y))) @@ -856,3 +868,20 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # weights = np.ones(x.shape) # extracted, weights = statistical_tool.extract_randomly([x,y], 10) # print(extracted) + + + n = 10000 + mean = [0, 0] + cov = [(2, .4), (.4, .2)] + rng = np.random.RandomState(0) + x, y = rng.multivariate_normal(mean, cov, n).T + + mu, sigma = 0, 4 # mean and standard deviation + z = np.random.normal(mu, sigma, n) + + statistical_tool = CoefficientsAnalysis() + # statistical_tool.visualize_correlation(x,y) + labels=['x','y','z'] + data=[x,y,z] + statistical_tool.visualize_many_correlations(data,labels) + plt.show() \ No newline at end of file From d35c267429e127db02517e7b74c4a5d1179475af Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 4 Mar 2024 17:17:29 +0100 Subject: [PATCH 087/111] Adding routines to compute weights based on Q-criterion + testing them in visualizing_residuals.py --- Proof_of_principle/visualizing_residuals.py | 268 +++++++++++++------- master_files/MesoModels.py | 149 +++++++++-- 2 files changed, 311 insertions(+), 106 deletions(-) diff --git a/Proof_of_principle/visualizing_residuals.py b/Proof_of_principle/visualizing_residuals.py index c9c211e..b014f23 100644 --- a/Proof_of_principle/visualizing_residuals.py +++ b/Proof_of_principle/visualizing_residuals.py @@ -49,99 +49,187 @@ model_points = meso_model.domain_vars['Points'] saving_directory = config['Directories']['figures_dir'] - print('Producing correlation plot for pi_res') - x = meso_model.meso_vars['shear_sq'] - y = meso_model.meso_vars['pi_res_sq'] - data = [x,y] - data = statistical_tool.trim_dataset(data, ranges, model_points) - preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} - data = statistical_tool.preprocess_data(data, preprocess_data) - data = statistical_tool.extract_randomly(data, 10000) - x, y = data[0], data[1] - xlabel = r'$\log(\tilde{\sigma}_{ab}\tilde{\sigma}^{ab})$' - ylabel = r'$\log(\tilde{\pi}_{ab}\tilde{\pi}^{ab})$' - g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) - saving_directory = config['Directories']['figures_dir'] - filename = '/pi_aniso_correlation.pdf' - plt.savefig(saving_directory + filename, format='pdf') - print(f'Finished correlation plot for pi_res, saved as {filename}\n\n') - - - print('Producing correlation plot for q_res') - x = meso_model.meso_vars['q_res_sq'] - y = meso_model.meso_vars['Theta_sq'] - data = [x,y] - data = statistical_tool.trim_dataset(data, ranges, model_points) - preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} - data = statistical_tool.preprocess_data(data, preprocess_data) - data = statistical_tool.extract_randomly(data, 10000) - x, y = data[0], data[1] - xlabel = r'$\log(\tilde{q}_a\tilde{q}^a)$' - ylabel = r'$\log(\tilde{\Theta}_a \tilde{\Theta}^a)$' - model_points = meso_model.domain_vars['Points'] - g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) - saving_directory = config['Directories']['figures_dir'] - filename = '/q_res_correlation.pdf' - plt.savefig(saving_directory + filename, format='pdf') - print(f'Finished correlation plot for q_res, saved as {filename}\n\n') - - - print('Producing correlation plot for Pi_res') - x = np.power(meso_model.meso_vars['Pi_res'], 2) - y = np.power(meso_model.meso_vars['exp_tilde'], 2) - data = [x,y] - data = statistical_tool.trim_dataset(data, ranges, model_points) - preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} - data = statistical_tool.preprocess_data(data, preprocess_data) - data = statistical_tool.extract_randomly(data, 10000) - x, y = data[0], data[1] - xlabel = r'$\log(\tilde{\Pi}^2)$' - ylabel = r'$\log(\tilde{\theta}^2)$' - model_points = meso_model.domain_vars['Points'] - g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) - saving_directory = config['Directories']['figures_dir'] - filename = '/Pi_res_correlation.pdf' - plt.savefig(saving_directory + filename, format='pdf') - print(f'Finished correlation plot for Pi_res, saved as {filename}\n\n') + # print('Producing correlation plot for pi_res') + # x = meso_model.meso_vars['shear_sq'] + # y = meso_model.meso_vars['pi_res_sq'] + # data = [x,y] + # data = statistical_tool.trim_dataset(data, ranges, model_points) + # preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} + # data = statistical_tool.preprocess_data(data, preprocess_data) + # data = statistical_tool.extract_randomly(data, 30000) + # x, y = data[0], data[1] + # xlabel = r'$\log(\tilde{\sigma}_{ab}\tilde{\sigma}^{ab})$' + # ylabel = r'$\log(\tilde{\pi}_{ab}\tilde{\pi}^{ab})$' + # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) + # saving_directory = config['Directories']['figures_dir'] + # filename = '/pi_aniso_correlation.pdf' + # plt.savefig(saving_directory + filename, format='pdf') + # print(f'Finished correlation plot for pi_res, saved as {filename}\n\n') + + + # print('Producing correlation plot for q_res') + # x = meso_model.meso_vars['q_res_sq'] + # y = meso_model.meso_vars['Theta_sq'] + # data = [x,y] + # data = statistical_tool.trim_dataset(data, ranges, model_points) + # preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} + # data = statistical_tool.preprocess_data(data, preprocess_data) + # data = statistical_tool.extract_randomly(data, 30000) + # x, y = data[0], data[1] + # xlabel = r'$\log(\tilde{q}_a\tilde{q}^a)$' + # ylabel = r'$\log(\tilde{\Theta}_a \tilde{\Theta}^a)$' + # model_points = meso_model.domain_vars['Points'] + # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) + # saving_directory = config['Directories']['figures_dir'] + # filename = '/q_res_correlation.pdf' + # plt.savefig(saving_directory + filename, format='pdf') + # print(f'Finished correlation plot for q_res, saved as {filename}\n\n') + + + # print('Producing correlation plot for Pi_res') + # x = np.power(meso_model.meso_vars['Pi_res'], 2) + # y = np.power(meso_model.meso_vars['exp_tilde'], 2) + # data = [x,y] + # data = statistical_tool.trim_dataset(data, ranges, model_points) + # preprocess_data = {"value_ranges": [[None, None], [None, None]], "log_abs": [1, 1]} + # data = statistical_tool.preprocess_data(data, preprocess_data) + # data = statistical_tool.extract_randomly(data, 30000) + # x, y = data[0], data[1] + # xlabel = r'$\log(\tilde{\Pi}^2)$' + # ylabel = r'$\log(\tilde{\theta}^2)$' + # model_points = meso_model.domain_vars['Points'] + # g2=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) + # saving_directory = config['Directories']['figures_dir'] + # filename = '/Pi_res_correlation.pdf' + # plt.savefig(saving_directory + filename, format='pdf') + # print(f'Finished correlation plot for Pi_res, saved as {filename}\n\n') # ############################################################# # PLOTTING RESIDUALS VS CORRESPONDING CLOSURE INGREDIENTS # ############################################################# - num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) - time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] - visualizer = Plotter_2D() - - vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] - norms = ['mysymlog', 'symlog', 'log'] - cmaps = ['seismic', 'seismic', 'plasma'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Bulk_viscosity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished bulk viscosity') - - vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] - norms = ['mysymlog', 'log', 'log'] - cmaps = ['seismic', 'plasma', 'plasma'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Shear_viscosity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished shear viscosity') - - vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] - norms = ['mysymlog', 'log', 'log'] - cmaps = ['seismic', 'plasma', 'plasma'] - fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) - fig.tight_layout() - time_for_filename = str(round(time_meso,2)) - filename = "/Heat_conductivity.pdf" - plt.savefig(saving_directory + filename, format = 'pdf') - print('Finished heat conductivity') - - - \ No newline at end of file + # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + # visualizer = Plotter_2D() + + # vars_strs = ['zeta', 'exp_tilde', 'Pi_res'] + # norms = ['mysymlog', 'symlog', 'log'] + # cmaps = ['seismic', 'seismic', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Bulk_viscosity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished bulk viscosity') + + # vars_strs = ['eta', 'shear_sq', 'pi_res_sq'] + # norms = ['mysymlog', 'log', 'log'] + # cmaps = ['seismic', 'plasma', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Shear_viscosity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished shear viscosity') + + # vars_strs = ['kappa', 'Theta_sq', 'q_res_sq'] + # norms = ['mysymlog', 'log', 'log'] + # cmaps = ['seismic', 'plasma', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # time_for_filename = str(round(time_meso,2)) + # filename = "/Heat_conductivity.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished heat conductivity') + + + # ############################################################ + # # PLOTTING Q1 AND Q2 AGAINST THE RESIDUALS SQUARED + # ############################################################ + + # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + # visualizer = Plotter_2D() + + # vars_strs = ['Pi_res_sq', 'Q1', 'Q2'] + # norms = ['log', 'mysymlog', 'log'] + # cmaps = ['plasma', 'coolwarm', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # filename = "/Pi_res_Q.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished Pi_res') + + # vars_strs = ['pi_res_sq', 'Q1', 'Q2'] + # norms = ['log', 'mysymlog', 'log'] + # cmaps = ['plasma', 'coolwarm', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # filename = "/an_pi_res_Q.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished pi_res') + + # vars_strs = ['q_res_sq', 'Q1', 'Q2'] + # norms = ['log', 'mysymlog', 'log'] + # cmaps = ['plasma', 'coolwarm', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # filename = "/q_res_Q.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished q_res') + + # ############################################################ + # # PLOTTING WEIGHING FUNCTIONS + # ############################################################ + + # num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + # time_meso = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + # visualizer = Plotter_2D() + + # d = {'Q1' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab} - \tilde{\omega}_{ab}\tilde{\omega}^{ab}$'} + # meso_model.update_labels_dict(d) + # d = {'Q2' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}/\tilde{\omega}_{ab}\tilde{\omega}^{ab}$'} + # meso_model.update_labels_dict(d) + # d = {'weights' : r'$w$'} + # meso_model.update_labels_dict(d) + + # print('Building skew weights based on Q1') + # meso_model.weights_Q1_skew() + # print('Finished re-building the weights\n') + + # vars_strs = ['weights', 'Q1'] + # norms = [None, 'mysymlog'] + # cmaps = ['plasma', 'coolwarm'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # filename = "/Q1_skew.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished Q1weights_plot\n') + + # print('Building non-negative weights based on Q1') + # meso_model.weights_Q1_non_neg() + # print('Finished re-building the weights\n') + + # vars_strs = ['weights', 'Q1'] + # norms = [None, 'mysymlog'] + # cmaps = ['plasma', 'coolwarm'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # filename = "/Q1_non_neg.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished Q1weights_plot') + + + # print('Building weights based on Q2') + # meso_model.weights_Q2() + # print('Finished re-building the weights\n') + + # vars_strs = ['weights', 'Q2'] + # norms = [None, 'log'] + # cmaps = ['plasma', 'plasma'] + # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, norms=norms, cmaps=cmaps) + # fig.tight_layout() + # filename = "/Q2_weights.pdf" + # plt.savefig(saving_directory + filename, format = 'pdf') + # print('Finished Q1weights_plot\n') \ No newline at end of file diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 8207c32..09c9189 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1219,12 +1219,14 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): 'Theta_sq': r'$\tilde{\Theta}_a\tilde{\Theta}^a$', 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$', 'det_shear': r'det$(\sigma)$', - 'vort_mod' : r'$|W|$', 'vort_sq' : r'$\omega_{ab}\omega^{ab}$', 'acc_mag': r'$|a|$', - 'U' : r'$U^a$'} + 'U' : r'$U^a$', + 'Q1' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab} - \tilde{\omega}_{ab}\tilde{\omega}^{ab}$', + 'Q2' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}/\tilde{\omega}_{ab}\tilde{\omega}^{ab}$', + 'weights' : r'$w$'} - def upgrade_labels_dict(self, entry_dict): + def update_labels_dict(self, entry_dict): """ pretty self-explanatory """ @@ -2304,33 +2306,32 @@ def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, metric[0,0] = -1 metric[1,1] = metric[2,2] = +1 + # Computing various invariants of the velocity gradients det_shear = np.linalg.det(shear) - shear_sq = np.einsum('ij,kl,ik,jl->', shear, shear, metric, metric) - var_names.append('det_shear') vars.append(det_shear) + + shear_sq = np.einsum('ij,kl,ik,jl->', shear, shear, metric, metric) var_names.append('shear_sq') vars.append(shear_sq) - # Q-criterion vort_sq = np.einsum('ij,kl,ik,jl->', vort, vort, metric, metric) var_names.append('vort_sq') vars.append(vort_sq) - Q_criterion = shear_sq - vort_sq - var_names.append('Q') - vars.append(Q_criterion) - - # Computing the magnitude of vorticity - vort_mod = np.sqrt(vort_sq / 2.) - var_names.append('vort_mod') - vars.append(vort_mod) - - #Computing the magnitude of the acceleration vector acc_mag = np.sqrt(np.einsum('i,ij,j->', acc, metric, acc)) var_names.append('acc_mag') vars.append(acc_mag) + # Computing quantities related to the Q-criterion + Q1 = shear_sq - vort_sq + var_names.append('Q1') + vars.append(Q1) + + Q2 = shear_sq/vort_sq + var_names.append('Q2') + vars.append(Q2) + #Computing squares of residuals and also of Theta_tilde Pi_res_sq = Pi_res * Pi_res var_names.append('Pi_res_sq') @@ -2386,6 +2387,122 @@ def modelling_coefficients_parallel(self, n_cpus): finally: self.meso_vars[key][tuple(grid_idxs)] = values[idx] + def weights_Q1_skew(self): + """ + Build weights for gridpoints: downplay points where shear is small, but take into + account both positive and negative values of Q1 + """ + def symlog(array): + return np.sign(array) * np.log10(np.abs(array)+1) + + Q1 = self.meso_vars['Q1'] + symlog_Q1 = symlog(Q1) + + M_pos = np.amax(symlog_Q1) + m_neg = np.amin(symlog_Q1) + + symlog_Q1_pos = np.ma.masked_where(symlog_Q1 < 0, symlog_Q1, copy=True).compressed() + symlog_Q1_neg = np.ma.masked_where(symlog_Q1 > 0, symlog_Q1, copy=True).compressed() + M_neg = np.amax(symlog_Q1_neg) + m_pos = np.amin(symlog_Q1_pos) + + print(f'm_neg, M_neg, m_pos, M_pos: {m_neg}, {M_neg}, {m_pos}, {M_pos}\n') + + c_pos = (M_pos + m_pos)/2. + c_neg = (M_neg + m_neg)/2. + + print(f'c_pos, c_neg: {c_pos}, {c_neg}') + + def get_weights(x, c_pos, c_neg): + if x >= 0: + result = (np.tanh(x-c_pos)+1)/2 + else: + result = (-np.tanh(x-c_neg)+1)/2 + return (result+1.)/2. + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + weights = np.zeros((Nt, Nx, Ny)) + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + weights[h,i,j] = get_weights(symlog_Q1[h,i,j], c_pos, c_neg) + + self.meso_vars.update({'weights' : weights}) + + def weights_Q1_non_neg(self): + """ + Build weights for gridpoints: downplay points where Q1 is negative + """ + def symlog(array): + return np.sign(array) * np.log10(np.abs(array)+1) + + Q1 = self.meso_vars['Q1'] + symlog_Q1 = symlog(Q1) + symlog_Q1_pos = np.ma.masked_where(symlog_Q1 < 0, symlog_Q1, copy=True).compressed() + symlog_Q1_neg = np.ma.masked_where(symlog_Q1 > 0, symlog_Q1, copy=True).compressed() + + M_pos = np.amax(symlog_Q1_pos) + m_pos = np.amin(symlog_Q1_pos) + M_neg = np.amax(symlog_Q1_neg) + m_neg = np.amin(symlog_Q1_neg) + + + print(f'm_neg, M_neg, m_pos, M_pos: {m_neg}, {M_neg}, {m_pos}, {M_pos}\n') + + c = (M_pos + m_neg)/2. + a_pos = (M_pos + m_pos)/2. + a_neg = np.abs(M_neg + m_neg)/2. + a = (a_pos + a_neg)/2. + + print(f'c, a: {c}, {a}') + + def get_weights(x, c, a): + result = np.tanh((x-c)/a) + return (result+1.)/2. + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + weights = np.zeros((Nt, Nx, Ny)) + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + weights[h,i,j] = get_weights(symlog_Q1[h,i,j], c, a) + + self.meso_vars.update({'weights' : weights}) + + def weights_Q2(self): + """ + Build weights for gridpoints based on Q2 + """ + Q2 = self.meso_vars['Q2'] + Q2 = np.log10(Q2) + + M = np.amax(Q2) + m = np.amin(Q2) + scale = (M+m)/2 + + def get_weights(x, scale): + result = np.tanh(x/scale) + result = (result +1)/2 + return result + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + weights = np.zeros((Nt, Nx, Ny)) + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + weights[h,i,j] = get_weights(Q2[h,i,j], scale) + + self.meso_vars.update({'weights' : weights}) + def EL_style_closure_regression(self, CoefficientAnalysis): """ Takes in a list of correlation quantities (strings? The regressors needed are computed From 6765f7b6f3ba3e0ab556e6b27f2beecc25220685 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 4 Mar 2024 18:28:24 +0100 Subject: [PATCH 088/111] Adding routines for weighed Pearson, and options for showing this on visualize_correlation plot --- master_files/Analysis.py | 137 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 130 insertions(+), 7 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 2579a52..7b4197f 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -325,6 +325,98 @@ def extract_randomly(self, list_of_data, num_extractions): return np.array(new_data) + def weighted_mean(self, data, weights): + """ + Return mean of data weighted wrt weights. + + Parameters: + ----------- + data: 1D array + weights: 1D array + + Returns: + -------- + Weighted mean + + Notes: + ------ + data and weights are assumed to be 1D arrays of same shape + """ + mean = 0. + tot_weights = 0. + + for i in range(len(data)): + mean += data[i] * weights[i] + tot_weights += weights[i] + + mean = mean/ tot_weights + return mean + + def weighted_covariance(self, x, y, weights): + """ + Return covariance of x vs y weighed wrt weights + + Parameters: + ----------- + x, y, weights: 1D arrays of same length + + Returns: + -------- + weighted covariance (float) + + Notes: + ------ + x, y, weights are assumed to be flattened and checked for compatibility already + + """ + x_mean = self.weighted_mean(x, weights) + y_mean = self.weighted_mean(y, weights) + + weighted_cov = 0. + tot_weights = 0. + for i in range(len(x)): + weighted_cov += weights[i] * (x[i]- x_mean) * (y[i] - y_mean) + tot_weights += weights[i] + + weighted_cov = weighted_cov / tot_weights + return weighted_cov + + def weigthed_pearson(self, x, y, weights=None): + """ + Return the weighted correlation coefficient + + Parameters: + ----------- + x, y , weights: nd.arrays + + Returns: + -------- + weighted correlation coefficient + """ + if x.shape != y.shape: + print('Arrays do not have the same shape, returning None\n') + return None + + if weights is not None: + if x.shape != weights.shape: + print('weights do not have the same shape as input arrays, setting these to 1 everywhere\n') + weights = np.ones(x.shape) + else: + print('No weigths passed, setting these to 1. everywhere') + weights = np.ones(x.shape) + + + X = x.flatten() + Y = y.flatten() + W = weights.flatten() + + corr_x = self.weighted_covariance(X, X, W) + corr_y = self.weighted_covariance(Y, Y, W) + corr_xy = self.weighted_covariance(X, Y, W) + + weighted_pearson = corr_xy / np.sqrt(corr_x * corr_y) + return weighted_pearson + def scalar_regression(self, y, X, weights=None, add_intercept=False): """ Routine to perform ordinary or weighted (multivariate) regression on some gridded data. @@ -494,7 +586,7 @@ def tensor_components_regression(self, y, X, spatial_dims, ranges=None, model_po results.append(self.scalar_regression(yc, Xc, ranges, model_points, weights, add_intercept)) return results - def visualize_correlation(self, x, y, xlabel=None, ylabel=None, hue_array=None, style_array=None, legend_dict=None, palette=None, markers=None): + def visualize_correlation(self, x, y, xlabel=None, ylabel=None, weights=None, hue_array=None, style_array=None, legend_dict=None, palette=None, markers=None): """ Method that returns an instance of JointGrid, with plotted the scatter plot and univariate distributions. Possibility to cut data to lie within ranges. @@ -522,8 +614,20 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, hue_array=None, print('Cannot check correlation: data is misaligned!') return None + if weights is not None: + if weights.shape != x.shape: + print('Weights not compatible with data: ignoring them.') + Weights = None + else: + Weights = weights + else: + Weights = None + X=x.flatten() Y=y.flatten() + if Weights is not None: + Weights = Weights.flatten() + if hue_array is not None: hue_array = hue_array.flatten() if style_array is not None: @@ -548,8 +652,13 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, hue_array=None, new_labels = [legend_dict[label] for label in labels] scatter.legend(handles, new_labels) - r, _ = stats.pearsonr(X,Y) - g.ax_joint.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=g.ax_joint.transAxes) + if Weights is not None: + rw = self.weigthed_pearson(X,Y,Weights) + g.ax_joint.annotate(r"$r_w = {:.2f}$".format(rw), xy=(.1, .9), xycoords=g.ax_joint.transAxes) + else: + r, _ = stats.pearsonr(X,Y) + g.ax_joint.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=g.ax_joint.transAxes) + return g @@ -604,6 +713,7 @@ def corrfunc(x, y, **kws): ax = plt.gca() ax.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=ax.transAxes) + with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') @@ -627,7 +737,6 @@ def hide_current_axis(*args, **kwds): # sns.kdeplot(x=X, y=Y, levels=5, ax=g.ax_joint, color="w", linewidths=1) g.fig.tight_layout() - g.map_lower(corrfunc) return g @@ -869,19 +978,33 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # extracted, weights = statistical_tool.extract_randomly([x,y], 10) # print(extracted) - + import random n = 10000 mean = [0, 0] cov = [(2, .4), (.4, .2)] rng = np.random.RandomState(0) x, y = rng.multivariate_normal(mean, cov, n).T + ws = [] + for i in range(len(x)): + ws.append(random.uniform(0.5,1.)) + ws = np.array(ws) + + # print(x.shape, ws.shape) + # r, _ = stats.pearsonr(x,y) + + statistical_tool = CoefficientsAnalysis() + + # wr = statistical_tool.weigthed_pearson(x,y,ws) + # print(f'stats.pearsonr: {r}') + # print(f'my_pearson: {wr}') + mu, sigma = 0, 4 # mean and standard deviation z = np.random.normal(mu, sigma, n) statistical_tool = CoefficientsAnalysis() - # statistical_tool.visualize_correlation(x,y) + # statistical_tool.visualize_correlation(x,y,weights=ws) labels=['x','y','z'] data=[x,y,z] - statistical_tool.visualize_many_correlations(data,labels) + statistical_tool.visualize_many_correlations(data, labels) plt.show() \ No newline at end of file From ef499ef6152f3996402a031e844b9e2a5a6332b2 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 5 Mar 2024 10:03:51 +0100 Subject: [PATCH 089/111] Adding weighted pearson to visualize_many_correlations plots --- master_files/Analysis.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 7b4197f..c8e794e 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -662,7 +662,7 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, weights=None, hu return g - def visualize_many_correlations(self, data, labels): + def visualize_many_correlations(self, data, labels, weights=None): """ Method that returns an instance of PairGrid of correlation plots for a list of vars. Plotted is: @@ -705,6 +705,16 @@ def visualize_many_correlations(self, data, labels): for i in range(len(data)): Data.append(data[i].flatten()) + if weights is not None: + if weights.shape != ref_shape: + print('The weights passed are not compatible with the data. Ignoring them and moving on') + Weights = None + else: + Weights = weights.flatten() + else: + Weights = None + + Data=np.column_stack(Data) Data_df = pd.DataFrame(Data, columns=labels) @@ -713,6 +723,13 @@ def corrfunc(x, y, **kws): ax = plt.gca() ax.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=ax.transAxes) + def corrfunc_weights(x, y, **kws): + X = np.array(x) + Y = np.array(y) + r_w = self.weigthed_pearson(X, Y, Weights) + ax = plt.gca() + ax.annotate(r"$r_w = {:.2f}$".format(r_w), xy=(.1, .9), xycoords=ax.transAxes) + with warnings.catch_warnings(): warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') @@ -736,8 +753,12 @@ def hide_current_axis(*args, **kwds): # sns.histplot(x=X, y=Y, bins=50, ax=g.ax_joint, pthresh=.1, cmap="mako") # sns.kdeplot(x=X, y=Y, levels=5, ax=g.ax_joint, color="w", linewidths=1) - g.fig.tight_layout() - g.map_lower(corrfunc) + # g.fig.tight_layout() + g.figure.tight_layout() + if Weights is not None: + g.map_lower(corrfunc_weights) + else: + g.map_lower(corrfunc) return g @@ -1006,5 +1027,5 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # statistical_tool.visualize_correlation(x,y,weights=ws) labels=['x','y','z'] data=[x,y,z] - statistical_tool.visualize_many_correlations(data, labels) + statistical_tool.visualize_many_correlations(data, labels, weights=ws) plt.show() \ No newline at end of file From 2127ec0c35c178d3ddddbb002a700501262fdeea Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 5 Mar 2024 15:30:18 +0100 Subject: [PATCH 090/111] Work on routines for adding weigths to regression etc --- Proof_of_principle/config_calibration.txt | 28 ++++----- Proof_of_principle/regressing_residual.py | 57 +++++++++++++++---- .../visualizing_correlations.py | 49 +++++++++++++--- master_files/Analysis.py | 14 ++--- master_files/MesoModels.py | 10 +--- 5 files changed, 110 insertions(+), 48 deletions(-) diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index 8b29c24..90e9d4f 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -5,7 +5,7 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8/fitting_eta/full_data +figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8/weights/fitting_eta/Q2 [Filenames] meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle @@ -15,20 +15,22 @@ mesogrid_T_slices_num = 3 [Ranges_for_analysis] -ranges = {"x_range": [0.04, 0.96], "y_range": [0.03, 0.96]} +ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.97]} [Visualize_correlations] -vars = ["eta", "T_tilde", "n_tilde"] -#preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} +vars = ["eta", "shear_sq", "vort_sq", "T_tilde", "n_tilde"] # Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[1e-6, null], [null, null], [null, null]], - "log_abs": [1, 1, 1]} +preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null]], + "log_abs": [1, 1, 1, 1, 1]} #set extractions 0 if you don't want to extract randomly from sample -extractions = 10000 +extractions = 30000 + +# possible options for building weights are: 'Q2' 'Q1_skew' 'Q1_non_neg' +weighing_func = Q1_non_neg @@ -38,7 +40,7 @@ dependent_var = zeta explanatory_vars = ["T_tilde", "rho_tilde"] pcs_num = 1 -regressors_2_reduce = ["T_tilde", "rho_tilde", "p_tilde"] +regressors_2_reduce = ["T_tilde", "rho_tilde", "n_tilde"] variance_wanted = 0.9 # Careful, only one dictionary here @@ -55,19 +57,17 @@ extractions = 0 [Regression_settings] dependent_var = eta -regressors = ["T_tilde", "n_tilde"] +regressors = ["n_tilde", "T_tilde"] add_intercept = 1 -#preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} - # Set values to null if you do not want to restrict the data preprocess_data = {"value_ranges": [[1e-6, null], [null, null], [null, null]], "log_abs": [1, 1, 1]} #set extractions 0 if you don't want to extract randomly from sample: extraction only in the scatter plot? Not in regression? -extractions = 10000 - - +extractions = 30000 +# possible options for building weights are: 'Q2' 'Q1_skew' 'Q1_non_neg' +weighing_func = Q2 diff --git a/Proof_of_principle/regressing_residual.py b/Proof_of_principle/regressing_residual.py index c877c18..dae8988 100644 --- a/Proof_of_principle/regressing_residual.py +++ b/Proof_of_principle/regressing_residual.py @@ -43,7 +43,7 @@ meso_model = pickle.load(filehandle) dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} - meso_model.upgrade_labels_dict(dict_to_add) + meso_model.update_labels_dict(dict_to_add) # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['Regression_settings']['dependent_var'] @@ -69,23 +69,52 @@ add_intercept = not not int(config['Regression_settings']['add_intercept']) extractions = int(config['Regression_settings']['extractions']) + # BUILDING THE WEIGHTS + weighing_func_str = config['Regression_settings']['weighing_func'] + if weighing_func_str == 'Q2': + meso_model.weights_Q2() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q2.__name__}\n') + elif weighing_func_str == 'Q1_skew': + meso_model.weights_Q1_skew() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q1_skew.__name__}\n') + elif weighing_func_str == 'Q1_non_neg': + meso_model.weights_Q1_non_neg() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q1_non_neg.__name__}\n') + else: + print(f'The string for building weights {weighing_func_str} does not match any of the implemented routines.\n') + weights = None + # PRE-PROCESSING and REGRESSING data = [dep_var] for i in range(len(regressors)): data.append(regressors[i]) statistical_tool = CoefficientsAnalysis() model_points = meso_model.domain_vars['Points'] - new_data = statistical_tool.trim_dataset(data, ranges, model_points) - new_data = statistical_tool.preprocess_data(new_data, preprocess_data) - # if extractions != 0: - # new_data = statistical_tool.extract_randomly(new_data, extractions) + + if weights is not None: + new_data = statistical_tool.trim_dataset(data + [weights], ranges, model_points) + weights = new_data[-1] + del new_data[-1] + new_data, weights = statistical_tool.preprocess_data(new_data, preprocess_data, weights=weights) + # if extractions != 0: + # new_data = statistical_tool.extract_randomly(new_data + [weights], extractions) + # weights = new_data[-1] + # del new_data[-1] + else: + new_data = statistical_tool.trim_dataset(data, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + # if extractions != 0: + # new_data = statistical_tool.extract_randomly(new_data, extractions) dep_var = new_data[0] regressors = [] for i in range(1,len(new_data)): regressors.append(new_data[i]) - coeffs, std_errors = statistical_tool.scalar_regression(dep_var, regressors, add_intercept=add_intercept) + coeffs, std_errors = statistical_tool.scalar_regression(dep_var, regressors, add_intercept=add_intercept, weights=weights) print('regression coeffs: {}\n'.format(coeffs)) print('Corresponding std errors: {}\n'.format(std_errors)) @@ -100,7 +129,13 @@ if extractions != 0: data_for_scatter = [dep_var, dep_var_model] - new_data = statistical_tool.extract_randomly(data_for_scatter, extractions) + if weights is not None: + new_data = statistical_tool.extract_randomly(data_for_scatter + [weights], extractions) + weights = new_data[-1] + del new_data[-1] + else: + new_data = statistical_tool.extract_randomly(data_for_scatter, extractions) + dep_var, dep_var_model = new_data[0], new_data[1] @@ -125,13 +160,15 @@ var_name = r"$\log($" + var_name + r"$)$" xlabel +=var_name - statistical_tool.visualize_correlation(dep_var_model, dep_var , xlabel, ylabel) + statistical_tool.visualize_correlation(dep_var_model, dep_var , xlabel, ylabel, weights = weights) saving_directory = config['Directories']['figures_dir'] + # if weights is not None: + # filename = '/' + weighing_func_str + '_Regress_' + # else: + # filename = '/Regress_' filename = f'/Regress_{dep_var_str}_vs' for i in range(1,len(regressors_strs)): filename += f'_{regressors_strs[i]}' - # if add_intercept: - # filename += "_intercept" filename += ".pdf" plt.savefig(saving_directory + filename, format='pdf') print(f'Finished regression and scatter plot for {dep_var_str}, saved as {filename}\n\n') diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py index b44af07..93ba96b 100644 --- a/Proof_of_principle/visualizing_correlations.py +++ b/Proof_of_principle/visualizing_correlations.py @@ -40,7 +40,7 @@ meso_model = pickle.load(filehandle) dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} - meso_model.upgrade_labels_dict(dict_to_add) + meso_model.update_labels_dict(dict_to_add) # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? var_strs = json.loads(config['Visualize_correlations']['vars']) @@ -61,14 +61,45 @@ preprocess_data = json.loads(config['Visualize_correlations']['preprocess_data']) extractions = int(config['Visualize_correlations']['extractions']) + # BUILDING THE WEIGHTS + weighing_func_str = config['Visualize_correlations']['weighing_func'] + if weighing_func_str == 'Q2': + meso_model.weights_Q2() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q2.__name__}\n') + elif weighing_func_str == 'Q1_skew': + meso_model.weights_Q1_skew() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q1_skew.__name__}\n') + elif weighing_func_str == 'Q1_non_neg': + meso_model.weights_Q1_non_neg() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q1_non_neg.__name__}\n') + else: + print(f'The string for building weights {weighing_func_str} does not match any of the implemented routines.\n') + weights = None + + # PRE-PROCESSING DATA statistical_tool = CoefficientsAnalysis() model_points = meso_model.domain_vars['Points'] - new_data = statistical_tool.trim_dataset(vars, ranges, model_points) - new_data = statistical_tool.preprocess_data(new_data, preprocess_data) - if extractions != 0: - new_data = statistical_tool.extract_randomly(new_data, extractions) - vars =new_data + if weights is not None: + new_data = statistical_tool.trim_dataset(vars + [weights], ranges, model_points) + weights = new_data[-1] + del new_data[-1] + new_data, weights = statistical_tool.preprocess_data(new_data, preprocess_data, weights=weights) + if extractions != 0: + new_data = statistical_tool.extract_randomly(new_data + [weights], extractions) + weights = new_data[-1] + del new_data[-1] + vars = new_data + + else: + new_data = statistical_tool.trim_dataset(vars, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + if extractions != 0: + new_data = statistical_tool.extract_randomly(new_data, extractions) + vars = new_data # FINALLY, THE CORRELATION PLOT labels = [] @@ -81,12 +112,14 @@ labels.append(label) if len(var_strs) ==2: - statistical_tool.visualize_correlation(vars[0], vars[1], xlabel=labels[0], ylabel=labels[1]) + statistical_tool.visualize_correlation(vars[0], vars[1], xlabel=labels[0], ylabel=labels[1], weights = weights) else: - statistical_tool.visualize_many_correlations(vars, labels) + statistical_tool.visualize_many_correlations(vars, labels, weights = weights) saving_directory = config['Directories']['figures_dir'] filename = '/Correlation' + if weights is not None: + filename = '/' + weighing_func_str + 'Correlation' for i in range(len(var_strs)): filename += "_" + var_strs[i] filename += ".pdf" diff --git a/master_files/Analysis.py b/master_files/Analysis.py index c8e794e..ca68ac2 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -281,7 +281,7 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): processed_list[i] = np.log10(np.abs(processed_list[i])) # removing corresponding entries from weights - if weights != None: + if weights is not None: Weights = np.ma.masked_array(weights, tot_mask).compressed() return processed_list, Weights else: @@ -323,7 +323,7 @@ def extract_randomly(self, list_of_data, num_extractions): shortened_array = np.array(shortened_list) new_data.append(shortened_array) - return np.array(new_data) + return new_data def weighted_mean(self, data, weights): """ @@ -457,16 +457,16 @@ def scalar_regression(self, y, X, weights=None, add_intercept=False): print('None of the passed regressors is compatible with dep data. Exiting.') return None - if weights: + if weights is not None: if np.shape(y) != np.shape(weights): print(f'The weights passed are not not aligned with data, setting these to 1') - weights=np.ones(y.shape) + Weights=np.ones(y.shape) # FLATTENING (IF NOT DONE YET IN PRE-PROCESSING) + FITTING Y = y.flatten() - if weights: - Weights = Weights.flatten() + if weights is not None: + Weights = weights.flatten() XX =[] if add_intercept: @@ -491,7 +491,7 @@ def scalar_regression(self, y, X, weights=None, add_intercept=False): # VERSION USING SKLEARN: model=LinearRegression(fit_intercept=False) - if weights: + if weights is not None: print('Fitting with weights') model.fit(XX, Y, sample_weight=Weights) else: diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 09c9189..d628774 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -2406,13 +2406,9 @@ def symlog(array): M_neg = np.amax(symlog_Q1_neg) m_pos = np.amin(symlog_Q1_pos) - print(f'm_neg, M_neg, m_pos, M_pos: {m_neg}, {M_neg}, {m_pos}, {M_pos}\n') - c_pos = (M_pos + m_pos)/2. c_neg = (M_neg + m_neg)/2. - print(f'c_pos, c_neg: {c_pos}, {c_neg}') - def get_weights(x, c_pos, c_neg): if x >= 0: result = (np.tanh(x-c_pos)+1)/2 @@ -2449,16 +2445,11 @@ def symlog(array): M_neg = np.amax(symlog_Q1_neg) m_neg = np.amin(symlog_Q1_neg) - - print(f'm_neg, M_neg, m_pos, M_pos: {m_neg}, {M_neg}, {m_pos}, {M_pos}\n') - c = (M_pos + m_neg)/2. a_pos = (M_pos + m_pos)/2. a_neg = np.abs(M_neg + m_neg)/2. a = (a_pos + a_neg)/2. - print(f'c, a: {c}, {a}') - def get_weights(x, c, a): result = np.tanh((x-c)/a) return (result+1.)/2. @@ -2710,3 +2701,4 @@ def EL_style_closure_regression(self, CoefficientAnalysis): print('Finished decomposing in parallel, time taken {}\n'.format(parallel_time)) # print('Speed-up factor: {}'.format(serial_time/parallel_time)) + \ No newline at end of file From 32034189f4d0937d6173d49d00a32ae8b7357019 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 7 Mar 2024 14:49:53 +0100 Subject: [PATCH 091/111] Updates to my_symlog plotting routines: taking care of values very close to 1. --- master_files/system/BaseFunctionality.py | 47 +++++++++++------------- 1 file changed, 22 insertions(+), 25 deletions(-) diff --git a/master_files/system/BaseFunctionality.py b/master_files/system/BaseFunctionality.py index aaea44b..52b224d 100644 --- a/master_files/system/BaseFunctionality.py +++ b/master_files/system/BaseFunctionality.py @@ -158,7 +158,10 @@ def symlog_num(num): -------- The symlog of the input num (separate copy) """ - result = np.sign(num) * np.log10(np.abs(num)+1) + if np.abs(num) +1. == 1.0: + result = num + else: + result = np.sign(num) * np.log10(np.abs(num)+1.) return result @staticmethod @@ -186,11 +189,11 @@ def symlog_var(var): count_zeros=0 temp = np.empty_like(var) for index in np.ndindex(var.shape): - if var[index] == 0: + value = var[index] + if value == 0: count_zeros +=1 else: - value = var[index] - temp[index] = np.sign(value) * np.log10(np.abs(value)+1) + temp[index] = MySymLogPlotting.symlog_num(value) if count_zeros >= 1: print('Careful: there are {} zeros in the data'.format(count_zeros)) return temp @@ -227,24 +230,15 @@ def get_mysymlog_var_ticks(var): ticks = [] ticks_labels = [] nodes = [] - - pos_var = np.where(var > 0, var, 0).flatten() - pos_var = pos_var[pos_var!=0] - - neg_var = np.where(var < 0, var, 0).flatten() - neg_var = neg_var[neg_var!=0] + - pos_var_small = np.where(pos_var < 1, pos_var, 0) - pos_var_small = pos_var_small[pos_var_small!=0] - - pos_var_large = np.where(pos_var >= 1, pos_var, 0) - pos_var_large = pos_var_large[pos_var_large!=0] - - neg_var_small = np.where(neg_var > -1 , neg_var, 0) - neg_var_small = neg_var_small[neg_var_small!=0] - - neg_var_large = np.where(neg_var <= -1, neg_var, 0) - neg_var_large = neg_var_large[neg_var_large!=0] + pos_var = np.ma.masked_where(var <0., var, copy=True).compressed() + pos_var_small = np.ma.masked_where(pos_var >=1., pos_var, copy=True).compressed() + pos_var_large = np.ma.masked_where(pos_var <1., pos_var, copy=True).compressed() + + neg_var = np.ma.masked_where(var >0., var, copy=True).compressed() + neg_var_small = np.ma.masked_where(neg_var <=-1., neg_var, copy=True).compressed() + neg_var_large = np.ma.masked_where(neg_var >-1., neg_var, copy=True).compressed() # Working out nodes, ticks and ticks_labels for the negative range if len(neg_var_large) >0: @@ -260,7 +254,7 @@ def get_mysymlog_var_ticks(var): if len(neg_var_small) == 0: - print('Actually: only negative large values', flush=True) + # print('Actually: only negative large values', flush=True) vmax = np.amax(neg_var_large) new_nodes = [MySymLogPlotting.symlog_num(vmax)] new_ticks = new_nodes @@ -290,7 +284,9 @@ def get_mysymlog_var_ticks(var): vmin = np.amin(neg_var_small) vmax = np.amax(neg_var_small) + # print(vmin, vmax, "\n") new_nodes = [MySymLogPlotting.symlog_num(vmin), MySymLogPlotting.symlog_num(vmax)] + # print(new_nodes) new_ticks = new_nodes new_ticks_labels = [r'$-10^{%d}$'%(int(d)) for d in np.log10([-vmin,-vmax])] @@ -310,7 +306,9 @@ def get_mysymlog_var_ticks(var): vmin = np.amin(pos_var_small) vmax = np.amax(pos_var_small) + # print(vmin, vmax) new_nodes = [MySymLogPlotting.symlog_num(vmin), MySymLogPlotting.symlog_num(vmax)] + # print(new_nodes) new_ticks = new_nodes new_ticks_labels = [r'$10^{%d}$'%(int(d)) for d in np.log10([vmin,vmax])] @@ -319,10 +317,10 @@ def get_mysymlog_var_ticks(var): nodes += new_nodes else: # len(pos_var_large) > 0: - # print('There are positive large values', flush=True) + print('There are positive large values', flush=True) if len(pos_var_small) >0: - # print('And also positive small values', flush=True) + print('And also positive small values', flush=True) vmin = np.amin(pos_var_small) vmax = np.amax(pos_var_small) @@ -361,7 +359,6 @@ def get_mysymlog_var_ticks(var): return ticks, ticks_labels, nodes - class MyThreeNodesNorm(mpl.colors.Normalize): """ Sub-classing colors.Normalize: the norm has three inner nodes plus the extrema. From b546fd9cc482896ee7c8b19d2bb74dabd7b8031c Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 7 Mar 2024 19:27:54 +0100 Subject: [PATCH 092/111] Adding residual weighing routine and modifying PoP files accordingly --- Proof_of_principle/config_calibration.txt | 70 ++++++++++------ Proof_of_principle/regressing_residual.py | 27 ++++++- .../visualizing_correlations.py | 27 ++++++- master_files/MesoModels.py | 81 ++++++++++++++++--- 4 files changed, 165 insertions(+), 40 deletions(-) diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt index 90e9d4f..33a9513 100644 --- a/Proof_of_principle/config_calibration.txt +++ b/Proof_of_principle/config_calibration.txt @@ -5,9 +5,10 @@ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled -figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8/weights/fitting_eta/Q2 +figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8/fitting/weights/fitting_eta/eta_pos/residual [Filenames] + meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle [Models_settings] @@ -15,23 +16,59 @@ mesogrid_T_slices_num = 3 [Ranges_for_analysis] -ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.97]} +ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} [Visualize_correlations] -vars = ["eta", "shear_sq", "vort_sq", "T_tilde", "n_tilde"] +vars = ["eta", "shear_sq", "vort_sq", "det_shear", "T_tilde", "n_tilde"] # Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null]], - "log_abs": [1, 1, 1, 1, 1]} +preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null], [null, null]], + "log_abs": [1, 1, 1, 1, 1, 1]} #set extractions 0 if you don't want to extract randomly from sample extractions = 30000 -# possible options for building weights are: 'Q2' 'Q1_skew' 'Q1_non_neg' -weighing_func = Q1_non_neg +#Options for building weights: 'Q2' 'Q1_skew' 'Q1_non_neg' 'residual_weights' 'denominator_weights'. Any other choice correspond to no weights + +weighing_func = residual_weights + +# this is only relevant when weights are built using 'residual_weights': specify a positive residual you want to consider for the weights +residual_str = pi_res_sq + +# this is only relevant when weights are built using 'denominator_weights': specify the positive quantity at the denominator of extracted coefficient +denominator_str = shear_sq + +# Set format_fig to either pdf or png +format_fig = png + + +[Regression_settings] + +dependent_var = eta +regressors = ["vort_sq", "det_shear"] +add_intercept = 1 + +# Set values to null if you do not want to restrict the data +preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null], [null, null]], + "log_abs": [1, 1, 1, 1, 1, 1]} + +#set extractions 0 if you don't want to extract randomly from sample: extraction only in the scatter plot? Not in regression? +extractions = 30000 + +#Options for building weights: 'Q2' 'Q1_skew' 'Q1_non_neg' 'residual_weights' 'denominator_weights'. Any other choice correspond to no weights + +weighing_func = residual_weights +# this is only relevant when weights are built using 'residual_weights': specify a positive residual you want to consider for the weights +residual_str = pi_res_sq + +# this is only relevant when weights are built using 'denominator_weights': specify the positive quantity at the denominator of extracted coefficient +denominator_str = shear_sq + +# Set format_fig to either pdf or png +format_fig = png [PCA_settings] @@ -52,22 +89,3 @@ preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null]], #set extractions 0 if you don't want to extract randomly from sample extractions = 0 - - -[Regression_settings] - -dependent_var = eta -regressors = ["n_tilde", "T_tilde"] -add_intercept = 1 - -# Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[1e-6, null], [null, null], [null, null]], - "log_abs": [1, 1, 1]} - -#set extractions 0 if you don't want to extract randomly from sample: extraction only in the scatter plot? Not in regression? -extractions = 30000 - -# possible options for building weights are: 'Q2' 'Q1_skew' 'Q1_non_neg' -weighing_func = Q2 - - diff --git a/Proof_of_principle/regressing_residual.py b/Proof_of_principle/regressing_residual.py index dae8988..4a92571 100644 --- a/Proof_of_principle/regressing_residual.py +++ b/Proof_of_principle/regressing_residual.py @@ -45,6 +45,9 @@ dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} meso_model.update_labels_dict(dict_to_add) + dict_to_add = {'det_shear' : r'$det(\sigma)$'} + meso_model.update_labels_dict(dict_to_add) + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['Regression_settings']['dependent_var'] dep_var = meso_model.meso_vars[dep_var_str] @@ -75,14 +78,29 @@ meso_model.weights_Q2() weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.weights_Q2.__name__}\n') + elif weighing_func_str == 'Q1_skew': meso_model.weights_Q1_skew() weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.weights_Q1_skew.__name__}\n') + elif weighing_func_str == 'Q1_non_neg': meso_model.weights_Q1_non_neg() weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.weights_Q1_non_neg.__name__}\n') + + elif weighing_func_str == "residual_weights": + residual_str = config['Regression_settings']['residual_str'] + meso_model.residual_weights(residual_str) + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.residual_weights.__name__}\n') + + elif weighing_func_str == "denominator_weights": + residual_str = config['Regression_settings']['denominator_str'] + meso_model.denominator_weights(residual_str) + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.denominator_weights.__name__}\n') + else: print(f'The string for building weights {weighing_func_str} does not match any of the implemented routines.\n') weights = None @@ -169,8 +187,13 @@ filename = f'/Regress_{dep_var_str}_vs' for i in range(1,len(regressors_strs)): filename += f'_{regressors_strs[i]}' - filename += ".pdf" - plt.savefig(saving_directory + filename, format='pdf') + + format = str(config['Regression_settings']['format_fig']) + filename += "." + format + dpi = None + if format == 'png': + dpi = 400 + plt.savefig(saving_directory + filename, format=format, dpi=dpi) print(f'Finished regression and scatter plot for {dep_var_str}, saved as {filename}\n\n') diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py index 93ba96b..bca48ef 100644 --- a/Proof_of_principle/visualizing_correlations.py +++ b/Proof_of_principle/visualizing_correlations.py @@ -42,6 +42,9 @@ dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} meso_model.update_labels_dict(dict_to_add) + dict_to_add = {'det_shear' : r'$det(\sigma)$'} + meso_model.update_labels_dict(dict_to_add) + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? var_strs = json.loads(config['Visualize_correlations']['vars']) vars = [] @@ -67,14 +70,29 @@ meso_model.weights_Q2() weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.weights_Q2.__name__}\n') + elif weighing_func_str == 'Q1_skew': meso_model.weights_Q1_skew() weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.weights_Q1_skew.__name__}\n') + elif weighing_func_str == 'Q1_non_neg': meso_model.weights_Q1_non_neg() weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.weights_Q1_non_neg.__name__}\n') + + elif weighing_func_str == "residual_weights": + residual_str = config['Regression_settings']['residual_str'] + meso_model.residual_weights(residual_str) + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.residual_weights.__name__}\n') + + elif weighing_func_str == "denominator_weights": + residual_str = config['Regression_settings']['denominator_str'] + meso_model.denominator_weights(residual_str) + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.denominator_weights.__name__}\n') + else: print(f'The string for building weights {weighing_func_str} does not match any of the implemented routines.\n') weights = None @@ -122,7 +140,12 @@ filename = '/' + weighing_func_str + 'Correlation' for i in range(len(var_strs)): filename += "_" + var_strs[i] - filename += ".pdf" - plt.savefig(saving_directory + filename, format='pdf') + + format = str(config['Regression_settings']['format_fig']) + filename += "." + format + dpi = None + if format == 'png': + dpi = 400 + plt.savefig(saving_directory + filename, format=format, dpi=dpi) print(f'Finished producing correlation plot for {var_strs}, saved as {filename}\n') diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index d628774..477aac7 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -13,6 +13,8 @@ import matplotlib.pyplot as plt import seaborn as sns import scipy.stats as scst +from scipy.linalg import det + from system.BaseFunctionality import * from MicroModels import * @@ -1218,7 +1220,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): 'shear_sq' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}$', 'Theta_sq': r'$\tilde{\Theta}_a\tilde{\Theta}^a$', 'q_res_sq': r'$\tilde{q}_a \tilde{q}^a$', - 'det_shear': r'det$(\sigma)$', + 'det_shear': r'$det(\sigma)$', 'vort_sq' : r'$\omega_{ab}\omega^{ab}$', 'acc_mag': r'$|a|$', 'U' : r'$U^a$', @@ -2307,7 +2309,8 @@ def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, metric[1,1] = metric[2,2] = +1 # Computing various invariants of the velocity gradients - det_shear = np.linalg.det(shear) + # det_shear = np.linalg.det(shear) + det_shear = det(shear) var_names.append('det_shear') vars.append(det_shear) @@ -2392,11 +2395,11 @@ def weights_Q1_skew(self): Build weights for gridpoints: downplay points where shear is small, but take into account both positive and negative values of Q1 """ - def symlog(array): - return np.sign(array) * np.log10(np.abs(array)+1) + # def symlog(array): + # return np.sign(array) * np.log10(np.abs(array)+1) Q1 = self.meso_vars['Q1'] - symlog_Q1 = symlog(Q1) + symlog_Q1 = MySymLogPlotting.symlog_var(Q1) M_pos = np.amax(symlog_Q1) m_neg = np.amin(symlog_Q1) @@ -2432,11 +2435,11 @@ def weights_Q1_non_neg(self): """ Build weights for gridpoints: downplay points where Q1 is negative """ - def symlog(array): - return np.sign(array) * np.log10(np.abs(array)+1) + # def symlog(array): + # return np.sign(array) * np.log10(np.abs(array)+1) Q1 = self.meso_vars['Q1'] - symlog_Q1 = symlog(Q1) + symlog_Q1 = MySymLogPlotting.symlog_var(Q1) symlog_Q1_pos = np.ma.masked_where(symlog_Q1 < 0, symlog_Q1, copy=True).compressed() symlog_Q1_neg = np.ma.masked_where(symlog_Q1 > 0, symlog_Q1, copy=True).compressed() @@ -2470,8 +2473,7 @@ def weights_Q2(self): """ Build weights for gridpoints based on Q2 """ - Q2 = self.meso_vars['Q2'] - Q2 = np.log10(Q2) + Q2 = np.log10(self.meso_vars['Q2']) M = np.amax(Q2) m = np.amin(Q2) @@ -2494,6 +2496,65 @@ def get_weights(x, scale): self.meso_vars.update({'weights' : weights}) + def residual_weights(self, residual_str): + """ + Build weights based on residual corresponding to input 'residual_str'. + Downplay points where this is small, that is those points where a closure is less required! + """ + + residual = np.log10(self.meso_vars[residual_str]) + + M = np.amax(residual) + m = np.amin(residual) + scale = (M+m)/2 + + def get_weights(x, scale): + result = np.tanh(x/scale) + result = (result +1)/2 + return result + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + weights = np.zeros((Nt, Nx, Ny)) + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + weights[h,i,j] = get_weights(residual[h,i,j], scale) + + self.meso_vars.update({'weights' : weights}) + + def denominator_weights(self, coeff_denominator_str): + """ + Build weights based on quantity corresponding to 'coeff_denominator_str'. + Downplay points where this is small, that is points where the extracted coefficient are less + trustworthy + """ + + denominator = np.log10(self.meso_vars[coeff_denominator_str]) + + M = np.amax(denominator) + m = np.amin(denominator) + scale = (M+m)/2 + + def get_weights(x, scale): + result = np.tanh(x/scale) + result = (result +1)/2 + return result + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + weights = np.zeros((Nt, Nx, Ny)) + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + weights[h,i,j] = get_weights(denominator[h,i,j], scale) + + self.meso_vars.update({'weights' : weights}) + def EL_style_closure_regression(self, CoefficientAnalysis): """ Takes in a list of correlation quantities (strings? The regressors needed are computed From 8448f037f51fbd923865e12f2d5b33d148624c37 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 7 Mar 2024 19:40:20 +0100 Subject: [PATCH 093/111] Adding residual weighing routine and modifying PoP files accordingly --- Proof_of_principle/visualizing_correlations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/visualizing_correlations.py index bca48ef..732d3a1 100644 --- a/Proof_of_principle/visualizing_correlations.py +++ b/Proof_of_principle/visualizing_correlations.py @@ -82,13 +82,13 @@ print(f'Finished building weights using {meso_model.weights_Q1_non_neg.__name__}\n') elif weighing_func_str == "residual_weights": - residual_str = config['Regression_settings']['residual_str'] + residual_str = config['Visualize_correlations']['residual_str'] meso_model.residual_weights(residual_str) weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.residual_weights.__name__}\n') elif weighing_func_str == "denominator_weights": - residual_str = config['Regression_settings']['denominator_str'] + residual_str = config['Visualize_correlations']['denominator_str'] meso_model.denominator_weights(residual_str) weights = meso_model.meso_vars['weights'] print(f'Finished building weights using {meso_model.denominator_weights.__name__}\n') @@ -137,7 +137,7 @@ saving_directory = config['Directories']['figures_dir'] filename = '/Correlation' if weights is not None: - filename = '/' + weighing_func_str + 'Correlation' + filename = '/Correlation' for i in range(len(var_strs)): filename += "_" + var_strs[i] From 8411c78e83cdcf4543f327db38beb7c97a9587fa Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 8 Mar 2024 14:56:26 +0100 Subject: [PATCH 094/111] Adding annotation box to hunting correlation scatter plot --- Proof_of_principle/hunting_correlations.py | 52 +++++++++++++--------- 1 file changed, 30 insertions(+), 22 deletions(-) diff --git a/Proof_of_principle/hunting_correlations.py b/Proof_of_principle/hunting_correlations.py index 88b1b30..3dae461 100644 --- a/Proof_of_principle/hunting_correlations.py +++ b/Proof_of_principle/hunting_correlations.py @@ -41,9 +41,6 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - # additional_entry = {'acc_mag': r'$|a|$'} - # meso_model.upgrade_labels_dict(additional_entry) - # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['PCA_settings']['dependent_var'] explanatory_vars_strs = json.loads(config['PCA_settings']['explanatory_vars']) @@ -65,7 +62,7 @@ # READING PREPROCESSING INFO FROM CONFIG FILE preprocess_data = json.loads(config['PCA_settings']['preprocess_data']) - extractions = int(config['PCA_settings']['extractions']) + extractions = int(config['PCA_settings']['extractions']) #This is only used for plotting, like in regression routines # PRE-PROCESSING data = [dep_var] @@ -75,8 +72,8 @@ model_points = meso_model.domain_vars['Points'] new_data = statistical_tool.trim_dataset(data, ranges, model_points) new_data = statistical_tool.preprocess_data(new_data, preprocess_data) - if extractions != 0: - new_data = statistical_tool.extract_randomly(new_data, extractions) + # if extractions != 0: + # new_data = statistical_tool.extract_randomly(new_data, extractions) dep_var = new_data[0] explanatory_vars = [] @@ -110,6 +107,9 @@ print('\nProducing correlation plot using info from PCA') x = dep_var_model y = dep_var + if extractions != 0: + new_data = statistical_tool.extract_randomly([x,y], extractions) + x, y = new_data[0], new_data[1] ylabel = dep_var_str if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): @@ -117,25 +117,33 @@ if preprocess_data['log_abs'][0] ==1: ylabel = r"$\log($" + ylabel + r"$)$" - xlabel = "" + g=statistical_tool.visualize_correlation(x, y, xlabel="highest PC model", ylabel=ylabel) + saving_directory = config['Directories']['figures_dir'] + filename = f'/PCA_hunt_{dep_var_str}' + format = str(config['Regression_settings']['format_fig']) + filename += "." + format + dpi = None + if format == 'png': + dpi = 400 + + # Building the text for the annotation box + text_for_box = "Coefficients:" + r"$(\log)$" + "\n" + legend_entries = [] for i in range(len(explanatory_vars_strs)): - sign, val = int(np.sign(coeffs[i])), str(round(np.abs(coeffs[i]),3)) - coeff_for_label = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) - text = explanatory_vars_strs[i] + label = explanatory_vars_strs[i] + " : " if hasattr(meso_model, 'labels_var_dict') and explanatory_vars_strs[i] in meso_model.labels_var_dict.keys(): - text = meso_model.labels_var_dict[explanatory_vars_strs[i]] - if preprocess_data['log_abs'][i+1]==1: - text = r"$\log($" + text + r"$)$" - xlabel += coeff_for_label + text + label = meso_model.labels_var_dict[explanatory_vars_strs[i]] + " : " + sign, val = int(np.sign(coeffs[i])), '%.3f' %np.abs(coeffs[i]) + coeff_for_label = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + label += coeff_for_label + "\n" + legend_entries.append(label) + for i in range(len(legend_entries)): + text_for_box += legend_entries[i] - model_points = meso_model.domain_vars['Points'] - g=statistical_tool.visualize_correlation(x, y, xlabel=xlabel, ylabel=ylabel) - saving_directory = config['Directories']['figures_dir'] - filename = f'/PCA_{dep_var_str}_vs' - for i in range(len(explanatory_vars_strs)): - filename += f'_{explanatory_vars_strs[i]}' - filename += ".pdf" - plt.savefig(saving_directory + filename, format='pdf') + bbox_args = dict(boxstyle="round", fc="0.95") + plt.annotate(text=text_for_box, xy = (0.99,0.1), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 9) + + plt.savefig(saving_directory + filename, format=format, dpi=dpi) print(f'Finished correlation plot for {dep_var_str}, saved as {filename}\n\n') \ No newline at end of file From 98b44497e7ad0c2adefce1d3455edfb5c912fe41 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 11 Mar 2024 11:40:39 +0100 Subject: [PATCH 095/111] Fixing bug in trim_data + optimizing the train test split --- master_files/Analysis.py | 136 +++++++++++++++++++++++++++++---------- 1 file changed, 101 insertions(+), 35 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index ca68ac2..5bc81ee 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -15,8 +15,10 @@ import pickle import seaborn as sns import warnings +import random from sklearn.linear_model import LinearRegression from sklearn.decomposition import PCA +from sklearn.model_selection import train_test_split from scipy import stats # import statsmodels.api as sm @@ -80,8 +82,9 @@ def trim_data(self, data, ranges, model_points): for i in range(len(ranges)): IdxsToRemove.append([ j for j in range(num_points[i]) if j < start_indices[i] or j > end_indices[i]]) - for i in range(len(ranges)): - newdata = np.delete(data, IdxsToRemove[i], axis=i) + newdata = np.delete(data, IdxsToRemove[0], axis=0) + for i in range(1, len(ranges)): + newdata = np.delete(newdata, IdxsToRemove[i], axis=i) return newdata @@ -215,9 +218,9 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): each value of the dictionary must be a list long as the arrays passed (checked for) Expected keys are: - 'pos_or_neg' is 1 (0) if you want to select positive (negative) values + 'value_ranges': value correspond to list of min, max to be considered - 'log_or_not' is 1 if you want to take the logarithm of the data + 'log_abs' is 1 if you want to take the logarithm of the (absolute) data weights: np.array, default to None the array with weights for each gridpoint. @@ -252,7 +255,7 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): else: return list_of_arrays - # USE THIS IF YOU WANNA REVERT TO PREVIOUS STUFF + # USE THIS IF YOU WANNA REVERT TO PREVIOUS STUFF: pos_or_neg key in preprocess dictionary # for i in range(len(list_of_arrays)): # pos_or_neg = preprocess_data['pos_or_neg'][i] # temp = self.get_pos_or_neg_mask(pos_or_neg, list_of_arrays[i]) @@ -287,6 +290,51 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): else: return processed_list + def split_train_test(self, list_of_arrays, test_percentage=0.2): + """ + Given a list of arrays, check compatibility among them, then each is split into + training set and test set (corresponding indices are selected from each array). + + Parameters: + ----------- + list_of_arrays: list of np.arrays, their compatibility is checked + + test_percentage: float, default to 0.2 + + Returns: + -------- + training_arrays, test_arrays + + Notes: + ------ + List of arrays can contain weights as well. + Yhese are treated on the same footage as the rest of the arrays. + """ + print('Splitting dataset into training and test set') + # Checking data is compatible + ref_shape = list_of_arrays[0].shape + new_data = [list_of_arrays[0].flatten()] + for i in range(1,len(list_of_arrays)): + if list_of_arrays[i].shape == ref_shape: + new_data.append(list_of_arrays[i].flatten()) + else: + print(f'The {i}th array in the dataset is not compatible with the first: ignoring it.') + + + # Using sklearn to do the split quickly. + len_array = len(new_data[0]) + available_indices = list(np.arange(0, len_array, dtype='int')) + + train_idxs, test_idxs = train_test_split(available_indices, test_size=test_percentage, shuffle=True) + train_data = [new_data[i][train_idxs] for i in range(len(new_data))] + test_data = [new_data[i][test_idxs] for i in range(len(new_data))] + + for i in range(len(train_data)): + train_data[i] = np.array(train_data[i]) + test_data[i] = np.array(test_data[i]) + + return train_data, test_data + def extract_randomly(self, list_of_data, num_extractions): """ Extract randomly from data. @@ -315,15 +363,15 @@ def extract_randomly(self, list_of_data, num_extractions): available_indices = list(np.arange(len(new_data[0]))) selected_indices = random.sample(available_indices, num_extractions) - new_data = [] - for i in range(len(list_of_data)): + extracted_data = [] + for i in range(len(new_data)): shortened_list = [] for j in range(len(selected_indices)): - shortened_list.append(list_of_data[i][selected_indices[j]]) + shortened_list.append(new_data[i][selected_indices[j]]) shortened_array = np.array(shortened_list) - new_data.append(shortened_array) + extracted_data.append(shortened_array) - return new_data + return extracted_data def weighted_mean(self, data, weights): """ @@ -825,7 +873,6 @@ def PCA_find_regressors_subset(self, data, var_wanted = 0.9): n_comp +=1 var_captured= exp_var_sum[n_comp] n_comp = n_comp+1 - # WORKING OUT THE COMPONENTS DECOMPOSITION var2comp = pca_model.components_ # shape: n_comp * n_var comp_decomp = [] @@ -833,7 +880,6 @@ def PCA_find_regressors_subset(self, data, var_wanted = 0.9): # should be the inverse but var2comp is unitary, so transpose comp_decomp.append(np.einsum('ij->ji', var2comp)[:,i]) - # # CREATING THE FIGURE WITH THE COMPONENTS PROFILE TO CHECK THEY'RE UNCORRELATED # # this block is temporary and will be removed later. # pc_profiles = np.einsum('ij,kj->ik', Data, var2comp) @@ -999,33 +1045,53 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # extracted, weights = statistical_tool.extract_randomly([x,y], 10) # print(extracted) - import random - n = 10000 - mean = [0, 0] - cov = [(2, .4), (.4, .2)] - rng = np.random.RandomState(0) - x, y = rng.multivariate_normal(mean, cov, n).T + # n = 500 + # mean = [0, 0] + # cov = [(2, .4), (.4, .2)] + # rng = np.random.RandomState(0) + # x, y = rng.multivariate_normal(mean, cov, n).T - ws = [] - for i in range(len(x)): - ws.append(random.uniform(0.5,1.)) - ws = np.array(ws) + # ws = [] + # for i in range(len(x)): + # ws.append(random.uniform(0.5,1.)) + # ws = np.array(ws) - # print(x.shape, ws.shape) - # r, _ = stats.pearsonr(x,y) + # # print(x.shape, ws.shape) + # # r, _ = stats.pearsonr(x,y) - statistical_tool = CoefficientsAnalysis() + # statistical_tool = CoefficientsAnalysis() + + # # wr = statistical_tool.weigthed_pearson(x,y,ws) + # # print(f'stats.pearsonr: {r}') + # # print(f'my_pearson: {wr}') + + # # mu, sigma = 0, 4 # mean and standard deviation + # # z = np.random.normal(mu, sigma, n) - # wr = statistical_tool.weigthed_pearson(x,y,ws) - # print(f'stats.pearsonr: {r}') - # print(f'my_pearson: {wr}') + # statistical_tool = CoefficientsAnalysis() + # g = statistical_tool.visualize_correlation(x,y,weights=ws) + # legend_labels = ['pippo', 'pluto'] + # text_for_box = r'$pippo$' + '\n' + '%.3f' %1.234454 + # # plt.text(0.85, 0.1, text_for_box, fontsize=12, transform=plt.gcf().transFigure) + # bbox_args = dict(boxstyle="round", fc="0.95") + # plt.annotate(text=text_for_box, xy = (0.99,0.1), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 9) + - mu, sigma = 0, 4 # mean and standard deviation - z = np.random.normal(mu, sigma, n) + # # plt.legend(bbox_to_anchor=(1.02, 0.2), loc='upper left', borderaxespad=0) + + # # labels=['x','y','z'] + # # data=[x,y,z] + # # statistical_tool.visualize_many_correlations(data, labels, weights=ws) + # plt.show() + + + a = np.arange(0,50,1) + b = np.arange(150,200,1) + data = [a,b] + print(data) statistical_tool = CoefficientsAnalysis() - # statistical_tool.visualize_correlation(x,y,weights=ws) - labels=['x','y','z'] - data=[x,y,z] - statistical_tool.visualize_many_correlations(data, labels, weights=ws) - plt.show() \ No newline at end of file + data_train, data_test = statistical_tool.split_train_test(data) + + print(f'data_train: {data_train}\n') + print(f'data_test: {data_test}\n') From d9e2a3752a3d79e01c355f812ba10eda3b4e615f Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 21 Mar 2024 15:36:40 +0100 Subject: [PATCH 096/111] Changes to origin for visualization.py, this was inconsistent with the extent used. --- master_files/Visualization.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index b2086a8..73bafc2 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -235,7 +235,7 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, components_indices=Non ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) mynorm = MyThreeNodesNorm(nodes) - im = ax.imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[i]) + im = ax.imshow(data_to_plot, extent=extent, origin='lower', norm=mynorm, cmap=cmaps[i]) divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) cbar = fig.colorbar(im, cax=cax, orientation='vertical') @@ -243,7 +243,7 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, components_indices=Non cbar.ax.set_yticklabels(labels) elif norms[i] != 'mysymlog': - im = ax.imshow(data_to_plot, extent=extent, norm=norms[i], cmap=cmaps[i]) + im = ax.imshow(data_to_plot, extent=extent, origin='lower', norm=norms[i], cmap=cmaps[i]) divider = make_axes_locatable(ax) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -366,7 +366,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) mynorm = MyThreeNodesNorm(nodes) - im = axes[i,j].imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[j][i]) + im = axes[i,j].imshow(data_to_plot, extent=extent, origin='lower', norm=mynorm, cmap=cmaps[j][i]) divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) cbar = fig.colorbar(im, cax=cax, orientation='vertical') @@ -374,7 +374,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com cbar.ax.set_yticklabels(labels) elif norms[j][i] != 'mysymlog': - im = axes[i,j].imshow(data_to_plot, extent=extent, norm=norms[j][i], cmap=cmaps[j][i]) + im = axes[i,j].imshow(data_to_plot, extent=extent, origin='lower', norm=norms[j][i], cmap=cmaps[j][i]) divider = make_axes_locatable(axes[i,j]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -385,7 +385,8 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com # fig.colorbar(im, cax=cax, orientation='vertical') # title = models[j].get_model_name() + "\n"+var_strs[j][i] - title = models[j].get_model_name() + '\n' + # title = models[j].get_model_name() + '\n' + title = '' if hasattr(models[j], 'labels_var_dict'): if var_strs[j][i] in models[j].labels_var_dict.keys(): title += models[j].labels_var_dict[var_strs[j][i]] @@ -416,7 +417,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) mynorm = MyThreeNodesNorm(nodes) - im = axes[i,2].imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[2][i]) + im = axes[i,2].imshow(data_to_plot, extent=extent, origin='lower', norm=mynorm, cmap=cmaps[2][i]) divider = make_axes_locatable(axes[i,2]) cax = divider.append_axes('right', size='5%', pad=0.05) cbar = fig.colorbar(im, cax=cax, orientation='vertical') @@ -424,7 +425,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com cbar.ax.set_yticklabels(labels) elif norms[2][i] != 'mysymlog': - im = axes[i,2].imshow(data_to_plot, extent=extent, norm=norms[2][i], cmap=cmaps[2][i]) + im = axes[i,2].imshow(data_to_plot, extent=extent, origin='lower', norm=norms[2][i], cmap=cmaps[2][i]) divider = make_axes_locatable(axes[i,2]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -464,7 +465,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(data_to_plot) data_to_plot = MySymLogPlotting.symlog_var(data_to_plot) mynorm = MyThreeNodesNorm(nodes) - im = axes[i,column].imshow(data_to_plot, extent=extent, norm=mynorm, cmap=cmaps[column][i]) + im = axes[i,column].imshow(data_to_plot, extent=extent, origin='lower', norm=mynorm, cmap=cmaps[column][i]) divider = make_axes_locatable(axes[i,column]) cax = divider.append_axes('right', size='5%', pad=0.05) cbar = fig.colorbar(im, cax=cax, orientation='vertical') @@ -472,7 +473,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com cbar.ax.set_yticklabels(labels) elif norms[column][i] != 'mysymlog': - im = axes[i,column].imshow(data_to_plot, extent=extent, norm=norms[column][i], cmap=cmaps[column][i]) + im = axes[i,column].imshow(data_to_plot, extent=extent, origin='lower', norm=norms[column][i], cmap=cmaps[column][i]) divider = make_axes_locatable(axes[i,column]) cax = divider.append_axes('right', size='5%', pad=0.05) fig.colorbar(im, cax=cax, orientation='vertical') @@ -495,7 +496,7 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com for i in range(len(models_names)): suptitle += models_names[i] + ", " suptitle += "models." - fig.suptitle(suptitle) + # fig.suptitle(suptitle) fig.tight_layout() # plt.subplot_tool() return fig From 5bf16606afd1c42c6afe808fec9883645579852a Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 22 Mar 2024 09:38:25 +0100 Subject: [PATCH 097/111] Updates to masterfiles --- master_files/Analysis.py | 135 ++++++++++++++++++++++++++++++++----- master_files/Filters.py | 45 ++++++++++--- master_files/MesoModels.py | 7 +- 3 files changed, 159 insertions(+), 28 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 5bc81ee..5ced715 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -19,7 +19,8 @@ from sklearn.linear_model import LinearRegression from sklearn.decomposition import PCA from sklearn.model_selection import train_test_split -from scipy import stats +# from scipy import stats +from scipy.stats import gaussian_kde, wasserstein_distance, pearsonr # import statsmodels.api as sm from system.BaseFunctionality import * @@ -169,8 +170,8 @@ def restrict_data_range_mask(self, var, min=None, max=None): if max is None: max = np.amax(var) - restricted_var = ma.masked_where(var <= min, var, copy = True) - restricted_var = ma.masked_where(restricted_var >= max, restricted_var, copy=True) + restricted_var = ma.masked_where(var < min, var, copy = True) + restricted_var = ma.masked_where(restricted_var > max, restricted_var, copy=True) restricted_var = restricted_var.compressed() return restricted_var @@ -199,6 +200,7 @@ def get_mask_min_max(self, array, min=None, max=None): masked_array = ma.masked_where(arraymax, masked_array, copy=True) + masked_array = ma.masked_where(masked_array == 0, masked_array, copy=True) mask = masked_array.mask @@ -373,6 +375,29 @@ def extract_randomly(self, list_of_data, num_extractions): return extracted_data + def centralize_dataset(self, list_of_arrays): + """ + """ + print('Centralizing the dataset') + # Checking data is compatible + ref_shape = list_of_arrays[0].shape + new_data = [list_of_arrays[0].flatten()] + for i in range(1,len(list_of_arrays)): + if list_of_arrays[i].shape == ref_shape: + new_data.append(list_of_arrays[i].flatten()) + else: + print(f'The {i}th array in the dataset is not compatible with the first: ignoring it.') + + means = [] + centralized_data = [] + for i in range(len(new_data)): + x = new_data[i] + x_mean = np.mean(x) + centralized_data.append(x - x_mean) + means.append(x_mean) + + return centralized_data, means + def weighted_mean(self, data, weights): """ Return mean of data weighted wrt weights. @@ -465,7 +490,7 @@ def weigthed_pearson(self, x, y, weights=None): weighted_pearson = corr_xy / np.sqrt(corr_x * corr_y) return weighted_pearson - def scalar_regression(self, y, X, weights=None, add_intercept=False): + def scalar_regression(self, y, X, weights=None, add_intercept=False, centralize=False): """ Routine to perform ordinary or weighted (multivariate) regression on some gridded data. @@ -525,6 +550,7 @@ def scalar_regression(self, y, X, weights=None, add_intercept=False): for i in range(len(X)): XX.append(X[i].flatten()) + XX = np.einsum('ij->ji', XX) # # VERSION USING STATSMODELS @@ -540,10 +566,10 @@ def scalar_regression(self, y, X, weights=None, add_intercept=False): # VERSION USING SKLEARN: model=LinearRegression(fit_intercept=False) if weights is not None: - print('Fitting with weights') + # print('Fitting with weights') model.fit(XX, Y, sample_weight=Weights) else: - print('Fitting without weights') + # print('Fitting without weights') model.fit(XX,Y) regress_coeff = list(model.coef_) @@ -704,7 +730,7 @@ def visualize_correlation(self, x, y, xlabel=None, ylabel=None, weights=None, hu rw = self.weigthed_pearson(X,Y,Weights) g.ax_joint.annotate(r"$r_w = {:.2f}$".format(rw), xy=(.1, .9), xycoords=g.ax_joint.transAxes) else: - r, _ = stats.pearsonr(X,Y) + r, _ = pearsonr(X,Y) g.ax_joint.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=g.ax_joint.transAxes) @@ -767,7 +793,7 @@ def visualize_many_correlations(self, data, labels, weights=None): Data_df = pd.DataFrame(Data, columns=labels) def corrfunc(x, y, **kws): - r, _ = stats.pearsonr(x, y) + r, _ = pearsonr(x, y) ax = plt.gca() ax.annotate(r"$r = {:.2f}$".format(r), xy=(.1, .9), xycoords=ax.transAxes) @@ -976,6 +1002,82 @@ def PCA_find_regressors(self, dependent_var, explanatory_vars, pcs_num=1): return highest_pcs_decomp, corresponding_scores + def wasserstein_distance(self, x, y, sample_points=100): + """ + Given two data arrays, first build the gaussian_kde of the distributions for each. + Then sample the distributions at 'sample_points' linearly distributed sample points, + and evaluate the Wasserstein distance between the two. + + Parameters: + ----------- + x, y: nd.array + + Returns: + -------- + the Wasserstein distance between the two distributions extracted from the sample data + """ + + if x.shape != y.shape: + print('The two arrays passed are not compatible, exiting') + return None + + X = x.flatten() + Y = y.flatten() + + kde_x = gaussian_kde(X, bw_method='scott' ) + kde_y = gaussian_kde(Y, bw_method='scott' ) + + min, max = np.amin([X, Y]), np.amax([X, Y]) + eval_points = np.linspace(min, max, sample_points) + + kde_sampled_x = kde_x(eval_points) + kde_sampled_y = kde_y(eval_points) + + w = wasserstein_distance(kde_sampled_x, kde_sampled_y) + return w + + def compare_distributions(self, x, y, xlabel=None, ylabel=None): + """ + Given two arrays x, y, compute and plot the respective distributions. + These are extracted from data using a gaussian_kde. + + Parameters: + ----------- + x, y: nd.array + + Returns: + -------- + the figure for later usage + """ + + if x.shape != y.shape: + print('The two arrays passed are not compatible, exiting') + return None + + X = x.flatten() + Y = y.flatten() + + kde_X = gaussian_kde(X, bw_method='scott' ) + kde_Y = gaussian_kde(Y, bw_method='scott' ) + + min, max = np.amin([X, Y]), np.amax([X, Y]) + + fig, ax = plt.subplots() + _, bins_X, _ = ax.hist(X, bins='scott', alpha=0.5, density=True, color='firebrick') + _, bins_Y, _ = ax.hist(Y, bins='scott', alpha=0.5, density=True, color='steelblue') + + num_eval_points = np.min([len(bins_X), len(bins_Y)]) + eval_points = np.linspace(min, max, num_eval_points) + pdf_X_eval = kde_X(eval_points) + pdf_Y_eval = kde_Y(eval_points) + + ax.plot(eval_points, pdf_X_eval, c='firebrick', label=xlabel) + ax.plot(eval_points, pdf_Y_eval, c='steelblue', label=ylabel) + + plt.legend() + + return fig + # Not sure about these two methods. def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ interp_dims, method, y_component_indices, x_component_indices): @@ -1085,13 +1187,14 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met # # statistical_tool.visualize_many_correlations(data, labels, weights=ws) # plt.show() - - a = np.arange(0,50,1) - b = np.arange(150,200,1) - data = [a,b] - print(data) + size = 10000 + a = np.random.normal(loc=0.0, scale=1, size=size) + b = np.random.normal(loc=1, scale=1.5, size=size) + statistical_tool = CoefficientsAnalysis() - data_train, data_test = statistical_tool.split_train_test(data) + dist = statistical_tool.wasserstein_distance(a,b) + print(f'wasserstein_distance: {dist}\n') + + fig = statistical_tool.compare_distributions(a,b,'pippo', 'pluto') + plt.show() - print(f'data_train: {data_train}\n') - print(f'data_test: {data_test}\n') diff --git a/master_files/Filters.py b/master_files/Filters.py index 336d379..75ec542 100644 --- a/master_files/Filters.py +++ b/master_files/Filters.py @@ -806,6 +806,20 @@ def __init__(self, micro_model, box_len): self.micro_model = micro_model self.L = box_len + def set_box_length(self, box_len): + """ + Method to change the width of the filter. + + Parameters: + ----------- + filter_width: float + + Returns: + -------- + None + """ + self.L = box_len + @staticmethod def get_tetrad_from_vels(spatial_vels): """ @@ -886,7 +900,7 @@ def initializer(L, grid, BC): # print('Initialized process in the pool', flush=True) @staticmethod - def find_observer_Gauss(point, pos_in_list_points): + def find_observer_Gauss(point, pos_in_list_points, initial_guess=None): """ CPU-bound task to be run in parralel. The routine combines what has been split into many in the serial @@ -921,15 +935,19 @@ def find_observer_Gauss(point, pos_in_list_points): # Building the initial guess # point = point_pos[0] # pos_in_list_points = point_pos[1] - guess = [] - BC_point = interpn(micro_grid, micro_BC, point)[0] - n_point = np.sqrt(-Base.Mink_dot(BC_point, BC_point)) - U_point = np.multiply( 1 / n_point , BC_point) - # U_point = np.multiply( 1 / self.micro_model.get_interpol_var('n', point), self.micro_model.get_interpol_var('BC', point) - for i in range(1, len(U_point)): - guess.append(U_point[i] / U_point[0]) - + # CHECK IF YOU REALLY WANT THIS: USEFUL FOR ROOTVSMIN.PY + if initial_guess is not None: + guess = initial_guess + else: + guess = [] + BC_point = interpn(micro_grid, micro_BC, point)[0] + n_point = np.sqrt(-Base.Mink_dot(BC_point, BC_point)) + U_point = np.multiply( 1 / n_point , BC_point) + # U_point = np.multiply( 1 / self.micro_model.get_interpol_var('n', point), self.micro_model.get_interpol_var('BC', point) + for i in range(1, len(U_point)): + guess.append(U_point[i] / U_point[0]) + # Routine to compute drift residual via Gauss-Legendre quadrature. def residual_gauss(spatial_vels, point): #, micro_model): # spatial_dims = micro_model.get_spatial_dims() @@ -965,7 +983,7 @@ def residual_gauss(spatial_vels, point): #, micro_model): if not sol.success: return sol.success, pos_in_list_points - def find_observers_parallel(self, points, n_cpus): + def find_observers_parallel(self, points, n_cpus, initial_guesses=None): """ Method to run find_observer_Gauss in parallel on a list of points @@ -1000,6 +1018,13 @@ def find_observers_parallel(self, points, n_cpus): BC = self.micro_model.vars['BC'] grid = self.micro_model.domain_vars['points'] args_for_pool = [ (points[i], i) for i in range(len(points))] + if initial_guesses is not None: + if len(initial_guesses) == len(points): + print('Using provided initial guesses') + args_for_pool = [ (points[i], i, initial_guesses[i]) for i in range(len(points))] + else: + args_for_pool = [ (points[i], i) for i in range(len(points))] + init = FindObs_root_parallel.initializer # initargs = (spatial_dims, L) initargs = (L, grid, BC) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 477aac7..419bc5c 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -2309,8 +2309,11 @@ def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, metric[1,1] = metric[2,2] = +1 # Computing various invariants of the velocity gradients - # det_shear = np.linalg.det(shear) - det_shear = det(shear) + + # This is required as the determinant is an invariant (does not change under coordinates changes) + # only when the tensor is written as a rank (1,1) + shear_rank11 = np.einsum('ij,jk->ik', shear, metric) + det_shear = det(shear_rank11) var_names.append('det_shear') vars.append(det_shear) From bf8dcc5c03275fcc714324bb5eb380eba99a544f Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 28 Mar 2024 10:44:55 +0100 Subject: [PATCH 098/111] Transposing the data etc so that plot axis are not flipped wrt usual --- master_files/Visualization.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 73bafc2..8b2401d 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -67,14 +67,19 @@ def get_var_data(self, model, var_str, t, x_range, y_range, component_indices=() Returns ------- data_to_plot : numpy array of floats - the (2D) data to be plotted by plt.imshow(). + the 2D data to be plotted by plt.imshow() extent: list of floats + Notes: ------ Logic: if method is raw_data, then no interp_dims are needed. - Better to have 'raw_data' and no interp_dims = None as default + Better to have 'raw_data' and interp_dims = None as default + + data_to_plot is transposed and extent is built to be used with: + origin=lower, extent=L,R,B,T by imshow + """ if var_str in model.get_all_var_strs(): @@ -104,7 +109,8 @@ def get_var_data(self, model, var_str, t, x_range, y_range, component_indices=() data_to_plot = np.zeros((nx, ny)) points = [t, xs, ys] - extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + extent = [points[1][0],points[1][-1],points[2][0],points[2][-1]] for i in range(nx): for j in range(ny): @@ -121,7 +127,8 @@ def get_var_data(self, model, var_str, t, x_range, y_range, component_indices=() gridpoints = model.get_gridpoints() points = [gridpoints[0][h], gridpoints[1][i_s:i_f+1], gridpoints[2][j_s:j_f+1]] - extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + # extent = [points[2][0],points[2][-1],points[1][0],points[1][-1]] + extent = [points[1][0],points[1][-1],points[2][0],points[2][-1]] data_shape = (i_f - i_s + 1, j_f - j_s + 1) data_to_plot = np.zeros(data_shape) @@ -134,6 +141,7 @@ def get_var_data(self, model, var_str, t, x_range, y_range, component_indices=() print('Data method is not a valid choice! Must be interpolate or raw_data.') return None # return data_to_plot, points + data_to_plot = np.transpose(data_to_plot) return data_to_plot, extent else: @@ -255,8 +263,8 @@ def plot_vars(self, model, var_strs, t, x_range, y_range, components_indices=Non if component_indices != (): title = title + ", {}-component".format(component_indices) ax.set_title(title) - ax.set_xlabel(r'$y$') - ax.set_ylabel(r'$x$') + ax.set_xlabel(r'$x$') + ax.set_ylabel(r'$y$') time_for_filename = str(round(t,2)) fig.suptitle('Snapshot from model {} at time {}'.format(model.get_model_name(), time_for_filename), fontsize = 12) @@ -398,8 +406,8 @@ def plot_vars_models_comparison(self, models, var_strs, t, x_range, y_range, com if components_indices[j][i] != (): title += " {}-component".format(components_indices[j][i]) axes[i,j].set_title(title) - axes[i,j].set_xlabel(r'$y$') - axes[i,j].set_ylabel(r'$x$') + axes[i,j].set_xlabel(r'$x$') + axes[i,j].set_ylabel(r'$y$') From a1b485d462440e12a73fb7ddd4d7abd3039e883d Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 8 Apr 2024 12:12:34 +0200 Subject: [PATCH 099/111] Updates to master files --- master_files/Analysis.py | 111 +++++++++++++++++------ master_files/MesoModels.py | 5 +- master_files/Visualization.py | 4 +- master_files/system/BaseFunctionality.py | 4 +- 4 files changed, 90 insertions(+), 34 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 5ced715..51d6d20 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -264,30 +264,39 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): # masks.append(temp) # Combining the masks of the different arrays - masks = [] - for i in range(len(list_of_arrays)): - min, max = preprocess_data['value_ranges'][i] - temp = self.get_mask_min_max(list_of_arrays[i], min, max) - masks.append(temp) - - tot_mask = masks[0] - for i in range(1,len(masks)): - tot_mask = np.logical_or(tot_mask, masks[i]) - - # Masking the full dataset with combined mask, then taking log - processed_list = [] - for i in range(len(list_of_arrays)): - temp = ma.masked_array(list_of_arrays[i], tot_mask) - processed_list.append(temp.compressed()) - #when array is compressed, this is automatically flattened! + if preprocess_data.__contains__('value_ranges'): + masks = [] + for i in range(len(list_of_arrays)): + min, max = preprocess_data['value_ranges'][i] + temp = self.get_mask_min_max(list_of_arrays[i], min, max) + masks.append(temp) + + tot_mask = masks[0] + for i in range(1,len(masks)): + tot_mask = np.logical_or(tot_mask, masks[i]) + + # Masking the full dataset with combined mask, then taking log + processed_list = [] + for i in range(len(list_of_arrays)): + temp = ma.masked_array(list_of_arrays[i], tot_mask) + processed_list.append(temp.compressed()) - for i in range(len(processed_list)): - if preprocess_data['log_abs'][i]: - processed_list[i] = np.log10(np.abs(processed_list[i])) + if weights is not None: + Weights = np.ma.masked_array(weights, tot_mask).compressed() + #when array is compressed, this is automatically flattened! + + if preprocess_data.__contains__('sqrt'): + for i in range(len(processed_list)): + if preprocess_data['sqrt'][i]: + processed_list[i] = np.sqrt(processed_list[i]) + + if preprocess_data.__contains__('log_abs'): + for i in range(len(processed_list)): + if preprocess_data['log_abs'][i]: + processed_list[i] = np.log10(np.abs(processed_list[i])) # removing corresponding entries from weights if weights is not None: - Weights = np.ma.masked_array(weights, tot_mask).compressed() return processed_list, Weights else: return processed_list @@ -573,14 +582,15 @@ def scalar_regression(self, y, X, weights=None, add_intercept=False, centralize= model.fit(XX,Y) regress_coeff = list(model.coef_) - # COMPUTING STD ERRORS ON REGRESSED COEFFICIENTS - n_data = len(Y) - Y_hat = model.predict(XX) - res = Y - Y_hat - res_sum_of_sq = np.dot(res,res) - sigma_res_sq = res_sum_of_sq / (n_data-n_reg) - var_beta = np.linalg.inv(np.einsum('ji,jl->il',XX,XX)) * sigma_res_sq - bse = [var_beta[i,i]**0.5 for i in range(n_reg)] + # # COMPUTING STD ERRORS ON REGRESSED COEFFICIENTS + # n_data = len(Y) + # Y_hat = model.predict(XX) + # res = Y - Y_hat + # res_sum_of_sq = np.dot(res,res) + # sigma_res_sq = res_sum_of_sq / (n_data-n_reg) + # var_beta = np.linalg.inv(np.einsum('ji,jl->il',XX,XX)) * sigma_res_sq + # bse = [var_beta[i,i]**0.5 for i in range(n_reg)] + bse=None return regress_coeff, bse @@ -1002,7 +1012,7 @@ def PCA_find_regressors(self, dependent_var, explanatory_vars, pcs_num=1): return highest_pcs_decomp, corresponding_scores - def wasserstein_distance(self, x, y, sample_points=100): + def wasserstein_distance(self, x, y, sample_points=200): """ Given two data arrays, first build the gaussian_kde of the distributions for each. Then sample the distributions at 'sample_points' linearly distributed sample points, @@ -1078,6 +1088,45 @@ def compare_distributions(self, x, y, xlabel=None, ylabel=None): return fig + def scatter_distrib_compare(self, x, y, figsize=[9,4]): + """ + WORK IN PROGRESS + """ + if x.shape != y.shape: + print('The two arrays passed are not compatible, exiting') + return None + + X = x.flatten() + Y = y.flatten() + + # fig, axes = plt.subplots(1,3, figsize=figsize) + fig, axes = plt.subplots(1,2, figsize=figsize) + axes = axes.flatten() + + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + + sns.set_theme(style="dark") + sns.scatterplot(x=X, y=Y, s=4, color=".15", ax=axes[0]) + sns.histplot(x=X, y=Y, bins=50, ax=axes[0], pthresh=.1, cmap="mako") + sns.kdeplot(x=X, y=Y, levels=5, ax=axes[0], color="w", linewidths=1) + + # axes[0].set_axis_labels(xlabel=xlabel, ylabel=ylabel) + + sns.histplot(X, stat='density', kde=True, color='firebrick', ax=axes[1], label='X') + sns.histplot(Y, stat='density', kde=True, color='steelblue', ax=axes[1], label='Y') + + # sns.histplot(X, stat='probability', kde=True, color='firebrick', ax=axes[2]) + # sns.histplot(Y, stat='probability', kde=True, color='steelblue', ax=axes[2]) + + # Can update legend afterwards using: h, _ = axes.get_legend_handles_labels() + axes.legend(h, updated_labels) + + fig.tight_layout() + return fig, axes + + # Not sure about these two methods. def JointPlot(self, model, y_var_str, x_var_str, t, x_range, y_range,\ interp_dims, method, y_component_indices, x_component_indices): @@ -1195,6 +1244,8 @@ def DistributionPlot(self, model, var_str, t, x_range, y_range, interp_dims, met dist = statistical_tool.wasserstein_distance(a,b) print(f'wasserstein_distance: {dist}\n') - fig = statistical_tool.compare_distributions(a,b,'pippo', 'pluto') + # fig = statistical_tool.compare_distributions(a,b,'pippo', 'pluto') + fig, _ = statistical_tool.scatter_distrib_compare(a,b) + plt.show() diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 419bc5c..e323c45 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -2388,7 +2388,10 @@ def modelling_coefficients_parallel(self, n_cpus): self.meso_vars[key] except KeyError: print('The key {} does not belong to meso_vars yet, adding it!'.format(key), flush=True) - shape = values[idx].shape + try: + shape = values[idx].shape + except AttributeError: + shape = [] self.meso_vars.update({key : np.zeros(([Nt,Nx,Ny]+ list(shape)))}) finally: self.meso_vars[key][tuple(grid_idxs)] = values[idx] diff --git a/master_files/Visualization.py b/master_files/Visualization.py index 8b2401d..e7ed228 100644 --- a/master_files/Visualization.py +++ b/master_files/Visualization.py @@ -7,9 +7,11 @@ import matplotlib.pyplot as plt +from matplotlib import colors +from mpl_toolkits.axes_grid1 import make_axes_locatable +from matplotlib.ticker import LogLocator import numpy as np import h5py -from mpl_toolkits.axes_grid1 import make_axes_locatable from scipy.interpolate import interpn from system.BaseFunctionality import * diff --git a/master_files/system/BaseFunctionality.py b/master_files/system/BaseFunctionality.py index 52b224d..58771b6 100644 --- a/master_files/system/BaseFunctionality.py +++ b/master_files/system/BaseFunctionality.py @@ -317,10 +317,10 @@ def get_mysymlog_var_ticks(var): nodes += new_nodes else: # len(pos_var_large) > 0: - print('There are positive large values', flush=True) + # print('There are positive large values', flush=True) if len(pos_var_small) >0: - print('And also positive small values', flush=True) + # print('And also positive small values', flush=True) vmin = np.amin(pos_var_small) vmax = np.amax(pos_var_small) From c129d16f0277d23319ce4829d5539cd64e1a457e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 8 Apr 2024 12:16:31 +0200 Subject: [PATCH 100/111] Restructuring PoP + adding scripts for draft --- .../calibration_scripts/.DS_Store | Bin 0 -> 6148 bytes .../config_calibration.txt | 135 ++++++++ .../calibration_scripts/find_best_fit.py | 289 +++++++++++++++++ .../hunting_correlations.py | 31 +- .../outputs/serial_5797466.out | 138 +++++++++ .../outputs/serial_5797515.out | 72 +++++ .../calibration_scripts/py_parallel.slurm | 34 ++ .../{ => calibration_scripts}/py_serial.slurm | 30 +- .../reducing_regressors.py | 26 +- .../regress+residual_check.py | 292 ++++++++++++++++++ .../regressing_residual.py | 260 ++++++++++++++++ .../visualizing_correlations.py | 11 +- Proof_of_principle/config_calibration.txt | 91 ------ Proof_of_principle/config_meso.txt | 27 -- Proof_of_principle/draft_scripts/.DS_Store | Bin 0 -> 6148 bytes .../draft_scripts/comparison_zoom.py | 162 ++++++++++ .../config_draft.txt} | 12 +- .../draft_scripts/const_coeff_all.py | 154 +++++++++ .../draft_scripts/draft_config.txt | 34 ++ .../draft_scripts/filter_scaling.py | 208 +++++++++++++ .../draft_scripts/outputs/.DS_Store | Bin 0 -> 6148 bytes .../draft_scripts/py_serial.slurm | 37 +++ Proof_of_principle/filter_scripts/.DS_Store | Bin 0 -> 6148 bytes .../filter_scripts/config_filter.txt | 48 +++ .../filter_scripts/filter_config.txt | 48 +++ .../filter_scripts/outputs/serial_5797282.out | 69 +++++ .../{ => filter_scripts}/pickling_meso.py | 53 +++- .../{ => filter_scripts}/py_parallel.slurm | 21 +- .../filter_scripts/py_serial.slurm | 42 +++ .../filter_scripts/visualizing_meso.py | 275 +++++++++++++++++ .../{ => filter_scripts}/visualizing_micro.py | 10 +- .../filter_scripts/visualizing_obs.py | 177 +++++++++++ .../visualizing_residuals.py | 94 +++--- Proof_of_principle/regressing_residual.py | 199 ------------ Proof_of_principle/visualizing_meso.py | 173 ----------- Proof_of_principle/visualizing_obs.py | 90 ------ 36 files changed, 2626 insertions(+), 716 deletions(-) create mode 100644 Proof_of_principle/calibration_scripts/.DS_Store create mode 100644 Proof_of_principle/calibration_scripts/config_calibration.txt create mode 100644 Proof_of_principle/calibration_scripts/find_best_fit.py rename Proof_of_principle/{ => calibration_scripts}/hunting_correlations.py (93%) create mode 100644 Proof_of_principle/calibration_scripts/outputs/serial_5797466.out create mode 100644 Proof_of_principle/calibration_scripts/outputs/serial_5797515.out create mode 100644 Proof_of_principle/calibration_scripts/py_parallel.slurm rename Proof_of_principle/{ => calibration_scripts}/py_serial.slurm (53%) rename Proof_of_principle/{ => calibration_scripts}/reducing_regressors.py (72%) create mode 100644 Proof_of_principle/calibration_scripts/regress+residual_check.py create mode 100644 Proof_of_principle/calibration_scripts/regressing_residual.py rename Proof_of_principle/{ => calibration_scripts}/visualizing_correlations.py (93%) delete mode 100644 Proof_of_principle/config_calibration.txt delete mode 100644 Proof_of_principle/config_meso.txt create mode 100644 Proof_of_principle/draft_scripts/.DS_Store create mode 100644 Proof_of_principle/draft_scripts/comparison_zoom.py rename Proof_of_principle/{config_visualizing.txt => draft_scripts/config_draft.txt} (81%) create mode 100644 Proof_of_principle/draft_scripts/const_coeff_all.py create mode 100644 Proof_of_principle/draft_scripts/draft_config.txt create mode 100644 Proof_of_principle/draft_scripts/filter_scaling.py create mode 100644 Proof_of_principle/draft_scripts/outputs/.DS_Store create mode 100644 Proof_of_principle/draft_scripts/py_serial.slurm create mode 100644 Proof_of_principle/filter_scripts/.DS_Store create mode 100644 Proof_of_principle/filter_scripts/config_filter.txt create mode 100644 Proof_of_principle/filter_scripts/filter_config.txt create mode 100644 Proof_of_principle/filter_scripts/outputs/serial_5797282.out rename Proof_of_principle/{ => filter_scripts}/pickling_meso.py (76%) rename Proof_of_principle/{ => filter_scripts}/py_parallel.slurm (51%) create mode 100644 Proof_of_principle/filter_scripts/py_serial.slurm create mode 100644 Proof_of_principle/filter_scripts/visualizing_meso.py rename Proof_of_principle/{ => filter_scripts}/visualizing_micro.py (89%) create mode 100644 Proof_of_principle/filter_scripts/visualizing_obs.py rename Proof_of_principle/{ => filter_scripts}/visualizing_residuals.py (75%) delete mode 100644 Proof_of_principle/regressing_residual.py delete mode 100644 Proof_of_principle/visualizing_meso.py delete mode 100644 Proof_of_principle/visualizing_obs.py diff --git a/Proof_of_principle/calibration_scripts/.DS_Store b/Proof_of_principle/calibration_scripts/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 + new_data = statistical_tool.extract_randomly([x,y], extractions) + File "/mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration/../../master_files/Analysis.py", line 376, in extract_randomly + selected_indices = random.sample(available_indices, num_extractions) + File "/home/tc2m23/.conda/envs/myenv/lib/python3.10/random.py", line 482, in sample + raise ValueError("Sample larger than population or is negative") +ValueError: Sample larger than population or is negative +================================================ +Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle +================================================ + + +Dependent var: eta, Explanatory vars: ['vort_sq', 'det_shear', 'n_tilde', 'T_tilde'] + +The string for building weights no_weights does not match any of the implemented routines. + +Trimming the dataset +Pre-processing dataset +Centralizing the dataset +Splitting dataset into training and test set +regression coeffs: [0.16266840064158283, 0.21363488531182281, 0.3895166980016922, 0.3895409159101455] + +Corresponding std errors: None + +Finished regression and scatter plot for eta, saved as /Regress_eta_vs_det_shear_n_tilde_T_tilde.png + + +============================================================================== +Running epilogue script on gold51. + +Submit time : 2024-04-08T10:54:51 +Start time : 2024-04-08T10:54:51 +End time : 2024-04-08T10:58:41 +Elapsed time : 00:03:50 (Timelimit=00:10:00) + +Job ID: 5797466 +Cluster: i5 +User/Group: tc2m23/pj +State: COMPLETED (exit code 0) +Cores: 1 +CPU Utilized: 00:03:18 +CPU Efficiency: 86.09% of 00:03:50 core-walltime +Job Wall-clock time: 00:03:50 +Memory Utilized: 4.14 GB +Memory Efficiency: 17.92% of 23.07 GB + diff --git a/Proof_of_principle/calibration_scripts/outputs/serial_5797515.out b/Proof_of_principle/calibration_scripts/outputs/serial_5797515.out new file mode 100644 index 0000000..72ff1e5 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/outputs/serial_5797515.out @@ -0,0 +1,72 @@ +Running SLURM prolog script on gold53.cluster.local +=============================================================================== +Job started on Mon Apr 8 11:00:53 BST 2024 +Job ID : 5797515 +Job name : py_serial.slurm +WorkDir : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration +Command : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration/py_serial.slurm +Partition : serial +Num hosts : 1 +Num cores : 1 +Num of tasks : 1 +Hosts allocated : gold53 +Job Output Follows ... +=============================================================================== +DeprecationWarning: 'source deactivate' is deprecated. Use 'conda deactivate'. +================================================ +Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle +================================================ + + +Dependent var: eta, Explanatory vars: ['n_tilde', 'T_tilde', 'det_shear', 'shear_sq', 'vort_sq', 'Q1', 'Q2'] + +Trimming the dataset +Pre-processing dataset +Preprocess_data dictionary is not compatible with list_of_arrays. Exiting +Scores of dependent var on principal components: +[-7.54696541e-12 -1.00000000e+00 1.46146095e-05 6.69383333e-07 + -3.16436543e-06 7.72403240e-06 -2.46691513e-14 8.40274830e-21] + +0-th highest PCs decomposition: +[-1.00000000e+00 1.46367506e-05 -7.10101218e-07 -7.54696540e-12 + 7.60710509e-06 -1.16321035e-07 3.32993677e-06 -6.61620063e-14] +Corresponding score: -0.9999999998581457 + +n_tilde-rescaled coefficient: 1.4636750621041325e-05 + +T_tilde-rescaled coefficient: -7.101012176048021e-07 + +det_shear-rescaled coefficient: -7.546965400152193e-12 + +shear_sq-rescaled coefficient: 7.6071050879069935e-06 + +vort_sq-rescaled coefficient: -1.1632103527774598e-07 + +Q1-rescaled coefficient: 3.3299367747304493e-06 + +Q2-rescaled coefficient: -6.616200627999648e-14 + + +Producing correlation plot using info from PCA +Finished correlation plot for eta, saved as /PCA_hunt_eta.png + + +============================================================================== +Running epilogue script on gold53. + +Submit time : 2024-04-08T11:00:52 +Start time : 2024-04-08T11:00:52 +End time : 2024-04-08T11:01:24 +Elapsed time : 00:00:32 (Timelimit=00:10:00) + +Job ID: 5797515 +Cluster: i5 +User/Group: tc2m23/pj +State: COMPLETED (exit code 0) +Cores: 1 +CPU Utilized: 00:00:20 +CPU Efficiency: 62.50% of 00:00:32 core-walltime +Job Wall-clock time: 00:00:32 +Memory Utilized: 4.12 GB +Memory Efficiency: 17.84% of 23.07 GB + diff --git a/Proof_of_principle/calibration_scripts/py_parallel.slurm b/Proof_of_principle/calibration_scripts/py_parallel.slurm new file mode 100644 index 0000000..a0d4b62 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/py_parallel.slurm @@ -0,0 +1,34 @@ +#!/bin/bash +#################################### +# JOB INFO: parallel, single node +#################################### + +#SBATCH --output=outputs/parallel_%A.out +#SBATCH --nodes=1 +#SBATCH --ntasks=40 +#SBATCH --time=01:00:00 + +#SBATCH --mail-type=begin +#SBATCH --mail-type=fail +#SBATCH --mail-type=end +#SBATCH --mail-user=celora@ice.csic.es + + +#################################### +# LOADING THE CONDA ENVIRONMENT +#################################### + +module load conda/py3-latest +# I get warnings to use 'conda deactivate' instead but that doesn't work +source deactivate +conda activate myenv + +#################################### +# LAUNCHING THE JOBS +#################################### + +python3 -u find_best_fit.py config_calibration.txt + + + + diff --git a/Proof_of_principle/py_serial.slurm b/Proof_of_principle/calibration_scripts/py_serial.slurm similarity index 53% rename from Proof_of_principle/py_serial.slurm rename to Proof_of_principle/calibration_scripts/py_serial.slurm index b7f583f..2e3dd7e 100644 --- a/Proof_of_principle/py_serial.slurm +++ b/Proof_of_principle/calibration_scripts/py_serial.slurm @@ -1,19 +1,13 @@ #!/bin/bash #################################### -# JOB INFO: +# JOB INFO: serial #################################### #SBATCH --nodes=1 #SBATCH --ntasks=1 -#SBATCH --time=02:30:00 -#SBATCH --output=outputs/visualize_slurm-%A.out -# #SBATCH --output=outputs/vis_residuals-%A.out -# #SBATCH --output=outputs/vis_calibration-%A.out - -# uncomment if array: -##SBATCH --array=1-3 -##SBATCH --output=outputs/slurm-%A_%a.out +#SBATCH --time=00:10:00 +#SBATCH --output=outputs/serial_%A.out #SBATCH --mail-type=begin # send email when job begins @@ -31,24 +25,16 @@ module load conda/py3-latest source deactivate conda activate myenv -#################################### -# RETRIEVING THE PARAMETERS FOR THE RUNS: OLD -#################################### - -# ET=$(awk -v var=${SLURM_ARRAY_TASK_ID} '$1==var {print $2}' config.txt) - #################################### # LAUNCHING THE JOBS ##################################### -# python3 -u visualizing_micro.py config_visualizing.txt -# python3 -u visualizing_obs.py config_visualizing.txt -# python3 -u visualizing_meso.py config_visualizing.txt - -python3 -u visualizing_residuals.py config_calibration.txt - +# python3 -u regress+residual_check.py config_calibration.txt # python3 -u visualizing_correlations.py config_calibration.txt # python3 -u reducing_regressors.py config_calibration.txt -# python3 -u hunting_correlations.py config_calibration.txt +python3 -u hunting_correlations.py config_calibration.txt # python3 -u regressing_residual.py config_calibration.txt + + + diff --git a/Proof_of_principle/reducing_regressors.py b/Proof_of_principle/calibration_scripts/reducing_regressors.py similarity index 72% rename from Proof_of_principle/reducing_regressors.py rename to Proof_of_principle/calibration_scripts/reducing_regressors.py index e41478b..da4a84c 100644 --- a/Proof_of_principle/reducing_regressors.py +++ b/Proof_of_principle/calibration_scripts/reducing_regressors.py @@ -1,6 +1,6 @@ import sys # import os -sys.path.append('../master_files/') +sys.path.append('../../master_files/') import pickle import configparser import json @@ -39,8 +39,6 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - # additional_entry = {'acc_mag': r'$|a|$'} - # meso_model.upgrade_labels_dict(additional_entry) # WHICH DATA YOU WANT TO RUN THE ROUTINE ON regressors_strs = json.loads(config['PCA_settings']['regressors_2_reduce']) @@ -51,10 +49,10 @@ print('Trying to reduce the following list to a smaller subset: {}\n'.format(regressors_strs)) # WHICH GRID-RANGES SHOULD WE CONSIDER? - ranges = json.loads(config['Ranges_for_analysis']['ranges']) + ranges = json.loads(config['PCA_settings']['ranges']) x_range = ranges['x_range'] y_range = ranges['y_range'] - num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + num_slices_meso = int(config['PCA_settings']['num_T_slices']) time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] @@ -67,13 +65,19 @@ model_points = meso_model.domain_vars['Points'] new_data = statistical_tool.trim_dataset(regressors, ranges, model_points) new_data = statistical_tool.preprocess_data(new_data, preprocess_data) - if extractions != 0: - new_data = statistical_tool.extract_randomly(new_data, extractions) + # if extractions != 0: + # new_data = statistical_tool.extract_randomly(new_data, extractions) # PERFORMING PCA TO EXTRACT PRINCIPAL COMPONENTS var_wanted = float(config['PCA_settings']['variance_wanted']) - comp_decomp = statistical_tool.PCA_find_regressors_subset(regressors, var_wanted=var_wanted) + comp_decomp = statistical_tool.PCA_find_regressors_subset(new_data, var_wanted=var_wanted) - print('The components decomposition are:') - for i in range(len(comp_decomp)): - print(f'The {i}-th component decomposition in terms of original vars is\n{comp_decomp[i]}\n') + saving_directory = config['Directories']['figures_dir'] + with open(saving_directory + '/Reducing_regressors.txt', 'w') as filehandle: + filehandle.write("===================================\nFeatures in the dataset:\n") + filehandle.write(regressors_strs[0]) + for i in range(1, len(regressors_strs)): + filehandle.write(", " + regressors_strs[i]) + filehandle.write("\n===================================\n") + for i in range(len(comp_decomp)): + filehandle.write(f'The {i}-th component decomposition in terms of original vars is\n\n{comp_decomp[i]}\n\n') diff --git a/Proof_of_principle/calibration_scripts/regress+residual_check.py b/Proof_of_principle/calibration_scripts/regress+residual_check.py new file mode 100644 index 0000000..91f8cf9 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/regress+residual_check.py @@ -0,0 +1,292 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +import math +from scipy import stats + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + ################################################ + # # REGRESSING THE COEFFICIENT + ################################################ + statistical_tool = CoefficientsAnalysis() + + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? + coeff_str = config['Regress+residual_check_settings']['coeff_str'] + coeff = meso_model.meso_vars[coeff_str] + coeff_regressors_strs = json.loads(config['Regress+residual_check_settings']['coeff_regressors_strs']) + regressors = [] + for i in range(len(coeff_regressors_strs)): + temp = meso_model.meso_vars[coeff_regressors_strs[i]] + regressors.append(temp) + print(f'Dependent var: {coeff_str}, Explanatory vars: {coeff_regressors_strs}\n') + + residual_str = config['Regress+residual_check_settings']['residual_str'] + closure_ingr_str = config['Regress+residual_check_settings']['closure_ingr_str'] + residual = meso_model.meso_vars[residual_str] + closure_ingr = meso_model.meso_vars[closure_ingr_str] + print(f'Residuals and closure ingredient for model check: {residual_str}, {closure_ingr_str}\n') + + # PREPARING THE DATA + preprocess_data = json.loads(config['Regress+residual_check_settings']['preprocess_data']) + add_intercept = not not int(config['Regress+residual_check_settings']['add_intercept']) + centralize = int(config['Regress+residual_check_settings']['centralize']) + test_percentage = float(config['Regress+residual_check_settings']['test_percentage']) + + # first step: trimming and pre-processing + regression_ranges = json.loads(config['Regress+residual_check_settings']['regression_ranges']) + x_range = regression_ranges['x_range'] + y_range = regression_ranges['y_range'] + idxs_time_slices = json.loads(config['Regress+residual_check_settings']['idxs_time_slices']) + times = [meso_model.domain_vars['T'][i] for i in idxs_time_slices] + t_range = [np.amin(times), np.amax(times)] + ranges = [t_range, x_range, y_range] + + data = [coeff] + for i in range(len(regressors)): + data.append(regressors[i]) + data.append(residual) + data.append(closure_ingr) + model_points = meso_model.domain_vars['Points'] + new_data = statistical_tool.trim_dataset(data, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + + # next step: centralizing the coefficients data + if centralize: + new_data, means = statistical_tool.centralize_dataset(new_data) + coeff_mean = means[0] + del means[0] + closure_ingr_mean = means[-1] + del means[-1] + residual_means = means[-1] + del means[-1] + regressors_means = means + + # second step: splitting into train and test + if test_percentage==0.: + coeff_train = new_data[0] + coeff_test = np.copy(new_data[0]) + del new_data[0] + closure_ingr_train = new_data[-1] + closure_ingr_test = np.copy(new_data[-1]) + del new_data[-1] + residual_train = new_data[-1] + residual_test = np.copy(new_data[-1]) + del new_data[-1] + regressors_train = new_data + regressors_test = [] + for elem in regressors_train: + regressors_test.append(np.copy(elem)) + else: + training_data, test_data = statistical_tool.split_train_test(new_data, test_percentage=test_percentage) + coeff_train = training_data[0] + coeff_test = test_data[0] + del training_data[0] + del test_data[0] + closure_ingr_train = training_data[-1] + closure_ingr_test = test_data[-1] + del training_data[-1] + del test_data[-1] + residual_train = training_data[-1] + residual_test = test_data[-1] + del training_data[-1] + del test_data[-1] + regressors_train = training_data + regressors_test = test_data + + # # fourth step: centralizing the coefficients data + # if centralize: + # tot_coeff = [np.stack((coeff_train, coeff_test))] + # tot_regress = [np.stack((regressors_test[i], regressors_train[i])) for i in range(len(regressors))] + + # _, means = statistical_tool.centralize_dataset(tot_coeff + tot_regress) + # coeff_mean = means[0] + # regressors_means = means[1:] + + # coeff_train -= coeff_mean + # for i in range(len(regressors_means)): + # regressors_train[i] -= regressors_means[i] + + coeffs, std_errors = statistical_tool.scalar_regression(coeff_train, regressors_train, add_intercept=add_intercept) + print('regression coeffs: {}\n'.format(coeffs)) + # print('Corresponding std errors: {}\n'.format(std_errors)) + + ###################################################### + # # RECONSTRUCTING PREDICTIONS FOR COEFF AND RESIDUAL + ###################################################### + # Note: the reconstruction depends on the order with which you pre-process above + + # building predictions for the coefficient + coeff_model = np.zeros(coeff_test.shape) + if centralize: + coeff_model += coeff_mean + coeff_test += coeff_mean + for i in range(len(regressors_test)): + # regressors_test[i] += regressors_means[i] + coeff_model += np.multiply(coeffs[i], regressors_test[i]) + else: + if add_intercept: + coeff_model += coeffs[0] + for i in range(len(regressors)): + coeff_model += np.multiply(coeffs[i+1], regressors_test[i]) + elif not add_intercept: + for i in range(len(regressors)): + coeff_model += np.multiply(coeffs[i], regressors_test[i]) + + # preprocessing and building model predictions for the residual + residual_model = np.zeros(residual_test.shape) + residual_model = coeff_model + closure_ingr_test + if centralize: + residual_model += closure_ingr_mean + residual_test += residual_means + + # FINALLY: PLOTTING + plt.rc("font", family="serif") + plt.rc("mathtext",fontset="cm") + plt.rc('font', size=8) + + fig, axes = plt.subplots(1,3, figsize=[13,4]) + axes = axes.flatten() + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + + sns.set_theme(style="dark") + sns.scatterplot(x=coeff_model, y=coeff_test, s=4, color=".15", ax=axes[0]) + sns.histplot(x=coeff_model, y=coeff_test, bins=50, ax=axes[0], pthresh=.1, cmap="mako") + sns.kdeplot(x=coeff_model, y=coeff_test, levels=5, ax=axes[0], color="w", linewidths=1) + + sns.histplot(coeff_model, stat='density', kde=True, color='steelblue', ax=axes[1], label='model') + sns.histplot(coeff_test, stat='density', kde=True, color='firebrick', ax=axes[1], label='sim. data') + + sns.histplot(residual_model, stat='density', kde=True, color='steelblue', ax=axes[2], label='model') + sns.histplot(residual_test, stat='density', kde=True, color='firebrick', ax=axes[2], label='sim. data') + + print('finished plot, now making it nice\n') + + # adding labels + coeff_label = coeff_str + if hasattr(meso_model, 'labels_var_dict') and coeff_str in meso_model.labels_var_dict.keys(): + coeff_label = meso_model.labels_var_dict[coeff_str] + if preprocess_data['log_abs'][0] == 1: + coeff_label = r"$\log($" + coeff_label + r"$)$" + + residual_label = residual_str + if hasattr(meso_model, 'labels_var_dict') and residual_str in meso_model.labels_var_dict.keys(): + residual_label = meso_model.labels_var_dict[residual_str] + # comment the following line if you don't take the log of the residual + residual_label = r"$\frac{1}{2}\log($" + residual_label + r"$)$" + + axes[0].set_xlabel('Regression model', fontsize=8) + axes[0].set_ylabel(coeff_label, fontsize=8) + axes[1].set_xlabel(coeff_label, fontsize=8) + axes[1].set_ylabel('pdf', fontsize=8) + axes[2].set_xlabel(residual_label, fontsize=8) + axes[2].set_ylabel('pdf', fontsize=8) + + fig.tight_layout() + + # Building the annotation box with the specifics of the model + if centralize: + text_for_box = r'$Means,$ $model$ $coeffs$:' + '\n' + add_text = coeff_str + " : " + if hasattr(meso_model, 'labels_var_dict') and coeff_str in meso_model.labels_var_dict.keys(): + add_text = meso_model.labels_var_dict[coeff_str] + " : " + sign, val = int(np.sign(coeff_mean)), '%.3f' %np.abs(coeff_mean) + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + add_text += coeff_for_text_box + text_for_box += add_text + + for i in range(len(coeff_regressors_strs)): + add_text = "\n" + coeff_regressors_strs[i] + " : " + if hasattr(meso_model, 'labels_var_dict') and coeff_regressors_strs[i] in meso_model.labels_var_dict.keys(): + add_text = "\n" + meso_model.labels_var_dict[coeff_regressors_strs[i]] + " : " + + sign, val = int(np.sign(regressors_means[i])), '%.3f' %np.abs(regressors_means[i]) + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r", $-{}$".format(val) + add_text += coeff_for_text_box + + sign, val = int(np.sign(coeffs[i])), '%.3f' %np.abs(coeffs[i]) + coeff_for_text_box = r", $+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + add_text += coeff_for_text_box + + text_for_box += add_text + + else: + text_for_box = r"$Model$ $coeff.s:$" + "\n" + if add_intercept: + sign, val = int(np.sign(coeffs[0])), '%.3f' %np.abs(coeffs[0]) + text_for_box += r'$offset$' +' : ' + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + text_for_box += coeff_for_text_box + else: + coeffs = [0] + coeffs + + for i in range(len(coeff_regressors_strs)): + sign, val = int(np.sign(coeffs[i+1])), '%.3f' %np.abs(coeffs[i+1]) + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + + add_text = "\n" + coeff_regressors_strs[i] + " : " + if hasattr(meso_model, 'labels_var_dict') and coeff_regressors_strs[i] in meso_model.labels_var_dict.keys(): + add_text = "\n" + meso_model.labels_var_dict[coeff_regressors_strs[i]] + " : " + + add_text += coeff_for_text_box + text_for_box += add_text + + + bbox_args = dict(boxstyle="round", fc="0.95") + plt.annotate(text=text_for_box, xy = (0.32,0.2), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 8) + + + # Adding legend to the distribution comparison panel + axes[1].legend(loc = 'best', prop={'size': 8}) + h, _ = axes[1].get_legend_handles_labels() + labels = ['Regression model', 'sim. data'] + axes[1].legend(h, labels, loc = 'best', prop={'size': 8, 'family' : 'serif'}) + + axes[2].legend(loc = 'best', prop={'size': 8}) + h, _ = axes[2].get_legend_handles_labels() + labels = ['Regression model', 'sim. data'] + axes[2].legend(h, labels, loc = 'best', prop={'size': 8, 'family' : 'serif'}) + + # Saving the figure + print(f'Finished plot, now saving...\n\n') + saving_directory = config['Directories']['figures_dir'] + filename = '/PROVA' + format = 'png' + filename += "." + format + dpi = 300 + plt.savefig(saving_directory + filename, format=format, dpi=dpi) + + + diff --git a/Proof_of_principle/calibration_scripts/regressing_residual.py b/Proof_of_principle/calibration_scripts/regressing_residual.py new file mode 100644 index 0000000..0e43ac4 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/regressing_residual.py @@ -0,0 +1,260 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +import math +from scipy import stats + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################## + # # RUN THE REGRESSION ROUTINE GIVEN DEPENDENT DATA AND EXPLANATORY + # # VARS. DATA IS PRE-PROCESSED + # ################################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? + dep_var_str = config['Regression_settings']['dependent_var'] + dep_var = meso_model.meso_vars[dep_var_str] + regressors_strs = json.loads(config['Regression_settings']['regressors']) + regressors = [] + for i in range(len(regressors_strs)): + temp = meso_model.meso_vars[regressors_strs[i]] + regressors.append(temp) + print(f'Dependent var: {dep_var_str}, Explanatory vars: {regressors_strs}\n') + + + # WHICH GRID-RANGES SHOULD WE CONSIDER? + regression_ranges = json.loads(config['Regression_settings']['ranges']) + x_range = regression_ranges['x_range'] + y_range = regression_ranges['y_range'] + num_slices_meso = int(config['Regression_settings']['num_T_slices']) + time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] + ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] + + # READING PREPROCESSING INFO FROM CONFIG FILE + preprocess_data = json.loads(config['Regression_settings']['preprocess_data']) + add_intercept = not not int(config['Regression_settings']['add_intercept']) + extractions = int(config['Regression_settings']['extractions']) + centralize = int(config['Regression_settings']['centralize']) + test_percentage = float(config['Regression_settings']['test_percentage']) + + # BUILDING THE WEIGHTS + weighing_func_str = config['Regression_settings']['weighing_func'] + if weighing_func_str == 'Q2': + meso_model.weights_Q2() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q2.__name__}\n') + + elif weighing_func_str == 'Q1_skew': + meso_model.weights_Q1_skew() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q1_skew.__name__}\n') + + elif weighing_func_str == 'Q1_non_neg': + meso_model.weights_Q1_non_neg() + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.weights_Q1_non_neg.__name__}\n') + + elif weighing_func_str == "residual_weights": + residual_str = config['Regression_settings']['residual_str'] + meso_model.residual_weights(residual_str) + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.residual_weights.__name__}\n') + + elif weighing_func_str == "denominator_weights": + residual_str = config['Regression_settings']['denominator_str'] + meso_model.denominator_weights(residual_str) + weights = meso_model.meso_vars['weights'] + print(f'Finished building weights using {meso_model.denominator_weights.__name__}\n') + + else: + print(f'The string for building weights {weighing_func_str} does not match any of the implemented routines.\n') + weights = None + + # PRE-PROCESSING: Trimming, pre-processing and splitting intro train + test set + data = [dep_var] + for i in range(len(regressors)): + data.append(regressors[i]) + statistical_tool = CoefficientsAnalysis() + model_points = meso_model.domain_vars['Points'] + + if weights is not None: + new_data = statistical_tool.trim_dataset(data + [weights], ranges, model_points) + weights = new_data[-1] + del new_data[-1] + new_data, weights = statistical_tool.preprocess_data(new_data, preprocess_data, weights=weights) + + if centralize: + new_data, means = statistical_tool.centralize_dataset(new_data) + dep_var_mean = means[0] + regressors_means = means[1:] + + training_data, test_data = statistical_tool.split_train_test(new_data + [weights], test_percentage=test_percentage) + dep_var_train = training_data[0] + dep_var_test = test_data[0] + del training_data[0] + del test_data[0] + weights_train = training_data[-1] + weights_test = test_data[-1] + del training_data[-1] + del test_data[-1] + regressors_train = training_data + regressors_test = test_data + + + else: + new_data = statistical_tool.trim_dataset(data, ranges, model_points) + new_data = statistical_tool.preprocess_data(new_data, preprocess_data) + + if centralize: + new_data, means = statistical_tool.centralize_dataset(new_data) + dep_var_mean = means[0] + regressors_means = means[1:] + + training_data, test_data = statistical_tool.split_train_test(new_data, test_percentage=test_percentage) + dep_var_train = training_data[0] + dep_var_test = test_data[0] + del training_data[0] + del test_data[0] + regressors_train = training_data + regressors_test = test_data + + weights_train = weights_test = None + + + coeffs, std_errors = statistical_tool.scalar_regression(dep_var_train, regressors_train, add_intercept=add_intercept, weights=weights_train) + print('regression coeffs: {}\n'.format(coeffs)) + print('Corresponding std errors: {}\n'.format(std_errors)) + + + # BUILDING TEST-DATA PREDICTIONs GIVEN THE REGRESSED MODEL + dep_var_model = np.zeros(dep_var_test.shape) + if centralize: + for i in range(len(regressors_test)): + dep_var_model += np.multiply(coeffs[i], regressors_test[i]) + + else: + if add_intercept: + dep_var_model += coeffs[0] + for i in range(len(regressors)): + dep_var_model += np.multiply(coeffs[i+1], regressors_test[i]) + elif not add_intercept: + for i in range(len(regressors)): + dep_var_model += np.multiply(coeffs[i], regressors_test[i]) + + # EXTRACTIONS ARE NOT NEEDED IF YOU SAVE THE FIG AS PNG + if extractions != 0: + data_for_scatter = [dep_var_test, dep_var_model] + new_data = statistical_tool.extract_randomly(data_for_scatter + [weights_test], extractions) + if weights is not None: + weights_test = new_data[-1] + del new_data[-1] + + dep_var_test, dep_var_model = new_data[0], new_data[1] + + + # FINALLY: PLOTTING + ylabel = dep_var_str + if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): + ylabel = meso_model.labels_var_dict[dep_var_str] + if preprocess_data['log_abs'][0] == 1: + ylabel = r"$\log($" + ylabel + r"$)$" + # statistical_tool.visualize_correlation(dep_var_model, dep_var_test , xlabel=r"$regression$ $model$", ylabel=ylabel, weights = weights) + fig = statistical_tool.compare_distributions(dep_var_model, dep_var_test, xlabel=r'$regression$ $model$', ylabel=ylabel) + + # Building the annotation box with the specifics of the model + if centralize: + text_for_box = r"$Model$ $coeff.s$, $means:$" + "\n" + add_text = dep_var_str + " : , " + if hasattr(meso_model, 'labels_var_dict') and dep_var_str in meso_model.labels_var_dict.keys(): + add_text = meso_model.labels_var_dict[dep_var_str] + " : , " + sign, val = int(np.sign(dep_var_mean)), '%.3f' %np.abs(dep_var_mean) + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + add_text += coeff_for_text_box + text_for_box += add_text + + for i in range(len(regressors_strs)): + add_text = "\n" + regressors_strs[i] + " : " + if hasattr(meso_model, 'labels_var_dict') and regressors_strs[i] in meso_model.labels_var_dict.keys(): + add_text = "\n" + meso_model.labels_var_dict[regressors_strs[i]] + " : " + + sign, val = int(np.sign(coeffs[i])), '%.3f' %np.abs(coeffs[i]) + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + add_text += coeff_for_text_box + + sign, val = int(np.sign(regressors_means[i])), '%.3f' %np.abs(regressors_means[i]) + coeff_for_text_box = r", $+{}$".format(val) if sign == 1 else r", $-{}$".format(val) + add_text += coeff_for_text_box + + text_for_box += add_text + + + else: + text_for_box = r"$Model$ $coeff.s:$" + "\n" + if add_intercept: + sign, val = int(np.sign(coeffs[0])), '%.3f' %np.abs(coeffs[0]) + text_for_box += r'$offset$' +' : ' + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + text_for_box += coeff_for_text_box + else: + coeffs = [0] + coeffs + + for i in range(len(regressors_strs)): + sign, val = int(np.sign(coeffs[i+1])), '%.3f' %np.abs(coeffs[i+1]) + coeff_for_text_box = r"$+{}$".format(val) if sign == 1 else r"$-{}$".format(val) + + add_text = "\n" + regressors_strs[i] + " : " + if hasattr(meso_model, 'labels_var_dict') and regressors_strs[i] in meso_model.labels_var_dict.keys(): + add_text = "\n" + meso_model.labels_var_dict[regressors_strs[i]] + " : " + + add_text += coeff_for_text_box + text_for_box += add_text + + bbox_args = dict(boxstyle="round", fc="0.95") + plt.annotate(text=text_for_box, xy = (0.99,0.1), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 8) + + + # Saving the figure + saving_directory = config['Directories']['figures_dir'] + filename = f'/Regress_{dep_var_str}_vs' + for i in range(1,len(regressors_strs)): + filename += f'_{regressors_strs[i]}' + format = str(config['Regression_settings']['format_fig']) + filename += "." + format + dpi = None + if format == 'png': + dpi = 400 + plt.savefig(saving_directory + filename, format=format, dpi=dpi) + print(f'Finished regression and scatter plot for {dep_var_str}, saved as {filename}\n\n') + + diff --git a/Proof_of_principle/visualizing_correlations.py b/Proof_of_principle/calibration_scripts/visualizing_correlations.py similarity index 93% rename from Proof_of_principle/visualizing_correlations.py rename to Proof_of_principle/calibration_scripts/visualizing_correlations.py index 732d3a1..8a4ec26 100644 --- a/Proof_of_principle/visualizing_correlations.py +++ b/Proof_of_principle/calibration_scripts/visualizing_correlations.py @@ -1,6 +1,6 @@ import sys # import os -sys.path.append('../master_files/') +sys.path.append('../../master_files/') import pickle import configparser import json @@ -39,11 +39,6 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) - dict_to_add = {'vort_sq' : r'$\omega_{ab}\omega^{ab}$'} - meso_model.update_labels_dict(dict_to_add) - - dict_to_add = {'det_shear' : r'$det(\sigma)$'} - meso_model.update_labels_dict(dict_to_add) # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? var_strs = json.loads(config['Visualize_correlations']['vars']) @@ -53,10 +48,10 @@ print(f'Producing scatter plot for the vars: {var_strs}\n') # WHICH GRID-RANGES SHOULD WE CONSIDER? - regression_ranges = json.loads(config['Ranges_for_analysis']['ranges']) + regression_ranges = json.loads(config['Visualize_correlations']['ranges']) x_range = regression_ranges['x_range'] y_range = regression_ranges['y_range'] - num_slices_meso = int(config['Models_settings']['mesogrid_T_slices_num']) + num_slices_meso = int(config['Visualize_correlations']['num_T_slices']) time_of_central_slice = meso_model.domain_vars['T'][int((num_slices_meso-1)/2)] ranges = [[time_of_central_slice, time_of_central_slice], x_range, y_range] diff --git a/Proof_of_principle/config_calibration.txt b/Proof_of_principle/config_calibration.txt deleted file mode 100644 index 33a9513..0000000 --- a/Proof_of_principle/config_calibration.txt +++ /dev/null @@ -1,91 +0,0 @@ -# CONFIGURATION FILE FOR SCRIPTS: -########################################################## - -[Directories] - -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled - -figures_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/figures/CG1/CG1_fw8/fitting/weights/fitting_eta/eta_pos/residual - -[Filenames] - -meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle - -[Models_settings] -mesogrid_T_slices_num = 3 - -[Ranges_for_analysis] - -ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} - - -[Visualize_correlations] - -vars = ["eta", "shear_sq", "vort_sq", "det_shear", "T_tilde", "n_tilde"] - -# Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null], [null, null]], - "log_abs": [1, 1, 1, 1, 1, 1]} - -#set extractions 0 if you don't want to extract randomly from sample -extractions = 30000 - -#Options for building weights: 'Q2' 'Q1_skew' 'Q1_non_neg' 'residual_weights' 'denominator_weights'. Any other choice correspond to no weights - -weighing_func = residual_weights - -# this is only relevant when weights are built using 'residual_weights': specify a positive residual you want to consider for the weights -residual_str = pi_res_sq - -# this is only relevant when weights are built using 'denominator_weights': specify the positive quantity at the denominator of extracted coefficient -denominator_str = shear_sq - -# Set format_fig to either pdf or png -format_fig = png - - -[Regression_settings] - -dependent_var = eta -regressors = ["vort_sq", "det_shear"] -add_intercept = 1 - -# Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null], [null, null]], - "log_abs": [1, 1, 1, 1, 1, 1]} - -#set extractions 0 if you don't want to extract randomly from sample: extraction only in the scatter plot? Not in regression? -extractions = 30000 - -#Options for building weights: 'Q2' 'Q1_skew' 'Q1_non_neg' 'residual_weights' 'denominator_weights'. Any other choice correspond to no weights - -weighing_func = residual_weights - -# this is only relevant when weights are built using 'residual_weights': specify a positive residual you want to consider for the weights -residual_str = pi_res_sq - -# this is only relevant when weights are built using 'denominator_weights': specify the positive quantity at the denominator of extracted coefficient -denominator_str = shear_sq - -# Set format_fig to either pdf or png -format_fig = png - - -[PCA_settings] - -dependent_var = zeta -explanatory_vars = ["T_tilde", "rho_tilde"] -pcs_num = 1 - -regressors_2_reduce = ["T_tilde", "rho_tilde", "n_tilde"] -variance_wanted = 0.9 - -# Careful, only one dictionary here -#preprocess_data = {"pos_or_neg": [1,1,1], "log_or_not": [1,1,1]} - -# Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null]], - "log_abs": [1, 1, 1]} - -#set extractions 0 if you don't want to extract randomly from sample -extractions = 0 diff --git a/Proof_of_principle/config_meso.txt b/Proof_of_principle/config_meso.txt deleted file mode 100644 index 630bacf..0000000 --- a/Proof_of_principle/config_meso.txt +++ /dev/null @@ -1,27 +0,0 @@ -# CONFIGURATION FILE FOR SCRIPT PICKLING_MESO.PY -################################################# - -[Directories] - -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/ - -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled - -[Micro_model_settings] -#snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. -#the options is activated using the first key, while the list refers to the index of the elements of -#ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) - -snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [6,7,8,9,10,11,12,13,14]} - -[Meso_model_settings] - -meso_grid = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97], "num_T_slices": 3, "coarse_grain_factor": 1, "coarse_grain_time": false} - -filtering_options = {"box_len_ratio": 8.0, "filter_width_ratio": 8.0} - -n_cpus = 40 - -[Pickled_filenames] - -meso_pickled = /rHD2d_nocg_fw=bl=8dx.pickle diff --git a/Proof_of_principle/draft_scripts/.DS_Store b/Proof_of_principle/draft_scripts/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..34db8d0df971cdec600ada5cd3db916b8b909682 GIT binary patch literal 6148 zcmeHKO-e&C5T4P358QOAyIi3+h~;^LUO;NK3N5cx3NCZDLiZj)S8hCjOTYQ4_-Gbx z6pgaiY@ zKrj#t1Oq?H0Pbv&=F~9yU?3O>20j^(^C6)LX2)WvM+Z7B0f6$1R)H?Hgv2Dr>{tw8 zfv|-FEtI{)U<=25a=+|Y3@x14ix2jd_r(kA>d2qeoj4mt9}ENoeFlzgIF$SU62DBf z$nS^5C>RI^{uu*2XeaF$ALV!J!RN_co6xS%M8vO%0)gIp1YjWN$e9|=o', pi_res[h,i,j], pi_res[h,i,j], metric, metric) + temp = np.log10(np.sqrt(temp)) + pi_res_sq[i,j] = temp + + temp = np.einsum('ij,kl,kj,li->',pi_res_model[h,i,j], pi_res_model[h,i,j], metric, metric) + temp = np.log10(np.sqrt(temp)) + pi_res_sq_mod[i,j] = temp + + temp = np.einsum('i,ij,j->', q_res[h,i,j], metric, q_res[h,i,j]) + temp = np.log10(np.sqrt(temp)) + q_res_sq[i,j] = temp + + temp = np.einsum('i,ij,j->', q_res_model[h,i,j], metric, q_res_model[h,i,j]) + temp = np.log10(np.sqrt(temp)) + q_res_sq_mod[i,j] = temp + + # CREATING SUBPLOTS WITH CORRESPONDING DISTRIBUTIONS + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(1,3, figsize=[13,4]) + axes = axes.flatten() + + data = [[Pi_res, Pi_res_model], [pi_res_sq, pi_res_sq_mod], [q_res_sq, q_res_sq_mod],] + + x_axis_labels = [] + x_axis_labels.append(r'$\log(\tilde\Pi)$') + x_axis_labels.append(r'$\log(\sqrt{\tilde\pi_{ab}\tilde\pi^{ab}})$') + x_axis_labels.append(r'$\log(\sqrt{\tilde q_{a}\tilde q^{a}})$') + + + + for i in range(len(axes)): + X = data[i][0] + Y = data[i][1] + if X.shape != Y.shape: + print(f'Careful: shapes not compatible. i={i}') + + X = X.flatten() + Y = Y.flatten() + + stat = 'probability' + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + sns.histplot(X, stat=stat, kde=True, color='firebrick', ax=axes[i], label='sim. data') + sns.histplot(Y, stat=stat, kde=True, color='steelblue', ax=axes[i], label='model') + + axes[i].legend(loc = 'best', prop={'size': 10}) + axes[i].set_xlabel(x_axis_labels[i], fontsize=10) + axes[i].set_ylabel(stat, fontsize=10) + + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = f'/Const_coeff_models_' + f'{stat}' + 'EOS' + format = 'png' + dpi = 300 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + + diff --git a/Proof_of_principle/draft_scripts/draft_config.txt b/Proof_of_principle/draft_scripts/draft_config.txt new file mode 100644 index 0000000..9e78f12 --- /dev/null +++ b/Proof_of_principle/draft_scripts/draft_config.txt @@ -0,0 +1,34 @@ +# CONFIGURATION FILE FOR SCRIPTS VISUALIZING_*.PY +################################################# + +[Directories] + +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data + +figures_dir = ./ + + +[Filenames] + +meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle + + +[Models_settings] +#snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. +#relevant only for visualizing_micro.py + +snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [9,10,11]} + +num_T_slices = 3 + +[Plot_settings] +# method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids +# interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid +# Else, set method to "raw_data" to use gridded data directly + +plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} + +inset_ranges = {"x_range": [0.75, 0.85], "y_range": [0.55, 0.65]} + +diff_plot_settings = {"method": "raw_data", "interp_dims": [300, 300]} + diff --git a/Proof_of_principle/draft_scripts/filter_scaling.py b/Proof_of_principle/draft_scripts/filter_scaling.py new file mode 100644 index 0000000..00e67be --- /dev/null +++ b/Proof_of_principle/draft_scripts/filter_scaling.py @@ -0,0 +1,208 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +from matplotlib import colors +from mpl_toolkits.axes_grid1.inset_locator import InsetPosition + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ############################################################################ + # # SCRIPT TO SHOW HOW THE IMPACT OF FILTERING SCALES WITH THE FILTER-SIZE + # ############################################################################ + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO AND MICRO MODELS + pickle_directory = config['Directories']['pickled_files_dir'] + + + print('=========================================================================') + print(f'Starting job on data from {pickle_directory}') + print('=========================================================================\n\n') + + # meso_pickled_filename = '/rHD2d_cg=fw=bl=2dx.pickle' + meso_pickled_filename = '/rHD2d_nocg_fw=2_bl=8dx.pickle' + MesoModelLoadFile = pickle_directory + meso_pickled_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_2 = pickle.load(filehandle) + micro_model = meso_2.micro_model + + # meso_pickled_filename = '/rHD2d_cg=fw=bl=4dx.pickle' + meso_pickled_filename = '/rHD2d_nocg_fw=4_bl=8dx.pickle' + MesoModelLoadFile = pickle_directory + meso_pickled_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_4 = pickle.load(filehandle) + + # meso_pickled_filename = '/rHD2d_cg=fw=bl=8dx.pickle' + meso_pickled_filename = '/rHD2d_nocg_fw=bl=8dx.pickle' + MesoModelLoadFile = pickle_directory + meso_pickled_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_8 = pickle.load(filehandle) + + + print('Finished reading pickled data\n') + + # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE + num_snaps = micro_model.domain_vars['nt'] + central_slice_num = int(num_snaps/2.) + time_micro = micro_model.domain_vars['t'][central_slice_num] + num_slices_meso = int(config['Models_settings']['num_T_slices']) + time_meso2 = meso_2.domain_vars['T'][int((num_slices_meso-1)/2)] + time_meso4 = meso_2.domain_vars['T'][int((num_slices_meso-1)/2)] + time_meso8 = meso_2.domain_vars['T'][int((num_slices_meso-1)/2)] + + compatible = True + if time_micro != time_meso2 or time_meso2 != time_meso4 or time_meso4 != time_meso8: + compatible = False + + if not compatible: + print("Slices of meso and micro models do not coincide. Careful!\n") + else: + print("Comparing data at same time-slice, hurray!\n") + + # # PLOT SETTINGS + plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) + x_range = plot_ranges['x_range'] + y_range = plot_ranges['y_range'] + saving_directory = config['Directories']['figures_dir'] + visualizer = Plotter_2D() + # diff_plot_settings =json.loads(config['Plot_settings']['diff_plot_settings']) + + # Building the data for showing the relative difference between the various plots + models = [micro_model, meso_2, meso_4, meso_8] + var_str = 'BC' + comp = (0,) + + # full_range = [0.04,0.96] + datamicro, extentmicro = visualizer.get_var_data(micro_model, var_str, time_micro, x_range, y_range, comp) + data2, extent2 = visualizer.get_var_data(meso_2, var_str, time_meso2, x_range, y_range, comp) + data4, extent4 = visualizer.get_var_data(meso_4, var_str, time_meso4, x_range, y_range, comp) + data8, extent8 = visualizer.get_var_data(meso_8, var_str, time_meso8, x_range, y_range, comp) + + + abs_diff2 = data2 - datamicro + abs_diff4 = data4 - datamicro + abs_diff8 = data8 - datamicro + + ar_mean = (np.abs(data2) + np.abs(datamicro))/2 + rel_diff2 = np.abs(datamicro -data2)/ar_mean + rel_diff2 = rel_diff2 / 2. + + ar_mean = (np.abs(data4) + np.abs(datamicro))/2 + rel_diff4 = np.abs(data4 -datamicro)/ar_mean + rel_diff4 = rel_diff4 / 4 + + ar_mean = (np.abs(data8) + np.abs(datamicro))/2 + rel_diff8 = np.abs(data8 - datamicro)/ar_mean + rel_diff8 = rel_diff8 /8 + + + # PLOTTING + fig = plt.figure(figsize=[13,4]) + ax1 = fig.add_subplot(1, 4, 1) + ax2 = fig.add_subplot(1, 4, 2, sharey=ax1) + ax3 = fig.add_subplot(1, 4, 3, sharey=ax1) + ax4 = fig.add_subplot(1, 4, 4, sharey=ax1) + axes = [ax1, ax2, ax3, ax4] + plt.setp(ax2.get_yticklabels(), visible=False) + plt.setp(ax3.get_yticklabels(), visible=False) + plt.setp(ax4.get_yticklabels(), visible=False) + + + + images =[] + + im = ax1.imshow(datamicro, extent=extentmicro, origin='lower', cmap='plasma') #, norm='log') + divider = make_axes_locatable(ax1) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = ax2.imshow(rel_diff2, extent=extent2, origin='lower', cmap='plasma') #, norm='log') + divider = make_axes_locatable(ax2) + cax = divider.append_axes('right', size='5%', pad=0.05) + cax.set_axis_off() + images.append(im) + + im = ax3.imshow(rel_diff4, extent=extent4, origin='lower', cmap='plasma') #, norm='log') + divider = make_axes_locatable(ax3) + cax = divider.append_axes('right', size='5%', pad=0.05) + cax.set_axis_off() + images.append(im) + + im = ax4.imshow(rel_diff8, extent=extent8, origin='lower', cmap='plasma') #, norm='log') + divider = make_axes_locatable(ax4) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + images.append(im) + + for i in range(len(axes)): + axes[i].set_xlabel(r'$x$') + axes[i].set_ylabel(r'$y$') + + title = micro_model.labels_var_dict[var_str] + title += r'$,$ $a=0$' + axes[0].set_title(title) + + title = r'$Scaled$ $rel.$ $diff.,$ $L=2dx$' + # title = meso_2.labels_var_dict[var_str] + # title += r'$,$ $a=0,$ $L=2dx$' + axes[1].set_title(title) + + title = r'$Scaled$ $rel.$ $diff.,$ $L=4dx$' + # title = meso_4.labels_var_dict[var_str] + # title += r'$,$ $a=0,$ $L=4dx$' + axes[2].set_title(title) + + title = r'$Scaled$ $rel.$ $diff.,$ $L=8dx$' + # title = meso_8.labels_var_dict[var_str] + # title += r'$,$ $a=0,$ $L=8dx$' + axes[3].set_title(title) + + fig.tight_layout() + + # COMMON COLORMAP + # Finding the global min and max and setting the shared colormap based on these. + vmin = min(image.get_array().min() for image in images) + vmax = max(image.get_array().max() for image in images) + norm = colors.Normalize(vmin=vmin, vmax=vmax) + for im in images: + im.set_norm(norm) + + # fig.colorbar(images[0], ax=axes, orientation='vertical', location='right', shrink=0.52, pad=0.025 ) + + # Make images respond to changes in the norm of other images (e.g. via the + # "edit axis, curves and images parameters" GUI on Qt), but be careful not to + # recurse infinitely! + def update(changed_image): + for im in images: + if (changed_image.get_cmap() != im.get_cmap() + or changed_image.get_clim() != im.get_clim()): + im.set_cmap(changed_image.get_cmap()) + im.set_clim(changed_image.get_clim()) + + for im in images: + im.callbacks.connect('changed', update) + + + format='png' + dpi=400 + filename = f"/Lin_scale_filt_{var_str}_{comp[0]}." + format + plt.savefig(saving_directory + filename, format = format, dpi=dpi) + + diff --git a/Proof_of_principle/draft_scripts/outputs/.DS_Store b/Proof_of_principle/draft_scripts/outputs/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..5008ddfcf53c02e82d7eee2e57c38e5672ef89f6 GIT binary patch literal 6148 zcmeH~Jr2S!425mzP>H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0H1@V-^m;4Wg<&0T*E43hX&L&p$$qDprKhvt+--jT7}7np#A3 zem<@ulZcFPQ@L2!n>{z**++&mCkOWA81W14cNZlEfg7;MkzE(HCqgga^y>{tEnwC%0;vJ&^%eQ zLs35+`xjp>T0 Date: Fri, 12 Apr 2024 13:11:52 +0200 Subject: [PATCH 101/111] No real updates, just polishing --- .../config_calibration.txt | 6 +- .../outputs/serial_5797466.out | 138 ------------------ .../outputs/serial_5797515.out | 72 --------- .../calibration_scripts/py_parallel.slurm | 4 +- .../calibration_scripts/py_serial.slurm | 4 +- .../regress+residual_check.py | 26 ++-- Proof_of_principle/draft_scripts/.DS_Store | Bin 6148 -> 6148 bytes .../draft_scripts/config_draft.txt | 4 +- .../filter_scripts/config_filter.txt | 4 +- .../filter_scripts/filter_config.txt | 48 ------ .../filter_scripts/outputs/serial_5797282.out | 69 --------- .../filter_scripts/py_serial.slurm | 6 +- .../filter_scripts/visualizing_meso.py | 17 ++- master_files/FileReaders.py | 10 +- 14 files changed, 44 insertions(+), 364 deletions(-) delete mode 100644 Proof_of_principle/calibration_scripts/outputs/serial_5797466.out delete mode 100644 Proof_of_principle/calibration_scripts/outputs/serial_5797515.out delete mode 100644 Proof_of_principle/filter_scripts/filter_config.txt delete mode 100644 Proof_of_principle/filter_scripts/outputs/serial_5797282.out diff --git a/Proof_of_principle/calibration_scripts/config_calibration.txt b/Proof_of_principle/calibration_scripts/config_calibration.txt index fa243fc..4f05c1c 100644 --- a/Proof_of_principle/calibration_scripts/config_calibration.txt +++ b/Proof_of_principle/calibration_scripts/config_calibration.txt @@ -19,7 +19,7 @@ ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} num_T_slices = 3 # Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[0, null], [null, null], [null, null], [null, null], [null, null], [null, null], [0, null], [null, null]], +preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null], [null, null], [null, null], [null, null], [null, null], [null, null]], "log_abs": [1, 1, 1, 1, 1, 1, 1, 1]} #set extractions 0 if you don't want to extract randomly from sample @@ -79,14 +79,14 @@ num_T_slices = 3 add_intercept = 0 centralize = 1 -test_percentage = 0.3 +test_percentage = 0.2 # Set values to null if you do not want to restrict the data preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null], [null, null], [null, null], [null, null], [null, null], [null, null]], "log_abs": [1, 1, 1, 1, 1, 1, 1, 1]} -n_cpus = 40 +n_cpus = 30 # Set format_fig to either pdf or png format_fig = png diff --git a/Proof_of_principle/calibration_scripts/outputs/serial_5797466.out b/Proof_of_principle/calibration_scripts/outputs/serial_5797466.out deleted file mode 100644 index aff40f8..0000000 --- a/Proof_of_principle/calibration_scripts/outputs/serial_5797466.out +++ /dev/null @@ -1,138 +0,0 @@ -Running SLURM prolog script on gold51.cluster.local -=============================================================================== -Job started on Mon Apr 8 10:54:51 BST 2024 -Job ID : 5797466 -Job name : py_serial.slurm -WorkDir : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration -Command : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration/py_serial.slurm -Partition : serial -Num hosts : 1 -Num cores : 1 -Num of tasks : 1 -Hosts allocated : gold51 -Job Output Follows ... -=============================================================================== -DeprecationWarning: 'source deactivate' is deprecated. Use 'conda deactivate'. -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle -================================================ - - -Dependent var: eta, Explanatory vars: ['det_shear', 'Q1', 'Q2'] - -Residuals and closure ingredient for model check: pi_res_sq, shear_sq - -Trimming the dataset -Pre-processing dataset -regression coeffs: [-1.6437584434006427, 0.1485971230179958, 0.045456745458183234, -0.1324133697499324] - -finished plot, now making it nice - -Finished plot, now saving... - - -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle -================================================ - - -Producing scatter plot for the vars: ['eta', 'shear_sq', 'vort_sq', 'det_shear', 'T_tilde', 'n_tilde', 'Q1', 'Q2'] - -The string for building weights nope does not match any of the implemented routines. - -Trimming the dataset -Pre-processing dataset -Finished producing correlation plot for ['eta', 'shear_sq', 'vort_sq', 'det_shear', 'T_tilde', 'n_tilde', 'Q1', 'Q2'], saved as /Correlation_eta_shear_sq_vort_sq_det_shear_T_tilde_n_tilde_Q1_Q2.png - -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle -================================================ - - -Trying to reduce the following list to a smaller subset: ['det_shear', 'shear_sq', 'vort_sq', 'Q1', 'Q2'] - -Trimming the dataset -Pre-processing dataset -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle -================================================ - - -Dependent var: eta, Explanatory vars: ['n_tilde', 'T_tilde', 'det_shear', 'shear_sq', 'vort_sq', 'Q1', 'Q2'] - -Trimming the dataset -Pre-processing dataset -Preprocess_data dictionary is not compatible with list_of_arrays. Exiting -Scores of dependent var on principal components: -[-7.54696541e-12 -1.00000000e+00 1.46146095e-05 6.69383333e-07 - -3.16436543e-06 7.72403240e-06 -2.46691513e-14 8.40274830e-21] - -0-th highest PCs decomposition: -[-1.00000000e+00 1.46367506e-05 -7.10101218e-07 -7.54696540e-12 - 7.60710509e-06 -1.16321035e-07 3.32993677e-06 -6.61620063e-14] -Corresponding score: -0.9999999998581457 - -n_tilde-rescaled coefficient: 1.4636750621041325e-05 - -T_tilde-rescaled coefficient: -7.101012176048021e-07 - -det_shear-rescaled coefficient: -7.546965400152193e-12 - -shear_sq-rescaled coefficient: 7.6071050879069935e-06 - -vort_sq-rescaled coefficient: -1.1632103527774598e-07 - -Q1-rescaled coefficient: 3.3299367747304493e-06 - -Q2-rescaled coefficient: -6.616200627999648e-14 - - -Producing correlation plot using info from PCA -Extracting randomly from dataset -Traceback (most recent call last): - File "/mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration/hunting_correlations.py", line 111, in - new_data = statistical_tool.extract_randomly([x,y], extractions) - File "/mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration/../../master_files/Analysis.py", line 376, in extract_randomly - selected_indices = random.sample(available_indices, num_extractions) - File "/home/tc2m23/.conda/envs/myenv/lib/python3.10/random.py", line 482, in sample - raise ValueError("Sample larger than population or is negative") -ValueError: Sample larger than population or is negative -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle -================================================ - - -Dependent var: eta, Explanatory vars: ['vort_sq', 'det_shear', 'n_tilde', 'T_tilde'] - -The string for building weights no_weights does not match any of the implemented routines. - -Trimming the dataset -Pre-processing dataset -Centralizing the dataset -Splitting dataset into training and test set -regression coeffs: [0.16266840064158283, 0.21363488531182281, 0.3895166980016922, 0.3895409159101455] - -Corresponding std errors: None - -Finished regression and scatter plot for eta, saved as /Regress_eta_vs_det_shear_n_tilde_T_tilde.png - - -============================================================================== -Running epilogue script on gold51. - -Submit time : 2024-04-08T10:54:51 -Start time : 2024-04-08T10:54:51 -End time : 2024-04-08T10:58:41 -Elapsed time : 00:03:50 (Timelimit=00:10:00) - -Job ID: 5797466 -Cluster: i5 -User/Group: tc2m23/pj -State: COMPLETED (exit code 0) -Cores: 1 -CPU Utilized: 00:03:18 -CPU Efficiency: 86.09% of 00:03:50 core-walltime -Job Wall-clock time: 00:03:50 -Memory Utilized: 4.14 GB -Memory Efficiency: 17.92% of 23.07 GB - diff --git a/Proof_of_principle/calibration_scripts/outputs/serial_5797515.out b/Proof_of_principle/calibration_scripts/outputs/serial_5797515.out deleted file mode 100644 index 72ff1e5..0000000 --- a/Proof_of_principle/calibration_scripts/outputs/serial_5797515.out +++ /dev/null @@ -1,72 +0,0 @@ -Running SLURM prolog script on gold53.cluster.local -=============================================================================== -Job started on Mon Apr 8 11:00:53 BST 2024 -Job ID : 5797515 -Job name : py_serial.slurm -WorkDir : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration -Command : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_calibration/py_serial.slurm -Partition : serial -Num hosts : 1 -Num cores : 1 -Num of tasks : 1 -Hosts allocated : gold53 -Job Output Follows ... -=============================================================================== -DeprecationWarning: 'source deactivate' is deprecated. Use 'conda deactivate'. -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data/rHD2d_cg=fw=bl=8dx.pickle -================================================ - - -Dependent var: eta, Explanatory vars: ['n_tilde', 'T_tilde', 'det_shear', 'shear_sq', 'vort_sq', 'Q1', 'Q2'] - -Trimming the dataset -Pre-processing dataset -Preprocess_data dictionary is not compatible with list_of_arrays. Exiting -Scores of dependent var on principal components: -[-7.54696541e-12 -1.00000000e+00 1.46146095e-05 6.69383333e-07 - -3.16436543e-06 7.72403240e-06 -2.46691513e-14 8.40274830e-21] - -0-th highest PCs decomposition: -[-1.00000000e+00 1.46367506e-05 -7.10101218e-07 -7.54696540e-12 - 7.60710509e-06 -1.16321035e-07 3.32993677e-06 -6.61620063e-14] -Corresponding score: -0.9999999998581457 - -n_tilde-rescaled coefficient: 1.4636750621041325e-05 - -T_tilde-rescaled coefficient: -7.101012176048021e-07 - -det_shear-rescaled coefficient: -7.546965400152193e-12 - -shear_sq-rescaled coefficient: 7.6071050879069935e-06 - -vort_sq-rescaled coefficient: -1.1632103527774598e-07 - -Q1-rescaled coefficient: 3.3299367747304493e-06 - -Q2-rescaled coefficient: -6.616200627999648e-14 - - -Producing correlation plot using info from PCA -Finished correlation plot for eta, saved as /PCA_hunt_eta.png - - -============================================================================== -Running epilogue script on gold53. - -Submit time : 2024-04-08T11:00:52 -Start time : 2024-04-08T11:00:52 -End time : 2024-04-08T11:01:24 -Elapsed time : 00:00:32 (Timelimit=00:10:00) - -Job ID: 5797515 -Cluster: i5 -User/Group: tc2m23/pj -State: COMPLETED (exit code 0) -Cores: 1 -CPU Utilized: 00:00:20 -CPU Efficiency: 62.50% of 00:00:32 core-walltime -Job Wall-clock time: 00:00:32 -Memory Utilized: 4.12 GB -Memory Efficiency: 17.84% of 23.07 GB - diff --git a/Proof_of_principle/calibration_scripts/py_parallel.slurm b/Proof_of_principle/calibration_scripts/py_parallel.slurm index a0d4b62..351d7be 100644 --- a/Proof_of_principle/calibration_scripts/py_parallel.slurm +++ b/Proof_of_principle/calibration_scripts/py_parallel.slurm @@ -5,8 +5,8 @@ #SBATCH --output=outputs/parallel_%A.out #SBATCH --nodes=1 -#SBATCH --ntasks=40 -#SBATCH --time=01:00:00 +#SBATCH --ntasks=30 +#SBATCH --time=00:10:00 #SBATCH --mail-type=begin #SBATCH --mail-type=fail diff --git a/Proof_of_principle/calibration_scripts/py_serial.slurm b/Proof_of_principle/calibration_scripts/py_serial.slurm index 2e3dd7e..3cf5524 100644 --- a/Proof_of_principle/calibration_scripts/py_serial.slurm +++ b/Proof_of_principle/calibration_scripts/py_serial.slurm @@ -32,9 +32,11 @@ conda activate myenv # python3 -u regress+residual_check.py config_calibration.txt # python3 -u visualizing_correlations.py config_calibration.txt # python3 -u reducing_regressors.py config_calibration.txt -python3 -u hunting_correlations.py config_calibration.txt +# python3 -u hunting_correlations.py config_calibration.txt # python3 -u regressing_residual.py config_calibration.txt +python3 -u L_dep_eta.py config_calibration.txt + diff --git a/Proof_of_principle/calibration_scripts/regress+residual_check.py b/Proof_of_principle/calibration_scripts/regress+residual_check.py index 91f8cf9..be5f12e 100644 --- a/Proof_of_principle/calibration_scripts/regress+residual_check.py +++ b/Proof_of_principle/calibration_scripts/regress+residual_check.py @@ -173,7 +173,7 @@ # FINALLY: PLOTTING plt.rc("font", family="serif") plt.rc("mathtext",fontset="cm") - plt.rc('font', size=8) + plt.rc('font', size=10) fig, axes = plt.subplots(1,3, figsize=[13,4]) axes = axes.flatten() @@ -207,12 +207,12 @@ # comment the following line if you don't take the log of the residual residual_label = r"$\frac{1}{2}\log($" + residual_label + r"$)$" - axes[0].set_xlabel('Regression model', fontsize=8) - axes[0].set_ylabel(coeff_label, fontsize=8) - axes[1].set_xlabel(coeff_label, fontsize=8) - axes[1].set_ylabel('pdf', fontsize=8) - axes[2].set_xlabel(residual_label, fontsize=8) - axes[2].set_ylabel('pdf', fontsize=8) + axes[0].set_xlabel('Regression model', fontsize=12) + axes[0].set_ylabel(coeff_label, fontsize=12) + axes[1].set_xlabel(coeff_label, fontsize=12) + axes[1].set_ylabel('pdf', fontsize=12) + axes[2].set_xlabel(residual_label, fontsize=12) + axes[2].set_ylabel('pdf', fontsize=12) fig.tight_layout() @@ -265,24 +265,24 @@ bbox_args = dict(boxstyle="round", fc="0.95") - plt.annotate(text=text_for_box, xy = (0.32,0.2), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 8) + plt.annotate(text=text_for_box, xy = (0.32,0.2), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 10) # Adding legend to the distribution comparison panel - axes[1].legend(loc = 'best', prop={'size': 8}) + axes[1].legend(loc = 'best', prop={'size': 10}) h, _ = axes[1].get_legend_handles_labels() labels = ['Regression model', 'sim. data'] - axes[1].legend(h, labels, loc = 'best', prop={'size': 8, 'family' : 'serif'}) + axes[1].legend(h, labels, loc = 'best', prop={'size': 10, 'family' : 'serif'}) - axes[2].legend(loc = 'best', prop={'size': 8}) + axes[2].legend(loc = 'best', prop={'size': 10}) h, _ = axes[2].get_legend_handles_labels() labels = ['Regression model', 'sim. data'] - axes[2].legend(h, labels, loc = 'best', prop={'size': 8, 'family' : 'serif'}) + axes[2].legend(h, labels, loc = 'best', prop={'size': 10, 'family' : 'serif'}) # Saving the figure print(f'Finished plot, now saving...\n\n') saving_directory = config['Directories']['figures_dir'] - filename = '/PROVA' + filename = '/Regress_and_check' format = 'png' filename += "." + format dpi = 300 diff --git a/Proof_of_principle/draft_scripts/.DS_Store b/Proof_of_principle/draft_scripts/.DS_Store index 34db8d0df971cdec600ada5cd3db916b8b909682..87ed1dd73670c92235d99fd97e19e263a65648a2 100644 GIT binary patch delta 166 zcmZoMXfc=&$y~}%GBIXreLe$_!B7Aui*wQqgOl@f3mCvaf(1yiGoYx*&3AE0%E?ax r%5Z$z+jRNc!=sMaRHYE1svyHKm>Xf8!sdmHX>1c4*fz6s{N)D#HmWZA delta 166 zcmZoMXfc=&$y~&cIx%KzeHue1Lpnn#1CYy5oRe-CoSdIqzyJal@_ztHb_Ns`x%nj7=~v diff --git a/Proof_of_principle/draft_scripts/config_draft.txt b/Proof_of_principle/draft_scripts/config_draft.txt index 9e78f12..854b28b 100644 --- a/Proof_of_principle/draft_scripts/config_draft.txt +++ b/Proof_of_principle/draft_scripts/config_draft.txt @@ -3,14 +3,14 @@ [Directories] -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data figures_dir = ./ [Filenames] -meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle +meso_pickled_filename = /rHD2d_cg=fw=bl=8dx.pickle [Models_settings] diff --git a/Proof_of_principle/filter_scripts/config_filter.txt b/Proof_of_principle/filter_scripts/config_filter.txt index 3377906..fa02072 100644 --- a/Proof_of_principle/filter_scripts/config_filter.txt +++ b/Proof_of_principle/filter_scripts/config_filter.txt @@ -5,14 +5,14 @@ hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/20dx/ -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data +pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data figures_dir = ./ [Filenames] -meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle +meso_pickled_filename = /rHD2d_cg=fw=bl=8dx.pickle micro_pickled_filename = diff --git a/Proof_of_principle/filter_scripts/filter_config.txt b/Proof_of_principle/filter_scripts/filter_config.txt deleted file mode 100644 index 3377906..0000000 --- a/Proof_of_principle/filter_scripts/filter_config.txt +++ /dev/null @@ -1,48 +0,0 @@ -# CONFIGURATION FILE FOR SCRIPT PICKLING_MESO.PY -################################################# - -[Directories] - -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/20dx/ - -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data - -figures_dir = ./ - - -[Filenames] - -meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle - -micro_pickled_filename = - - -[Micro_model_settings] -#snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. -#the options is activated using the first key, while the list refers to the index of the elements of -#ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) - -snapshots_opts = {"fewer_snaps_required": true, - "smaller_list": [5,6,7,8,9,10,11,12,13,14,15]} - -[Meso_model_settings] - -meso_grid = {"x_range": [0.3, 0.4], "y_range": [0.3, 0.4], "num_T_slices": 3, - "coarse_grain_factor": 4, "coarse_grain_time": false} - -filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 4.0} - -n_cpus = 40 - - -[Plot_settings] -# method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids -# interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid -# Else, set method to "raw_data" to use gridded data directly - -plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} - -diff_plot_settings = {"method": "raw_data", "interp_dims": [300, 300]} - - - diff --git a/Proof_of_principle/filter_scripts/outputs/serial_5797282.out b/Proof_of_principle/filter_scripts/outputs/serial_5797282.out deleted file mode 100644 index a57ea8f..0000000 --- a/Proof_of_principle/filter_scripts/outputs/serial_5797282.out +++ /dev/null @@ -1,69 +0,0 @@ -Running SLURM prolog script on gold51.cluster.local -=============================================================================== -Job started on Mon Apr 8 09:21:34 BST 2024 -Job ID : 5797282 -Job name : py_serial.slurm -WorkDir : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_filter -Command : /mainfs/home/tc2m23/Filtering/Proof_of_principle/scripts_filter/py_serial.slurm -Partition : serial -Num hosts : 1 -Num cores : 1 -Num of tasks : 1 -Hosts allocated : gold51 -Job Output Follows ... -=============================================================================== -DeprecationWarning: 'source deactivate' is deprecated. Use 'conda deactivate'. -========================================================================= -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/20dx/ -========================================================================= - - -Finished reading micro data from hdf5, structures also set up. -No list of components indices passed: setting this to an empty list. -========================================================================= -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data/rHD2d_nocg_fw=bl=8dx.pickle -========================================================================= - - -Finished reading data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data/rHD2d_nocg_fw=bl=8dx.pickle -Comparing data at same time-slice, hurray! -Finished plotting the observers -========================================================================= -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data/rHD2d_nocg_fw=bl=8dx.pickle -========================================================================= - - -Finished reading pickled data - -Comparing data at same time-slice, hurray! - -================================================ -Starting job on data from /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data/rHD2d_nocg_fw=bl=8dx.pickle -================================================ - - -No list of components indices passed: setting this to an empty list. -Finished bulk viscosity -No list of components indices passed: setting this to an empty list. -Finished shear viscosity -No list of components indices passed: setting this to an empty list. -Finished heat conductivity -============================================================================== -Running epilogue script on gold51. - -Submit time : 2024-04-08T09:21:33 -Start time : 2024-04-08T09:21:34 -End time : 2024-04-08T09:25:01 -Elapsed time : 00:03:27 (Timelimit=00:10:00) - -Job ID: 5797282 -Cluster: i5 -User/Group: tc2m23/pj -State: COMPLETED (exit code 0) -Cores: 1 -CPU Utilized: 00:03:18 -CPU Efficiency: 95.65% of 00:03:27 core-walltime -Job Wall-clock time: 00:03:27 -Memory Utilized: 3.79 GB -Memory Efficiency: 16.41% of 23.07 GB - diff --git a/Proof_of_principle/filter_scripts/py_serial.slurm b/Proof_of_principle/filter_scripts/py_serial.slurm index a0a3e1a..c00fa73 100644 --- a/Proof_of_principle/filter_scripts/py_serial.slurm +++ b/Proof_of_principle/filter_scripts/py_serial.slurm @@ -31,10 +31,10 @@ conda activate myenv # LAUNCHING THE JOBS ##################################### -python3 -u visualizing_micro.py config_filter.txt -python3 -u visualizing_obs.py config_filter.txt +# python3 -u visualizing_micro.py config_filter.txt +# python3 -u visualizing_obs.py config_filter.txt python3 -u visualizing_meso.py config_filter.txt -python3 -u visualizing_residuals.py config_filter.txt +# python3 -u visualizing_residuals.py config_filter.txt diff --git a/Proof_of_principle/filter_scripts/visualizing_meso.py b/Proof_of_principle/filter_scripts/visualizing_meso.py index 9f4a1b4..619dc78 100644 --- a/Proof_of_principle/filter_scripts/visualizing_meso.py +++ b/Proof_of_principle/filter_scripts/visualizing_meso.py @@ -207,7 +207,10 @@ norms = ['log', 'log', 'log'] cmaps = ['plasma', 'plasma', 'plasma'] - fig, axes = plt.subplots(1,3, squeeze=False, figsize=[10,3], sharey=True) + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(1,3, squeeze=False, figsize=[12,4], sharey=True) axes = axes.flatten() images = [] @@ -232,12 +235,12 @@ # fig.colorbar(im, cax=cax, orientation='vertical') images.append(im) - axes[i].set_xlabel(r'$x$') - axes[i].set_ylabel(r'$y$') + axes[i].set_xlabel(r'$x$', fontsize=10) + axes[i].set_ylabel(r'$y$', fontsize=10) - axes[0].set_title(r'$\tilde{\Pi}$') - axes[1].set_title(r'$\sqrt{\tilde{\pi}_{ab}\tilde{\pi}^{ab}}$') - axes[2].set_title(r'$\sqrt{\tilde{q}_{a}\tilde{q}^{a}}$') + axes[0].set_title(r'$\tilde{\Pi}$', fontsize=14) + axes[1].set_title(r'$\sqrt{\tilde{\pi}_{ab}\tilde{\pi}^{ab}}$', fontsize=14) + axes[2].set_title(r'$\sqrt{\tilde{q}_{a}\tilde{q}^{a}}$', fontsize=14) fig.tight_layout() @@ -253,7 +256,7 @@ for im in images: im.set_norm(norm) - fig.colorbar(images[0], ax=axes.ravel().tolist(), orientation='vertical', location='right', shrink=1) + fig.colorbar(images[0], ax=axes.ravel().tolist(), orientation='vertical', location='right', shrink=0.9) # Make images respond to changes in the norm of other images (e.g. via the # "edit axis, curves and images parameters" GUI on Qt), but be careful not to diff --git a/master_files/FileReaders.py b/master_files/FileReaders.py index bd5dbcf..6ed3e9d 100644 --- a/master_files/FileReaders.py +++ b/master_files/FileReaders.py @@ -205,13 +205,15 @@ def read_in_data_HDF5_missing_xy(self, micro_model): micro_model.domain_vars['x'] = np.zeros(micro_model.domain_vars['nx']) for i in range(len(micro_model.domain_vars['x'])): - micro_model.domain_vars['x'][i] = (micro_model.domain_vars['xmax']- micro_model.domain_vars['xmin']) / (2 * micro_model.domain_vars['nx']) + \ - i * micro_model.domain_vars['dx'] + # micro_model.domain_vars['x'][i] = (micro_model.domain_vars['xmax']- micro_model.domain_vars['xmin']) / (2 * micro_model.domain_vars['nx']) + \ + # i * micro_model.domain_vars['dx'] + micro_model.domain_vars['x'][i] = micro_model.domain_vars['xmin'] + i * micro_model.domain_vars['dx'] micro_model.domain_vars['y'] = np.zeros(micro_model.domain_vars['ny']) for i in range(len(micro_model.domain_vars['y'])): - micro_model.domain_vars['y'][i] = (micro_model.domain_vars['ymax']- micro_model.domain_vars['ymin']) / (2 * micro_model.domain_vars['ny']) + \ - i * micro_model.domain_vars['dy'] + # micro_model.domain_vars['y'][i] = (micro_model.domain_vars['ymax']- micro_model.domain_vars['ymin']) / (2 * micro_model.domain_vars['ny']) + \ + # i * micro_model.domain_vars['dy'] + micro_model.domain_vars['y'][i] = micro_model.domain_vars['ymin'] + i * micro_model.domain_vars['dy'] micro_model.domain_vars['points'] = [micro_model.domain_vars['t'], micro_model.domain_vars['x'], \ micro_model.domain_vars['y']] From 0dc1ef6db46c6899c680b36cc2e2b11a8424fcb4 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Fri, 12 Apr 2024 13:12:55 +0200 Subject: [PATCH 102/111] Routine to check for residual scaling --- .../calibration_scripts/L_dep_eta.py | 138 ++++++++++++++++++ 1 file changed, 138 insertions(+) create mode 100644 Proof_of_principle/calibration_scripts/L_dep_eta.py diff --git a/Proof_of_principle/calibration_scripts/L_dep_eta.py b/Proof_of_principle/calibration_scripts/L_dep_eta.py new file mode 100644 index 0000000..f494109 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/L_dep_eta.py @@ -0,0 +1,138 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +import math +from scipy import stats +from itertools import product +import multiprocessing as mp +from sklearn.metrics import mean_absolute_error + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + + print('================================================') + print(f'Starting job on data from {pickle_directory}') + print('================================================\n\n') + + meso_filename_8 = '/rHD2d_cg=fw=bl=8dx.pickle' + meso_filename_4 = '/rHD2d_cg=fw=bl=4dx.pickle' + meso_filename_2 = '/rHD2d_cg=fw=bl=2dx.pickle' + + MesoModelLoadFile = pickle_directory + meso_filename_2 + with open(MesoModelLoadFile, 'rb') as filehandle: + meso2 = pickle.load(filehandle) + + MesoModelLoadFile = pickle_directory + meso_filename_4 + with open(MesoModelLoadFile, 'rb') as filehandle: + meso4 = pickle.load(filehandle) + + MesoModelLoadFile = pickle_directory + meso_filename_8 + with open(MesoModelLoadFile, 'rb') as filehandle: + meso8 = pickle.load(filehandle) + + eta2 = np.abs(meso2.meso_vars['eta']) + eta4 = np.abs(meso4.meso_vars['eta']) + eta8 = np.abs(meso8.meso_vars['eta']) + + etas = [eta2, eta4, eta8] + filter_sizes = [2,4,8] + + log_etas = [np.log10(elem) for elem in etas] + means = [np.mean(elem) for elem in log_etas] + print(f'The log-mean values of eta are: {means[0]}, {means[1]}, {means[2]}') + + etas_rescaled = [etas[i]/(filter_sizes[i]**2) for i in range(len(etas))] + log_etas_rescaled = [np.log10(elem) for elem in etas_rescaled] + + shear_sq2 = meso2.meso_vars['shear_sq'] + shear_sq4 = meso4.meso_vars['shear_sq'] + shear_sq8 = meso8.meso_vars['shear_sq'] + + pi_res_sq2 = meso2.meso_vars['pi_res_sq'] + pi_res_sq4 = meso4.meso_vars['pi_res_sq'] + pi_res_sq8 = meso8.meso_vars['pi_res_sq'] + + shear_sqs = [np.log10(elem) for elem in [shear_sq2, shear_sq4, shear_sq8]] + pi_res_sqs = [np.log10(elem) for elem in [pi_res_sq2, pi_res_sq4, pi_res_sq8]] + + means = [np.mean(elem) for elem in pi_res_sqs] + print(f'The log-mean value of pi_res_sq are: {means[0]}, {means[1]}, {means[2]}') + + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(1,3, figsize=[12,4]) + axes = axes.flatten() + + stat = 'density' + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + + sns.histplot(shear_sqs[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[0], label=r'$L=2dx$') + sns.histplot(shear_sqs[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[0], label=r'$L=4dx$') + sns.histplot(shear_sqs[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[0], label=r'$L=8dx$') + + sns.histplot(pi_res_sqs[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[1], label=r'$L=2dx$') + sns.histplot(pi_res_sqs[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[1], label=r'$L=4dx$') + sns.histplot(pi_res_sqs[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[1], label=r'$L=8dx$') + + sns.histplot(log_etas_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[2], label=r'$L=2dx$') + sns.histplot(log_etas_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[2], label=r'$L=4dx$') + sns.histplot(log_etas_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[2], label=r'$L=8dx$') + + + axes[0].legend(loc = 'best', prop={'size': 10}) + axes[1].legend(loc = 'best', prop={'size': 10}) + axes[2].legend(loc = 'best', prop={'size': 10}) + + axes[0].set_ylabel('pdf', fontsize=12) + axes[1].set_ylabel('pdf', fontsize=12) + axes[2].set_ylabel('pdf', fontsize=12) + + + xlabel = meso2.labels_var_dict['shear_sq'] + xlabel = r'$\log($' + xlabel + r'$)$' + axes[0].set_xlabel(xlabel, fontsize=12) + + xlabel = meso2.labels_var_dict['pi_res_sq'] + xlabel = r'$\log($' + xlabel + r'$)$' + axes[1].set_xlabel(xlabel, fontsize=12) + + axes[2].set_xlabel(r'$\eta/\tilde{L}^2,\qquad \tilde{L} = L/dx$', fontsize=12) + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = 'L_dep_eta' + format = 'png' + dpi = 400 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + + + + + + + From 1083f7b9ffdcc4d1fa6cb09ec1dc23d35f6f88f6 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Mon, 22 Apr 2024 17:43:54 +0200 Subject: [PATCH 103/111] Adding routine to check residual scaling with filter size --- .../config_calibration.txt | 25 +-- .../calibration_scripts/find_best_fit.py | 1 - .../fs_residual_dependence.py | 146 ++++++++++++++++++ .../regress+residual_check.py | 10 +- 4 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 Proof_of_principle/calibration_scripts/fs_residual_dependence.py diff --git a/Proof_of_principle/calibration_scripts/config_calibration.txt b/Proof_of_principle/calibration_scripts/config_calibration.txt index 4f05c1c..d9de56c 100644 --- a/Proof_of_principle/calibration_scripts/config_calibration.txt +++ b/Proof_of_principle/calibration_scripts/config_calibration.txt @@ -38,6 +38,11 @@ denominator_str = shear_sq # Set format_fig to either pdf or png format_fig = png +[Fs_residual_dependence] + +coeff = zeta +residual = Pi_res +EL_force = exp_tilde [Regression_settings] @@ -72,8 +77,8 @@ format_fig = png [Find_best_fit_settings] -var_to_model = eta -regressors_strs = ["vort_sq", "det_shear", "shear_sq", "T_tilde", "n_tilde", "Q1", "Q2"] +var_to_model = zeta +regressors_strs = ["vort_sq", "det_shear", "shear_sq", "T_tilde", "n_tilde", "Q1", "Q2", "acc_mag", "Theta_sq"] ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} num_T_slices = 3 @@ -83,8 +88,7 @@ test_percentage = 0.2 # Set values to null if you do not want to restrict the data -preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null], [null, null], [null, null], [null, null], [null, null], [null, null]], - "log_abs": [1, 1, 1, 1, 1, 1, 1, 1]} +preprocess_data = {"log_abs": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]} n_cpus = 30 @@ -93,10 +97,10 @@ format_fig = png [Regress+residual_check_settings] -coeff_str = eta -coeff_regressors_strs = ["det_shear", "Q1", "Q2"] -residual_str = pi_res_sq -closure_ingr_str = shear_sq +coeff_str = zeta +coeff_regressors_strs = ["n_tilde", "T_tilde"] +residual_str = Pi_res +closure_ingr_str = exp_tilde regression_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} idxs_time_slices = [1] @@ -106,9 +110,8 @@ centralize = 0 #set the following 0 if you don't want to split into train and test test_percentage = 0. -preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null], [null, null], [null, null], [null, null]], - "log_abs": [1, 1, 1, 1, 1, 1], - "sqrt": [0, 0, 0, 0, 1, 1]} +preprocess_data = { "log_abs": [1, 1, 1, 1, 1], + "sqrt": [0, 0, 0, 0, 0]} [PCA_settings] diff --git a/Proof_of_principle/calibration_scripts/find_best_fit.py b/Proof_of_principle/calibration_scripts/find_best_fit.py index c517029..9e58d5b 100644 --- a/Proof_of_principle/calibration_scripts/find_best_fit.py +++ b/Proof_of_principle/calibration_scripts/find_best_fit.py @@ -171,7 +171,6 @@ def parall_regress_task(comb_regressors): ordering_idx = np.argsort(wassersteins) - # Change this: output also the coefficients? Yes but for check! # for i in range(-1,-6,-1): for i in range(0,6,1): index = ordering_idx[i] diff --git a/Proof_of_principle/calibration_scripts/fs_residual_dependence.py b/Proof_of_principle/calibration_scripts/fs_residual_dependence.py new file mode 100644 index 0000000..1e67321 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/fs_residual_dependence.py @@ -0,0 +1,146 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +import math +from scipy import stats +from itertools import product +import multiprocessing as mp +from sklearn.metrics import mean_absolute_error + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + + print('================================================') + print(f'Starting job on data from {pickle_directory}') + print('================================================\n\n') + + meso_filename_8 = '/rHD2d_cg=fw=bl=8dx.pickle' + meso_filename_4 = '/rHD2d_cg=fw=bl=4dx.pickle' + meso_filename_2 = '/rHD2d_cg=fw=bl=2dx.pickle' + + MesoModelLoadFile = pickle_directory + meso_filename_2 + with open(MesoModelLoadFile, 'rb') as filehandle: + meso2 = pickle.load(filehandle) + + MesoModelLoadFile = pickle_directory + meso_filename_4 + with open(MesoModelLoadFile, 'rb') as filehandle: + meso4 = pickle.load(filehandle) + + MesoModelLoadFile = pickle_directory + meso_filename_8 + with open(MesoModelLoadFile, 'rb') as filehandle: + meso8 = pickle.load(filehandle) + + # READING INFO ON RESIDUALS FROM CONFIG FILE + coeff_str = config['Fs_residual_dependence']['coeff'] + residual_str = config['Fs_residual_dependence']['residual'] + EL_force_str = config['Fs_residual_dependence']['EL_force'] + + print(f'coeff_str: {coeff_str}\nresidual_str: {residual_str}\nEL_force_str: {EL_force_str}\n') + + coeff2 = np.abs(meso2.meso_vars[coeff_str]) + coeff4 = np.abs(meso4.meso_vars[coeff_str]) + coeff8 = np.abs(meso8.meso_vars[coeff_str]) + + coeffs = [coeff2, coeff4, coeff8] + filter_sizes = [2,4,8] + + log_coeffs = [np.log10(elem) for elem in coeffs] + means = [np.mean(elem) for elem in log_coeffs] + print(f'The log-mean values of {coeff_str} are: {means[0]}, {means[1]}, {means[2]}') + + coeffs_rescaled = [coeffs[i]/(filter_sizes[i]**2) for i in range(len(coeffs))] + log_coeffs_rescaled = [np.log10(elem) for elem in coeffs_rescaled] + + EL_force2 = np.abs(meso2.meso_vars[EL_force_str]) + EL_force4 = np.abs(meso4.meso_vars[EL_force_str]) + EL_force8 = np.abs(meso8.meso_vars[EL_force_str]) + + residual2 = meso2.meso_vars[residual_str] + residual4 = meso4.meso_vars[residual_str] + residual8 = meso8.meso_vars[residual_str] + + log_residuals = [np.log10(elem) for elem in [residual2, residual4, residual8]] + log_EL_forces = [np.log10(elem) for elem in [EL_force2, EL_force4, EL_force8]] + + means = [np.mean(elem) for elem in log_residuals] + print(f'The log-mean value of pi_res_sq are: {means[0]}, {means[1]}, {means[2]}') + + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(1,3, figsize=[12,4]) + axes = axes.flatten() + + stat = 'density' + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + + sns.histplot(log_EL_forces[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[0], label=r'$L=2dx$') + sns.histplot(log_EL_forces[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[0], label=r'$L=4dx$') + sns.histplot(log_EL_forces[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[0], label=r'$L=8dx$') + + sns.histplot(log_residuals[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[1], label=r'$L=2dx$') + sns.histplot(log_residuals[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[1], label=r'$L=4dx$') + sns.histplot(log_residuals[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[1], label=r'$L=8dx$') + + sns.histplot(log_coeffs_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[2], label=r'$L=2dx$') + sns.histplot(log_coeffs_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[2], label=r'$L=4dx$') + sns.histplot(log_coeffs_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[2], label=r'$L=8dx$') + + + axes[0].legend(loc = 'best', prop={'size': 10}) + axes[1].legend(loc = 'best', prop={'size': 10}) + axes[2].legend(loc = 'best', prop={'size': 10}) + + axes[0].set_ylabel('pdf', fontsize=12) + axes[1].set_ylabel('pdf', fontsize=12) + axes[2].set_ylabel('pdf', fontsize=12) + + xlabel = meso2.labels_var_dict[EL_force_str] + xlabel = r'$\log(|$' + xlabel + r'$|)$' + axes[0].set_xlabel(xlabel, fontsize=12) + + xlabel = meso2.labels_var_dict[residual_str] + xlabel = r'$\log($' + xlabel + r'$)$' + axes[1].set_xlabel(xlabel, fontsize=12) + + xlabel = meso2.labels_var_dict[coeff_str] + xlabel = xlabel + r'$/\tilde{L}^2,\qquad \tilde{L} = L/dx$' + axes[2].set_xlabel(xlabel, fontsize=12) + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = 'fs_residual_dependence' + format = 'png' + dpi = 400 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + + + + + + + diff --git a/Proof_of_principle/calibration_scripts/regress+residual_check.py b/Proof_of_principle/calibration_scripts/regress+residual_check.py index be5f12e..af2ddd9 100644 --- a/Proof_of_principle/calibration_scripts/regress+residual_check.py +++ b/Proof_of_principle/calibration_scripts/regress+residual_check.py @@ -181,6 +181,11 @@ warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + coeff_model = coeff_model.flatten() + coeff_test = coeff_test.flatten() + residual_model = residual_model.flatten() + residual_test = residual_test.flatten() + sns.set_theme(style="dark") sns.scatterplot(x=coeff_model, y=coeff_test, s=4, color=".15", ax=axes[0]) sns.histplot(x=coeff_model, y=coeff_test, bins=50, ax=axes[0], pthresh=.1, cmap="mako") @@ -205,7 +210,8 @@ if hasattr(meso_model, 'labels_var_dict') and residual_str in meso_model.labels_var_dict.keys(): residual_label = meso_model.labels_var_dict[residual_str] # comment the following line if you don't take the log of the residual - residual_label = r"$\frac{1}{2}\log($" + residual_label + r"$)$" + # residual_label = r"$\frac{1}{2}\log($" + residual_label + r"$)$" + residual_label = r"$\log($" + residual_label + r"$)$" axes[0].set_xlabel('Regression model', fontsize=12) axes[0].set_ylabel(coeff_label, fontsize=12) @@ -265,7 +271,7 @@ bbox_args = dict(boxstyle="round", fc="0.95") - plt.annotate(text=text_for_box, xy = (0.32,0.2), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 10) + plt.annotate(text=text_for_box, xy = (0.18,0.95), xycoords='figure fraction', bbox=bbox_args, ha="right", va="top", fontsize = 10) # Adding legend to the distribution comparison panel From d70e9264eb72c93317439a356c46a95016d7c9e4 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 25 Apr 2024 09:54:56 +0200 Subject: [PATCH 104/111] Adding routines to compute coeff componentwise in parallel --- master_files/MesoModels.py | 109 ++++++++++++++++++++++++++++++++++++- 1 file changed, 106 insertions(+), 3 deletions(-) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index e323c45..d2a400e 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -2213,7 +2213,7 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): metric[1,1] = metric[2,2] = +1 # CALCULATING BULK VISCOUS COEFF - zeta = Pi_res / exp + zeta = - Pi_res / exp coefficients_names.append('zeta') coefficients.append(zeta) @@ -2227,7 +2227,7 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): transformed_shear = np.einsum('ji,jk,kl', pi_eigv, shear, pi_eigv) abs_pi_eig = np.abs(pi_eig) pos = list(abs_pi_eig).index(np.max(abs_pi_eig)) - eta = eta * np.sign(pi_eig[pos] / transformed_shear[pos,pos]) + eta = eta * np.sign( - pi_eig[pos] / transformed_shear[pos,pos]) coefficients_names.append('eta') coefficients.append(eta) @@ -2235,7 +2235,7 @@ def EL_style_closure_task(Pi_res, exp, pi_res, shear, q_res, Theta, h, i, j): q_res_sq = np.einsum('i,ij,j', q_res, metric, q_res) Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) cos_angle = np.einsum('i,ij,j', q_res, metric, Theta) / (q_res_sq * Theta_sq) - sign = np.sign(cos_angle) + sign = - np.sign(cos_angle) kappa = sign * np.sqrt(q_res_sq / Theta_sq) coefficients_names.append('kappa') coefficients.append(kappa) @@ -2295,6 +2295,109 @@ def EL_style_closure_parallel(self, n_cpus): finally: self.meso_vars[key][tuple(grid_idxs)] = values[idx] + @staticmethod + def EL_componentwise_task(pi_res, shear, h, i, j): + """ + Task for computing the shear coefficients componentwise + + Parameters: + ----------- + pi_res: nd.array + the anisotropic stress residual at gridpoint h,i,j + + shear: nd.array + the (Favre-observer) shear tensor at gridpoint h,i,j + + Returns: + -------- + list of strs: + the names by which you want to store data in the class instance + + nd.array: + the componentwise values of the coefficient, for now eta only + + list: + the gridpoint indices on the meso-grid + + Notes: + ------ + """ + eta_componentwise = np.zeros(pi_res.shape) + eta_componentwise = - np.divide(pi_res, shear) + coeff_name = 'eta_cw' + + return [coeff_name], [eta_componentwise], [h, i, j] + + def EL_componentwise_parallel(self, n_cpus, store=False): + """ + Wrapper of EL_componentwise_task: launch the task in parallel with n_cpus + + Parameters: + ----------- + n_cpus: int + number of processes for parallelization + + store: bool + If true, the meso_vars dictionary is extended to include these + Otherwise, the dictionary is created and returned + """ + args_for_pool=[] + + Nt = self.domain_vars['Nt'] + Nx = self.domain_vars['Nx'] + Ny = self.domain_vars['Ny'] + + printshit = self.meso_vars['shear_tilde'].shape + print(f'shear_tilde.shape: {printshit}') + printshit = self.meso_vars['pi_res'].shape + print(f'pi_res.shape: {printshit}') + + for h in range(Nt): + for i in range(Nx): + for j in range(Ny): + shear_t = self.meso_vars['shear_tilde'][h,i,j] + pi_res = self.meso_vars['pi_res'][h,i,j] + + args_for_pool.append((pi_res, shear_t, h, i, j)) + + with mp.Pool(processes=n_cpus) as pool: + print('Computing vars for modelling coefficients with {} processes'.format(pool._processes), flush=True) + + for result in pool.starmap(resHD2D.EL_componentwise_task, args_for_pool): + keys, values, grid_idxs = result + + if store: + for idx, key in enumerate(keys): + try: + self.meso_vars[key] + except KeyError: + print('The key {} does not belong to meso_vars yet, adding it!'.format(key), flush=True) + try: + shape = values[idx].shape + except AttributeError: + shape = [] + self.meso_vars.update({key : np.zeros(([Nt,Nx,Ny]+ list(shape)))}) + finally: + self.meso_vars[key][tuple(grid_idxs)] = values[idx] + + else: + for i, key in enumerate(keys): + try: + results_dictionary[key] + except UnboundLocalError: + results_dictionary = dict.fromkeys([key]) + shape = values[i].shape + results_dictionary[key] = np.zeros(([Nt, Nx, Ny] + list(shape))) + except KeyError: + shape = values[i].shape + results_dictionary.update( {key : np.zeros(([Nt, Nx, Ny] + list(shape)))} ) + + finally: + results_dictionary[key][tuple(grid_idxs)] = values[i] + + if not store: + return results_dictionary + @staticmethod def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, h, i, j): """ From 595ce3559ef10c0fecfbdd9ef9781db5c8ef1b64 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 25 Apr 2024 15:10:57 +0200 Subject: [PATCH 105/111] Adding more closure ingredients --- master_files/Analysis.py | 10 +++--- master_files/MesoModels.py | 69 +++++++++++++++++++++++++++----------- 2 files changed, 54 insertions(+), 25 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 51d6d20..0c5cbae 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -284,17 +284,17 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): if weights is not None: Weights = np.ma.masked_array(weights, tot_mask).compressed() #when array is compressed, this is automatically flattened! - - if preprocess_data.__contains__('sqrt'): - for i in range(len(processed_list)): - if preprocess_data['sqrt'][i]: - processed_list[i] = np.sqrt(processed_list[i]) if preprocess_data.__contains__('log_abs'): for i in range(len(processed_list)): if preprocess_data['log_abs'][i]: processed_list[i] = np.log10(np.abs(processed_list[i])) + if preprocess_data.__contains__('sqrt'): + for i in range(len(processed_list)): + if preprocess_data['sqrt'][i]: + processed_list[i] = np.sqrt(processed_list[i]) + # removing corresponding entries from weights if weights is not None: return processed_list, Weights diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index d2a400e..9796fca 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1172,7 +1172,7 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): # dictionary with non-local quantities (keys must match one of meso_vars or structure) - self.nonlocal_vars_strs = ['u_tilde', 'T_tilde'] + self.nonlocal_vars_strs = ['u_tilde', 'T_tilde', 'n_tilde', 'eps_tilde'] Dstrs = ['D_' + i for i in self.nonlocal_vars_strs] self.deriv_vars = dict.fromkeys(Dstrs) @@ -1195,19 +1195,26 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): self.labels_var_dict = {'SET' : r'$$', 'BC' : r'$$', + 'U' : r'$U^a$', + 'Gamma' : r'$\Gamma$', + 'eos_res' : r'$M$', + 'Pi_res' : r'$\tilde{\Pi}$', + 'q_res' : r'$\tilde{q}^a$', + 'pi_res' : r'$\tilde{\pi}^{ab}$', 'eps_tilde' : r'$\tilde{\varepsilon}$', 'n_tilde' : r'$\tilde{n}$', 'p_tilde' : r'$\tilde{p}$', 'p_filt' : r'$

$', - 'eos_res' : r'$M$', - 'Pi_res' : r'$\tilde{\Pi}$', 'T_tilde' : r'$\tilde{T}$', 'u_tilde' : r'$\tilde{u}^a$', 'D_u_tilde' : r'$\nabla_{a}\tilde{u}^b$', 'D_T_tilde' : r'$\nabla_{a}\tilde{T}$', - 'q_res' : r'$\tilde{q}^a$', - 'pi_res' : r'$\tilde{\pi}^{ab}$', - 'Gamma' : r'$\Gamma$', + 'D_n_tilde' : r'$\nabla_{a}\tilde{n}$', + 'D_eps_tilde' : r'$\nabla_{a}\tilde{\varepsilon}$', + 'n_tilde_dot' : r'$\dot{\tilde{n}}$', + 'T_tilde_dot' : r'$\dot{\tilde{T}}$', + 'sD_T_tilde': r'$D_{a}\tilde{T}$', + 'sD_n_tilde': r'$D_{a}\tilde{n}$', 'shear_tilde' : r'$\tilde{\sigma}^{ab}$', 'acc_tilde' : r'$\tilde{a}^a$', 'exp_tilde' : r'$\tilde{\theta}$', @@ -1223,7 +1230,8 @@ def __init__(self, micro_model, find_obs, filter, interp_method = 'linear'): 'det_shear': r'$det(\sigma)$', 'vort_sq' : r'$\omega_{ab}\omega^{ab}$', 'acc_mag': r'$|a|$', - 'U' : r'$U^a$', + 'sD_n_tilde_sq' : r'$D_{a}\tilde{n}D^{a}\tilde{n}$', + 'dot_Dn_Theta' : r'$D_{a}\tilde{n}\Theta^{a}$', 'Q1' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab} - \tilde{\omega}_{ab}\tilde{\omega}^{ab}$', 'Q2' : r'$\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}/\tilde{\omega}_{ab}\tilde{\omega}^{ab}$', 'weights' : r'$w$'} @@ -1984,7 +1992,7 @@ def closure_ingredients(self): self.meso_vars[key][h,i,j] = values[idx] @staticmethod - def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): + def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, nabla_n, h, i, j): """ Task to decompose Favre obs + Temperature derivatives These will be used in EL_style_closure to extract the @@ -2056,15 +2064,22 @@ def closure_ingredients_task(u_t, nabla_u, T_t, nabla_T, h, i, j): # vort_t = np.multiply(1/2., Daub - np.einsum('ij->ji', Daub)) - # CLOSURE INGREDIENTS: HEAT FLUX + # CLOSURE INGREDIENTS: DERIVATIVES OF N_TILDE AND T_TILDE projector = np.zeros((3,3)) np.fill_diagonal(projector, 1) projector += np.einsum('i,j->ij',u_t_cov, u_t) #This is different from the projector above: look at indices! - DaT = np.einsum('ij,j->i', projector, nabla_T) - Theta_t = DaT + np.multiply(T_t, np.einsum('ij,j->i', metric, acc_t)) - closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'vort_tilde','Theta_tilde'] - closure_vars = [shear_t, exp_t, acc_t, vort_t, Theta_t] + sD_n_tilde = np.einsum('ij,j->i', projector, nabla_n) + sD_T_tilde = np.einsum('ij,j->i', projector, nabla_T) + + n_tilde_dot = np.einsum('i,i->', u_t, nabla_n) + T_tilde_dot = np.einsum('i,i->', u_t, nabla_T) + + Theta_t = sD_T_tilde + np.multiply(T_t, np.einsum('ij,j->i', metric, acc_t)) + + closure_vars_strs = ['shear_tilde', 'exp_tilde', 'acc_tilde', 'vort_tilde','Theta_tilde', 'n_tilde_dot', \ + 'T_tilde_dot', 'sD_n_tilde', 'sD_T_tilde' ] + closure_vars = [shear_t, exp_t, acc_t, vort_t, Theta_t, n_tilde_dot, T_tilde_dot, sD_n_tilde, sD_T_tilde] return closure_vars_strs, closure_vars, [h,i,j] def closure_ingredients_parallel(self, n_cpus): @@ -2092,7 +2107,8 @@ def closure_ingredients_parallel(self, n_cpus): nabla_u = self.deriv_vars['D_u_tilde'][h,i,j] T_t = self.meso_vars['T_tilde'][h,i,j] nabla_T = self.deriv_vars['D_T_tilde'][h,i,j] - args_for_pool.append((u_t, nabla_u, T_t, nabla_T, h, i, j)) + nabla_n = self.deriv_vars['D_n_tilde'][h,i,j] + args_for_pool.append((u_t, nabla_u, T_t, nabla_T, nabla_n, h, i, j)) with mp.Pool(processes=n_cpus) as pool: print('Computing closure ingredients with {} processes'.format(pool._processes), flush=True) @@ -2399,7 +2415,7 @@ def EL_componentwise_parallel(self, n_cpus, store=False): return results_dictionary @staticmethod - def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, h, i, j): + def modelling_coefficients_task(shear, vort, acc, Theta, sD_n_tilde, Pi_res, pi_res, q_res, h, i, j): """ Task (to be used in parallel) to compute various quantities that will be used later for modelling the extracted closure coefficients. @@ -2441,19 +2457,31 @@ def modelling_coefficients_task(shear, vort, acc, Theta, Pi_res, pi_res, q_res, var_names.append('Q2') vars.append(Q2) - #Computing squares of residuals and also of Theta_tilde + # Computing scalars out of thermo gradients + Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) + var_names.append('Theta_sq') + vars.append(Theta_sq) + + sD_n_tilde_sq = np.einsum('i,ij,j->', sD_n_tilde, metric, sD_n_tilde) + var_names.append('sD_n_tilde_sq') + vars.append(sD_n_tilde_sq) + + dot_Dn_Theta = np.einsum('i,ij,j->', sD_n_tilde, metric, Theta ) + var_names.append('dot_Dn_Theta') + vars.append(dot_Dn_Theta) + + # Computing squares of residuals and also of Theta_tilde Pi_res_sq = Pi_res * Pi_res var_names.append('Pi_res_sq') vars.append(Pi_res_sq) + pi_res_sq = np.einsum('ij,kl,ik,jl->', pi_res, pi_res, metric, metric) var_names.append('pi_res_sq') vars.append(pi_res_sq) + q_res_sq = np.einsum('i,ij,j', q_res, metric, q_res) var_names.append('q_res_sq') vars.append(q_res_sq) - Theta_sq = np.einsum('i,ij,j', Theta, metric, Theta) - var_names.append('Theta_sq') - vars.append(Theta_sq) return var_names, vars, [h,i,j] @@ -2474,11 +2502,12 @@ def modelling_coefficients_parallel(self, n_cpus): vort_t = self.meso_vars['vort_tilde'][h,i,j] acc_t = self.meso_vars['acc_tilde'][h,i,j] Theta_t = self.meso_vars['Theta_tilde'][h,i,j] + sD_n_tilde = self.meso_vars['sD_n_tilde'][h,i,j] Pi_res = self.meso_vars['Pi_res'][h,i,j] pi_res = self.meso_vars['pi_res'][h,i,j] q_res = self.meso_vars['q_res'][h,i,j] - args_for_pool.append((shear_t, vort_t, acc_t, Theta_t, Pi_res, pi_res, q_res, h, i, j)) + args_for_pool.append((shear_t, vort_t, acc_t, Theta_t, sD_n_tilde, Pi_res, pi_res, q_res, h, i, j)) with mp.Pool(processes=n_cpus) as pool: print('Computing vars for modelling coefficients with {} processes'.format(pool._processes), flush=True) From 2017eb62de2433d8c82fa2c3bb81aed6e8d88e4d Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Thu, 25 Apr 2024 16:33:28 +0200 Subject: [PATCH 106/111] going back to computing the EoS residual separately --- master_files/MesoModels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index 9796fca..cf0e110 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1772,6 +1772,7 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): s_ab_tracefree = s_ab - np.multiply(s, metric + np.einsum('i,j->ij', u_t, u_t)) p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) + # rechange this: this was just to test the impact of EOS residuals onto the modelling of zeta # Pi_res = s - p_t Pi_res = s - p_filt EOS_res = p_filt - p_t From f1acb752bc39b4dd91316177c371025274e092b1 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 7 May 2024 11:43:22 +0200 Subject: [PATCH 107/111] cleaning repo --- .../calibration_scripts/L_dep_eta.py | 138 ------------------ .../draft_scripts/draft_config.txt | 34 ----- 2 files changed, 172 deletions(-) delete mode 100644 Proof_of_principle/calibration_scripts/L_dep_eta.py delete mode 100644 Proof_of_principle/draft_scripts/draft_config.txt diff --git a/Proof_of_principle/calibration_scripts/L_dep_eta.py b/Proof_of_principle/calibration_scripts/L_dep_eta.py deleted file mode 100644 index f494109..0000000 --- a/Proof_of_principle/calibration_scripts/L_dep_eta.py +++ /dev/null @@ -1,138 +0,0 @@ -import sys -# import os -sys.path.append('../../master_files/') -import pickle -import configparser -import json -import time -import math -from scipy import stats -from itertools import product -import multiprocessing as mp -from sklearn.metrics import mean_absolute_error - -from FileReaders import * -from MicroModels import * -from MesoModels import * -from Visualization import * -from Analysis import * - -if __name__ == '__main__': - - # READING SIMULATION SETTINGS FROM CONFIG FILE - if len(sys.argv) == 1: - print(f"You must pass the configuration file for the simulations.") - raise Exception() - - config = configparser.ConfigParser() - config.read(sys.argv[1]) - - # LOADING MESO MODEL - pickle_directory = config['Directories']['pickled_files_dir'] - - print('================================================') - print(f'Starting job on data from {pickle_directory}') - print('================================================\n\n') - - meso_filename_8 = '/rHD2d_cg=fw=bl=8dx.pickle' - meso_filename_4 = '/rHD2d_cg=fw=bl=4dx.pickle' - meso_filename_2 = '/rHD2d_cg=fw=bl=2dx.pickle' - - MesoModelLoadFile = pickle_directory + meso_filename_2 - with open(MesoModelLoadFile, 'rb') as filehandle: - meso2 = pickle.load(filehandle) - - MesoModelLoadFile = pickle_directory + meso_filename_4 - with open(MesoModelLoadFile, 'rb') as filehandle: - meso4 = pickle.load(filehandle) - - MesoModelLoadFile = pickle_directory + meso_filename_8 - with open(MesoModelLoadFile, 'rb') as filehandle: - meso8 = pickle.load(filehandle) - - eta2 = np.abs(meso2.meso_vars['eta']) - eta4 = np.abs(meso4.meso_vars['eta']) - eta8 = np.abs(meso8.meso_vars['eta']) - - etas = [eta2, eta4, eta8] - filter_sizes = [2,4,8] - - log_etas = [np.log10(elem) for elem in etas] - means = [np.mean(elem) for elem in log_etas] - print(f'The log-mean values of eta are: {means[0]}, {means[1]}, {means[2]}') - - etas_rescaled = [etas[i]/(filter_sizes[i]**2) for i in range(len(etas))] - log_etas_rescaled = [np.log10(elem) for elem in etas_rescaled] - - shear_sq2 = meso2.meso_vars['shear_sq'] - shear_sq4 = meso4.meso_vars['shear_sq'] - shear_sq8 = meso8.meso_vars['shear_sq'] - - pi_res_sq2 = meso2.meso_vars['pi_res_sq'] - pi_res_sq4 = meso4.meso_vars['pi_res_sq'] - pi_res_sq8 = meso8.meso_vars['pi_res_sq'] - - shear_sqs = [np.log10(elem) for elem in [shear_sq2, shear_sq4, shear_sq8]] - pi_res_sqs = [np.log10(elem) for elem in [pi_res_sq2, pi_res_sq4, pi_res_sq8]] - - means = [np.mean(elem) for elem in pi_res_sqs] - print(f'The log-mean value of pi_res_sq are: {means[0]}, {means[1]}, {means[2]}') - - - plt.rc("font",family="serif") - plt.rc("mathtext",fontset="cm") - fig, axes = plt.subplots(1,3, figsize=[12,4]) - axes = axes.flatten() - - stat = 'density' - with warnings.catch_warnings(): - warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') - warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') - - sns.histplot(shear_sqs[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[0], label=r'$L=2dx$') - sns.histplot(shear_sqs[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[0], label=r'$L=4dx$') - sns.histplot(shear_sqs[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[0], label=r'$L=8dx$') - - sns.histplot(pi_res_sqs[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[1], label=r'$L=2dx$') - sns.histplot(pi_res_sqs[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[1], label=r'$L=4dx$') - sns.histplot(pi_res_sqs[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[1], label=r'$L=8dx$') - - sns.histplot(log_etas_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[2], label=r'$L=2dx$') - sns.histplot(log_etas_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[2], label=r'$L=4dx$') - sns.histplot(log_etas_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[2], label=r'$L=8dx$') - - - axes[0].legend(loc = 'best', prop={'size': 10}) - axes[1].legend(loc = 'best', prop={'size': 10}) - axes[2].legend(loc = 'best', prop={'size': 10}) - - axes[0].set_ylabel('pdf', fontsize=12) - axes[1].set_ylabel('pdf', fontsize=12) - axes[2].set_ylabel('pdf', fontsize=12) - - - xlabel = meso2.labels_var_dict['shear_sq'] - xlabel = r'$\log($' + xlabel + r'$)$' - axes[0].set_xlabel(xlabel, fontsize=12) - - xlabel = meso2.labels_var_dict['pi_res_sq'] - xlabel = r'$\log($' + xlabel + r'$)$' - axes[1].set_xlabel(xlabel, fontsize=12) - - axes[2].set_xlabel(r'$\eta/\tilde{L}^2,\qquad \tilde{L} = L/dx$', fontsize=12) - - fig.tight_layout() - - fig_directory = config['Directories']['figures_dir'] - filename = 'L_dep_eta' - format = 'png' - dpi = 400 - filename += "." + format - plt.savefig(fig_directory + filename, format=format, dpi=dpi) - - - - - - - diff --git a/Proof_of_principle/draft_scripts/draft_config.txt b/Proof_of_principle/draft_scripts/draft_config.txt deleted file mode 100644 index 9e78f12..0000000 --- a/Proof_of_principle/draft_scripts/draft_config.txt +++ /dev/null @@ -1,34 +0,0 @@ -# CONFIGURATION FILE FOR SCRIPTS VISUALIZING_*.PY -################################################# - -[Directories] - -pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/20dx_data - -figures_dir = ./ - - -[Filenames] - -meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle - - -[Models_settings] -#snapshots_opts are useful in case not all snapshots from METHOD in the folder are required. -#relevant only for visualizing_micro.py - -snapshots_opts = {"fewer_snaps_required": false, "smaller_list": [9,10,11]} - -num_T_slices = 3 - -[Plot_settings] -# method sets that used for producing difference plots: set it to 'interpolate' when plotting models with different grids -# interp_dims is relevant when 'interpolate' method is used: should be set to the dims of the coarser grid -# Else, set method to "raw_data" to use gridded data directly - -plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} - -inset_ranges = {"x_range": [0.75, 0.85], "y_range": [0.55, 0.65]} - -diff_plot_settings = {"method": "raw_data", "interp_dims": [300, 300]} - From 2daf41aa09153841ece20aecb16968989afc3cd8 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 7 May 2024 11:44:10 +0200 Subject: [PATCH 108/111] minor updates to master classes --- master_files/Analysis.py | 10 +++++----- master_files/MesoModels.py | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/master_files/Analysis.py b/master_files/Analysis.py index 0c5cbae..30d67cf 100644 --- a/master_files/Analysis.py +++ b/master_files/Analysis.py @@ -285,16 +285,16 @@ def preprocess_data(self, list_of_arrays, preprocess_data, weights=None): Weights = np.ma.masked_array(weights, tot_mask).compressed() #when array is compressed, this is automatically flattened! - if preprocess_data.__contains__('log_abs'): - for i in range(len(processed_list)): - if preprocess_data['log_abs'][i]: - processed_list[i] = np.log10(np.abs(processed_list[i])) - if preprocess_data.__contains__('sqrt'): for i in range(len(processed_list)): if preprocess_data['sqrt'][i]: processed_list[i] = np.sqrt(processed_list[i]) + if preprocess_data.__contains__('log_abs'): + for i in range(len(processed_list)): + if preprocess_data['log_abs'][i]: + processed_list[i] = np.log10(np.abs(processed_list[i])) + # removing corresponding entries from weights if weights is not None: return processed_list, Weights diff --git a/master_files/MesoModels.py b/master_files/MesoModels.py index cf0e110..0ec1b94 100644 --- a/master_files/MesoModels.py +++ b/master_files/MesoModels.py @@ -1470,6 +1470,8 @@ def setup_meso_grid(self, patch_bdrs, coarse_factor = 1, coarse_time = False): # Setup arrays for derivatives of the model. self.deriv_vars['D_u_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1, self.spatial_dims+1)) self.deriv_vars['D_T_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.deriv_vars['D_eps_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) + self.deriv_vars['D_n_tilde'] = np.zeros((Nt, Nx, Ny, self.spatial_dims+1)) def find_observers(self): """ @@ -1768,8 +1770,8 @@ def decompose_structures_task(BC, SET , p_filt, h, i, j): q_a = - np.einsum('ij,jk,kl,l->i', h_ab, SET, metric, u_t) s_ab = np.einsum('ij,kl,jl->ik', h_ab, h_ab, SET) s = np.einsum('ij,ji->', s_ab, metric) - s = np.multiply(1/spatial_dims, s) - s_ab_tracefree = s_ab - np.multiply(s, metric + np.einsum('i,j->ij', u_t, u_t)) + # s = np.multiply(1/spatial_dims, s) + s_ab_tracefree = s_ab - np.multiply(s/3, metric + np.einsum('i,j->ij', u_t, u_t)) p_t = resHD2D.p_Gamma_law(eps_t, n_t, 4.0/3.0) # rechange this: this was just to test the impact of EOS residuals onto the modelling of zeta From 7a66f2035aa9520b30fe35356e9386cd6c16374e Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 7 May 2024 11:47:12 +0200 Subject: [PATCH 109/111] Polishing filter scripts folder --- .../filter_scripts/config_filter.txt | 16 +-- .../filter_scripts/pickling_meso.py | 15 ++- .../filter_scripts/visualizing_meso.py | 108 ++++++++++++++---- .../filter_scripts/visualizing_micro.py | 13 ++- .../filter_scripts/visualizing_obs.py | 2 +- 5 files changed, 110 insertions(+), 44 deletions(-) diff --git a/Proof_of_principle/filter_scripts/config_filter.txt b/Proof_of_principle/filter_scripts/config_filter.txt index fa02072..3026fa3 100644 --- a/Proof_of_principle/filter_scripts/config_filter.txt +++ b/Proof_of_principle/filter_scripts/config_filter.txt @@ -3,18 +3,18 @@ [Directories] -hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/20dx/ +hdf5_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/METHOD_output/50dx/ pickled_files_dir = /scratch/tc2m23/KHIRandom/hydro/new_data/800X800/ET10/pickled/50dx_data -figures_dir = ./ +figures_dir = . [Filenames] -meso_pickled_filename = /rHD2d_cg=fw=bl=8dx.pickle +meso_pickled_filename = /rHD2d_nocg_fw=bl=8dx.pickle -micro_pickled_filename = +micro_pickled_filename = / [Micro_model_settings] @@ -23,14 +23,14 @@ micro_pickled_filename = #ordered filenames' list to be retained (list ordered with glob in FileReaders.METHOD_HDF5 ) snapshots_opts = {"fewer_snaps_required": true, - "smaller_list": [5,6,7,8,9,10,11,12,13,14,15]} + "smaller_list": [15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35]} [Meso_model_settings] -meso_grid = {"x_range": [0.3, 0.4], "y_range": [0.3, 0.4], "num_T_slices": 3, - "coarse_grain_factor": 4, "coarse_grain_time": false} +meso_grid = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97], "num_T_slices": 3, + "coarse_grain_factor": 1, "coarse_grain_time": false} -filtering_options = {"box_len_ratio": 4.0, "filter_width_ratio": 4.0} +filtering_options = {"box_len_ratio": 8.0, "filter_width_ratio": 2.0} n_cpus = 40 diff --git a/Proof_of_principle/filter_scripts/pickling_meso.py b/Proof_of_principle/filter_scripts/pickling_meso.py index 154dd3b..dcff019 100644 --- a/Proof_of_principle/filter_scripts/pickling_meso.py +++ b/Proof_of_principle/filter_scripts/pickling_meso.py @@ -60,6 +60,9 @@ x_range = meso_grid['x_range'] y_range = meso_grid['y_range'] + ts = micro_model.domain_vars['t'][:] + print(f'ts: {ts}\n') + box_len_ratio = float(filtering_options['box_len_ratio']) filter_width_ratio = float(filtering_options['filter_width_ratio']) @@ -146,12 +149,12 @@ # time_taken = time.perf_counter() - start_time # print('Finished computing weights in serial, time taken: {}\n'.format(time_taken), flush=True) - # # PICKLING THE CLASS INSTANCE FOR FUTURE USE - # pickle_directory = config['Directories']['pickled_files_dir'] - # filename = config['Filenames']['meso_pickled_filename'] - # MesoModelPickleDumpFile = pickle_directory + filename - # with open(MesoModelPickleDumpFile, 'wb') as filehandle: - # pickle.dump(meso_model, filehandle) + # PICKLING THE CLASS INSTANCE FOR FUTURE USE + pickle_directory = config['Directories']['pickled_files_dir'] + filename = config['Filenames']['meso_pickled_filename'] + MesoModelPickleDumpFile = pickle_directory + filename + with open(MesoModelPickleDumpFile, 'wb') as filehandle: + pickle.dump(meso_model, filehandle) diff --git a/Proof_of_principle/filter_scripts/visualizing_meso.py b/Proof_of_principle/filter_scripts/visualizing_meso.py index 619dc78..7ffc655 100644 --- a/Proof_of_principle/filter_scripts/visualizing_meso.py +++ b/Proof_of_principle/filter_scripts/visualizing_meso.py @@ -58,7 +58,7 @@ x_range = plot_ranges['x_range'] y_range = plot_ranges['y_range'] saving_directory = config['Directories']['figures_dir'] - visualizer = Plotter_2D([10, 3]) + visualizer = Plotter_2D([12, 8]) diff_plot_settings =json.loads(config['Plot_settings']['diff_plot_settings']) diff_method = diff_plot_settings['method'] interp_dims = diff_plot_settings['interp_dims'] @@ -100,28 +100,28 @@ # time_taken = time.perf_counter() - start_time # print(f'Finished plotting model comparison: time taken (X2) ={time_taken}\n') - # # # # PLOTTING THE DECOMPOSED SET - # # # ############################# - # vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] - # norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] - # cmaps = ['seismic','seismic','seismic','seismic','seismic','seismic'] - # components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/DecomposedSET_1.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - - # vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] - # norms = ['mysymlog', 'mysymlog', 'mysymlog', 'log', 'log', 'log'] - # cmaps = ['seismic','seismic','seismic', 'plasma', 'plasma', 'plasma'] - # components = [(0,), (1,), (2,), (), (), ()] - # fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) - # fig.tight_layout() - # time_for_filename = str(round(time_meso,2)) - # filename = "/DecomposedSET_2.pdf" - # plt.savefig(saving_directory + filename, format = 'pdf') - # print('Finished plotting decomposition of SET\n') + # # # PLOTTING THE DECOMPOSED SET + # # ############################# + vars_strs = ['pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res', 'pi_res'] + norms = ['mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog', 'mysymlog'] + cmaps = ['Spectral_r','Spectral_r','Spectral_r','Spectral_r','Spectral_r','Spectral_r'] + components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/DecomposedSET_1.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + + vars_strs = ['q_res', 'q_res', 'q_res', 'Pi_res', 'p_tilde', 'p_filt'] + norms = ['mysymlog', 'mysymlog', 'mysymlog', 'log', 'log', 'log'] + cmaps = ['Spectral_r','Spectral_r','Spectral_r', 'plasma', 'plasma', 'plasma'] + components = [(0,), (1,), (2,), (), (), ()] + fig = visualizer.plot_vars(meso_model, vars_strs, time_meso, x_range, y_range, components_indices=components, norms=norms, cmaps=cmaps) + fig.tight_layout() + time_for_filename = str(round(time_meso,2)) + filename = "/DecomposedSET_2.pdf" + plt.savefig(saving_directory + filename, format = 'pdf') + print('Finished plotting decomposition of SET\n') # # # # PLOTTING THE DERIVATIVES OF FAVRE VEL AND TEMPERATURE # # ######################################################### @@ -191,7 +191,7 @@ # plt.savefig(saving_directory + filename, format = 'pdf') # print('Finished plotting vorticity\n', flush=True) - + # # SUMMARY PLOT OF THE RESIDUALS vars_strs = ['Pi_res', 'pi_res_sq', 'q_res_sq'] vars = [] extents = [] @@ -275,4 +275,64 @@ def update(changed_image): dpi=400 filename = f"/Residuals." + format plt.savefig(saving_directory + filename, format = format, dpi=dpi) + + + # # NOW SUMMARY PLOT FOR THE CLOSURE INGREDIENTS + + vars_strs = ['exp_tilde', 'shear_sq', 'Theta_sq'] + vars = [] + extents = [] + for var_str in vars_strs: + temp_var, temp_extent = visualizer.get_var_data(meso_model, var_str, time_meso, x_range, y_range) + if var_str != 'exp_tilde': + temp_var = np.sqrt(temp_var) + vars.append(temp_var) + extents.append(temp_extent) + + + + norms = ['symlog', 'log', 'log'] + cmaps = ['Spectral_r', 'plasma', 'plasma'] + + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(1,3, squeeze=False, figsize=[12,4], sharey=True) + axes = axes.flatten() + + images = [] + + for i in range(len(axes)): + + if norms[i] == 'mysymlog': + ticks, labels, nodes = MySymLogPlotting.get_mysymlog_var_ticks(vars[i]) + data_to_plot = MySymLogPlotting.symlog_var(vars[i]) + mynorm = MyThreeNodesNorm(nodes) + im = axes[i].imshow(data_to_plot, extent=extents[i], origin='lower', norm=mynorm, cmap=cmaps[i]) + divider = make_axes_locatable(axes[i]) + cax = divider.append_axes('right', size='5%', pad=0.05) + cbar = fig.colorbar(im, cax=cax, orientation='vertical') + cbar.set_ticks(ticks) + cbar.ax.set_yticklabels(labels) + + else: + im = axes[i].imshow(vars[i], extent=extents[i], origin='lower', cmap=cmaps[i], norm=norms[i]) + divider = make_axes_locatable(axes[i]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + images.append(im) + + axes[i].set_xlabel(r'$x$', fontsize=10) + axes[i].set_ylabel(r'$y$', fontsize=10) + + axes[0].set_title(r'$\tilde{\theta}$', fontsize=14) + axes[1].set_title(r'$\sqrt{\tilde{\sigma}_{ab}\tilde{\sigma}^{ab}}$', fontsize=14) + axes[2].set_title(r'$\sqrt{\tilde{\Theta}_{a}\tilde{\Theta}^{a}}$', fontsize=14) + + fig.tight_layout() + + format='png' + dpi=400 + filename = f"/Gradients." + format + plt.savefig(saving_directory + filename, format = format, dpi=dpi) diff --git a/Proof_of_principle/filter_scripts/visualizing_micro.py b/Proof_of_principle/filter_scripts/visualizing_micro.py index 887ce13..78dc253 100644 --- a/Proof_of_principle/filter_scripts/visualizing_micro.py +++ b/Proof_of_principle/filter_scripts/visualizing_micro.py @@ -24,7 +24,7 @@ config.read(sys.argv[1]) # LOADING MICRO DATA FROM HDF5 OR PICKLE - micro_from_hdf5 = True + micro_from_hdf5 = False if micro_from_hdf5: hdf5_directory = config['Directories']['hdf5_dir'] @@ -50,7 +50,10 @@ print(f'Starting job on data from {MicroModelLoadFile}') print('=========================================================================\n\n') with open(MicroModelLoadFile, 'rb') as filehandle: - micro_model = pickle.load(filehandle) + meso_model = pickle.load(filehandle) + micro_model = meso_model.micro_model + # micro_model = pickle.load(filehandle) + # PLOT SETTINGS plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) @@ -68,7 +71,7 @@ # Plotting the baryon current vars = ['BC', 'BC', 'BC', 'W', 'vx', 'vy'] norms= ['log', 'symlog', 'symlog', 'log', 'symlog', 'symlog'] - cmaps = [None, 'seismic', 'seismic', None, 'seismic', 'seismic'] + cmaps = [None, 'Spectral_r', 'Spectral_r', None, 'Spectral_r', 'Spectral_r'] components = [(0,), (1,), (2,), (), (), ()] fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components, norms=norms, cmaps=cmaps) fig.tight_layout() @@ -80,7 +83,7 @@ vars = ['SET', 'SET', 'SET', 'SET', 'SET', 'SET'] components = [(0,0), (0,1), (0,2), (1,1), (1,2), (2,2)] norms= ['log', 'symlog', 'symlog', 'log', 'symlog', 'log'] - cmaps = [None, 'seismic', 'seismic', None, 'seismic', None] + cmaps = [None, 'Spectral_r', 'Spectral_r', None, 'Spectral_r', None] fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, components_indices = components, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(plot_time,2)) @@ -90,7 +93,7 @@ # plotting primitive quantities vars = ['W', 'vx', 'vy', 'n', 'p', 'e'] norms= ['log', 'symlog', 'symlog', 'log', 'log', 'log'] - cmaps = [None, 'seismic', 'seismic', None, None, None] + cmaps = [None, 'Spectral_r', 'Spectral_r', None, None, None] fig=visualizer.plot_vars(micro_model, vars, plot_time, x_range, y_range, norms=norms, cmaps=cmaps) fig.tight_layout() time_for_filename = str(round(plot_time,2)) diff --git a/Proof_of_principle/filter_scripts/visualizing_obs.py b/Proof_of_principle/filter_scripts/visualizing_obs.py index 7a1786c..1da6c46 100644 --- a/Proof_of_principle/filter_scripts/visualizing_obs.py +++ b/Proof_of_principle/filter_scripts/visualizing_obs.py @@ -100,7 +100,7 @@ # axes[1].set_title(meso_model.labels_var_dict['U'] + r"$,$ $a=0$") # axes[2].set_title(r"$Relative$ $difference$") - comp = (0,) + comp = (1,) bar_vel_data, extent_micro = visualizer.get_var_data(micro_model, 'bar_vel', time_meso, x_range, y_range, comp) obs_data, extent_meso = visualizer.get_var_data(meso_model, 'U', time_meso, x_range, y_range, comp) ar_mean = (np.abs(bar_vel_data) + np.abs(obs_data))/2 From 2b9c5d9188a6cef5f4f0ff630bcc1361d6b6c92a Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 7 May 2024 11:50:15 +0200 Subject: [PATCH 110/111] cleaning calibration and draft scripts folders --- .../config_calibration.txt | 32 ++- .../calibration_scripts/find_best_fit.py | 64 +++++- .../fs_residual_dependence.py | 157 ++++++++++---- .../regress+residual_check.py | 6 +- .../draft_scripts/comparison_zoom.py | 6 +- .../draft_scripts/config_draft.txt | 2 +- .../draft_scripts/filter_scaling.py | 202 +++++++++++++++++- 7 files changed, 392 insertions(+), 77 deletions(-) diff --git a/Proof_of_principle/calibration_scripts/config_calibration.txt b/Proof_of_principle/calibration_scripts/config_calibration.txt index d9de56c..aaffb85 100644 --- a/Proof_of_principle/calibration_scripts/config_calibration.txt +++ b/Proof_of_principle/calibration_scripts/config_calibration.txt @@ -44,6 +44,14 @@ coeff = zeta residual = Pi_res EL_force = exp_tilde +[compare_eta_cw] + +n_cpus = 30 +components = [[0,0], [0,1], [0,2], [1,1], [1,2]] +preprocess_data = {"value_ranges": [[null, null], [null, null], [null, null], [null, null], [null, null]], + "log_abs": [1, 1, 1, 1, 1]} + + [Regression_settings] dependent_var = eta @@ -77,8 +85,10 @@ format_fig = png [Find_best_fit_settings] -var_to_model = zeta -regressors_strs = ["vort_sq", "det_shear", "shear_sq", "T_tilde", "n_tilde", "Q1", "Q2", "acc_mag", "Theta_sq"] +var_to_model = eta +regressors_strs = ["vort_sq", "det_shear", "shear_sq", "T_tilde", "n_tilde", "Q1", "Q2", "acc_mag", + "Theta_sq", "n_tilde_dot", "T_tilde_dot", "sD_n_tilde_sq", "dot_Dn_Theta", "exp_tilde"] + ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} num_T_slices = 3 @@ -88,21 +98,21 @@ test_percentage = 0.2 # Set values to null if you do not want to restrict the data -preprocess_data = {"log_abs": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]} +preprocess_data = {"log_abs": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]} -n_cpus = 30 +n_cpus = 40 # Set format_fig to either pdf or png format_fig = png [Regress+residual_check_settings] -coeff_str = zeta -coeff_regressors_strs = ["n_tilde", "T_tilde"] -residual_str = Pi_res -closure_ingr_str = exp_tilde +coeff_str = kappa +coeff_regressors_strs = ["shear_sq", "n_tilde", "Q1", "sD_n_tilde_sq", "dot_Dn_Theta"] +residual_str = q_res_sq +closure_ingr_str = Theta_sq -regression_ranges = {"x_range": [0.03, 0.97], "y_range": [0.03, 0.97]} +regression_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} idxs_time_slices = [1] add_intercept = 1 @@ -110,8 +120,8 @@ centralize = 0 #set the following 0 if you don't want to split into train and test test_percentage = 0. -preprocess_data = { "log_abs": [1, 1, 1, 1, 1], - "sqrt": [0, 0, 0, 0, 0]} +preprocess_data = { "log_abs": [1, 1, 1, 1, 1, 1, 1, 1], + "sqrt": [0, 0, 0, 0, 0, 0, 1, 1]} [PCA_settings] diff --git a/Proof_of_principle/calibration_scripts/find_best_fit.py b/Proof_of_principle/calibration_scripts/find_best_fit.py index 9e58d5b..cec5a40 100644 --- a/Proof_of_principle/calibration_scripts/find_best_fit.py +++ b/Proof_of_principle/calibration_scripts/find_best_fit.py @@ -46,6 +46,60 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso_model = pickle.load(filehandle) + # # RE-COMPUTING DERIVATIVES AND STUFF FOR MODELLING COEFFICIENTS + # # adding labels to dictionary for better figures + # entry_dic = {'D_n_tilde' : r'$\nabla_{a}\tilde{n}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'D_eps_tilde' : r'$\nabla_{a}\tilde{\varepsilon}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'n_tilde_dot' : r'$\dot{\tilde{n}}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'T_tilde_dot' : r'$\dot{\tilde{T}}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'sD_T_tilde': r'$D_{a}\tilde{T}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'sD_n_tilde': r'$D_{a}\tilde{n}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'sD_n_tilde_sq' : r'$D_{a}\tilde{n}D^{a}\tilde{n}$'} + # meso_model.update_labels_dict(entry_dic) + # entry_dic = {'dot_Dn_Theta' : r'$D_{a}\tilde{n}\Theta^{a}$'} + # meso_model.update_labels_dict(entry_dic) + + # Nt = meso_model.domain_vars['Nt'] + # Nx = meso_model.domain_vars['Nx'] + # Ny = meso_model.domain_vars['Ny'] + + # meso_model.nonlocal_vars_strs = ['u_tilde', 'T_tilde', 'n_tilde', 'eps_tilde'] + # meso_model.deriv_vars.update({'D_n_tilde' : np.zeros((Nt,Nx,Ny,3))}) + # meso_model.deriv_vars.update({'D_eps_tilde' : np.zeros((Nt,Nx,Ny,3))}) + + # n_cpus = int(config['Find_best_fit_settings']['n_cpus']) + # start_time = time.perf_counter() + # meso_model.calculate_derivatives() + # time_taken = time.perf_counter() - start_time + # print('Finished computing derivatives (serial), time taken: {}\n'.format(time_taken), flush=True) + + # start_time = time.perf_counter() + # meso_model.closure_ingredients_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Finished computing the closure ingredients in parallel, time taken: {}\n'.format(time_taken), flush=True) + + # start_time = time.perf_counter() + # meso_model.EL_style_closure_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Finished computing the EL_style closure in parallel, time taken: {}\n'.format(time_taken), flush=True) + + # start_time = time.perf_counter() + # meso_model.modelling_coefficients_parallel(n_cpus) + # time_taken = time.perf_counter() - start_time + # print('Finished computing quantities to model extracted coefficients, time taken: {}\n'.format(time_taken), flush=True) + + # pickle_directory = config['Directories']['pickled_files_dir'] + # filename = config['Filenames']['meso_pickled_filename'] + # MesoModelPickleDumpFile = pickle_directory + filename + # with open(MesoModelPickleDumpFile, 'wb') as filehandle: + # pickle.dump(meso_model, filehandle) + # WHICH DATA YOU WANT TO RUN THE ROUTINE ON? dep_var_str = config['Find_best_fit_settings']['var_to_model'] @@ -55,7 +109,7 @@ for i in range(len(regressors_strs)): temp = meso_model.meso_vars[regressors_strs[i]] regressors.append(temp) - print(f'Dependent var: {dep_var_str}, Explanatory vars: {regressors_strs}\n') + print(f'Dependent var: {dep_var_str},\n Explanatory vars: {regressors_strs}\n') # WHICH GRID-RANGES SHOULD WE CONSIDER? @@ -135,9 +189,11 @@ def parall_regress_task(comb_regressors): dep_var_model += np.multiply(coeffs[i], actual_regressors[i]) # r, _ = stats.pearsonr(dep_var_test, dep_var_model) + # return r, coeffs , comb_regressors + # mean_error = mean_absolute_error(dep_var_test, dep_var_model) - # return r, mean_error, coeffs , comb_regressors - + # return mean_error, coeffs , comb_regressors + w = statistical_tool.wasserstein_distance(dep_var_test, dep_var_model, sample_points=300) return w, coeffs, comb_regressors @@ -152,7 +208,7 @@ def parall_regress_task(comb_regressors): print('Performing all possible regression of {} in parallel with {} processes\n'.format(dep_var_str, pool._processes), flush=True) for result in pool.map(parall_regress_task, bool_regressors_combs): # pearsons.append(result[0]) - # mean_errors.append(result[1]) + # mean_errors.append(result[0]) wassersteins.append(result[0]) fitted_coeffs.append(result[1]) regressors_combinations.append(result[2]) diff --git a/Proof_of_principle/calibration_scripts/fs_residual_dependence.py b/Proof_of_principle/calibration_scripts/fs_residual_dependence.py index 1e67321..e294b7c 100644 --- a/Proof_of_principle/calibration_scripts/fs_residual_dependence.py +++ b/Proof_of_principle/calibration_scripts/fs_residual_dependence.py @@ -50,6 +50,94 @@ with open(MesoModelLoadFile, 'rb') as filehandle: meso8 = pickle.load(filehandle) + # # READING INFO ON RESIDUALS FROM CONFIG FILE + # coeff_str = config['Fs_residual_dependence']['coeff'] + # residual_str = config['Fs_residual_dependence']['residual'] + # EL_force_str = config['Fs_residual_dependence']['EL_force'] + + # print(f'coeff_str: {coeff_str}\nresidual_str: {residual_str}\nEL_force_str: {EL_force_str}\n') + + # coeff2 = np.abs(meso2.meso_vars[coeff_str]) + # coeff4 = np.abs(meso4.meso_vars[coeff_str]) + # coeff8 = np.abs(meso8.meso_vars[coeff_str]) + + # coeffs = [coeff2, coeff4, coeff8] + # filter_sizes = [2,4,8] + + # log_coeffs = [np.log10(elem) for elem in coeffs] + # means = [np.mean(elem) for elem in log_coeffs] + # print(f'The log-mean values of {coeff_str} are: {means[0]}, {means[1]}, {means[2]}') + + # coeffs_rescaled = [coeffs[i]/(filter_sizes[i]**2) for i in range(len(coeffs))] + # log_coeffs_rescaled = [np.log10(elem) for elem in coeffs_rescaled] + + # EL_force2 = np.abs(meso2.meso_vars[EL_force_str]) + # EL_force4 = np.abs(meso4.meso_vars[EL_force_str]) + # EL_force8 = np.abs(meso8.meso_vars[EL_force_str]) + + # residual2 = meso2.meso_vars[residual_str] + # residual4 = meso4.meso_vars[residual_str] + # residual8 = meso8.meso_vars[residual_str] + + # log_residuals = [np.log10(elem) for elem in [residual2, residual4, residual8]] + # log_EL_forces = [np.log10(elem) for elem in [EL_force2, EL_force4, EL_force8]] + + # means = [np.mean(elem) for elem in log_residuals] + # print(f'The log-mean value of pi_res_sq are: {means[0]}, {means[1]}, {means[2]}') + + + # plt.rc("font",family="serif") + # plt.rc("mathtext",fontset="cm") + # fig, axes = plt.subplots(1,3, figsize=[12,4]) + # axes = axes.flatten() + + # stat = 'density' + # with warnings.catch_warnings(): + # warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + # warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + + # sns.histplot(log_EL_forces[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[0], label=r'$L=2dx$') + # sns.histplot(log_EL_forces[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[0], label=r'$L=4dx$') + # sns.histplot(log_EL_forces[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[0], label=r'$L=8dx$') + + # sns.histplot(log_residuals[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[1], label=r'$L=2dx$') + # sns.histplot(log_residuals[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[1], label=r'$L=4dx$') + # sns.histplot(log_residuals[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[1], label=r'$L=8dx$') + + # sns.histplot(log_coeffs_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[2], label=r'$L=2dx$') + # sns.histplot(log_coeffs_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[2], label=r'$L=4dx$') + # sns.histplot(log_coeffs_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[2], label=r'$L=8dx$') + + + # axes[0].legend(loc = 'best', prop={'size': 10}) + # axes[1].legend(loc = 'best', prop={'size': 10}) + # axes[2].legend(loc = 'best', prop={'size': 10}) + + # axes[0].set_ylabel('pdf', fontsize=12) + # axes[1].set_ylabel('pdf', fontsize=12) + # axes[2].set_ylabel('pdf', fontsize=12) + + # xlabel = meso2.labels_var_dict[EL_force_str] + # xlabel = r'$\log(|$' + xlabel + r'$|)$' + # axes[0].set_xlabel(xlabel, fontsize=12) + + # xlabel = meso2.labels_var_dict[residual_str] + # xlabel = r'$\log($' + xlabel + r'$)$' + # axes[1].set_xlabel(xlabel, fontsize=12) + + # xlabel = meso2.labels_var_dict[coeff_str] + # xlabel = xlabel + r'$/\tilde{L}^2,\qquad \tilde{L} = L/dx$' + # axes[2].set_xlabel(xlabel, fontsize=12) + + # fig.tight_layout() + + # fig_directory = config['Directories']['figures_dir'] + # filename = 'fs_residual_dependence' + # format = 'png' + # dpi = 400 + # filename += "." + format + # plt.savefig(fig_directory + filename, format=format, dpi=dpi) + # READING INFO ON RESIDUALS FROM CONFIG FILE coeff_str = config['Fs_residual_dependence']['coeff'] residual_str = config['Fs_residual_dependence']['residual'] @@ -57,38 +145,27 @@ print(f'coeff_str: {coeff_str}\nresidual_str: {residual_str}\nEL_force_str: {EL_force_str}\n') - coeff2 = np.abs(meso2.meso_vars[coeff_str]) - coeff4 = np.abs(meso4.meso_vars[coeff_str]) - coeff8 = np.abs(meso8.meso_vars[coeff_str]) - - coeffs = [coeff2, coeff4, coeff8] - filter_sizes = [2,4,8] + zeta2 = np.abs(meso2.meso_vars['zeta']) + zeta4 = np.abs(meso4.meso_vars['zeta']) + zeta8 = np.abs(meso8.meso_vars['zeta']) + zetas = [zeta2, zeta4, zeta8] - log_coeffs = [np.log10(elem) for elem in coeffs] - means = [np.mean(elem) for elem in log_coeffs] - print(f'The log-mean values of {coeff_str} are: {means[0]}, {means[1]}, {means[2]}') - - coeffs_rescaled = [coeffs[i]/(filter_sizes[i]**2) for i in range(len(coeffs))] - log_coeffs_rescaled = [np.log10(elem) for elem in coeffs_rescaled] + kappa2 = np.abs(meso2.meso_vars['kappa']) + kappa4 = np.abs(meso4.meso_vars['kappa']) + kappa8 = np.abs(meso8.meso_vars['kappa']) + kappas = [kappa2, kappa4, kappa8] - EL_force2 = np.abs(meso2.meso_vars[EL_force_str]) - EL_force4 = np.abs(meso4.meso_vars[EL_force_str]) - EL_force8 = np.abs(meso8.meso_vars[EL_force_str]) - - residual2 = meso2.meso_vars[residual_str] - residual4 = meso4.meso_vars[residual_str] - residual8 = meso8.meso_vars[residual_str] - - log_residuals = [np.log10(elem) for elem in [residual2, residual4, residual8]] - log_EL_forces = [np.log10(elem) for elem in [EL_force2, EL_force4, EL_force8]] + filter_sizes = [2,4,8] - means = [np.mean(elem) for elem in log_residuals] - print(f'The log-mean value of pi_res_sq are: {means[0]}, {means[1]}, {means[2]}') + zetas_rescaled = [zetas[i]/(filter_sizes[i]**2) for i in range(len(zetas))] + kappas_rescaled = [kappas[i]/(filter_sizes[i]**2) for i in range(len(kappas))] + log_zetas_rescaled = [np.log10(elem) for elem in zetas_rescaled] + log_kappas_rescaled = [np.log10(elem) for elem in kappas_rescaled] plt.rc("font",family="serif") plt.rc("mathtext",fontset="cm") - fig, axes = plt.subplots(1,3, figsize=[12,4]) + fig, axes = plt.subplots(1,2, figsize=[8,4]) axes = axes.flatten() stat = 'density' @@ -96,38 +173,28 @@ warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') - sns.histplot(log_EL_forces[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[0], label=r'$L=2dx$') - sns.histplot(log_EL_forces[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[0], label=r'$L=4dx$') - sns.histplot(log_EL_forces[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[0], label=r'$L=8dx$') - - sns.histplot(log_residuals[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[1], label=r'$L=2dx$') - sns.histplot(log_residuals[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[1], label=r'$L=4dx$') - sns.histplot(log_residuals[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[1], label=r'$L=8dx$') + sns.histplot(log_zetas_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[0], label=r'$L=2dx$') + sns.histplot(log_zetas_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[0], label=r'$L=4dx$') + sns.histplot(log_zetas_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[0], label=r'$L=8dx$') - sns.histplot(log_coeffs_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[2], label=r'$L=2dx$') - sns.histplot(log_coeffs_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[2], label=r'$L=4dx$') - sns.histplot(log_coeffs_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[2], label=r'$L=8dx$') + sns.histplot(log_kappas_rescaled[0].flatten(), stat=stat, kde=True, color='firebrick', ax=axes[1], label=r'$L=2dx$') + sns.histplot(log_kappas_rescaled[1].flatten(), stat=stat, kde=True, color='steelblue', ax=axes[1], label=r'$L=4dx$') + sns.histplot(log_kappas_rescaled[2].flatten(), stat=stat, kde=True, color='olive', ax=axes[1], label=r'$L=8dx$') axes[0].legend(loc = 'best', prop={'size': 10}) axes[1].legend(loc = 'best', prop={'size': 10}) - axes[2].legend(loc = 'best', prop={'size': 10}) axes[0].set_ylabel('pdf', fontsize=12) axes[1].set_ylabel('pdf', fontsize=12) - axes[2].set_ylabel('pdf', fontsize=12) - xlabel = meso2.labels_var_dict[EL_force_str] - xlabel = r'$\log(|$' + xlabel + r'$|)$' + xlabel = meso2.labels_var_dict['zeta'] + xlabel = xlabel + r'$/\tilde{L}^2,\qquad \tilde{L} = L/dx$' axes[0].set_xlabel(xlabel, fontsize=12) - xlabel = meso2.labels_var_dict[residual_str] - xlabel = r'$\log($' + xlabel + r'$)$' - axes[1].set_xlabel(xlabel, fontsize=12) - - xlabel = meso2.labels_var_dict[coeff_str] + xlabel = meso2.labels_var_dict['kappa'] xlabel = xlabel + r'$/\tilde{L}^2,\qquad \tilde{L} = L/dx$' - axes[2].set_xlabel(xlabel, fontsize=12) + axes[1].set_xlabel(xlabel, fontsize=12) fig.tight_layout() diff --git a/Proof_of_principle/calibration_scripts/regress+residual_check.py b/Proof_of_principle/calibration_scripts/regress+residual_check.py index af2ddd9..95bd4b1 100644 --- a/Proof_of_principle/calibration_scripts/regress+residual_check.py +++ b/Proof_of_principle/calibration_scripts/regress+residual_check.py @@ -210,8 +210,8 @@ if hasattr(meso_model, 'labels_var_dict') and residual_str in meso_model.labels_var_dict.keys(): residual_label = meso_model.labels_var_dict[residual_str] # comment the following line if you don't take the log of the residual - # residual_label = r"$\frac{1}{2}\log($" + residual_label + r"$)$" - residual_label = r"$\log($" + residual_label + r"$)$" + residual_label = r"$\frac{1}{2}\log($" + residual_label + r"$)$" + # residual_label = r"$\log($" + residual_label + r"$)$" axes[0].set_xlabel('Regression model', fontsize=12) axes[0].set_ylabel(coeff_label, fontsize=12) @@ -271,7 +271,7 @@ bbox_args = dict(boxstyle="round", fc="0.95") - plt.annotate(text=text_for_box, xy = (0.18,0.95), xycoords='figure fraction', bbox=bbox_args, ha="right", va="top", fontsize = 10) + plt.annotate(text=text_for_box, xy = (0.34,0.2), xycoords='figure fraction', bbox=bbox_args, ha="right", va="bottom", fontsize = 9) # Adding legend to the distribution comparison panel diff --git a/Proof_of_principle/draft_scripts/comparison_zoom.py b/Proof_of_principle/draft_scripts/comparison_zoom.py index bc57f69..fe8292c 100644 --- a/Proof_of_principle/draft_scripts/comparison_zoom.py +++ b/Proof_of_principle/draft_scripts/comparison_zoom.py @@ -106,15 +106,15 @@ axes[i].set_ylabel(r'$y$') title = r'$Rel.$ $difference$' - ax1.set_title(title) + ax1.set_title(title, fontsize=10) title = micro_model.labels_var_dict[var_str] title += r'$,$ $a=0$' - ax2.set_title(title) + ax2.set_title(title, fontsize=10) title = meso_model.labels_var_dict[var_str] title += r'$,$ $a=0$' - ax3.set_title(title) + ax3.set_title(title, fontsize=10) # fig.tight_layout() divider = make_axes_locatable(ax2) diff --git a/Proof_of_principle/draft_scripts/config_draft.txt b/Proof_of_principle/draft_scripts/config_draft.txt index 854b28b..4be187e 100644 --- a/Proof_of_principle/draft_scripts/config_draft.txt +++ b/Proof_of_principle/draft_scripts/config_draft.txt @@ -28,7 +28,7 @@ num_T_slices = 3 plot_ranges = {"x_range": [0.04, 0.96], "y_range": [0.04, 0.96]} -inset_ranges = {"x_range": [0.75, 0.85], "y_range": [0.55, 0.65]} +inset_ranges = {"x_range": [0.15, 0.35], "y_range": [0.5, 0.7]} diff_plot_settings = {"method": "raw_data", "interp_dims": [300, 300]} diff --git a/Proof_of_principle/draft_scripts/filter_scaling.py b/Proof_of_principle/draft_scripts/filter_scaling.py index 00e67be..f33c22c 100644 --- a/Proof_of_principle/draft_scripts/filter_scaling.py +++ b/Proof_of_principle/draft_scripts/filter_scaling.py @@ -16,9 +16,9 @@ if __name__ == '__main__': - # ############################################################################ - # # SCRIPT TO SHOW HOW THE IMPACT OF FILTERING SCALES WITH THE FILTER-SIZE - # ############################################################################ + # ############################################################################################### + # # SCRIPT TO SHOW HOW THE IMPACT OF FILTERING SCALES WITH THE FILTER-SIZE: REL DIFFERENCES + # ############################################################################################### # READING SIMULATION SETTINGS FROM CONFIG FILE if len(sys.argv) == 1: @@ -36,20 +36,17 @@ print(f'Starting job on data from {pickle_directory}') print('=========================================================================\n\n') - # meso_pickled_filename = '/rHD2d_cg=fw=bl=2dx.pickle' meso_pickled_filename = '/rHD2d_nocg_fw=2_bl=8dx.pickle' MesoModelLoadFile = pickle_directory + meso_pickled_filename with open(MesoModelLoadFile, 'rb') as filehandle: meso_2 = pickle.load(filehandle) micro_model = meso_2.micro_model - # meso_pickled_filename = '/rHD2d_cg=fw=bl=4dx.pickle' meso_pickled_filename = '/rHD2d_nocg_fw=4_bl=8dx.pickle' MesoModelLoadFile = pickle_directory + meso_pickled_filename with open(MesoModelLoadFile, 'rb') as filehandle: meso_4 = pickle.load(filehandle) - # meso_pickled_filename = '/rHD2d_cg=fw=bl=8dx.pickle' meso_pickled_filename = '/rHD2d_nocg_fw=bl=8dx.pickle' MesoModelLoadFile = pickle_directory + meso_pickled_filename with open(MesoModelLoadFile, 'rb') as filehandle: @@ -157,22 +154,22 @@ title = micro_model.labels_var_dict[var_str] title += r'$,$ $a=0$' - axes[0].set_title(title) + axes[0].set_title(title, fontsize=10) title = r'$Scaled$ $rel.$ $diff.,$ $L=2dx$' # title = meso_2.labels_var_dict[var_str] # title += r'$,$ $a=0,$ $L=2dx$' - axes[1].set_title(title) + axes[1].set_title(title, fontsize=10) title = r'$Scaled$ $rel.$ $diff.,$ $L=4dx$' # title = meso_4.labels_var_dict[var_str] # title += r'$,$ $a=0,$ $L=4dx$' - axes[2].set_title(title) + axes[2].set_title(title, fontsize=10) title = r'$Scaled$ $rel.$ $diff.,$ $L=8dx$' # title = meso_8.labels_var_dict[var_str] # title += r'$,$ $a=0,$ $L=8dx$' - axes[3].set_title(title) + axes[3].set_title(title, fontsize=10) fig.tight_layout() @@ -206,3 +203,188 @@ def update(changed_image): plt.savefig(saving_directory + filename, format = format, dpi=dpi) + + # # ################################################################################################## + # # # SCRIPT TO SHOW HOW THE IMPACT OF FILTERING SCALES WITH THE FILTER-SIZE: COARSE-GRAINING + # # ################################################################################################## + + # # READING SIMULATION SETTINGS FROM CONFIG FILE + # if len(sys.argv) == 1: + # print(f"You must pass the configuration file for the simulations.") + # raise Exception() + + # config = configparser.ConfigParser() + # config.read(sys.argv[1]) + + # # LOADING MESO AND MICRO MODELS + # pickle_directory = config['Directories']['pickled_files_dir'] + + + # print('=========================================================================') + # print(f'Starting job on data from {pickle_directory}') + # print('=========================================================================\n\n') + + # meso_pickled_filename = '/rHD2d_cg=fw=bl=2dx.pickle' + # MesoModelLoadFile = pickle_directory + meso_pickled_filename + # with open(MesoModelLoadFile, 'rb') as filehandle: + # meso_2 = pickle.load(filehandle) + # micro_model = meso_2.micro_model + + # meso_pickled_filename = '/rHD2d_cg=fw=bl=4dx.pickle' + # MesoModelLoadFile = pickle_directory + meso_pickled_filename + # with open(MesoModelLoadFile, 'rb') as filehandle: + # meso_4 = pickle.load(filehandle) + + # meso_pickled_filename = '/rHD2d_cg=fw=bl=8dx.pickle' + # MesoModelLoadFile = pickle_directory + meso_pickled_filename + # with open(MesoModelLoadFile, 'rb') as filehandle: + # meso_8 = pickle.load(filehandle) + + + # print('Finished reading pickled data\n') + + # # CHECKING WE ARE COMPARING DATA FROM THE SAME TIME-SLICE + # num_snaps = micro_model.domain_vars['nt'] + # central_slice_num = int(num_snaps/2.) + # time_micro = micro_model.domain_vars['t'][central_slice_num] + # num_slices_meso = int(config['Models_settings']['num_T_slices']) + # time_meso2 = meso_2.domain_vars['T'][int((num_slices_meso-1)/2)] + # time_meso4 = meso_2.domain_vars['T'][int((num_slices_meso-1)/2)] + # time_meso8 = meso_2.domain_vars['T'][int((num_slices_meso-1)/2)] + + # compatible = True + # if time_micro != time_meso2 or time_meso2 != time_meso4 or time_meso4 != time_meso8: + # compatible = False + + # if not compatible: + # print("Slices of meso and micro models do not coincide. Careful!\n") + # else: + # print("Comparing data at same time-slice, hurray!\n") + + # # # PLOT SETTINGS + # plot_ranges = json.loads(config['Plot_settings']['plot_ranges']) + # x_range = plot_ranges['x_range'] + # y_range = plot_ranges['y_range'] + # saving_directory = config['Directories']['figures_dir'] + # visualizer = Plotter_2D() + # # diff_plot_settings =json.loads(config['Plot_settings']['diff_plot_settings']) + + # # Building the data for showing the relative difference between the various plots + # models = [micro_model, meso_2, meso_4, meso_8] + # var_str = 'BC' + # comp = (0,) + + # inset_ranges = json.loads(config['Plot_settings']['inset_ranges']) + # inset_x_range = inset_ranges['x_range'] + # inset_y_range = inset_ranges['y_range'] + + # # full_range = [0.04,0.96] + # datamicro, extentmicro = visualizer.get_var_data(micro_model, var_str, time_micro, x_range, y_range, comp) + # data2, extent2 = visualizer.get_var_data(meso_2, var_str, time_meso2, inset_x_range, inset_y_range, comp) + # data4, extent4 = visualizer.get_var_data(meso_4, var_str, time_meso4, inset_x_range, inset_y_range, comp) + # data8, extent8 = visualizer.get_var_data(meso_8, var_str, time_meso8, inset_x_range, inset_y_range, comp) + + + # # PLOTTING + # fig = plt.figure(figsize=[13,4]) + # ax1 = fig.add_subplot(1, 4, 1) + # ax2 = fig.add_subplot(1, 4, 2, sharey=ax1) + # ax3 = fig.add_subplot(1, 4, 3, sharey=ax1) + # ax4 = fig.add_subplot(1, 4, 4, sharey=ax1) + # axes = [ax1, ax2, ax3, ax4] + # plt.setp(ax3.get_yticklabels(), visible=False) + # plt.setp(ax4.get_yticklabels(), visible=False) + + + # images =[] + + # im = ax1.imshow(datamicro, extent=extentmicro, origin='lower', cmap='plasma') #, norm='log') + # divider = make_axes_locatable(ax1) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # images.append(im) + + # im = ax2.imshow(rel_diff2, extent=extent2, origin='lower', cmap='plasma') #, norm='log') + # divider = make_axes_locatable(ax2) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # cax.set_axis_off() + # images.append(im) + + # im = ax3.imshow(rel_diff4, extent=extent4, origin='lower', cmap='plasma') #, norm='log') + # divider = make_axes_locatable(ax3) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # cax.set_axis_off() + # images.append(im) + + # im = ax4.imshow(rel_diff8, extent=extent8, origin='lower', cmap='plasma') #, norm='log') + # divider = make_axes_locatable(ax4) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # fig.colorbar(im, cax=cax, orientation='vertical') + # images.append(im) + + # for i in range(len(axes)): + # axes[i].set_xlabel(r'$x$') + # axes[i].set_ylabel(r'$y$') + + # title = micro_model.labels_var_dict[var_str] + # title += r'$,$ $a=0$' + # axes[0].set_title(title) + + # # title = r'$Scaled$ $rel.$ $diff.,$ $L=2dx$' + # title = meso_2.labels_var_dict[var_str] + # title += r'$,$ $a=0,$ $L=2dx$' + # axes[1].set_title(title) + + # # title = r'$Scaled$ $rel.$ $diff.,$ $L=4dx$' + # title = meso_4.labels_var_dict[var_str] + # title += r'$,$ $a=0,$ $L=4dx$' + # axes[2].set_title(title) + + # # title = r'$Scaled$ $rel.$ $diff.,$ $L=8dx$' + # title = meso_8.labels_var_dict[var_str] + # title += r'$,$ $a=0,$ $L=8dx$' + # axes[3].set_title(title) + + # fig.tight_layout() + + # # COMMON COLORMAP + # # Finding the global min and max and setting the shared colormap based on these. + # vmin = min(image.get_array().min() for image in images) + # vmax = max(image.get_array().max() for image in images) + # norm = colors.Normalize(vmin=vmin, vmax=vmax) + # for im in images: + # im.set_norm(norm) + + # # fig.colorbar(images[0], ax=axes, orientation='vertical', location='right', shrink=0.52, pad=0.025 ) + + # # Make images respond to changes in the norm of other images (e.g. via the + # # "edit axis, curves and images parameters" GUI on Qt), but be careful not to + # # recurse infinitely! + # def update(changed_image): + # for im in images: + # if (changed_image.get_cmap() != im.get_cmap() + # or changed_image.get_clim() != im.get_clim()): + # im.set_cmap(changed_image.get_cmap()) + # im.set_clim(changed_image.get_clim()) + + # for im in images: + # im.callbacks.connect('changed', update) + + + # # Adding boundaries of zoomed in regions + # A = np.array([inset_x_range[0], inset_y_range[0]]) + # B = np.array([inset_x_range[1], inset_y_range[0]]) + # C = np.array([inset_x_range[1], inset_y_range[1]]) + # D = np.array([inset_x_range[0], inset_y_range[1]]) + # arrows_starts = [A, B, C, D] + # arrows_increments = [arrows_starts[i] - arrows_starts[i-1] for i in range(1, len(arrows_starts))] + # arrows_increments.append(A-D) + # for i in range(len(arrows_starts)): + # ax1.arrow(arrows_starts[i][0], arrows_starts[i][1], arrows_increments[i][0], arrows_increments[i][1], \ + # width=0.005,color='white',head_length=0.0,head_width=0.0) + + # fig.tight_layout() + + # format='png' + # dpi=400 + # filename = f"/Filter_scale_BC_bl=fw." + format + # plt.savefig(saving_directory + filename, format = format, dpi=dpi) \ No newline at end of file From 7bb072367321e28abbf6fabdc525ba641f152446 Mon Sep 17 00:00:00 2001 From: ThomasCelora Date: Tue, 7 May 2024 13:30:23 +0200 Subject: [PATCH 111/111] tracking newest scripts --- .../calibration_scripts/compare_eta_cw.py | 236 +++++++++++++++ .../draft_scripts/gamma_interp.py | 272 ++++++++++++++++++ 2 files changed, 508 insertions(+) create mode 100644 Proof_of_principle/calibration_scripts/compare_eta_cw.py create mode 100644 Proof_of_principle/draft_scripts/gamma_interp.py diff --git a/Proof_of_principle/calibration_scripts/compare_eta_cw.py b/Proof_of_principle/calibration_scripts/compare_eta_cw.py new file mode 100644 index 0000000..b16e129 --- /dev/null +++ b/Proof_of_principle/calibration_scripts/compare_eta_cw.py @@ -0,0 +1,236 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +import math +from scipy import stats +from itertools import product +import multiprocessing as mp +from sklearn.metrics import mean_absolute_error + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + + print('================================================') + print(f'Starting job on data from {pickle_directory}') + print('================================================\n\n') + + meso_filename = config['Filenames']['meso_pickled_filename'] + + MesoModelLoadFile = pickle_directory + meso_filename + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + n_cpus = int(config['compare_eta_cw']['n_cpus']) + cw_dict = meso_model.EL_componentwise_parallel(n_cpus, store=False) + print('Finished computing the coefficients componentwise\n') + print(f'Keys in the dictionary: {cw_dict.keys()}') + + # meso_model.EL_style_closure_parallel(n_cpus) + # print('Finished recomputing the coefficients covariantly, now saving\n') + # pickle_directory = config['Directories']['pickled_files_dir'] + # filename = config['Filenames']['meso_pickled_filename'] + # MesoModelPickleDumpFile = pickle_directory + filename + # with open(MesoModelPickleDumpFile, 'wb') as filehandle: + # pickle.dump(meso_model, filehandle) + + eta_cw = cw_dict['eta_cw'] + print(f'eta_cw.shape: {eta_cw.shape}\n') + + eta_cw_shape = eta_cw[0,0,0].shape + Nt = meso_model.domain_vars['Nt'] + Nx = meso_model.domain_vars['Nx'] + Ny = meso_model.domain_vars['Ny'] + new_shape = tuple(list(eta_cw_shape) + [Nt, Nx, Ny]) + print(f'new_shape: {new_shape}\n') + eta_cw = eta_cw.reshape(new_shape) + + components = json.loads(config['compare_eta_cw']['components']) + components = [tuple(comp) for comp in components] + preprocess_data = json.loads(config['compare_eta_cw']['preprocess_data']) + statistical_tool = CoefficientsAnalysis() + + eta_comps = [eta_cw[comp] for comp in components] + eta_comps = statistical_tool.preprocess_data(eta_comps, preprocess_data) + + eta_quadratic = meso_model.meso_vars['eta'] + eta_quadratic = np.log10(np.abs(eta_quadratic)) + + print('Finished preparing data for plottting distributions\n') + + # # #################################################### + # # PLOTTING THE COMPONENTS DISTRIBUTIONS ALL AT ONCE + # # #################################################### + # plt.rc("font",family="serif") + # plt.rc("mathtext",fontset="cm") + # fig, ax = plt.subplots(1,1, figsize=[6,4]) + + # with warnings.catch_warnings(): + # warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + # warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + # for i in range(len(eta_comps)): + # label = 'comp = ' + str(components[i]) + # sns.histplot(eta_comps[i].flatten(), ax=ax, stat='density', kde=True, label=label) + + # label = 'squaring' + # sns.histplot(eta_quadratic.flatten(), ax=ax, stat='density', color='black', kde=True, label=label) + + # ax.set_ylabel('pdf', fontsize=10) + # xlabel = r'$\log(|\eta|)$' + # ax.set_xlabel(xlabel, fontsize=10) + # ax.legend(loc = 'best', prop={'size': 10}) + + + # fig.tight_layout() + + # fig_directory = config['Directories']['figures_dir'] + # filename = 'eta_cw_distrib' + # format = 'png' + # dpi = 400 + # filename += "." + format + # plt.savefig(fig_directory + filename, format=format, dpi=dpi) + # plt.close() + + # # ################################################################################## + # SINGLE PLOT WITH 6 PANELS AND HISTOGRAMS OF POSITIVE AND NEGATIVE VALUES SEPARATELY + # # ################################################################################## + components = [(0,0), (0,1), (0,2), (1,1), (1,2)] + eta_comps = [np.array(eta_cw[comp]) for comp in components] + + eta_quadratic = meso_model.meso_vars['eta'] + eta_quad_pos = ma.masked_where(eta_quadratic<0, eta_quadratic, copy=True).compressed() + eta_quad_neg = ma.masked_where(eta_quadratic>0, eta_quadratic, copy=True).compressed() + eta_quad_pos = np.log10(np.abs(eta_quad_pos)) + eta_quad_neg = np.log10(np.abs(eta_quad_neg)) + eta_quad_counts = [len(eta_quad_pos), len(eta_quad_neg)] + + tot_counts = [] + for i in range(len(eta_comps)): + eta_pos = ma.masked_where(eta_comps[i]<0, eta_comps[i], copy=True) + eta_neg = ma.masked_where(eta_comps[i]>0, eta_comps[i], copy=True) + eta_pos = eta_pos.compressed() + eta_neg = eta_neg.compressed() + eta_pos = np.log10(eta_pos) + eta_neg = np.log10(np.abs(eta_neg)) + eta_comps[i] = [eta_pos, eta_neg] + + counts = [len(eta_pos), len(eta_neg)] + tot_counts.append(counts) + tot_number = len((eta_cw[components[i]]).flatten()) + print(f'len(eta_comps[i].flatten()): {tot_number}') + + + # Computing the min and max to set a common range in the plot + min, max = np.amin(eta_quad_pos), np.amax(eta_quad_pos) + if np.amin(eta_quad_neg) < np.amin(eta_quad_pos): + min = np.amin(eta_quad_neg) + if np.amax(eta_quad_neg) > np.amax(eta_quad_pos): + max = np.amax(eta_quad_neg) + + for i in range(len(eta_comps)): + m1, M1 = np.amin(eta_comps[i][0]), np.amax(eta_comps[i][0]) + m2, M2 = np.amin(eta_comps[i][1]), np.amax(eta_comps[i][1]) + m, M = np.amin([m1,m2]), np.amax([M1,M2]) + if m < min: + min = m + if M > max: + max = M + if max >0: + max = 0 + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(2,3, figsize=[12,8]) + axes = axes.flatten() + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", message='is_categorical_dtype is deprecated') + warnings.filterwarnings('ignore', message='use_inf_as_na option is deprecated') + + for i in range(5): + # sns.histplot(eta_comps[i][0], ax=axes[i], stat='density', kde=True, label=r'$\eta_{+}$') + # sns.histplot(eta_comps[i][1], ax=axes[i], stat='density', kde=True, label=r'$\eta_{-}$') + label = r'$\eta_{+}$' + ', #=' + str(tot_counts[i][0]) + sns.histplot(eta_comps[i][0], ax=axes[i], label=label) + label = r'$\eta_{-}$' + ', #=' + str(tot_counts[i][1]) + sns.histplot(eta_comps[i][1], ax=axes[i], label=label) + + xlabel = r'$\log(|\eta|)$, comp=' + str(components[i]) + axes[i].set_xlabel(xlabel, fontsize=10) + axes[i].set_ylabel('counts', fontsize=10) + axes[i].set_xlim([min, max]) + + axes[i].legend(loc = 'best', prop={'size': 10}) + + label = r'$\eta_{+}$' + ', #=' + str(eta_quad_counts[0]) + sns.histplot(eta_quad_pos, ax=axes[5], label=label) + label = r'$\eta_{-}$' + ', #=' + str(eta_quad_counts[1]) + sns.histplot(eta_quad_neg, ax=axes[5], label=label) + + xlabel = r'$\log(|\eta|)$' + ', squaring' + axes[5].set_xlabel(xlabel, fontsize=10) + axes[5].set_ylabel('counts', fontsize=10) + axes[5].legend(loc = 'best', prop={'size': 10}) + axes[5].set_xlim([min, max]) + + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = 'pos_neg_histos' + format = 'png' + dpi = 400 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + plt.close() + + # ################################################# + # # PLOTTING THE SIGNS + # ################################################# + # for i in range(len(axes)): + # quantity = eta_cw[components[i]] + # quantity = np.sign(quantity) + # im = axes[i].imshow(quantity, origin='lower', cmap='Spectral_r') + # divider = make_axes_locatable(axes[i]) + # cax = divider.append_axes('right', size='5%', pad=0.05) + # fig.colorbar(im, cax=cax, orientation='vertical') + + # xlabel = r'$|\eta|/\eta$, comp=' + str(components[i]) + # axes[i].set_xlabel(xlabel, fontsize=12) + + # fig.tight_layout() + + # fig_directory = config['Directories']['figures_dir'] + # filename = 'eta_cw_signs' + # format = 'png' + # dpi = 400 + # filename += "." + format + # plt.savefig(fig_directory + filename, format=format, dpi=dpi) + # plt.close() + + + + + + + + diff --git a/Proof_of_principle/draft_scripts/gamma_interp.py b/Proof_of_principle/draft_scripts/gamma_interp.py new file mode 100644 index 0000000..992738b --- /dev/null +++ b/Proof_of_principle/draft_scripts/gamma_interp.py @@ -0,0 +1,272 @@ +import sys +# import os +sys.path.append('../../master_files/') +import pickle +import configparser +import json +import time +import math +from scipy import stats +from scipy.interpolate import SmoothBivariateSpline + +from matplotlib.ticker import FuncFormatter + +from FileReaders import * +from MicroModels import * +from MesoModels import * +from Visualization import * +from Analysis import * + +if __name__ == '__main__': + + # ################################################################## + # EXTRACTING THE 1ST ADIABATIC COEFFICIENT LOCALLY + # ################################################################## + + # READING SIMULATION SETTINGS FROM CONFIG FILE + if len(sys.argv) == 1: + print(f"You must pass the configuration file for the simulations.") + raise Exception() + + config = configparser.ConfigParser() + config.read(sys.argv[1]) + + # LOADING MESO MODEL + pickle_directory = config['Directories']['pickled_files_dir'] + meso_pickled_filename = config['Filenames']['meso_pickled_filename'] + MesoModelLoadFile = pickle_directory + meso_pickled_filename + + print('================================================') + print(f'Starting job on data from {MesoModelLoadFile}') + print('================================================\n\n') + + with open(MesoModelLoadFile, 'rb') as filehandle: + meso_model = pickle.load(filehandle) + + micro_model = meso_model.micro_model + + nx, ny = micro_model.domain_vars['nx'], micro_model.domain_vars['ny'] + central_slice_idx = int((micro_model.domain_vars['nt']-1)/2) + time_micro = micro_model.domain_vars['t'][central_slice_idx] + x_range = micro_model.domain_vars['xmin'], micro_model.domain_vars['xmax'] + y_range = micro_model.domain_vars['ymin'], micro_model.domain_vars['ymax'] + + visualizer = Plotter_2D() + + p_micro, micro_extent = visualizer.get_var_data(micro_model, 'p', time_micro, x_range, y_range) + n_micro, _ = visualizer.get_var_data(micro_model, 'n', time_micro, x_range, y_range) + int_en_micro, _ = visualizer.get_var_data(micro_model, 'e', time_micro, x_range, y_range) + + eps_micro = n_micro + np.multiply(n_micro, int_en_micro) + + p_micro_flat = p_micro.flatten() + n_micro_flat = n_micro.flatten() + eps_micro_flat = eps_micro.flatten() + + micro_SBS_p = SmoothBivariateSpline(n_micro_flat, eps_micro_flat, p_micro_flat) + micro_SBS_p_n = micro_SBS_p.partial_derivative(1, 0) + micro_SBS_p_eps = micro_SBS_p.partial_derivative(0, 1) + + def compute_Gamma(p, n,eps, partial_n_p, partial_eps_p): + """ + Given some values for the number density, the energy density and the pressure + Compute the corresponding adiabatic index (1st) using the cubic spline interpolations above + """ + Gamma = partial_n_p + (p+eps)/n * partial_eps_p + Gamma *= n/p + + return Gamma + + gamma_micro = np.zeros_like(p_micro) + rel_diff_gamma_micro = np.zeros_like(p_micro) + for i in range(len(p_micro[:,0])): + for j in range(len(p_micro[0,:])): + p = p_micro[i, j] + n = n_micro[i, j] + eps = eps_micro[i, j] + + partial_n_p = micro_SBS_p_n(n, eps) + partial_eps_p = micro_SBS_p_eps(n, eps) + gamma_micro[i,j] = compute_Gamma(p, n , eps, partial_n_p, partial_eps_p) -4./3. + rel_diff_gamma_micro[i,j] = np.abs(gamma_micro[i,j]) / (4/3) + + + Nt, Nx, Ny = meso_model.domain_vars['Nt'], meso_model.domain_vars['Nx'], meso_model.domain_vars['Ny'] + central_slice_idx = int((Nt-1)/2) + + time_meso = meso_model.domain_vars['T'][central_slice_idx] + x_range = meso_model.domain_vars['Xmin'], meso_model.domain_vars['Xmax'] + y_range = meso_model.domain_vars['Ymin'], meso_model.domain_vars['Ymax'] + + p_filt, meso_extent = visualizer.get_var_data(meso_model, 'p_filt', time_meso, x_range, y_range) + n_tilde, _ = visualizer.get_var_data(meso_model, 'n_tilde', time_meso, x_range, y_range) + eps_tilde, _ = visualizer.get_var_data(meso_model, 'eps_tilde', time_meso, x_range, y_range) + + p_filt_flat = p_filt.flatten() + n_tilde_flat = n_tilde.flatten() + eps_tilde_flat = eps_tilde.flatten() + + meso_SBS_p = SmoothBivariateSpline(n_tilde_flat, eps_tilde_flat, p_filt_flat) + meso_SBS_p_n = meso_SBS_p.partial_derivative(1, 0) + meso_SBS_p_eps = meso_SBS_p.partial_derivative(0, 1) + + gamma_meso = np.zeros_like(p_filt) + rel_diff_gamma_meso = np.zeros_like(p_filt) + for i in range(len(p_filt[:,0])): + for j in range(len(p_filt[0,:])): + p = p_filt[i, j] + n = n_tilde[i, j] + eps = eps_tilde[i, j] + + partial_n_p = meso_SBS_p_n(n, eps) + partial_eps_p = meso_SBS_p_eps(n, eps) + + gamma_meso[i,j] = compute_Gamma(p, n , eps, partial_n_p, partial_eps_p) -4./3. + rel_diff_gamma_meso[i,j] = np.abs(gamma_meso[i,j]) / (4/3.) + + + ############################################################################################### + # PLOTTING THE DIFFERENCE BETWEEN THE EXTRACTED ADIABATIC INDEX AND THE EXPECTED VALUE OF 4/3 + ############################################################################################### + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(nrows=1, ncols=2, figsize=[9,4]) + axes = axes.flatten() + + im = axes[0].imshow(gamma_micro, extent=micro_extent, origin='lower', cmap='Spectral_r', norm='symlog') + divider = make_axes_locatable(axes[0]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[1].imshow(gamma_meso, extent=meso_extent, origin='lower', cmap='Spectral_r', norm='symlog') + divider = make_axes_locatable(axes[1]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + axes[0].set_title(r"$\Gamma_{micro} - 4/3$", fontsize =10) + axes[1].set_title(r"$\Gamma_{meso} - 4/3$", fontsize =10) + + for ax in axes: + ax.set_xlabel(r'$x$') + ax.set_ylabel(r'$y$') + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = 'Interp_Gamma' + format = 'png' + dpi = 300 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + plt.close() + + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + fig, axes = plt.subplots(nrows=1, ncols=2, figsize=[9,4]) + axes = axes.flatten() + + im = axes[0].imshow(rel_diff_gamma_micro, extent=micro_extent, origin='lower', cmap='plasma', norm='log') + divider = make_axes_locatable(axes[0]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[1].imshow(rel_diff_gamma_meso, extent=meso_extent, origin='lower', cmap='plasma', norm='log') + divider = make_axes_locatable(axes[1]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + axes[0].set_title(r"$\frac{|\Gamma_{micro} - 4/3|}{4/3}$", fontsize =12) + axes[1].set_title(r"$\frac{|\Gamma_{meso} - 4/3|}{4/3}$", fontsize =12) + + for ax in axes: + ax.set_xlabel(r'$x$') + ax.set_ylabel(r'$y$') + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = 'Interp_Gamma_reldiff' + format = 'png' + dpi = 300 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + plt.close() + + + ################################################################################## + # PLOTTING THE INTERPOLATED P AND ITS DERIVATIVES: CHECK ON IMPACT OF ARTIFACTS + ################################################################################## + + p_filt_interp = np.zeros_like(p_filt) + d_n_p = np.zeros_like(p_filt) + d_eps_p = np.zeros_like(p_filt) + for i in range(len(p_filt[:,0])): + for j in range(len(p_filt[0,:])): + n = n_tilde[i, j] + eps = eps_tilde[i, j] + + p_filt_interp[i,j] = meso_SBS_p(n, eps) + d_n_p[i,j] = meso_SBS_p_n(n, eps) + d_eps_p[i,j] = meso_SBS_p_eps(n, eps) + + plt.rc("font",family="serif") + plt.rc("mathtext",fontset="cm") + + + fig, axes = plt.subplots(nrows=2, ncols=3, figsize=[13,8], sharex = True, sharey = True) + axes = axes.flatten() + + im = axes[0].imshow(p_filt, extent=meso_extent, origin='lower', cmap='plasma') + divider = make_axes_locatable(axes[0]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[1].imshow(n_tilde, extent=meso_extent, origin='lower', cmap='plasma') + divider = make_axes_locatable(axes[1]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[2].imshow(eps_tilde, extent=meso_extent, origin='lower', cmap='plasma') + divider = make_axes_locatable(axes[2]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[3].imshow(p_filt_interp, extent=meso_extent, origin='lower', cmap='plasma') + divider = make_axes_locatable(axes[3]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[4].imshow(d_n_p, extent=meso_extent, origin='lower', cmap='plasma') + divider = make_axes_locatable(axes[4]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + im = axes[5].imshow(d_eps_p, extent=meso_extent, origin='lower', cmap='plasma') + divider = make_axes_locatable(axes[5]) + cax = divider.append_axes('right', size='5%', pad=0.05) + fig.colorbar(im, cax=cax, orientation='vertical') + + + for ax in axes: + ax.set_xlabel(r'$x$') + ax.set_ylabel(r'$y$') + + + axes[0].set_title(r"$

$", fontsize=12) + axes[1].set_title(r"$\tilde n$", fontsize=12) + axes[2].set_title(r"$\tilde \varepsilon$", fontsize=12) + axes[3].set_title(r"$

_{interpolated}$", fontsize=12) + axes[4].set_title(r"$\partial_{\tilde{n}}

$", fontsize=14) + axes[5].set_title(r"$\partial_{\tilde{\varepsilon}}

$", fontsize=14) + + + fig.tight_layout() + + fig_directory = config['Directories']['figures_dir'] + filename = 'Interpolation_test' + format = 'png' + dpi = 300 + filename += "." + format + plt.savefig(fig_directory + filename, format=format, dpi=dpi) + plt.close() \ No newline at end of file