Neurons example, pt. 2: large datasets
Contents
Neurons example, pt. 2: large datasets¶
import numpyro
from bayes_window import models, fake_spikes_explore, BayesWindow, BayesConditions, BayesRegression
from bayes_window.generative_models import generate_fake_spikes
import numpy as np
from importlib import reload
import altair as alt
alt.data_transformers.disable_max_rows()
try:
alt.renderers.enable('altair_saver', fmts=['png'])
except Exception:
pass
df, df_monster, index_cols, firing_rates = generate_fake_spikes(n_trials=20,
n_neurons=6,
n_mice=3,
dur=5,
mouse_response_slope=40,
overall_stim_response_strength=5)
Step by step¶
1. Firing rate¶
bw1 = BayesConditions(df=df_monster, y='isi',treatment='stim', condition=['neuron_x_mouse','i_trial'], group='mouse')
bw1.fit(dist_y='gamma');
# bw.plot_model_quality()
2021-11-30 15:43:46.937780: E external/org_tensorflow/tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
Untransformed dimension in ['neuron_x_mouse', 'i_trial'] may be a problem. If you made a new numpyro model, look in utils.rename_posterior()
<xarray.DataArray 'slope' (chain: 4, neuron_x_mouse: 18, i_trial: 20, draw: 200)>
array([[[[ 2.05496639e-01, -1.23332642e-01, 5.72049767e-02, ...,
1.35442659e-01, 1.94491595e-02, -4.14554775e-02],
[ 2.65519530e-01, -1.15224615e-01, 3.47977281e-02, ...,
1.28003508e-01, -1.37674987e-01, 1.38172805e-01],
[-1.26889199e-02, 3.64163369e-01, -1.88080207e-01, ...,
2.10027397e-02, 8.41769874e-02, -1.58533901e-02],
...,
[-2.48875335e-01, -4.86549921e-03, -1.04419798e-01, ...,
-2.27378830e-01, 1.76198035e-02, -3.14136118e-01],
[-1.57199949e-01, -2.33469084e-01, 1.06595501e-01, ...,
-2.59016573e-01, 5.60912713e-02, -1.21270530e-01],
[ 1.20249018e-01, -1.07145742e-01, 1.90651089e-01, ...,
-7.81020522e-02, 1.20727688e-01, -8.28569978e-02]],
[[-1.00893274e-01, 2.90834680e-02, -2.19155848e-01, ...,
7.74909556e-03, -1.19591244e-02, -9.88359600e-02],
[-5.47480062e-02, -2.18393505e-01, -2.01403350e-02, ...,
-7.83368945e-02, -1.72045887e-01, 1.56961679e-02],
[ 2.81938374e-01, 1.19365260e-01, 3.39779854e-01, ...,
2.92986751e-01, 2.40052968e-01, 1.07049696e-01],
...
3.85448813e-01, -3.20130885e-01, -1.21005788e-01],
[-1.37223423e-01, -2.61590183e-02, -6.42637014e-02, ...,
5.61146326e-02, -1.68350384e-01, -7.38538578e-02],
[-3.15864563e-01, -2.04520047e-01, -1.55076757e-01, ...,
-1.03898562e-01, -2.36955464e-01, -2.09580064e-01]],
[[-6.30551502e-02, -6.51129335e-02, -8.43590721e-02, ...,
-1.17666081e-01, 6.72511086e-02, -5.27364388e-02],
[-6.94911256e-02, -1.36902541e-01, 2.11670995e-05, ...,
-2.96028048e-01, 6.78422302e-02, 6.22219518e-02],
[-9.19345766e-03, -2.33238935e-03, 1.21193975e-01, ...,
1.23459108e-01, 3.15059572e-02, 9.11614150e-02],
...,
[ 1.96272999e-01, -5.35045192e-02, -6.59052283e-02, ...,
-9.75256190e-02, 6.70770630e-02, -3.30348574e-02],
[-2.69988477e-01, 6.11172803e-03, -2.77296901e-01, ...,
-1.52591243e-01, -2.34488785e-01, -2.53550678e-01],
[ 1.61253318e-01, 1.51031688e-01, -1.22800015e-01, ...,
3.94184813e-02, 3.89426798e-02, 2.21719556e-02]]]],
dtype=float32)
Coordinates:
* chain (chain) int64 0 1 2 3
* draw (draw) int64 0 1 2 3 4 5 6 7 ... 193 194 195 196 197 198 199
* neuron_x_mouse (neuron_x_mouse) object 'm0bayes0' 'm0bayes1' ... 'm2bayes5'
* i_trial (i_trial) float64 0.0 1.0 2.0 3.0 ... 16.0 17.0 18.0 19.0
('chain', 'neuron_x_mouse', 'i_trial', 'draw')
i_trial
2. Regression¶
bw2 = BayesRegression(df=bw1.posterior['mu_per_condition'],
y='center interval', treatment='stim', condition=['neuron_x_mouse'], group='mouse')
bw2.fit(model=models.model_hierarchical,
do_make_change='subtract',
dist_y='student',
robust_slopes=False,
add_group_intercept=False,
add_group_slope=False,
fold_change_index_cols=('stim', 'mouse', 'neuron_x_mouse'))
bw2.plot_model_quality()
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
/tmp/ipykernel_2833256/2990684204.py in <module>
----> 1 bw2 = BayesRegression(df=bw1.posterior['mu_per_condition'],
2 y='center interval', treatment='stim', condition=['neuron_x_mouse'], group='mouse')
3 bw2.fit(model=models.model_hierarchical,
4 do_make_change='subtract',
5 dist_y='student',
~/mmy/bayes-window/bayes_window/slopes.py in __init__(self, window, add_data, **kwargs)
37
38 def __init__(self, window=None, add_data=True, **kwargs):
---> 39 window = copy(window) if window is not None else BayesWindow(**kwargs)
40 window.add_data = add_data
41 self.window = window
~/mmy/bayes-window/bayes_window/workflow.py in __init__(self, df, y, treatment, condition, group, group2, detail, add_data)
43 assert treatment in df.columns
44 if group:
---> 45 assert group in df.columns
46 self.add_data = add_data
47 self.treatment = treatment # if type(treatment)=='list' else [treatment] # self.levels[2]
AssertionError:
bw2.chart
Each neuron separately via SVI¶
1. Firing rate¶
from tqdm import tqdm
gb='neuron_x_mouse'
step1_res=[]
for i, df_m_n in tqdm(df_monster.groupby(gb)):
bw1 = BayesConditions(df_m_n, y='isi',treatment='stim', condition=['i_trial'], group='mouse'
).fit(dist_y='gamma',
# fit_fn=fitting.fit_svi
)
posterior=bw1.posterior['mu_per_condition'].copy()
posterior[gb] = i
step1_res.append(posterior)
2. Regression¶
TODO add sigma to step 2 inputs
import pandas as pd
bw2 = BayesRegression(pd.concat(step1_res),
y='center interval', treatment='stim', condition=['neuron_x_mouse'], group='mouse',
detail='i_trial')
bw2.fit(model=models.model_hierarchical,
do_make_change='subtract',
dist_y='student',
robust_slopes=False,
add_group_intercept=False,
add_group_slope=False)
bw2.chart
NUTS 1-step GLM¶
# Gamma GLM
bw = BayesRegression(df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical,
progress_bar=True,
do_make_change='subtract',
dist_y='gamma',
robust_slopes=False,
add_group_intercept=False,
add_group_slope=False,
fold_change_index_cols=('stim', 'mouse', 'neuron','neuron_x_mouse'))
bw.plot_model_quality()
import altair as alt
alt.data_transformers.disable_max_rows()
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
NUTS student¶
bw = BayesRegression(df=df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical,
progress_bar=True,
do_make_change='subtract',
dist_y='student',
robust_slopes=False,
add_group_intercept=False,
add_group_slope=False,
fold_change_index_cols=('stim', 'mouse', 'neuron','neuron_x_mouse'))
bw.plot_model_quality()
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
bw = BayesRegression(df_monster, y='isi', treatment='stim', condition='neuron_x_mouse', group='mouse')
bw.fit(model=models.model_hierarchical, do_make_change='subtract',
progress_bar=True,
dist_y='student',
use_gpu=True,
num_chains=1,
num_warmup=500,
add_group_slope=True, add_group_intercept=False,
fold_change_index_cols=('stim', 'mouse', 'neuron','neuron_x_mouse'))
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
#bw.explore_models(use_gpu=True)
NUTS Lognormal¶
bw = BayesRegression(df=df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical, do_make_change='subtract',
progress_bar=True,
use_gpu=True, num_chains=1, n_draws=1500, num_warmup=1500,
dist_y='lognormal',
add_group_slope=True, add_group_intercept=True,
fold_change_index_cols=('stim', 'mouse', 'neuron'))
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
bw.explore_models(add_group_slope=True)
BarkerMH¶
%%time
from bayes_window import fitting
from importlib import reload
reload(fitting)
bw = BayesRegression(df=df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical, do_make_change='subtract',
sampler=numpyro.infer.BarkerMH,
# progress_bar=True,
use_gpu=False, num_chains=1, n_draws=5000, num_warmup=3000,
dist_y='student',
add_group_slope=True, add_group_intercept=True,
fold_change_index_cols=('stim', 'mouse', 'neuron'),
fit_method=fitting.fit_numpyro,
)
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
Fit using SVI¶
%%time
from bayes_window import fitting
bw = BayesRegression(df=df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical, do_make_change='subtract',
n_draws=5000,
dist_y='gamma',
add_group_slope=True, add_group_intercept=False,
fold_change_index_cols=('stim', 'mouse', 'neuron'),
fit_method=fitting.fit_svi,
)
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
%%time
from bayes_window import fitting
import numpyro
from importlib import reload
reload(fitting)
bw = BayesRegression(df=df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical, do_make_change='subtract',
autoguide=numpyro.infer.autoguide.AutoLaplaceApproximation,
optim=numpyro.optim.Adam(step_size=0.0005),
loss=numpyro.infer.Trace_ELBO(),
dist_y='lognormal',
add_group_slope=True, add_group_intercept=True,
fold_change_index_cols=('stim', 'mouse', 'neuron'),
fit_method=fitting.fit_svi,
n_draws=int(1e5),
num_warmup=int(1e5),
)
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
%%time
from bayes_window import fitting
import numpyro
#numpyro.enable_validation(False)
bw = BayesRegression(df=df_monster, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.model_hierarchical, do_make_change='subtract',
#use_gpu=True,
autoguide=numpyro.infer.autoguide.AutoLaplaceApproximation,
optim=numpyro.optim.Adam(1),
loss=numpyro.infer.Trace_ELBO(),
dist_y='lognormal',
add_group_slope=True, add_group_intercept=False,
fold_change_index_cols=('stim', 'mouse', 'neuron'),
fit_method=fitting.fit_svi,
#progress_bar=False,
n_draws=int(1e5),
num_warmup=int(1e5),
)
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
Pretty model
reload(models)
bw = BayesRegression(df=df, y='isi', treatment='stim', condition=['neuron', 'mouse'], group='mouse')
bw.fit(model=models.reparam_model(models.model_hierarchical_for_render), do_make_change='subtract',
progress_bar=True,
use_gpu=False, num_chains=1, n_draws=1500, num_warmup=1500,
dist_y='normal',
add_group_slope=True, add_group_intercept=True,
fold_change_index_cols=('stim', 'mouse', 'neuron'))
bw.plot(x='neuron', color='mouse', independent_axes=True, finalize=True)
bw.facet(column='mouse',width=200,height=200).display()
#!pip install git+https://github.com/pyro-ppl/numpyro.git
from numpyro.contrib.render import render_model
reload(models)
render_model(models.model_hierarchical_for_render, model_args=(1, 1, 1, 1,
'gamma', True,
True),
render_distributions=True)
#!pip install git+https://github.com/pyro-ppl/numpyro.git
from numpyro.contrib.render import render_model
reload(models)
render_model(models.model_hierarchical_for_render, model_args=(1, 1, 1, 1,
'gamma', True,
False),
render_distributions=True)
#!pip install git+https://github.com/pyro-ppl/numpyro.git
from numpyro.contrib.render import render_model
reload(models)
render_model(models.model_hier_stim_one_codition, model_args=(1, 1, 1,
'gamma',
),
render_distributions=True)
n(Divergences) = 2
Two-step¶
Packaged version 1¶
Separate levels
# bw = BayesRegression(df=df_monster, y='isi',treatment='stim', condition=['neuron_x_mouse'],
# group='mouse', detail='i_trial')
# bw=bw.fit_twostep_by_group(dist_y_step_one='gamma', dist_y='student')
# bw.chart
[Parallel(n_jobs=12)]: Using backend LokyBackend with 12 concurrent workers.
2021-11-30 15:30:36.703976: E external/org_tensorflow/tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE: forward compatibility was attempted on non supported HW
[Parallel(n_jobs=12)]: Done 14 out of 18 | elapsed: 5.5min remaining: 1.6min
[Parallel(n_jobs=12)]: Done 18 out of 18 | elapsed: 6.3min finished
---------------------------------------------------------------------------
AssertionError Traceback (most recent call last)
/tmp/ipykernel_2767917/122053733.py in <module>
1 bw = BayesRegression(df=df_monster, y='isi',treatment='stim', condition=['neuron_x_mouse'],
2 group='mouse', detail='i_trial')
----> 3 bw=bw.fit_twostep_by_group(dist_y_step_one='gamma', dist_y='student')
4
5 bw.chart
~/mmy/bayes-window/bayes_window/slopes.py in fit_twostep_by_group(self, dist_y_step_one, groupby, dist_y, parallel, **kwargs)
461 step1_res = [fit_subset(df_m_n, i) for i, df_m_n in tqdm(self.window.data.groupby(groupby))]
462
--> 463 window_step_two = BayesRegression(df=pd.concat(step1_res).rename({'center interval': self.window.y}, axis=1),
464 y=self.window.y, treatment=self.window.treatment,
465 condition=list(
~/mmy/bayes-window/bayes_window/slopes.py in __init__(self, window, add_data, **kwargs)
37
38 def __init__(self, window=None, add_data=True, **kwargs):
---> 39 window = copy(window) if window is not None else BayesWindow(**kwargs)
40 window.add_data = add_data
41 self.window = window
~/mmy/bayes-window/bayes_window/workflow.py in __init__(self, df, y, treatment, condition, group, group2, detail, add_data)
43 assert treatment in df.columns
44 if group:
---> 45 assert group in df.columns
46 self.add_data = add_data
47 self.treatment = treatment # if type(treatment)=='list' else [treatment] # self.levels[2]
AssertionError:
Packaged version 2¶
No grouping in first step
# bw = BayesRegression(df=df_monster, y='isi',treatment='stim', condition=['neuron_x_mouse'], group='mouse', detail='i_trial')
# bw=bw.fit_twostep(dist_y_step_one='gamma', dist_y='student')
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
/tmp/ipykernel_2833256/143605735.py in <module>
1 bw = BayesRegression(df=df_monster, y='isi',treatment='stim', condition=['neuron_x_mouse'], group='mouse', detail='i_trial')
----> 2 bw=bw.fit_twostep(dist_y_step_one='gamma', dist_y='student')
~/mmy/bayes-window/bayes_window/slopes.py in fit_twostep(self, dist_y_step_one, **kwargs)
423 if self.window.detail not in self.window.condition:
424 self.window.condition += [self.window.detail]
--> 425 window_step_one = BayesConditions()
426 window_step_one.fit(dist_y=dist_y_step_one)
427
~/mmy/bayes-window/bayes_window/conditions.py in __init__(self, window, add_data, **kwargs)
33
34 def __init__(self, window=None, add_data=True, **kwargs):
---> 35 window = copy(window) if window is not None else BayesWindow(**kwargs)
36 window.add_data = add_data
37 self.window = window
TypeError: __init__() missing 3 required positional arguments: 'df', 'y', and 'treatment'
# bw.chart