diff --git a/analysis/matlab/variations/naive_boxa_d0_10/analyze.m b/analysis/matlab/variations/naive_boxa_d0_10/analyze.m index 82da5ed..0d40402 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/analyze.m +++ b/analysis/matlab/variations/naive_boxa_d0_10/analyze.m @@ -1,30 +1,58 @@ -% Variation analysis -- the paper's linear mixed model on the successful-reach -% COUNT, fit on this folder's curated data subset. +% Variation analysis -- the paper's linear mixed model on this folder's data, +% for BOTH metrics: +% metric = count : behavior = # successes -> result.txt +% metric = rate : behavior = success / attempts -> result_rate.txt +% (rate uses only sessions with attempts > 0) % % model: behavior ~ stim + day + stim:day + (1|rat) -% behavior = successful reaches (count per session) -% stim = 1 for the treatment group(s), 0 for the control group(s) -% day = training day within this window (0 = first analyzed day) -% rat = subject (random intercept) -% -% Self-contained: reads data.csv beside this script and writes result.txt. -% Run headless from this folder with: matlab -batch "analyze" -% (This is a copy of analysis/matlab/variation_analyze.m; see make_variations.m.) +% stim = 1 treatment / 0 control; day = training day within window (0 = +% first analyzed day); rat = subject (random intercept). +% For the interaction we report residual DF, Satterthwaite DF, and the honest +% per-animal random-slope test. Self-contained: reads data.csv beside this +% script. Run headless with: matlab -batch "analyze" +% (Copy of analysis/matlab/variation_analyze.m; see make_variations.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end -vname = regexprep(here, '.*[/\\]', ''); % folder name = variation id +vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -tbl = table(D.success, D.day - min(D.day), double(D.stim), categorical(D.subject), ... +Rc = localAnalyze(D, 'count', here, vname); +Rr = localAnalyze(D, 'rate', here, vname); + +% Machine-readable handoff for SUMMARY.csv (count drives it; rate appended). +VARRESULT = struct('name', vname, 'nRats', Rc.nRats, 'nObs', Rc.nObs, ... + 'interP', Rc.interP, 'interEst', Rc.interEst, ... + 'interPsatt', Rc.interPsatt, 'interPrs', Rc.interPrs, ... + 'stimP', Rc.stimP, 'dayP', Rc.dayP, 'covEqual', Rc.covEqual, ... + 'interPrate', Rr.interP, 'interEstRate', Rr.interEst, 'interPrsRate', Rr.interPrs); + +% ------------------------------------------------------------------ helper +function R = localAnalyze(D, metric, here, vname) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); + beh = D.success ./ D.total; + mlabel = 'success RATE (success/attempts)'; suffix = '_rate'; +else + beh = D.success; + mlabel = 'success COUNT'; suffix = ''; +end +R = struct('interP', NaN, 'interEst', NaN, 'interPsatt', NaN, 'interPrs', NaN, ... + 'stimP', NaN, 'dayP', NaN, 'nRats', numel(unique(D.subject)), ... + 'nObs', height(D), 'covEqual', false); +if numel(unique(D.stim)) < 2 || numel(unique(D.day)) < 2 + localWrite(sprintf('VARIATION: %s [metric: %s]\nInsufficient data for this metric.\n', ... + vname, mlabel), here, suffix); + return +end + +tbl = table(beh, D.day - min(D.day), double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); m = fitlme(tbl, 'behavior ~ stim + day + stim:day + (1|rat)'); C = m.Coefficients; A = anova(m); ci = coefCI(m); -As = anova(m, 'DFMethod', 'satterthwaite'); % Satterthwaite denominator DF +As = anova(m, 'DFMethod', 'satterthwaite'); -% Honest test: refit with a per-animal random SLOPE so the interaction DF -% collapses toward the animal count (guarded -- may not converge in short windows). rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; wst = warning('off', 'all'); try @@ -43,6 +71,7 @@ row = @(nm, t) sprintf('%-26s t(%d)=%6.2f F(%d)=%7.3f p=%.4g p=%.4g (df=%.0f C.DF(gi(t)), C.tStat(gi(t)), A.DF1(ga(t)), A.FStat(ga(t)), C.pValue(gi(t)), ... As.pValue(gs(t)), As.DF2(gs(t))); +ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); maxT = max(D.day(D.stim == 1)); minT = min(D.day(D.stim == 1)); maxC = max(D.day(D.stim == 0)); minC = min(D.day(D.stim == 0)); if abs(maxT - maxC) > 2 @@ -50,20 +79,14 @@ if abs(maxT - maxC) > 2 else cov = '(equal day coverage over this window)'; end - -ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); -if pI >= 0.05 - verdict = 'n.s. -- slopes parallel (no differential learning rate)'; -elseif eI > 0 - verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; -else - verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; -end +if pI >= 0.05; verdict = 'n.s. -- slopes parallel (no differential learning rate)'; +elseif eI > 0; verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; +else; verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; end bar = repmat('=', 1, 78); raw = regexprep(evalc('disp(m)'), '', ''); -s = sprintf('%s\nVARIATION: %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT)\n')]; +s = sprintf('%s\nVARIATION: %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('day = training day within window (0 = first analyzed day)\n')]; s = [s sprintf('treatment (stim=1): %s\n', strjoin(cellstr(unique(D.group(D.stim == 1))), ', '))]; s = [s sprintf('control (stim=0): %s\n', strjoin(cellstr(unique(D.group(D.stim == 0))), ', '))]; @@ -74,7 +97,7 @@ s = [s sprintf('%-26s %-18s %-12s %s\n%s\n', 'effect', 't(df) / F(df1)', 'p (res s = [s row('stim x day (interaction)', 'day:stim')]; s = [s row('day (learning)', 'day')]; s = [s row('stim (main, window start)', 'stim')]; -s = [s sprintf('interaction 95%% CI: [%+.2f, %+.2f]\n', ci(ii, 1), ci(ii, 2))]; +s = [s sprintf('interaction 95%% CI: [%+.4g, %+.4g]\n', ci(ii, 1), ci(ii, 2))]; if rsOk s = [s sprintf('HONEST LME (per-animal random slope, day|rat): interaction F(1,%.1f)=%.2f, p=%.4g\n', rsDf, rsF, rsP)]; else @@ -82,17 +105,18 @@ else end s = [s sprintf([' (Satterthwaite DF ~= residual on this random-intercept model; the random-slope\n' ... ' model above is the honest learning-rate test -- DF collapses toward the animal count.)\n'])]; -s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.2f)\n', verdict, pI, eI)]; -s = [s sprintf('Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.4g)\n', verdict, pI, eI)]; +s = [s sprintf('Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +localWrite(s, here, suffix); + +R = struct('interP', pI, 'interEst', eI, 'interPsatt', As.pValue(gs('day:stim')), ... + 'interPrs', rsP, 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... + 'nRats', numel(unique(D.subject)), 'nObs', height(D), 'covEqual', abs(maxT - maxC) <= 2); +end + +function localWrite(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'result.txt'), 'w'); -fprintf(fid, '%s', s); -fclose(fid); - -% Machine-readable handoff for the summary table (see make_variations.m). -VARRESULT = struct('name', vname, 'nRats', numel(unique(D.subject)), ... - 'nObs', height(D), 'interP', pI, 'interEst', eI, ... - 'interPsatt', As.pValue(gs('day:stim')), 'interPrs', rsP, ... - 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... - 'covEqual', abs(maxT - maxC) <= 2); +fid = fopen(fullfile(here, ['result' suffix '.txt']), 'w'); +fprintf(fid, '%s', s); fclose(fid); +end diff --git a/analysis/matlab/variations/naive_boxa_d0_10/learning_curve.png b/analysis/matlab/variations/naive_boxa_d0_10/learning_curve.png index 58d5ff5..dd07c2b 100644 Binary files a/analysis/matlab/variations/naive_boxa_d0_10/learning_curve.png and b/analysis/matlab/variations/naive_boxa_d0_10/learning_curve.png differ diff --git a/analysis/matlab/variations/naive_boxa_d0_10/learning_curve_rate.png b/analysis/matlab/variations/naive_boxa_d0_10/learning_curve_rate.png new file mode 100644 index 0000000..bb7956e Binary files /dev/null and b/analysis/matlab/variations/naive_boxa_d0_10/learning_curve_rate.png differ diff --git a/analysis/matlab/variations/naive_boxa_d0_10/logpower_result.txt b/analysis/matlab/variations/naive_boxa_d0_10/logpower_result.txt index 60b6e42..3af794c 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/logpower_result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_10/logpower_result.txt @@ -1,7 +1,7 @@ ============================================================================== -LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_10 +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_10 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT) +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = # successes (count)) log(day) uses 1-indexed training day (our day 0 = paper "Day 1") observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 @@ -10,9 +10,9 @@ stim x log(day) interaction: F(1,111)=3.705 p(resid)=0.05681 p(Satt)=0.05696 ( honest per-animal random slope (log-day): F(1,8.2)=2.03 p=0.1913 Cohen's f (interaction, partial eta^2=0.007) = 0.086 (small; f: .10 small, .25 medium, .40 large) ---- power simulation (log-day ground truth: stim:logday=+8.43, ratSD=8.13, resSD=14.95) --- +--- power simulation (log-day ground truth: stim:logday=+8.428, ratSD=8.126, resSD=14.95) --- - true stim:log(day) = +8.43 (100% of observed) + true stim:log(day) = +8.428 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.17 | 0.35 <- observed @@ -22,7 +22,7 @@ Cohen's f (interaction, partial eta^2=0.007) = 0.086 (small; f: .10 small, .25 16 | 0.95 | 0.96 24 | 0.99 | 1.00 - true stim:log(day) = +4.21 (50% of observed) + true stim:log(day) = +4.214 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.07 | 0.07 <- observed @@ -32,5 +32,4 @@ Cohen's f (interaction, partial eta^2=0.007) = 0.086 (small; f: .10 small, .25 16 | 0.55 | 0.53 24 | 0.63 | 0.62 -Read the per-animal column as the honest power; the LME column matches the -paper's power code (anova interaction p, observation-level DF) and is optimistic. +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d0_10/logpower_result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_10/logpower_result_rate.txt new file mode 100644 index 0000000..f3a0479 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_10/logpower_result_rate.txt @@ -0,0 +1,35 @@ +============================================================================== +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_10 [metric: success RATE] +============================================================================== +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = success RATE) +log(day) uses 1-indexed training day (our day 0 = paper "Day 1") +observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 + +--- fitted on real data --- +stim x log(day) interaction: F(1,109)=4.842 p(resid)=0.02988 p(Satt)=0.03001 (df=103) + honest per-animal random slope (log-day): F(1,7.9)=3.23 p=0.1106 +Cohen's f (interaction, partial eta^2=0.013) = 0.116 (small-medium; f: .10 small, .25 medium, .40 large) + +--- power simulation (log-day ground truth: stim:logday=+0.06771, ratSD=0.04956, resSD=0.1034) --- + + true stim:log(day) = +0.06771 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.23 | 0.47 <- observed + 5 | 0.57 | 0.71 + 8 | 0.85 | 0.87 <- observed + 12 | 0.98 | 0.99 + 16 | 1.00 | 1.00 + 24 | 1.00 | 1.00 + + true stim:log(day) = +0.03385 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.07 | 0.12 <- observed + 5 | 0.12 | 0.17 + 8 | 0.32 | 0.39 <- observed + 12 | 0.36 | 0.39 + 16 | 0.63 | 0.66 + 24 | 0.75 | 0.78 + +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d0_10/logpowersim.m b/analysis/matlab/variations/naive_boxa_d0_10/logpowersim.m index e0686ee..5e38720 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/logpowersim.m +++ b/analysis/matlab/variations/naive_boxa_d0_10/logpowersim.m @@ -1,50 +1,52 @@ -% Variation log-day analysis + Cohen's f + power simulation. +% Variation log-day analysis + Cohen's f + power simulation, for BOTH metrics: +% metric = count : behavior = # successes -> logpower_result.txt +% metric = rate : behavior = success / attempts -> logpower_result_rate.txt % -% The paper's power code models behavior against LOG training day, not raw day: +% The paper's power code models behavior against LOG training day: % behavior ~ stim + log(day) + stim:log(day) + (1|rat). -% Their day is 1-indexed (1..10); our data.csv day is 0-indexed (day 0 = paper -% "Day 1"), so log(day + 1) reproduces their transform exactly. -% -% This script (a) refits that log-day model on data.csv, (b) reports the -% interaction (residual DF, Satterthwaite DF, and the honest per-animal -% random-slope test) and Cohen's f -- the partial-eta^2 effect size of the -% interaction, var(fitted_full) - var(fitted_no_interaction) over var(behavior) -% -- and (c) runs the Monte-Carlo power simulation on the log-day model, -% scoring per-animal (cluster-honest) and LME power across N. -% Writes logpower_result.txt. Run: matlab -batch "logpowersim" +% Their day is 1-indexed; our data.csv day is 0-indexed, so log(day + 1) +% reproduces their transform (our day 0 = paper "Day 1"). For each metric this +% refits that model, reports the interaction (residual / Satterthwaite / honest +% per-animal random-slope DF) and Cohen's f (partial-eta^2 effect size), then +% runs the Monte-Carlo power sim (per-animal cluster-honest + LME power). +% Run: matlab -batch "logpowersim" % (Copy of analysis/matlab/variation_logpower.m; see make_variation_logpower.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' column = log(day+1) -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; -warnState = warning('off', 'all'); -rng(1); +localLogPower(D, 'count', here, vname); +localLogPower(D, 'rate', here, vname); -logday = log(D.day + 1); % 0-indexed day -> their log(1-indexed day) -tbl0 = table(D.success, logday, double(D.stim), categorical(D.subject), ... +% ---------------------------------------------------------------- per metric +function localLogPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; +FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' = log(day+1) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); +logday = log(D.day + 1); +tbl0 = table(beh, logday, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT)\n')]; +s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('log(day) uses 1-indexed training day (our day 0 = paper "Day 1")\n')]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for this analysis (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for this analysis.\n')]; + localFinish(s, here, suffix); warning(warnState); return end -% ---- fitted model on the real data ---- full = fitlme(tbl0, FORMULA); An = anova(full); Asatt = anova(full, 'DFMethod', 'satterthwaite'); ii = strcmp(An.Term, 'day:stim'); is = strcmp(Asatt.Term, 'day:stim'); @@ -52,7 +54,6 @@ reduced = fitlme(tbl0, 'behavior ~ stim + day + (1|rat)'); eta2part = max((var(fitted(full)) - var(fitted(reduced))) / var(tbl0.behavior), 0); cohenf = sqrt(eta2part / (1 - eta2part)); -% honest per-animal random-slope interaction rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; try mr = fitlme(tbl0, 'behavior ~ stim + day + stim:day + (day|rat)'); @@ -76,19 +77,18 @@ end s = [s sprintf('Cohen''s f (interaction, partial eta^2=%.3f) = %.3f (%s; f: .10 small, .25 medium, .40 large)\n', ... eta2part, cohenf, mag)]; -% ---- power simulation on the log-day ground truth ---- cn = full.CoefficientNames; be = full.fixedEffects; b0 = be(strcmp(cn, '(Intercept)')); bStim = be(strcmp(cn, 'stim')); bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(full); sRat = sqrt(psi{1}); sRes = sqrt(full.MSE); -days = unique(tbl0.day); % the log(day) grid +days = unique(tbl0.day); -s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.2f, ratSD=%.2f, resSD=%.2f) ---\n', ... +s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.4g, ratSD=%.4g, resSD=%.4g) ---\n', ... bInt, sRat, sRes)]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:log(day) = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:log(day) = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -102,19 +102,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end -s = [s sprintf(['\nRead the per-animal column as the honest power; the LME column matches the\n' ... - 'paper''s power code (anova interaction p, observation-level DF) and is optimistic.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead per-animal as the honest power; LME matches the paper''s power code (optimistic).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'logpower_result.txt'), 'w'); +fid = fopen(fullfile(here, ['logpower_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d0_10/plotcurve.m b/analysis/matlab/variations/naive_boxa_d0_10/plotcurve.m index bfc1386..193a44b 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/plotcurve.m +++ b/analysis/matlab/variations/naive_boxa_d0_10/plotcurve.m @@ -1,11 +1,12 @@ -% Variation learning-curve plot, in the style of the paper: +% Variation learning-curve plots, in the style of the paper: % "Lines indicate mean (and SEM) across animals in the anodal (red) and % control (blue) groups." -% Plots mean +/- SEM successful reaches per training day for the treatment / -% anodal group (stim = 1, red) and the control group (stim = 0, blue), reading -% this folder's data.csv and saving learning_curve.png. The per-group N is read -% from the data (each variation pools different groups), so the legend shows the -% actual counts. Training day is 1-indexed (our day 0 = the paper's "Day 1"). +% Produces TWO figures from this folder's data.csv: +% learning_curve.png # successes (count) per training day +% learning_curve_rate.png success rate (success/attempts) per training day +% anodal / treatment = stim 1 (red); control = stim 0 (blue). Per-group N is +% read from the data. Training day is 1-indexed (our day 0 = paper "Day 1") and +% the x-axis tick labels are drawn vertically. % Run: matlab -batch "plotcurve" % (Copy of analysis/matlab/variation_plot.m; see make_variation_plot.m.) @@ -14,15 +15,20 @@ if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -days = unique(D.day); % 0-indexed -xd = days + 1; % plot as 1-indexed training day (paper axis) - +days = unique(D.day); +xd = days + 1; % plot as 1-indexed training day (paper axis) red = [0.85 0.10 0.10]; blue = [0.10 0.30 0.85]; -[Ma, Sa, na] = localCurve(D, 1, days); % anodal / treatment (stim = 1) -[Mc, Sc, nc] = localCurve(D, 0, days); % control (stim = 0) +localPlot(D, days, xd, 'count', '# successes', ... + fullfile(here, 'learning_curve.png'), vname, red, blue); +localPlot(D, days, xd, 'rate', 'success rate', ... + fullfile(here, 'learning_curve_rate.png'), vname, red, blue); +% ------------------------------------------------------------------ helpers +function localPlot(D, days, xd, metric, ylab, outFile, vname, red, blue) +[Ma, Sa, na] = localCurve(D, 1, days, metric); % anodal / treatment +[Mc, Sc, nc] = localCurve(D, 0, days, metric); % control fig = figure('Visible', 'off', 'Color', 'w', 'Position', [100 100 560 460]); hold on e1 = errorbar(xd, Ma, Sa, '-o', 'Color', red, 'MarkerFaceColor', red, 'LineWidth', 2); @@ -30,26 +36,30 @@ e2 = errorbar(xd, Mc, Sc, '-o', 'Color', blue, 'MarkerFaceColor', blue, 'LineWid hold off legend([e1 e2], {sprintf('anodal, N = %d', na), sprintf('control, N = %d', nc)}, ... 'Location', 'northwest', 'Box', 'off'); -xlabel('training day'); -ylabel('# successes'); +xlabel('training day'); ylabel(ylab); title(vname, 'Interpreter', 'none'); set(gca, 'XTick', xd, 'FontName', 'Arial', 'FontSize', 13, 'LineWidth', 1.5, 'Box', 'off'); - -outFile = fullfile(here, 'learning_curve.png'); +xtickangle(90); % vertical x-axis tick labels exportgraphics(fig, outFile, 'Resolution', 150); close(fig); -fprintf('%s: wrote learning_curve.png (anodal N=%d, control N=%d)\n', vname, na, nc); +fprintf('%s: wrote %s (anodal N=%d, control N=%d)\n', vname, outFile, na, nc); +end -% ------------------------------------------------------------------ helper -function [M, S, n] = localCurve(D, stimVal, days) -%LOCALCURVE Per-day mean and SEM of successes across the animals in a group. +function [M, S, n] = localCurve(D, stimVal, days, metric) +%LOCALCURVE Per-day mean and SEM across the animals in a group, for a metric. subs = unique(D.subject(D.stim == stimVal)); n = numel(subs); X = nan(numel(days), n); for j = 1:n for i = 1:numel(days) r = D.subject == subs(j) & D.day == days(i); - if any(r); X(i, j) = mean(D.success(r)); end + if ~any(r); continue; end + if strcmp(metric, 'rate') + tot = sum(D.total(r)); + if tot > 0; X(i, j) = sum(D.success(r)) / tot; end + else + X(i, j) = mean(D.success(r)); + end end end M = mean(X, 2, 'omitnan'); diff --git a/analysis/matlab/variations/naive_boxa_d0_10/power_result.txt b/analysis/matlab/variations/naive_boxa_d0_10/power_result.txt index 64b5503..eb8153d 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/power_result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_10/power_result.txt @@ -1,11 +1,11 @@ ============================================================================== -POWER SIMULATION -- naive_boxa_d0_10 +POWER SIMULATION -- naive_boxa_d0_10 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + day + stim:day + (1|rat) (success COUNT; day within-window) +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = # successes (count); day within-window) observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 -ground truth: stim:day=+1.18/day, rat SD=8.67, residual SD=13.71, days=11 +ground truth: stim:day=+1.178/day, rat SD=8.671, residual SD=13.71, days=11 - true stim:day interaction = +1.18 (100% of observed) + true stim:day interaction = +1.178 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.07 | 0.17 <- observed @@ -15,7 +15,7 @@ ground truth: stim:day=+1.18/day, rat SD=8.67, residual SD=13.71, days=11 16 | 0.68 | 0.70 24 | 0.88 | 0.88 - true stim:day interaction = +0.59 (50% of observed) + true stim:day interaction = +0.589 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.05 | 0.07 <- observed @@ -25,5 +25,4 @@ ground truth: stim:day=+1.18/day, rat SD=8.67, residual SD=13.71, days=11 16 | 0.28 | 0.27 24 | 0.28 | 0.29 -Read the per-animal column as the honest power. At the observed N this study -is typically underpowered; per-animal power reaches ~0.8 only at larger N. +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d0_10/power_result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_10/power_result_rate.txt new file mode 100644 index 0000000..017911b --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_10/power_result_rate.txt @@ -0,0 +1,28 @@ +============================================================================== +POWER SIMULATION -- naive_boxa_d0_10 [metric: success RATE] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE; day within-window) +observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 +ground truth: stim:day=+0.008893/day, rat SD=0.05104, residual SD=0.09808, days=11 + + true stim:day interaction = +0.008893 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.08 | 0.16 <- observed + 5 | 0.21 | 0.25 + 8 | 0.36 | 0.43 <- observed + 12 | 0.62 | 0.68 + 16 | 0.72 | 0.73 + 24 | 0.88 | 0.88 + + true stim:day interaction = +0.004446 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.07 | 0.08 <- observed + 5 | 0.05 | 0.08 + 8 | 0.11 | 0.11 <- observed + 12 | 0.17 | 0.19 + 16 | 0.30 | 0.29 + 24 | 0.33 | 0.35 + +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d0_10/powersim.m b/analysis/matlab/variations/naive_boxa_d0_10/powersim.m index 554d032..ecd9c6c 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/powersim.m +++ b/analysis/matlab/variations/naive_boxa_d0_10/powersim.m @@ -1,45 +1,49 @@ % Variation power simulation -- Monte-Carlo power for the paper's stim x day -% interaction, using THIS folder's data as the ground truth. +% interaction, using THIS folder's data as the ground truth, for BOTH metrics: +% metric = count : behavior = # successes -> power_result.txt +% metric = rate : behavior = success / attempts -> power_result_rate.txt % % Ground truth: fitlme(behavior ~ stim + day + stim:day + (1|rat)) on data.csv -% (success COUNT; day within-window). Its fixed effects, per-rat intercept SD, -% and residual SD generate NREP synthetic datasets at each rats-per-group N and -% each true-effect multiplier (1 = observed slope, 0.5 = half). Each dataset is -% scored at alpha = 0.05 two ways: -% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest -- the -% honest power, matching the random-slope / per-animal inference) +% (day within-window). Its fixed effects, per-rat intercept SD, and residual SD +% generate NREP synthetic datasets at each rats-per-group N and each true-effect +% multiplier (1 = observed, 0.5 = half). Each is scored at alpha=0.05 by: +% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest power) % LME : the fitlme stim:day p (observation-level DF -- optimistic) -% Writes power_result.txt beside this script. Run: matlab -batch "powersim" +% Writes power_result[_rate].txt. Run: matlab -batch "powersim" % (Copy of analysis/matlab/variation_power.m; see make_variation_power.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); + +localPower(D, 'count', here, vname); +localPower(D, 'rate', here, vname); + +% ---------------------------------------------------------------- per metric +function localPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; - -warnState = warning('off', 'all'); -rng(1); - +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); day0 = min(D.day); -tbl0 = table(D.success, D.day - day0, double(D.stim), categorical(D.subject), ... +tbl0 = table(beh, D.day - day0, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); - nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nPOWER SIMULATION -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: %s (success COUNT; day within-window)\n', FORMULA)]; +s = sprintf('%s\nPOWER SIMULATION -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: %s (behavior = %s; day within-window)\n', FORMULA, mlabel)]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for a power simulation (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for a power simulation.\n')]; + localFinish(s, here, suffix); warning(warnState); return end lme = fitlme(tbl0, FORMULA); @@ -49,14 +53,13 @@ bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(lme); sRat = sqrt(psi{1}); sRes = sqrt(lme.MSE); days = (0:max(tbl0.day))'; -s = [s sprintf('ground truth: stim:day=%+.2f/day, rat SD=%.2f, residual SD=%.2f, days=%d\n', ... +s = [s sprintf('ground truth: stim:day=%+.4g/day, rat SD=%.4g, residual SD=%.4g, days=%d\n', ... bInt, sRat, sRes, numel(days))]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:day interaction = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n', 'N/group')]; %#ok - s = [s sprintf(' %s\n', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:day interaction = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -70,20 +73,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end - -s = [s sprintf(['\nRead the per-animal column as the honest power. At the observed N this study\n' ... - 'is typically underpowered; per-animal power reaches ~0.8 only at larger N.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead the per-animal column as the honest power; LME is optimistic (obs-level DF).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'power_result.txt'), 'w'); +fid = fopen(fullfile(here, ['power_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d0_10/result.txt b/analysis/matlab/variations/naive_boxa_d0_10/result.txt index a972e49..67d4c5a 100644 --- a/analysis/matlab/variations/naive_boxa_d0_10/result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_10/result.txt @@ -1,5 +1,5 @@ ============================================================================== -VARIATION: naive_boxa_d0_10 +VARIATION: naive_boxa_d0_10 [metric: success COUNT] ============================================================================== model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT) day = training day within window (0 = first analyzed day) @@ -60,9 +60,9 @@ effect t(df) / F(df1) p (resid) Satterthwaite: p (df) stim x day (interaction) t(111)= 1.31 F(1)= 1.707 p=0.1941 p=0.1943 (df=104) day (learning) t(111)= 16.91 F(1)=285.824 p=1.743e-32 p=6.758e-32 (df=106) stim (main, window start) t(111)= 1.55 F(1)= 2.392 p=0.1248 p=0.1361 (df=22) -interaction 95% CI: [-0.61, +2.96] +interaction 95% CI: [-0.6088, +2.965] HONEST LME (per-animal random slope, day|rat): interaction F(1,10.1)=0.83, p=0.3836 (Satterthwaite DF ~= residual on this random-intercept model; the random-slope model above is the honest learning-rate test -- DF collapses toward the animal count.) -INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.1941, slope diff=+1.18) -Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008. +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.1941, slope diff=+1.178) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d0_10/result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_10/result_rate.txt new file mode 100644 index 0000000..5f1e72e --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_10/result_rate.txt @@ -0,0 +1,68 @@ +============================================================================== +VARIATION: naive_boxa_d0_10 [metric: success RATE (success/attempts)] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE (success/attempts)) +day = training day within window (0 = first analyzed day) +treatment (stim=1): Electrode-Box-B2 +control (stim=0): Electrode-Box-A, Electrode-Box-A2, Naive +N = 11 rats, 113 sessions raw day coverage: treat 0..10, control 0..10 +(equal day coverage over this window) + +============================================================================== +FULL MODEL SUMMARY -- fitlme +============================================================================== + +Linear mixed-effects model fit by ML + +Model information: + Number of observations 113 + Fixed effects coefficients 4 + Random effects coefficients 11 + Covariance parameters 2 + +Formula: + behavior ~ 1 + day*stim + (1 | rat) + +Model fit statistics: + AIC BIC LogLikelihood Deviance + -177.53 -161.16 94.764 -189.53 + +Fixed effects coefficients (95% CIs): + Name Estimate SE tStat DF pValue + {'(Intercept)'} 0.19077 0.027729 6.8796 109 3.9606e-10 + {'day' } 0.044032 0.0036365 12.108 109 6.709e-22 + {'stim' } 0.064146 0.051551 1.2443 109 0.21605 + {'day:stim' } 0.0088928 0.0065096 1.3661 109 0.17471 + + + Lower Upper + 0.13581 0.24572 + 0.036824 0.051239 + -0.038027 0.16632 + -0.0040089 0.021795 + +Random effects covariance parameters (95% CIs): +Group: rat (11 Levels) + Name1 Name2 Type Estimate + {'(Intercept)'} {'(Intercept)'} {'std'} 0.05104 + + + Lower Upper + 0.028096 0.092721 + +Group: Error + Name Estimate Lower Upper + {'Res Std'} 0.09808 0.085453 0.11257 + + +effect t(df) / F(df1) p (resid) Satterthwaite: p (df) +---------------------------------------------------------------------------- +stim x day (interaction) t(109)= 1.37 F(1)= 1.866 p=0.1747 p=0.1749 (df=103) +day (learning) t(109)= 12.11 F(1)=146.612 p=6.709e-22 p=1.131e-21 (df=105) +stim (main, window start) t(109)= 1.24 F(1)= 1.548 p=0.2161 p=0.2244 (df=26) +interaction 95% CI: [-0.004009, +0.02179] +HONEST LME (per-animal random slope, day|rat): interaction F(1,9.6)=1.38, p=0.2681 + (Satterthwaite DF ~= residual on this random-intercept model; the random-slope + model above is the honest learning-rate test -- DF collapses toward the animal count.) +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.1747, slope diff=+0.008893) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d0_13/analyze.m b/analysis/matlab/variations/naive_boxa_d0_13/analyze.m index 82da5ed..0d40402 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/analyze.m +++ b/analysis/matlab/variations/naive_boxa_d0_13/analyze.m @@ -1,30 +1,58 @@ -% Variation analysis -- the paper's linear mixed model on the successful-reach -% COUNT, fit on this folder's curated data subset. +% Variation analysis -- the paper's linear mixed model on this folder's data, +% for BOTH metrics: +% metric = count : behavior = # successes -> result.txt +% metric = rate : behavior = success / attempts -> result_rate.txt +% (rate uses only sessions with attempts > 0) % % model: behavior ~ stim + day + stim:day + (1|rat) -% behavior = successful reaches (count per session) -% stim = 1 for the treatment group(s), 0 for the control group(s) -% day = training day within this window (0 = first analyzed day) -% rat = subject (random intercept) -% -% Self-contained: reads data.csv beside this script and writes result.txt. -% Run headless from this folder with: matlab -batch "analyze" -% (This is a copy of analysis/matlab/variation_analyze.m; see make_variations.m.) +% stim = 1 treatment / 0 control; day = training day within window (0 = +% first analyzed day); rat = subject (random intercept). +% For the interaction we report residual DF, Satterthwaite DF, and the honest +% per-animal random-slope test. Self-contained: reads data.csv beside this +% script. Run headless with: matlab -batch "analyze" +% (Copy of analysis/matlab/variation_analyze.m; see make_variations.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end -vname = regexprep(here, '.*[/\\]', ''); % folder name = variation id +vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -tbl = table(D.success, D.day - min(D.day), double(D.stim), categorical(D.subject), ... +Rc = localAnalyze(D, 'count', here, vname); +Rr = localAnalyze(D, 'rate', here, vname); + +% Machine-readable handoff for SUMMARY.csv (count drives it; rate appended). +VARRESULT = struct('name', vname, 'nRats', Rc.nRats, 'nObs', Rc.nObs, ... + 'interP', Rc.interP, 'interEst', Rc.interEst, ... + 'interPsatt', Rc.interPsatt, 'interPrs', Rc.interPrs, ... + 'stimP', Rc.stimP, 'dayP', Rc.dayP, 'covEqual', Rc.covEqual, ... + 'interPrate', Rr.interP, 'interEstRate', Rr.interEst, 'interPrsRate', Rr.interPrs); + +% ------------------------------------------------------------------ helper +function R = localAnalyze(D, metric, here, vname) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); + beh = D.success ./ D.total; + mlabel = 'success RATE (success/attempts)'; suffix = '_rate'; +else + beh = D.success; + mlabel = 'success COUNT'; suffix = ''; +end +R = struct('interP', NaN, 'interEst', NaN, 'interPsatt', NaN, 'interPrs', NaN, ... + 'stimP', NaN, 'dayP', NaN, 'nRats', numel(unique(D.subject)), ... + 'nObs', height(D), 'covEqual', false); +if numel(unique(D.stim)) < 2 || numel(unique(D.day)) < 2 + localWrite(sprintf('VARIATION: %s [metric: %s]\nInsufficient data for this metric.\n', ... + vname, mlabel), here, suffix); + return +end + +tbl = table(beh, D.day - min(D.day), double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); m = fitlme(tbl, 'behavior ~ stim + day + stim:day + (1|rat)'); C = m.Coefficients; A = anova(m); ci = coefCI(m); -As = anova(m, 'DFMethod', 'satterthwaite'); % Satterthwaite denominator DF +As = anova(m, 'DFMethod', 'satterthwaite'); -% Honest test: refit with a per-animal random SLOPE so the interaction DF -% collapses toward the animal count (guarded -- may not converge in short windows). rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; wst = warning('off', 'all'); try @@ -43,6 +71,7 @@ row = @(nm, t) sprintf('%-26s t(%d)=%6.2f F(%d)=%7.3f p=%.4g p=%.4g (df=%.0f C.DF(gi(t)), C.tStat(gi(t)), A.DF1(ga(t)), A.FStat(ga(t)), C.pValue(gi(t)), ... As.pValue(gs(t)), As.DF2(gs(t))); +ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); maxT = max(D.day(D.stim == 1)); minT = min(D.day(D.stim == 1)); maxC = max(D.day(D.stim == 0)); minC = min(D.day(D.stim == 0)); if abs(maxT - maxC) > 2 @@ -50,20 +79,14 @@ if abs(maxT - maxC) > 2 else cov = '(equal day coverage over this window)'; end - -ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); -if pI >= 0.05 - verdict = 'n.s. -- slopes parallel (no differential learning rate)'; -elseif eI > 0 - verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; -else - verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; -end +if pI >= 0.05; verdict = 'n.s. -- slopes parallel (no differential learning rate)'; +elseif eI > 0; verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; +else; verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; end bar = repmat('=', 1, 78); raw = regexprep(evalc('disp(m)'), '', ''); -s = sprintf('%s\nVARIATION: %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT)\n')]; +s = sprintf('%s\nVARIATION: %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('day = training day within window (0 = first analyzed day)\n')]; s = [s sprintf('treatment (stim=1): %s\n', strjoin(cellstr(unique(D.group(D.stim == 1))), ', '))]; s = [s sprintf('control (stim=0): %s\n', strjoin(cellstr(unique(D.group(D.stim == 0))), ', '))]; @@ -74,7 +97,7 @@ s = [s sprintf('%-26s %-18s %-12s %s\n%s\n', 'effect', 't(df) / F(df1)', 'p (res s = [s row('stim x day (interaction)', 'day:stim')]; s = [s row('day (learning)', 'day')]; s = [s row('stim (main, window start)', 'stim')]; -s = [s sprintf('interaction 95%% CI: [%+.2f, %+.2f]\n', ci(ii, 1), ci(ii, 2))]; +s = [s sprintf('interaction 95%% CI: [%+.4g, %+.4g]\n', ci(ii, 1), ci(ii, 2))]; if rsOk s = [s sprintf('HONEST LME (per-animal random slope, day|rat): interaction F(1,%.1f)=%.2f, p=%.4g\n', rsDf, rsF, rsP)]; else @@ -82,17 +105,18 @@ else end s = [s sprintf([' (Satterthwaite DF ~= residual on this random-intercept model; the random-slope\n' ... ' model above is the honest learning-rate test -- DF collapses toward the animal count.)\n'])]; -s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.2f)\n', verdict, pI, eI)]; -s = [s sprintf('Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.4g)\n', verdict, pI, eI)]; +s = [s sprintf('Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +localWrite(s, here, suffix); + +R = struct('interP', pI, 'interEst', eI, 'interPsatt', As.pValue(gs('day:stim')), ... + 'interPrs', rsP, 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... + 'nRats', numel(unique(D.subject)), 'nObs', height(D), 'covEqual', abs(maxT - maxC) <= 2); +end + +function localWrite(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'result.txt'), 'w'); -fprintf(fid, '%s', s); -fclose(fid); - -% Machine-readable handoff for the summary table (see make_variations.m). -VARRESULT = struct('name', vname, 'nRats', numel(unique(D.subject)), ... - 'nObs', height(D), 'interP', pI, 'interEst', eI, ... - 'interPsatt', As.pValue(gs('day:stim')), 'interPrs', rsP, ... - 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... - 'covEqual', abs(maxT - maxC) <= 2); +fid = fopen(fullfile(here, ['result' suffix '.txt']), 'w'); +fprintf(fid, '%s', s); fclose(fid); +end diff --git a/analysis/matlab/variations/naive_boxa_d0_13/learning_curve.png b/analysis/matlab/variations/naive_boxa_d0_13/learning_curve.png index 026ae49..fa988ed 100644 Binary files a/analysis/matlab/variations/naive_boxa_d0_13/learning_curve.png and b/analysis/matlab/variations/naive_boxa_d0_13/learning_curve.png differ diff --git a/analysis/matlab/variations/naive_boxa_d0_13/learning_curve_rate.png b/analysis/matlab/variations/naive_boxa_d0_13/learning_curve_rate.png new file mode 100644 index 0000000..1962484 Binary files /dev/null and b/analysis/matlab/variations/naive_boxa_d0_13/learning_curve_rate.png differ diff --git a/analysis/matlab/variations/naive_boxa_d0_13/logpower_result.txt b/analysis/matlab/variations/naive_boxa_d0_13/logpower_result.txt index 264e541..c0b8d44 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/logpower_result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_13/logpower_result.txt @@ -1,7 +1,7 @@ ============================================================================== -LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_13 +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_13 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT) +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = # successes (count)) log(day) uses 1-indexed training day (our day 0 = paper "Day 1") observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 @@ -10,9 +10,9 @@ stim x log(day) interaction: F(1,134)=4.865 p(resid)=0.02911 p(Satt)=0.02918 ( honest per-animal random slope (log-day): F(1,7.9)=3.22 p=0.1109 Cohen's f (interaction, partial eta^2=0.007) = 0.083 (small; f: .10 small, .25 medium, .40 large) ---- power simulation (log-day ground truth: stim:logday=+8.27, ratSD=8.74, resSD=14.51) --- +--- power simulation (log-day ground truth: stim:logday=+8.265, ratSD=8.74, resSD=14.51) --- - true stim:log(day) = +8.27 (100% of observed) + true stim:log(day) = +8.265 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.21 | 0.55 <- observed @@ -22,7 +22,7 @@ Cohen's f (interaction, partial eta^2=0.007) = 0.083 (small; f: .10 small, .25 16 | 0.98 | 0.98 24 | 1.00 | 1.00 - true stim:log(day) = +4.13 (50% of observed) + true stim:log(day) = +4.133 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.08 | 0.17 <- observed @@ -32,5 +32,4 @@ Cohen's f (interaction, partial eta^2=0.007) = 0.083 (small; f: .10 small, .25 16 | 0.64 | 0.68 24 | 0.76 | 0.78 -Read the per-animal column as the honest power; the LME column matches the -paper's power code (anova interaction p, observation-level DF) and is optimistic. +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d0_13/logpower_result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_13/logpower_result_rate.txt new file mode 100644 index 0000000..21868f6 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_13/logpower_result_rate.txt @@ -0,0 +1,35 @@ +============================================================================== +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_13 [metric: success RATE] +============================================================================== +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = success RATE) +log(day) uses 1-indexed training day (our day 0 = paper "Day 1") +observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 + +--- fitted on real data --- +stim x log(day) interaction: F(1,132)=5.963 p(resid)=0.01593 p(Satt)=0.01598 (df=127) + honest per-animal random slope (log-day): F(1,8.1)=4.16 p=0.07502 +Cohen's f (interaction, partial eta^2=0.012) = 0.111 (small-medium; f: .10 small, .25 medium, .40 large) + +--- power simulation (log-day ground truth: stim:logday=+0.06399, ratSD=0.05139, resSD=0.1) --- + + true stim:log(day) = +0.06399 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.26 | 0.66 <- observed + 5 | 0.57 | 0.78 + 8 | 0.93 | 0.96 <- observed + 12 | 1.00 | 1.00 + 16 | 0.99 | 0.99 + 24 | 1.00 | 1.00 + + true stim:log(day) = +0.032 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.09 | 0.23 <- observed + 5 | 0.23 | 0.28 + 8 | 0.48 | 0.53 <- observed + 12 | 0.55 | 0.61 + 16 | 0.76 | 0.78 + 24 | 0.85 | 0.82 + +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d0_13/logpowersim.m b/analysis/matlab/variations/naive_boxa_d0_13/logpowersim.m index e0686ee..5e38720 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/logpowersim.m +++ b/analysis/matlab/variations/naive_boxa_d0_13/logpowersim.m @@ -1,50 +1,52 @@ -% Variation log-day analysis + Cohen's f + power simulation. +% Variation log-day analysis + Cohen's f + power simulation, for BOTH metrics: +% metric = count : behavior = # successes -> logpower_result.txt +% metric = rate : behavior = success / attempts -> logpower_result_rate.txt % -% The paper's power code models behavior against LOG training day, not raw day: +% The paper's power code models behavior against LOG training day: % behavior ~ stim + log(day) + stim:log(day) + (1|rat). -% Their day is 1-indexed (1..10); our data.csv day is 0-indexed (day 0 = paper -% "Day 1"), so log(day + 1) reproduces their transform exactly. -% -% This script (a) refits that log-day model on data.csv, (b) reports the -% interaction (residual DF, Satterthwaite DF, and the honest per-animal -% random-slope test) and Cohen's f -- the partial-eta^2 effect size of the -% interaction, var(fitted_full) - var(fitted_no_interaction) over var(behavior) -% -- and (c) runs the Monte-Carlo power simulation on the log-day model, -% scoring per-animal (cluster-honest) and LME power across N. -% Writes logpower_result.txt. Run: matlab -batch "logpowersim" +% Their day is 1-indexed; our data.csv day is 0-indexed, so log(day + 1) +% reproduces their transform (our day 0 = paper "Day 1"). For each metric this +% refits that model, reports the interaction (residual / Satterthwaite / honest +% per-animal random-slope DF) and Cohen's f (partial-eta^2 effect size), then +% runs the Monte-Carlo power sim (per-animal cluster-honest + LME power). +% Run: matlab -batch "logpowersim" % (Copy of analysis/matlab/variation_logpower.m; see make_variation_logpower.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' column = log(day+1) -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; -warnState = warning('off', 'all'); -rng(1); +localLogPower(D, 'count', here, vname); +localLogPower(D, 'rate', here, vname); -logday = log(D.day + 1); % 0-indexed day -> their log(1-indexed day) -tbl0 = table(D.success, logday, double(D.stim), categorical(D.subject), ... +% ---------------------------------------------------------------- per metric +function localLogPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; +FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' = log(day+1) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); +logday = log(D.day + 1); +tbl0 = table(beh, logday, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT)\n')]; +s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('log(day) uses 1-indexed training day (our day 0 = paper "Day 1")\n')]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for this analysis (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for this analysis.\n')]; + localFinish(s, here, suffix); warning(warnState); return end -% ---- fitted model on the real data ---- full = fitlme(tbl0, FORMULA); An = anova(full); Asatt = anova(full, 'DFMethod', 'satterthwaite'); ii = strcmp(An.Term, 'day:stim'); is = strcmp(Asatt.Term, 'day:stim'); @@ -52,7 +54,6 @@ reduced = fitlme(tbl0, 'behavior ~ stim + day + (1|rat)'); eta2part = max((var(fitted(full)) - var(fitted(reduced))) / var(tbl0.behavior), 0); cohenf = sqrt(eta2part / (1 - eta2part)); -% honest per-animal random-slope interaction rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; try mr = fitlme(tbl0, 'behavior ~ stim + day + stim:day + (day|rat)'); @@ -76,19 +77,18 @@ end s = [s sprintf('Cohen''s f (interaction, partial eta^2=%.3f) = %.3f (%s; f: .10 small, .25 medium, .40 large)\n', ... eta2part, cohenf, mag)]; -% ---- power simulation on the log-day ground truth ---- cn = full.CoefficientNames; be = full.fixedEffects; b0 = be(strcmp(cn, '(Intercept)')); bStim = be(strcmp(cn, 'stim')); bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(full); sRat = sqrt(psi{1}); sRes = sqrt(full.MSE); -days = unique(tbl0.day); % the log(day) grid +days = unique(tbl0.day); -s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.2f, ratSD=%.2f, resSD=%.2f) ---\n', ... +s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.4g, ratSD=%.4g, resSD=%.4g) ---\n', ... bInt, sRat, sRes)]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:log(day) = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:log(day) = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -102,19 +102,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end -s = [s sprintf(['\nRead the per-animal column as the honest power; the LME column matches the\n' ... - 'paper''s power code (anova interaction p, observation-level DF) and is optimistic.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead per-animal as the honest power; LME matches the paper''s power code (optimistic).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'logpower_result.txt'), 'w'); +fid = fopen(fullfile(here, ['logpower_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d0_13/plotcurve.m b/analysis/matlab/variations/naive_boxa_d0_13/plotcurve.m index bfc1386..193a44b 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/plotcurve.m +++ b/analysis/matlab/variations/naive_boxa_d0_13/plotcurve.m @@ -1,11 +1,12 @@ -% Variation learning-curve plot, in the style of the paper: +% Variation learning-curve plots, in the style of the paper: % "Lines indicate mean (and SEM) across animals in the anodal (red) and % control (blue) groups." -% Plots mean +/- SEM successful reaches per training day for the treatment / -% anodal group (stim = 1, red) and the control group (stim = 0, blue), reading -% this folder's data.csv and saving learning_curve.png. The per-group N is read -% from the data (each variation pools different groups), so the legend shows the -% actual counts. Training day is 1-indexed (our day 0 = the paper's "Day 1"). +% Produces TWO figures from this folder's data.csv: +% learning_curve.png # successes (count) per training day +% learning_curve_rate.png success rate (success/attempts) per training day +% anodal / treatment = stim 1 (red); control = stim 0 (blue). Per-group N is +% read from the data. Training day is 1-indexed (our day 0 = paper "Day 1") and +% the x-axis tick labels are drawn vertically. % Run: matlab -batch "plotcurve" % (Copy of analysis/matlab/variation_plot.m; see make_variation_plot.m.) @@ -14,15 +15,20 @@ if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -days = unique(D.day); % 0-indexed -xd = days + 1; % plot as 1-indexed training day (paper axis) - +days = unique(D.day); +xd = days + 1; % plot as 1-indexed training day (paper axis) red = [0.85 0.10 0.10]; blue = [0.10 0.30 0.85]; -[Ma, Sa, na] = localCurve(D, 1, days); % anodal / treatment (stim = 1) -[Mc, Sc, nc] = localCurve(D, 0, days); % control (stim = 0) +localPlot(D, days, xd, 'count', '# successes', ... + fullfile(here, 'learning_curve.png'), vname, red, blue); +localPlot(D, days, xd, 'rate', 'success rate', ... + fullfile(here, 'learning_curve_rate.png'), vname, red, blue); +% ------------------------------------------------------------------ helpers +function localPlot(D, days, xd, metric, ylab, outFile, vname, red, blue) +[Ma, Sa, na] = localCurve(D, 1, days, metric); % anodal / treatment +[Mc, Sc, nc] = localCurve(D, 0, days, metric); % control fig = figure('Visible', 'off', 'Color', 'w', 'Position', [100 100 560 460]); hold on e1 = errorbar(xd, Ma, Sa, '-o', 'Color', red, 'MarkerFaceColor', red, 'LineWidth', 2); @@ -30,26 +36,30 @@ e2 = errorbar(xd, Mc, Sc, '-o', 'Color', blue, 'MarkerFaceColor', blue, 'LineWid hold off legend([e1 e2], {sprintf('anodal, N = %d', na), sprintf('control, N = %d', nc)}, ... 'Location', 'northwest', 'Box', 'off'); -xlabel('training day'); -ylabel('# successes'); +xlabel('training day'); ylabel(ylab); title(vname, 'Interpreter', 'none'); set(gca, 'XTick', xd, 'FontName', 'Arial', 'FontSize', 13, 'LineWidth', 1.5, 'Box', 'off'); - -outFile = fullfile(here, 'learning_curve.png'); +xtickangle(90); % vertical x-axis tick labels exportgraphics(fig, outFile, 'Resolution', 150); close(fig); -fprintf('%s: wrote learning_curve.png (anodal N=%d, control N=%d)\n', vname, na, nc); +fprintf('%s: wrote %s (anodal N=%d, control N=%d)\n', vname, outFile, na, nc); +end -% ------------------------------------------------------------------ helper -function [M, S, n] = localCurve(D, stimVal, days) -%LOCALCURVE Per-day mean and SEM of successes across the animals in a group. +function [M, S, n] = localCurve(D, stimVal, days, metric) +%LOCALCURVE Per-day mean and SEM across the animals in a group, for a metric. subs = unique(D.subject(D.stim == stimVal)); n = numel(subs); X = nan(numel(days), n); for j = 1:n for i = 1:numel(days) r = D.subject == subs(j) & D.day == days(i); - if any(r); X(i, j) = mean(D.success(r)); end + if ~any(r); continue; end + if strcmp(metric, 'rate') + tot = sum(D.total(r)); + if tot > 0; X(i, j) = sum(D.success(r)) / tot; end + else + X(i, j) = mean(D.success(r)); + end end end M = mean(X, 2, 'omitnan'); diff --git a/analysis/matlab/variations/naive_boxa_d0_13/power_result.txt b/analysis/matlab/variations/naive_boxa_d0_13/power_result.txt index 37e92dd..8282c2b 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/power_result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_13/power_result.txt @@ -1,11 +1,11 @@ ============================================================================== -POWER SIMULATION -- naive_boxa_d0_13 +POWER SIMULATION -- naive_boxa_d0_13 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + day + stim:day + (1|rat) (success COUNT; day within-window) +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = # successes (count); day within-window) observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 -ground truth: stim:day=+1.31/day, rat SD=8.76, residual SD=15.18, days=14 +ground truth: stim:day=+1.306/day, rat SD=8.76, residual SD=15.18, days=14 - true stim:day interaction = +1.31 (100% of observed) + true stim:day interaction = +1.306 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.12 | 0.32 <- observed @@ -15,7 +15,7 @@ ground truth: stim:day=+1.31/day, rat SD=8.76, residual SD=15.18, days=14 16 | 0.95 | 0.96 24 | 0.97 | 0.98 - true stim:day interaction = +0.65 (50% of observed) + true stim:day interaction = +0.6532 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.07 | 0.17 <- observed @@ -25,5 +25,4 @@ ground truth: stim:day=+1.31/day, rat SD=8.76, residual SD=15.18, days=14 16 | 0.42 | 0.51 24 | 0.62 | 0.60 -Read the per-animal column as the honest power. At the observed N this study -is typically underpowered; per-animal power reaches ~0.8 only at larger N. +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d0_13/power_result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_13/power_result_rate.txt new file mode 100644 index 0000000..9ebdf75 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_13/power_result_rate.txt @@ -0,0 +1,28 @@ +============================================================================== +POWER SIMULATION -- naive_boxa_d0_13 [metric: success RATE] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE; day within-window) +observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 +ground truth: stim:day=+0.009461/day, rat SD=0.04773, residual SD=0.1046, days=14 + + true stim:day interaction = +0.009461 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.13 | 0.37 <- observed + 5 | 0.40 | 0.52 + 8 | 0.70 | 0.72 <- observed + 12 | 0.96 | 0.96 + 16 | 0.97 | 0.97 + 24 | 0.99 | 0.99 + + true stim:day interaction = +0.00473 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.07 | 0.18 <- observed + 5 | 0.18 | 0.20 + 8 | 0.34 | 0.32 <- observed + 12 | 0.38 | 0.39 + 16 | 0.48 | 0.53 + 24 | 0.64 | 0.63 + +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d0_13/powersim.m b/analysis/matlab/variations/naive_boxa_d0_13/powersim.m index 554d032..ecd9c6c 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/powersim.m +++ b/analysis/matlab/variations/naive_boxa_d0_13/powersim.m @@ -1,45 +1,49 @@ % Variation power simulation -- Monte-Carlo power for the paper's stim x day -% interaction, using THIS folder's data as the ground truth. +% interaction, using THIS folder's data as the ground truth, for BOTH metrics: +% metric = count : behavior = # successes -> power_result.txt +% metric = rate : behavior = success / attempts -> power_result_rate.txt % % Ground truth: fitlme(behavior ~ stim + day + stim:day + (1|rat)) on data.csv -% (success COUNT; day within-window). Its fixed effects, per-rat intercept SD, -% and residual SD generate NREP synthetic datasets at each rats-per-group N and -% each true-effect multiplier (1 = observed slope, 0.5 = half). Each dataset is -% scored at alpha = 0.05 two ways: -% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest -- the -% honest power, matching the random-slope / per-animal inference) +% (day within-window). Its fixed effects, per-rat intercept SD, and residual SD +% generate NREP synthetic datasets at each rats-per-group N and each true-effect +% multiplier (1 = observed, 0.5 = half). Each is scored at alpha=0.05 by: +% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest power) % LME : the fitlme stim:day p (observation-level DF -- optimistic) -% Writes power_result.txt beside this script. Run: matlab -batch "powersim" +% Writes power_result[_rate].txt. Run: matlab -batch "powersim" % (Copy of analysis/matlab/variation_power.m; see make_variation_power.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); + +localPower(D, 'count', here, vname); +localPower(D, 'rate', here, vname); + +% ---------------------------------------------------------------- per metric +function localPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; - -warnState = warning('off', 'all'); -rng(1); - +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); day0 = min(D.day); -tbl0 = table(D.success, D.day - day0, double(D.stim), categorical(D.subject), ... +tbl0 = table(beh, D.day - day0, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); - nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nPOWER SIMULATION -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: %s (success COUNT; day within-window)\n', FORMULA)]; +s = sprintf('%s\nPOWER SIMULATION -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: %s (behavior = %s; day within-window)\n', FORMULA, mlabel)]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for a power simulation (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for a power simulation.\n')]; + localFinish(s, here, suffix); warning(warnState); return end lme = fitlme(tbl0, FORMULA); @@ -49,14 +53,13 @@ bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(lme); sRat = sqrt(psi{1}); sRes = sqrt(lme.MSE); days = (0:max(tbl0.day))'; -s = [s sprintf('ground truth: stim:day=%+.2f/day, rat SD=%.2f, residual SD=%.2f, days=%d\n', ... +s = [s sprintf('ground truth: stim:day=%+.4g/day, rat SD=%.4g, residual SD=%.4g, days=%d\n', ... bInt, sRat, sRes, numel(days))]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:day interaction = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n', 'N/group')]; %#ok - s = [s sprintf(' %s\n', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:day interaction = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -70,20 +73,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end - -s = [s sprintf(['\nRead the per-animal column as the honest power. At the observed N this study\n' ... - 'is typically underpowered; per-animal power reaches ~0.8 only at larger N.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead the per-animal column as the honest power; LME is optimistic (obs-level DF).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'power_result.txt'), 'w'); +fid = fopen(fullfile(here, ['power_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d0_13/result.txt b/analysis/matlab/variations/naive_boxa_d0_13/result.txt index 506fa7e..0f3226c 100644 --- a/analysis/matlab/variations/naive_boxa_d0_13/result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_13/result.txt @@ -1,5 +1,5 @@ ============================================================================== -VARIATION: naive_boxa_d0_13 +VARIATION: naive_boxa_d0_13 [metric: success COUNT] ============================================================================== model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT) day = training day within window (0 = first analyzed day) @@ -60,9 +60,9 @@ effect t(df) / F(df1) p (resid) Satterthwaite: p (df) stim x day (interaction) t(134)= 1.74 F(1)= 3.015 p=0.08481 p=0.08489 (df=129) day (learning) t(134)= 16.74 F(1)=280.201 p=1.213e-34 p=2.161e-34 (df=131) stim (main, window start) t(134)= 1.48 F(1)= 2.198 p=0.1405 p=0.1519 (df=23) -interaction 95% CI: [-0.18, +2.79] +interaction 95% CI: [-0.1817, +2.794] HONEST LME (per-animal random slope, day|rat): interaction F(1,25.3)=2.32, p=0.1403 (Satterthwaite DF ~= residual on this random-intercept model; the random-slope model above is the honest learning-rate test -- DF collapses toward the animal count.) -INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.08481, slope diff=+1.31) -Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008. +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.08481, slope diff=+1.306) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d0_13/result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_13/result_rate.txt new file mode 100644 index 0000000..24c125d --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_13/result_rate.txt @@ -0,0 +1,68 @@ +============================================================================== +VARIATION: naive_boxa_d0_13 [metric: success RATE (success/attempts)] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE (success/attempts)) +day = training day within window (0 = first analyzed day) +treatment (stim=1): Electrode-Box-B2 +control (stim=0): Electrode-Box-A, Electrode-Box-A2, Naive +N = 11 rats, 136 sessions raw day coverage: treat 0..13, control 0..13 +(equal day coverage over this window) + +============================================================================== +FULL MODEL SUMMARY -- fitlme +============================================================================== + +Linear mixed-effects model fit by ML + +Model information: + Number of observations 136 + Fixed effects coefficients 4 + Random effects coefficients 11 + Covariance parameters 2 + +Formula: + behavior ~ 1 + day*stim + (1 | rat) + +Model fit statistics: + AIC BIC LogLikelihood Deviance + -202.35 -184.87 107.17 -214.35 + +Fixed effects coefficients (95% CIs): + Name Estimate SE tStat DF pValue + {'(Intercept)'} 0.22695 0.026364 8.6084 132 1.9298e-14 + {'day' } 0.034099 0.0028279 12.058 132 4.9136e-23 + {'stim' } 0.061777 0.049323 1.2525 132 0.2126 + {'day:stim' } 0.009461 0.0052125 1.8151 132 0.071786 + + + Lower Upper + 0.1748 0.2791 + 0.028506 0.039693 + -0.035788 0.15934 + -0.00084987 0.019772 + +Random effects covariance parameters (95% CIs): +Group: rat (11 Levels) + Name1 Name2 Type Estimate + {'(Intercept)'} {'(Intercept)'} {'std'} 0.047732 + + + Lower Upper + 0.026198 0.086966 + +Group: Error + Name Estimate Lower Upper + {'Res Std'} 0.10455 0.092342 0.11837 + + +effect t(df) / F(df1) p (resid) Satterthwaite: p (df) +---------------------------------------------------------------------------- +stim x day (interaction) t(132)= 1.82 F(1)= 3.294 p=0.07179 p=0.07185 (df=128) +day (learning) t(132)= 12.06 F(1)=145.397 p=4.914e-23 p=5.237e-23 (df=131) +stim (main, window start) t(132)= 1.25 F(1)= 1.569 p=0.2126 p=0.2207 (df=28) +interaction 95% CI: [-0.0008499, +0.01977] +HONEST LME (per-animal random slope, day|rat): interaction F(1,10.0)=2.86, p=0.1216 + (Satterthwaite DF ~= residual on this random-intercept model; the random-slope + model above is the honest learning-rate test -- DF collapses toward the animal count.) +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.07179, slope diff=+0.009461) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d0_5/analyze.m b/analysis/matlab/variations/naive_boxa_d0_5/analyze.m index 82da5ed..0d40402 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/analyze.m +++ b/analysis/matlab/variations/naive_boxa_d0_5/analyze.m @@ -1,30 +1,58 @@ -% Variation analysis -- the paper's linear mixed model on the successful-reach -% COUNT, fit on this folder's curated data subset. +% Variation analysis -- the paper's linear mixed model on this folder's data, +% for BOTH metrics: +% metric = count : behavior = # successes -> result.txt +% metric = rate : behavior = success / attempts -> result_rate.txt +% (rate uses only sessions with attempts > 0) % % model: behavior ~ stim + day + stim:day + (1|rat) -% behavior = successful reaches (count per session) -% stim = 1 for the treatment group(s), 0 for the control group(s) -% day = training day within this window (0 = first analyzed day) -% rat = subject (random intercept) -% -% Self-contained: reads data.csv beside this script and writes result.txt. -% Run headless from this folder with: matlab -batch "analyze" -% (This is a copy of analysis/matlab/variation_analyze.m; see make_variations.m.) +% stim = 1 treatment / 0 control; day = training day within window (0 = +% first analyzed day); rat = subject (random intercept). +% For the interaction we report residual DF, Satterthwaite DF, and the honest +% per-animal random-slope test. Self-contained: reads data.csv beside this +% script. Run headless with: matlab -batch "analyze" +% (Copy of analysis/matlab/variation_analyze.m; see make_variations.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end -vname = regexprep(here, '.*[/\\]', ''); % folder name = variation id +vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -tbl = table(D.success, D.day - min(D.day), double(D.stim), categorical(D.subject), ... +Rc = localAnalyze(D, 'count', here, vname); +Rr = localAnalyze(D, 'rate', here, vname); + +% Machine-readable handoff for SUMMARY.csv (count drives it; rate appended). +VARRESULT = struct('name', vname, 'nRats', Rc.nRats, 'nObs', Rc.nObs, ... + 'interP', Rc.interP, 'interEst', Rc.interEst, ... + 'interPsatt', Rc.interPsatt, 'interPrs', Rc.interPrs, ... + 'stimP', Rc.stimP, 'dayP', Rc.dayP, 'covEqual', Rc.covEqual, ... + 'interPrate', Rr.interP, 'interEstRate', Rr.interEst, 'interPrsRate', Rr.interPrs); + +% ------------------------------------------------------------------ helper +function R = localAnalyze(D, metric, here, vname) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); + beh = D.success ./ D.total; + mlabel = 'success RATE (success/attempts)'; suffix = '_rate'; +else + beh = D.success; + mlabel = 'success COUNT'; suffix = ''; +end +R = struct('interP', NaN, 'interEst', NaN, 'interPsatt', NaN, 'interPrs', NaN, ... + 'stimP', NaN, 'dayP', NaN, 'nRats', numel(unique(D.subject)), ... + 'nObs', height(D), 'covEqual', false); +if numel(unique(D.stim)) < 2 || numel(unique(D.day)) < 2 + localWrite(sprintf('VARIATION: %s [metric: %s]\nInsufficient data for this metric.\n', ... + vname, mlabel), here, suffix); + return +end + +tbl = table(beh, D.day - min(D.day), double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); m = fitlme(tbl, 'behavior ~ stim + day + stim:day + (1|rat)'); C = m.Coefficients; A = anova(m); ci = coefCI(m); -As = anova(m, 'DFMethod', 'satterthwaite'); % Satterthwaite denominator DF +As = anova(m, 'DFMethod', 'satterthwaite'); -% Honest test: refit with a per-animal random SLOPE so the interaction DF -% collapses toward the animal count (guarded -- may not converge in short windows). rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; wst = warning('off', 'all'); try @@ -43,6 +71,7 @@ row = @(nm, t) sprintf('%-26s t(%d)=%6.2f F(%d)=%7.3f p=%.4g p=%.4g (df=%.0f C.DF(gi(t)), C.tStat(gi(t)), A.DF1(ga(t)), A.FStat(ga(t)), C.pValue(gi(t)), ... As.pValue(gs(t)), As.DF2(gs(t))); +ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); maxT = max(D.day(D.stim == 1)); minT = min(D.day(D.stim == 1)); maxC = max(D.day(D.stim == 0)); minC = min(D.day(D.stim == 0)); if abs(maxT - maxC) > 2 @@ -50,20 +79,14 @@ if abs(maxT - maxC) > 2 else cov = '(equal day coverage over this window)'; end - -ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); -if pI >= 0.05 - verdict = 'n.s. -- slopes parallel (no differential learning rate)'; -elseif eI > 0 - verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; -else - verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; -end +if pI >= 0.05; verdict = 'n.s. -- slopes parallel (no differential learning rate)'; +elseif eI > 0; verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; +else; verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; end bar = repmat('=', 1, 78); raw = regexprep(evalc('disp(m)'), '', ''); -s = sprintf('%s\nVARIATION: %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT)\n')]; +s = sprintf('%s\nVARIATION: %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('day = training day within window (0 = first analyzed day)\n')]; s = [s sprintf('treatment (stim=1): %s\n', strjoin(cellstr(unique(D.group(D.stim == 1))), ', '))]; s = [s sprintf('control (stim=0): %s\n', strjoin(cellstr(unique(D.group(D.stim == 0))), ', '))]; @@ -74,7 +97,7 @@ s = [s sprintf('%-26s %-18s %-12s %s\n%s\n', 'effect', 't(df) / F(df1)', 'p (res s = [s row('stim x day (interaction)', 'day:stim')]; s = [s row('day (learning)', 'day')]; s = [s row('stim (main, window start)', 'stim')]; -s = [s sprintf('interaction 95%% CI: [%+.2f, %+.2f]\n', ci(ii, 1), ci(ii, 2))]; +s = [s sprintf('interaction 95%% CI: [%+.4g, %+.4g]\n', ci(ii, 1), ci(ii, 2))]; if rsOk s = [s sprintf('HONEST LME (per-animal random slope, day|rat): interaction F(1,%.1f)=%.2f, p=%.4g\n', rsDf, rsF, rsP)]; else @@ -82,17 +105,18 @@ else end s = [s sprintf([' (Satterthwaite DF ~= residual on this random-intercept model; the random-slope\n' ... ' model above is the honest learning-rate test -- DF collapses toward the animal count.)\n'])]; -s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.2f)\n', verdict, pI, eI)]; -s = [s sprintf('Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.4g)\n', verdict, pI, eI)]; +s = [s sprintf('Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +localWrite(s, here, suffix); + +R = struct('interP', pI, 'interEst', eI, 'interPsatt', As.pValue(gs('day:stim')), ... + 'interPrs', rsP, 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... + 'nRats', numel(unique(D.subject)), 'nObs', height(D), 'covEqual', abs(maxT - maxC) <= 2); +end + +function localWrite(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'result.txt'), 'w'); -fprintf(fid, '%s', s); -fclose(fid); - -% Machine-readable handoff for the summary table (see make_variations.m). -VARRESULT = struct('name', vname, 'nRats', numel(unique(D.subject)), ... - 'nObs', height(D), 'interP', pI, 'interEst', eI, ... - 'interPsatt', As.pValue(gs('day:stim')), 'interPrs', rsP, ... - 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... - 'covEqual', abs(maxT - maxC) <= 2); +fid = fopen(fullfile(here, ['result' suffix '.txt']), 'w'); +fprintf(fid, '%s', s); fclose(fid); +end diff --git a/analysis/matlab/variations/naive_boxa_d0_5/learning_curve.png b/analysis/matlab/variations/naive_boxa_d0_5/learning_curve.png index c59672a..ec2dd12 100644 Binary files a/analysis/matlab/variations/naive_boxa_d0_5/learning_curve.png and b/analysis/matlab/variations/naive_boxa_d0_5/learning_curve.png differ diff --git a/analysis/matlab/variations/naive_boxa_d0_5/learning_curve_rate.png b/analysis/matlab/variations/naive_boxa_d0_5/learning_curve_rate.png new file mode 100644 index 0000000..a7b3c3b Binary files /dev/null and b/analysis/matlab/variations/naive_boxa_d0_5/learning_curve_rate.png differ diff --git a/analysis/matlab/variations/naive_boxa_d0_5/logpower_result.txt b/analysis/matlab/variations/naive_boxa_d0_5/logpower_result.txt index 02048c0..51e2c1f 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/logpower_result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_5/logpower_result.txt @@ -1,7 +1,7 @@ ============================================================================== -LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_5 +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_5 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT) +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = # successes (count)) log(day) uses 1-indexed training day (our day 0 = paper "Day 1") observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 @@ -10,9 +10,9 @@ stim x log(day) interaction: F(1,61)=8.016 p(resid)=0.006273 p(Satt)=0.006505 honest per-animal random slope (log-day): F(1,15.5)=6.72 p=0.01995 Cohen's f (interaction, partial eta^2=0.048) = 0.223 (small-medium; f: .10 small, .25 medium, .40 large) ---- power simulation (log-day ground truth: stim:logday=+18.30, ratSD=8.82, resSD=13.99) --- +--- power simulation (log-day ground truth: stim:logday=+18.3, ratSD=8.823, resSD=13.99) --- - true stim:log(day) = +18.30 (100% of observed) + true stim:log(day) = +18.3 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.33 | 0.66 <- observed @@ -22,7 +22,7 @@ Cohen's f (interaction, partial eta^2=0.048) = 0.223 (small-medium; f: .10 smal 16 | 1.00 | 1.00 24 | 1.00 | 1.00 - true stim:log(day) = +9.15 (50% of observed) + true stim:log(day) = +9.151 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.11 | 0.24 <- observed @@ -32,5 +32,4 @@ Cohen's f (interaction, partial eta^2=0.048) = 0.223 (small-medium; f: .10 smal 16 | 0.74 | 0.78 24 | 0.93 | 0.94 -Read the per-animal column as the honest power; the LME column matches the -paper's power code (anova interaction p, observation-level DF) and is optimistic. +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d0_5/logpower_result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_5/logpower_result_rate.txt new file mode 100644 index 0000000..705f41e --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_5/logpower_result_rate.txt @@ -0,0 +1,35 @@ +============================================================================== +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d0_5 [metric: success RATE] +============================================================================== +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = success RATE) +log(day) uses 1-indexed training day (our day 0 = paper "Day 1") +observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 + +--- fitted on real data --- +stim x log(day) interaction: F(1,59)=12.241 p(resid)=0.0008959 p(Satt)=0.000969 (df=52) + honest per-animal random slope (log-day): F(1,13.9)=10.72 p=0.005603 +Cohen's f (interaction, partial eta^2=0.104) = 0.340 (medium; f: .10 small, .25 medium, .40 large) + +--- power simulation (log-day ground truth: stim:logday=+0.1706, ratSD=0.04897, resSD=0.1038) --- + + true stim:log(day) = +0.1706 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.45 | 0.84 <- observed + 5 | 0.87 | 0.96 + 8 | 1.00 | 1.00 <- observed + 12 | 1.00 | 1.00 + 16 | 1.00 | 1.00 + 24 | 1.00 | 1.00 + + true stim:log(day) = +0.08531 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.12 | 0.31 <- observed + 5 | 0.36 | 0.48 + 8 | 0.58 | 0.67 <- observed + 12 | 0.76 | 0.78 + 16 | 0.89 | 0.91 + 24 | 0.99 | 0.99 + +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d0_5/logpowersim.m b/analysis/matlab/variations/naive_boxa_d0_5/logpowersim.m index e0686ee..5e38720 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/logpowersim.m +++ b/analysis/matlab/variations/naive_boxa_d0_5/logpowersim.m @@ -1,50 +1,52 @@ -% Variation log-day analysis + Cohen's f + power simulation. +% Variation log-day analysis + Cohen's f + power simulation, for BOTH metrics: +% metric = count : behavior = # successes -> logpower_result.txt +% metric = rate : behavior = success / attempts -> logpower_result_rate.txt % -% The paper's power code models behavior against LOG training day, not raw day: +% The paper's power code models behavior against LOG training day: % behavior ~ stim + log(day) + stim:log(day) + (1|rat). -% Their day is 1-indexed (1..10); our data.csv day is 0-indexed (day 0 = paper -% "Day 1"), so log(day + 1) reproduces their transform exactly. -% -% This script (a) refits that log-day model on data.csv, (b) reports the -% interaction (residual DF, Satterthwaite DF, and the honest per-animal -% random-slope test) and Cohen's f -- the partial-eta^2 effect size of the -% interaction, var(fitted_full) - var(fitted_no_interaction) over var(behavior) -% -- and (c) runs the Monte-Carlo power simulation on the log-day model, -% scoring per-animal (cluster-honest) and LME power across N. -% Writes logpower_result.txt. Run: matlab -batch "logpowersim" +% Their day is 1-indexed; our data.csv day is 0-indexed, so log(day + 1) +% reproduces their transform (our day 0 = paper "Day 1"). For each metric this +% refits that model, reports the interaction (residual / Satterthwaite / honest +% per-animal random-slope DF) and Cohen's f (partial-eta^2 effect size), then +% runs the Monte-Carlo power sim (per-animal cluster-honest + LME power). +% Run: matlab -batch "logpowersim" % (Copy of analysis/matlab/variation_logpower.m; see make_variation_logpower.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' column = log(day+1) -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; -warnState = warning('off', 'all'); -rng(1); +localLogPower(D, 'count', here, vname); +localLogPower(D, 'rate', here, vname); -logday = log(D.day + 1); % 0-indexed day -> their log(1-indexed day) -tbl0 = table(D.success, logday, double(D.stim), categorical(D.subject), ... +% ---------------------------------------------------------------- per metric +function localLogPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; +FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' = log(day+1) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); +logday = log(D.day + 1); +tbl0 = table(beh, logday, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT)\n')]; +s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('log(day) uses 1-indexed training day (our day 0 = paper "Day 1")\n')]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for this analysis (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for this analysis.\n')]; + localFinish(s, here, suffix); warning(warnState); return end -% ---- fitted model on the real data ---- full = fitlme(tbl0, FORMULA); An = anova(full); Asatt = anova(full, 'DFMethod', 'satterthwaite'); ii = strcmp(An.Term, 'day:stim'); is = strcmp(Asatt.Term, 'day:stim'); @@ -52,7 +54,6 @@ reduced = fitlme(tbl0, 'behavior ~ stim + day + (1|rat)'); eta2part = max((var(fitted(full)) - var(fitted(reduced))) / var(tbl0.behavior), 0); cohenf = sqrt(eta2part / (1 - eta2part)); -% honest per-animal random-slope interaction rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; try mr = fitlme(tbl0, 'behavior ~ stim + day + stim:day + (day|rat)'); @@ -76,19 +77,18 @@ end s = [s sprintf('Cohen''s f (interaction, partial eta^2=%.3f) = %.3f (%s; f: .10 small, .25 medium, .40 large)\n', ... eta2part, cohenf, mag)]; -% ---- power simulation on the log-day ground truth ---- cn = full.CoefficientNames; be = full.fixedEffects; b0 = be(strcmp(cn, '(Intercept)')); bStim = be(strcmp(cn, 'stim')); bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(full); sRat = sqrt(psi{1}); sRes = sqrt(full.MSE); -days = unique(tbl0.day); % the log(day) grid +days = unique(tbl0.day); -s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.2f, ratSD=%.2f, resSD=%.2f) ---\n', ... +s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.4g, ratSD=%.4g, resSD=%.4g) ---\n', ... bInt, sRat, sRes)]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:log(day) = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:log(day) = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -102,19 +102,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end -s = [s sprintf(['\nRead the per-animal column as the honest power; the LME column matches the\n' ... - 'paper''s power code (anova interaction p, observation-level DF) and is optimistic.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead per-animal as the honest power; LME matches the paper''s power code (optimistic).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'logpower_result.txt'), 'w'); +fid = fopen(fullfile(here, ['logpower_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d0_5/plotcurve.m b/analysis/matlab/variations/naive_boxa_d0_5/plotcurve.m index bfc1386..193a44b 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/plotcurve.m +++ b/analysis/matlab/variations/naive_boxa_d0_5/plotcurve.m @@ -1,11 +1,12 @@ -% Variation learning-curve plot, in the style of the paper: +% Variation learning-curve plots, in the style of the paper: % "Lines indicate mean (and SEM) across animals in the anodal (red) and % control (blue) groups." -% Plots mean +/- SEM successful reaches per training day for the treatment / -% anodal group (stim = 1, red) and the control group (stim = 0, blue), reading -% this folder's data.csv and saving learning_curve.png. The per-group N is read -% from the data (each variation pools different groups), so the legend shows the -% actual counts. Training day is 1-indexed (our day 0 = the paper's "Day 1"). +% Produces TWO figures from this folder's data.csv: +% learning_curve.png # successes (count) per training day +% learning_curve_rate.png success rate (success/attempts) per training day +% anodal / treatment = stim 1 (red); control = stim 0 (blue). Per-group N is +% read from the data. Training day is 1-indexed (our day 0 = paper "Day 1") and +% the x-axis tick labels are drawn vertically. % Run: matlab -batch "plotcurve" % (Copy of analysis/matlab/variation_plot.m; see make_variation_plot.m.) @@ -14,15 +15,20 @@ if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -days = unique(D.day); % 0-indexed -xd = days + 1; % plot as 1-indexed training day (paper axis) - +days = unique(D.day); +xd = days + 1; % plot as 1-indexed training day (paper axis) red = [0.85 0.10 0.10]; blue = [0.10 0.30 0.85]; -[Ma, Sa, na] = localCurve(D, 1, days); % anodal / treatment (stim = 1) -[Mc, Sc, nc] = localCurve(D, 0, days); % control (stim = 0) +localPlot(D, days, xd, 'count', '# successes', ... + fullfile(here, 'learning_curve.png'), vname, red, blue); +localPlot(D, days, xd, 'rate', 'success rate', ... + fullfile(here, 'learning_curve_rate.png'), vname, red, blue); +% ------------------------------------------------------------------ helpers +function localPlot(D, days, xd, metric, ylab, outFile, vname, red, blue) +[Ma, Sa, na] = localCurve(D, 1, days, metric); % anodal / treatment +[Mc, Sc, nc] = localCurve(D, 0, days, metric); % control fig = figure('Visible', 'off', 'Color', 'w', 'Position', [100 100 560 460]); hold on e1 = errorbar(xd, Ma, Sa, '-o', 'Color', red, 'MarkerFaceColor', red, 'LineWidth', 2); @@ -30,26 +36,30 @@ e2 = errorbar(xd, Mc, Sc, '-o', 'Color', blue, 'MarkerFaceColor', blue, 'LineWid hold off legend([e1 e2], {sprintf('anodal, N = %d', na), sprintf('control, N = %d', nc)}, ... 'Location', 'northwest', 'Box', 'off'); -xlabel('training day'); -ylabel('# successes'); +xlabel('training day'); ylabel(ylab); title(vname, 'Interpreter', 'none'); set(gca, 'XTick', xd, 'FontName', 'Arial', 'FontSize', 13, 'LineWidth', 1.5, 'Box', 'off'); - -outFile = fullfile(here, 'learning_curve.png'); +xtickangle(90); % vertical x-axis tick labels exportgraphics(fig, outFile, 'Resolution', 150); close(fig); -fprintf('%s: wrote learning_curve.png (anodal N=%d, control N=%d)\n', vname, na, nc); +fprintf('%s: wrote %s (anodal N=%d, control N=%d)\n', vname, outFile, na, nc); +end -% ------------------------------------------------------------------ helper -function [M, S, n] = localCurve(D, stimVal, days) -%LOCALCURVE Per-day mean and SEM of successes across the animals in a group. +function [M, S, n] = localCurve(D, stimVal, days, metric) +%LOCALCURVE Per-day mean and SEM across the animals in a group, for a metric. subs = unique(D.subject(D.stim == stimVal)); n = numel(subs); X = nan(numel(days), n); for j = 1:n for i = 1:numel(days) r = D.subject == subs(j) & D.day == days(i); - if any(r); X(i, j) = mean(D.success(r)); end + if ~any(r); continue; end + if strcmp(metric, 'rate') + tot = sum(D.total(r)); + if tot > 0; X(i, j) = sum(D.success(r)) / tot; end + else + X(i, j) = mean(D.success(r)); + end end end M = mean(X, 2, 'omitnan'); diff --git a/analysis/matlab/variations/naive_boxa_d0_5/power_result.txt b/analysis/matlab/variations/naive_boxa_d0_5/power_result.txt index 6223e39..e584fe8 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/power_result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_5/power_result.txt @@ -1,11 +1,11 @@ ============================================================================== -POWER SIMULATION -- naive_boxa_d0_5 +POWER SIMULATION -- naive_boxa_d0_5 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + day + stim:day + (1|rat) (success COUNT; day within-window) +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = # successes (count); day within-window) observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 -ground truth: stim:day=+6.40/day, rat SD=9.10, residual SD=12.48, days=6 +ground truth: stim:day=+6.403/day, rat SD=9.103, residual SD=12.48, days=6 - true stim:day interaction = +6.40 (100% of observed) + true stim:day interaction = +6.403 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.36 | 0.77 <- observed @@ -15,7 +15,7 @@ ground truth: stim:day=+6.40/day, rat SD=9.10, residual SD=12.48, days=6 16 | 1.00 | 1.00 24 | 1.00 | 1.00 - true stim:day interaction = +3.20 (50% of observed) + true stim:day interaction = +3.202 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.11 | 0.24 <- observed @@ -25,5 +25,4 @@ ground truth: stim:day=+6.40/day, rat SD=9.10, residual SD=12.48, days=6 16 | 0.82 | 0.84 24 | 0.95 | 0.94 -Read the per-animal column as the honest power. At the observed N this study -is typically underpowered; per-animal power reaches ~0.8 only at larger N. +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d0_5/power_result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_5/power_result_rate.txt new file mode 100644 index 0000000..72c4d9a --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_5/power_result_rate.txt @@ -0,0 +1,28 @@ +============================================================================== +POWER SIMULATION -- naive_boxa_d0_5 [metric: success RATE] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE; day within-window) +observed groups: stim n=3, control n=8 nrep=120, alpha=0.05 +ground truth: stim:day=+0.05713/day, rat SD=0.05041, residual SD=0.09994, days=6 + + true stim:day interaction = +0.05713 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.42 | 0.85 <- observed + 5 | 0.86 | 0.96 + 8 | 1.00 | 1.00 <- observed + 12 | 1.00 | 1.00 + 16 | 1.00 | 1.00 + 24 | 1.00 | 1.00 + + true stim:day interaction = +0.02856 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.12 | 0.32 <- observed + 5 | 0.32 | 0.44 + 8 | 0.57 | 0.66 <- observed + 12 | 0.74 | 0.79 + 16 | 0.89 | 0.88 + 24 | 0.97 | 0.99 + +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d0_5/powersim.m b/analysis/matlab/variations/naive_boxa_d0_5/powersim.m index 554d032..ecd9c6c 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/powersim.m +++ b/analysis/matlab/variations/naive_boxa_d0_5/powersim.m @@ -1,45 +1,49 @@ % Variation power simulation -- Monte-Carlo power for the paper's stim x day -% interaction, using THIS folder's data as the ground truth. +% interaction, using THIS folder's data as the ground truth, for BOTH metrics: +% metric = count : behavior = # successes -> power_result.txt +% metric = rate : behavior = success / attempts -> power_result_rate.txt % % Ground truth: fitlme(behavior ~ stim + day + stim:day + (1|rat)) on data.csv -% (success COUNT; day within-window). Its fixed effects, per-rat intercept SD, -% and residual SD generate NREP synthetic datasets at each rats-per-group N and -% each true-effect multiplier (1 = observed slope, 0.5 = half). Each dataset is -% scored at alpha = 0.05 two ways: -% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest -- the -% honest power, matching the random-slope / per-animal inference) +% (day within-window). Its fixed effects, per-rat intercept SD, and residual SD +% generate NREP synthetic datasets at each rats-per-group N and each true-effect +% multiplier (1 = observed, 0.5 = half). Each is scored at alpha=0.05 by: +% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest power) % LME : the fitlme stim:day p (observation-level DF -- optimistic) -% Writes power_result.txt beside this script. Run: matlab -batch "powersim" +% Writes power_result[_rate].txt. Run: matlab -batch "powersim" % (Copy of analysis/matlab/variation_power.m; see make_variation_power.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); + +localPower(D, 'count', here, vname); +localPower(D, 'rate', here, vname); + +% ---------------------------------------------------------------- per metric +function localPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; - -warnState = warning('off', 'all'); -rng(1); - +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); day0 = min(D.day); -tbl0 = table(D.success, D.day - day0, double(D.stim), categorical(D.subject), ... +tbl0 = table(beh, D.day - day0, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); - nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nPOWER SIMULATION -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: %s (success COUNT; day within-window)\n', FORMULA)]; +s = sprintf('%s\nPOWER SIMULATION -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: %s (behavior = %s; day within-window)\n', FORMULA, mlabel)]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for a power simulation (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for a power simulation.\n')]; + localFinish(s, here, suffix); warning(warnState); return end lme = fitlme(tbl0, FORMULA); @@ -49,14 +53,13 @@ bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(lme); sRat = sqrt(psi{1}); sRes = sqrt(lme.MSE); days = (0:max(tbl0.day))'; -s = [s sprintf('ground truth: stim:day=%+.2f/day, rat SD=%.2f, residual SD=%.2f, days=%d\n', ... +s = [s sprintf('ground truth: stim:day=%+.4g/day, rat SD=%.4g, residual SD=%.4g, days=%d\n', ... bInt, sRat, sRes, numel(days))]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:day interaction = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n', 'N/group')]; %#ok - s = [s sprintf(' %s\n', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:day interaction = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -70,20 +73,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end - -s = [s sprintf(['\nRead the per-animal column as the honest power. At the observed N this study\n' ... - 'is typically underpowered; per-animal power reaches ~0.8 only at larger N.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead the per-animal column as the honest power; LME is optimistic (obs-level DF).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'power_result.txt'), 'w'); +fid = fopen(fullfile(here, ['power_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d0_5/result.txt b/analysis/matlab/variations/naive_boxa_d0_5/result.txt index f8f2c4c..23ba3d9 100644 --- a/analysis/matlab/variations/naive_boxa_d0_5/result.txt +++ b/analysis/matlab/variations/naive_boxa_d0_5/result.txt @@ -1,5 +1,5 @@ ============================================================================== -VARIATION: naive_boxa_d0_5 +VARIATION: naive_boxa_d0_5 [metric: success COUNT] ============================================================================== model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT) day = training day within window (0 = first analyzed day) @@ -60,9 +60,9 @@ effect t(df) / F(df1) p (resid) Satterthwaite: p (df) stim x day (interaction) t(61)= 3.15 F(1)= 9.909 p=0.002545 p=0.002679 (df=54) day (learning) t(61)= 8.14 F(1)= 66.293 p=2.506e-11 p=5.765e-11 (df=54) stim (main, window start) t(61)= 0.15 F(1)= 0.022 p=0.8814 p=0.8822 (df=24) -interaction 95% CI: [+2.34, +10.47] +interaction 95% CI: [+2.336, +10.47] HONEST LME (per-animal random slope, day|rat): interaction F(1,10.5)=5.80, p=0.03561 (Satterthwaite DF ~= residual on this random-intercept model; the random-slope model above is the honest learning-rate test -- DF collapses toward the animal count.) -INTERPRETATION: stim x day interaction SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates) (p=0.002545, slope diff=+6.40) -Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008. +INTERPRETATION: stim x day interaction SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates) (p=0.002545, slope diff=+6.403) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d0_5/result_rate.txt b/analysis/matlab/variations/naive_boxa_d0_5/result_rate.txt new file mode 100644 index 0000000..f869ce0 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d0_5/result_rate.txt @@ -0,0 +1,68 @@ +============================================================================== +VARIATION: naive_boxa_d0_5 [metric: success RATE (success/attempts)] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE (success/attempts)) +day = training day within window (0 = first analyzed day) +treatment (stim=1): Electrode-Box-B2 +control (stim=0): Electrode-Box-A, Electrode-Box-A2, Naive +N = 11 rats, 63 sessions raw day coverage: treat 0..5, control 0..5 +(equal day coverage over this window) + +============================================================================== +FULL MODEL SUMMARY -- fitlme +============================================================================== + +Linear mixed-effects model fit by ML + +Model information: + Number of observations 63 + Fixed effects coefficients 4 + Random effects coefficients 11 + Covariance parameters 2 + +Formula: + behavior ~ 1 + day*stim + (1 | rat) + +Model fit statistics: + AIC BIC LogLikelihood Deviance + -89.552 -76.693 50.776 -101.55 + +Fixed effects coefficients (95% CIs): + Name Estimate SE tStat DF + {'(Intercept)'} 0.20761 0.03353 6.1919 59 + {'day' } 0.035262 0.0090474 3.8975 59 + {'stim' } -0.042773 0.060953 -0.70173 59 + {'day:stim' } 0.057126 0.016495 3.4631 59 + + + pValue Lower Upper + 6.1978e-08 0.14052 0.27471 + 0.000251 0.017158 0.053365 + 0.4856 -0.16474 0.079194 + 0.0010002 0.024119 0.090133 + +Random effects covariance parameters (95% CIs): +Group: rat (11 Levels) + Name1 Name2 Type Estimate + {'(Intercept)'} {'(Intercept)'} {'std'} 0.050413 + + + Lower Upper + 0.02388 0.10643 + +Group: Error + Name Estimate Lower Upper + {'Res Std'} 0.099938 0.082348 0.12129 + + +effect t(df) / F(df1) p (resid) Satterthwaite: p (df) +---------------------------------------------------------------------------- +stim x day (interaction) t(59)= 3.46 F(1)= 11.993 p=0.001 p=0.00108 (df=52) +day (learning) t(59)= 3.90 F(1)= 15.190 p=0.000251 p=0.0002756 (df=53) +stim (main, window start) t(59)= -0.70 F(1)= 0.492 p=0.4856 p=0.4879 (df=32) +interaction 95% CI: [+0.02412, +0.09013] +HONEST LME (per-animal random slope, day|rat): interaction F(1,12.0)=9.63, p=0.009127 + (Satterthwaite DF ~= residual on this random-intercept model; the random-slope + model above is the honest learning-rate test -- DF collapses toward the animal count.) +INTERPRETATION: stim x day interaction SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates) (p=0.001, slope diff=+0.05713) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d6_10/analyze.m b/analysis/matlab/variations/naive_boxa_d6_10/analyze.m index 82da5ed..0d40402 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/analyze.m +++ b/analysis/matlab/variations/naive_boxa_d6_10/analyze.m @@ -1,30 +1,58 @@ -% Variation analysis -- the paper's linear mixed model on the successful-reach -% COUNT, fit on this folder's curated data subset. +% Variation analysis -- the paper's linear mixed model on this folder's data, +% for BOTH metrics: +% metric = count : behavior = # successes -> result.txt +% metric = rate : behavior = success / attempts -> result_rate.txt +% (rate uses only sessions with attempts > 0) % % model: behavior ~ stim + day + stim:day + (1|rat) -% behavior = successful reaches (count per session) -% stim = 1 for the treatment group(s), 0 for the control group(s) -% day = training day within this window (0 = first analyzed day) -% rat = subject (random intercept) -% -% Self-contained: reads data.csv beside this script and writes result.txt. -% Run headless from this folder with: matlab -batch "analyze" -% (This is a copy of analysis/matlab/variation_analyze.m; see make_variations.m.) +% stim = 1 treatment / 0 control; day = training day within window (0 = +% first analyzed day); rat = subject (random intercept). +% For the interaction we report residual DF, Satterthwaite DF, and the honest +% per-animal random-slope test. Self-contained: reads data.csv beside this +% script. Run headless with: matlab -batch "analyze" +% (Copy of analysis/matlab/variation_analyze.m; see make_variations.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end -vname = regexprep(here, '.*[/\\]', ''); % folder name = variation id +vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -tbl = table(D.success, D.day - min(D.day), double(D.stim), categorical(D.subject), ... +Rc = localAnalyze(D, 'count', here, vname); +Rr = localAnalyze(D, 'rate', here, vname); + +% Machine-readable handoff for SUMMARY.csv (count drives it; rate appended). +VARRESULT = struct('name', vname, 'nRats', Rc.nRats, 'nObs', Rc.nObs, ... + 'interP', Rc.interP, 'interEst', Rc.interEst, ... + 'interPsatt', Rc.interPsatt, 'interPrs', Rc.interPrs, ... + 'stimP', Rc.stimP, 'dayP', Rc.dayP, 'covEqual', Rc.covEqual, ... + 'interPrate', Rr.interP, 'interEstRate', Rr.interEst, 'interPrsRate', Rr.interPrs); + +% ------------------------------------------------------------------ helper +function R = localAnalyze(D, metric, here, vname) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); + beh = D.success ./ D.total; + mlabel = 'success RATE (success/attempts)'; suffix = '_rate'; +else + beh = D.success; + mlabel = 'success COUNT'; suffix = ''; +end +R = struct('interP', NaN, 'interEst', NaN, 'interPsatt', NaN, 'interPrs', NaN, ... + 'stimP', NaN, 'dayP', NaN, 'nRats', numel(unique(D.subject)), ... + 'nObs', height(D), 'covEqual', false); +if numel(unique(D.stim)) < 2 || numel(unique(D.day)) < 2 + localWrite(sprintf('VARIATION: %s [metric: %s]\nInsufficient data for this metric.\n', ... + vname, mlabel), here, suffix); + return +end + +tbl = table(beh, D.day - min(D.day), double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); m = fitlme(tbl, 'behavior ~ stim + day + stim:day + (1|rat)'); C = m.Coefficients; A = anova(m); ci = coefCI(m); -As = anova(m, 'DFMethod', 'satterthwaite'); % Satterthwaite denominator DF +As = anova(m, 'DFMethod', 'satterthwaite'); -% Honest test: refit with a per-animal random SLOPE so the interaction DF -% collapses toward the animal count (guarded -- may not converge in short windows). rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; wst = warning('off', 'all'); try @@ -43,6 +71,7 @@ row = @(nm, t) sprintf('%-26s t(%d)=%6.2f F(%d)=%7.3f p=%.4g p=%.4g (df=%.0f C.DF(gi(t)), C.tStat(gi(t)), A.DF1(ga(t)), A.FStat(ga(t)), C.pValue(gi(t)), ... As.pValue(gs(t)), As.DF2(gs(t))); +ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); maxT = max(D.day(D.stim == 1)); minT = min(D.day(D.stim == 1)); maxC = max(D.day(D.stim == 0)); minC = min(D.day(D.stim == 0)); if abs(maxT - maxC) > 2 @@ -50,20 +79,14 @@ if abs(maxT - maxC) > 2 else cov = '(equal day coverage over this window)'; end - -ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); -if pI >= 0.05 - verdict = 'n.s. -- slopes parallel (no differential learning rate)'; -elseif eI > 0 - verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; -else - verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; -end +if pI >= 0.05; verdict = 'n.s. -- slopes parallel (no differential learning rate)'; +elseif eI > 0; verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; +else; verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; end bar = repmat('=', 1, 78); raw = regexprep(evalc('disp(m)'), '', ''); -s = sprintf('%s\nVARIATION: %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT)\n')]; +s = sprintf('%s\nVARIATION: %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('day = training day within window (0 = first analyzed day)\n')]; s = [s sprintf('treatment (stim=1): %s\n', strjoin(cellstr(unique(D.group(D.stim == 1))), ', '))]; s = [s sprintf('control (stim=0): %s\n', strjoin(cellstr(unique(D.group(D.stim == 0))), ', '))]; @@ -74,7 +97,7 @@ s = [s sprintf('%-26s %-18s %-12s %s\n%s\n', 'effect', 't(df) / F(df1)', 'p (res s = [s row('stim x day (interaction)', 'day:stim')]; s = [s row('day (learning)', 'day')]; s = [s row('stim (main, window start)', 'stim')]; -s = [s sprintf('interaction 95%% CI: [%+.2f, %+.2f]\n', ci(ii, 1), ci(ii, 2))]; +s = [s sprintf('interaction 95%% CI: [%+.4g, %+.4g]\n', ci(ii, 1), ci(ii, 2))]; if rsOk s = [s sprintf('HONEST LME (per-animal random slope, day|rat): interaction F(1,%.1f)=%.2f, p=%.4g\n', rsDf, rsF, rsP)]; else @@ -82,17 +105,18 @@ else end s = [s sprintf([' (Satterthwaite DF ~= residual on this random-intercept model; the random-slope\n' ... ' model above is the honest learning-rate test -- DF collapses toward the animal count.)\n'])]; -s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.2f)\n', verdict, pI, eI)]; -s = [s sprintf('Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.4g)\n', verdict, pI, eI)]; +s = [s sprintf('Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +localWrite(s, here, suffix); + +R = struct('interP', pI, 'interEst', eI, 'interPsatt', As.pValue(gs('day:stim')), ... + 'interPrs', rsP, 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... + 'nRats', numel(unique(D.subject)), 'nObs', height(D), 'covEqual', abs(maxT - maxC) <= 2); +end + +function localWrite(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'result.txt'), 'w'); -fprintf(fid, '%s', s); -fclose(fid); - -% Machine-readable handoff for the summary table (see make_variations.m). -VARRESULT = struct('name', vname, 'nRats', numel(unique(D.subject)), ... - 'nObs', height(D), 'interP', pI, 'interEst', eI, ... - 'interPsatt', As.pValue(gs('day:stim')), 'interPrs', rsP, ... - 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... - 'covEqual', abs(maxT - maxC) <= 2); +fid = fopen(fullfile(here, ['result' suffix '.txt']), 'w'); +fprintf(fid, '%s', s); fclose(fid); +end diff --git a/analysis/matlab/variations/naive_boxa_d6_10/learning_curve.png b/analysis/matlab/variations/naive_boxa_d6_10/learning_curve.png index 4e66928..604d88b 100644 Binary files a/analysis/matlab/variations/naive_boxa_d6_10/learning_curve.png and b/analysis/matlab/variations/naive_boxa_d6_10/learning_curve.png differ diff --git a/analysis/matlab/variations/naive_boxa_d6_10/learning_curve_rate.png b/analysis/matlab/variations/naive_boxa_d6_10/learning_curve_rate.png new file mode 100644 index 0000000..a572fc9 Binary files /dev/null and b/analysis/matlab/variations/naive_boxa_d6_10/learning_curve_rate.png differ diff --git a/analysis/matlab/variations/naive_boxa_d6_10/logpower_result.txt b/analysis/matlab/variations/naive_boxa_d6_10/logpower_result.txt index 08f1714..e09ca2d 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/logpower_result.txt +++ b/analysis/matlab/variations/naive_boxa_d6_10/logpower_result.txt @@ -1,7 +1,7 @@ ============================================================================== -LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d6_10 +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d6_10 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT) +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = # successes (count)) log(day) uses 1-indexed training day (our day 0 = paper "Day 1") observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 @@ -10,7 +10,7 @@ stim x log(day) interaction: F(1,46)=0.613 p(resid)=0.4376 p(Satt)=0.4382 (df= honest per-animal random slope (log-day): F(1,10.0)=0.43 p=0.5247 Cohen's f (interaction, partial eta^2=0.005) = 0.069 (small; f: .10 small, .25 medium, .40 large) ---- power simulation (log-day ground truth: stim:logday=-15.28, ratSD=11.24, resSD=10.10) --- +--- power simulation (log-day ground truth: stim:logday=-15.28, ratSD=11.24, resSD=10.1) --- true stim:log(day) = -15.28 (100% of observed) N/group | per-animal power | LME power @@ -32,5 +32,4 @@ Cohen's f (interaction, partial eta^2=0.005) = 0.069 (small; f: .10 small, .25 16 | 0.10 | 0.11 24 | 0.13 | 0.14 -Read the per-animal column as the honest power; the LME column matches the -paper's power code (anova interaction p, observation-level DF) and is optimistic. +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d6_10/logpower_result_rate.txt b/analysis/matlab/variations/naive_boxa_d6_10/logpower_result_rate.txt new file mode 100644 index 0000000..a2be137 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d6_10/logpower_result_rate.txt @@ -0,0 +1,35 @@ +============================================================================== +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d6_10 [metric: success RATE] +============================================================================== +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = success RATE) +log(day) uses 1-indexed training day (our day 0 = paper "Day 1") +observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 + +--- fitted on real data --- +stim x log(day) interaction: F(1,46)=2.393 p(resid)=0.1287 p(Satt)=0.1297 (df=40) + honest per-animal random slope (log-day): F(1,10.0)=1.69 p=0.2224 +Cohen's f (interaction, partial eta^2=0.021) = 0.147 (small-medium; f: .10 small, .25 medium, .40 large) + +--- power simulation (log-day ground truth: stim:logday=-0.1921, ratSD=0.0649, resSD=0.06429) --- + + true stim:log(day) = -0.1921 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.12 | 0.25 <- observed + 5 | 0.24 | 0.36 + 8 | 0.49 | 0.56 + 12 | 0.78 | 0.80 + 16 | 0.88 | 0.89 + 24 | 0.97 | 0.95 + + true stim:log(day) = -0.09607 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.03 | 0.09 <- observed + 5 | 0.07 | 0.12 + 8 | 0.17 | 0.21 + 12 | 0.17 | 0.17 + 16 | 0.33 | 0.37 + 24 | 0.36 | 0.38 + +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d6_10/logpowersim.m b/analysis/matlab/variations/naive_boxa_d6_10/logpowersim.m index e0686ee..5e38720 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/logpowersim.m +++ b/analysis/matlab/variations/naive_boxa_d6_10/logpowersim.m @@ -1,50 +1,52 @@ -% Variation log-day analysis + Cohen's f + power simulation. +% Variation log-day analysis + Cohen's f + power simulation, for BOTH metrics: +% metric = count : behavior = # successes -> logpower_result.txt +% metric = rate : behavior = success / attempts -> logpower_result_rate.txt % -% The paper's power code models behavior against LOG training day, not raw day: +% The paper's power code models behavior against LOG training day: % behavior ~ stim + log(day) + stim:log(day) + (1|rat). -% Their day is 1-indexed (1..10); our data.csv day is 0-indexed (day 0 = paper -% "Day 1"), so log(day + 1) reproduces their transform exactly. -% -% This script (a) refits that log-day model on data.csv, (b) reports the -% interaction (residual DF, Satterthwaite DF, and the honest per-animal -% random-slope test) and Cohen's f -- the partial-eta^2 effect size of the -% interaction, var(fitted_full) - var(fitted_no_interaction) over var(behavior) -% -- and (c) runs the Monte-Carlo power simulation on the log-day model, -% scoring per-animal (cluster-honest) and LME power across N. -% Writes logpower_result.txt. Run: matlab -batch "logpowersim" +% Their day is 1-indexed; our data.csv day is 0-indexed, so log(day + 1) +% reproduces their transform (our day 0 = paper "Day 1"). For each metric this +% refits that model, reports the interaction (residual / Satterthwaite / honest +% per-animal random-slope DF) and Cohen's f (partial-eta^2 effect size), then +% runs the Monte-Carlo power sim (per-animal cluster-honest + LME power). +% Run: matlab -batch "logpowersim" % (Copy of analysis/matlab/variation_logpower.m; see make_variation_logpower.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' column = log(day+1) -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; -warnState = warning('off', 'all'); -rng(1); +localLogPower(D, 'count', here, vname); +localLogPower(D, 'rate', here, vname); -logday = log(D.day + 1); % 0-indexed day -> their log(1-indexed day) -tbl0 = table(D.success, logday, double(D.stim), categorical(D.subject), ... +% ---------------------------------------------------------------- per metric +function localLogPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; +FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' = log(day+1) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); +logday = log(D.day + 1); +tbl0 = table(beh, logday, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT)\n')]; +s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('log(day) uses 1-indexed training day (our day 0 = paper "Day 1")\n')]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for this analysis (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for this analysis.\n')]; + localFinish(s, here, suffix); warning(warnState); return end -% ---- fitted model on the real data ---- full = fitlme(tbl0, FORMULA); An = anova(full); Asatt = anova(full, 'DFMethod', 'satterthwaite'); ii = strcmp(An.Term, 'day:stim'); is = strcmp(Asatt.Term, 'day:stim'); @@ -52,7 +54,6 @@ reduced = fitlme(tbl0, 'behavior ~ stim + day + (1|rat)'); eta2part = max((var(fitted(full)) - var(fitted(reduced))) / var(tbl0.behavior), 0); cohenf = sqrt(eta2part / (1 - eta2part)); -% honest per-animal random-slope interaction rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; try mr = fitlme(tbl0, 'behavior ~ stim + day + stim:day + (day|rat)'); @@ -76,19 +77,18 @@ end s = [s sprintf('Cohen''s f (interaction, partial eta^2=%.3f) = %.3f (%s; f: .10 small, .25 medium, .40 large)\n', ... eta2part, cohenf, mag)]; -% ---- power simulation on the log-day ground truth ---- cn = full.CoefficientNames; be = full.fixedEffects; b0 = be(strcmp(cn, '(Intercept)')); bStim = be(strcmp(cn, 'stim')); bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(full); sRat = sqrt(psi{1}); sRes = sqrt(full.MSE); -days = unique(tbl0.day); % the log(day) grid +days = unique(tbl0.day); -s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.2f, ratSD=%.2f, resSD=%.2f) ---\n', ... +s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.4g, ratSD=%.4g, resSD=%.4g) ---\n', ... bInt, sRat, sRes)]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:log(day) = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:log(day) = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -102,19 +102,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end -s = [s sprintf(['\nRead the per-animal column as the honest power; the LME column matches the\n' ... - 'paper''s power code (anova interaction p, observation-level DF) and is optimistic.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead per-animal as the honest power; LME matches the paper''s power code (optimistic).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'logpower_result.txt'), 'w'); +fid = fopen(fullfile(here, ['logpower_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d6_10/plotcurve.m b/analysis/matlab/variations/naive_boxa_d6_10/plotcurve.m index bfc1386..193a44b 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/plotcurve.m +++ b/analysis/matlab/variations/naive_boxa_d6_10/plotcurve.m @@ -1,11 +1,12 @@ -% Variation learning-curve plot, in the style of the paper: +% Variation learning-curve plots, in the style of the paper: % "Lines indicate mean (and SEM) across animals in the anodal (red) and % control (blue) groups." -% Plots mean +/- SEM successful reaches per training day for the treatment / -% anodal group (stim = 1, red) and the control group (stim = 0, blue), reading -% this folder's data.csv and saving learning_curve.png. The per-group N is read -% from the data (each variation pools different groups), so the legend shows the -% actual counts. Training day is 1-indexed (our day 0 = the paper's "Day 1"). +% Produces TWO figures from this folder's data.csv: +% learning_curve.png # successes (count) per training day +% learning_curve_rate.png success rate (success/attempts) per training day +% anodal / treatment = stim 1 (red); control = stim 0 (blue). Per-group N is +% read from the data. Training day is 1-indexed (our day 0 = paper "Day 1") and +% the x-axis tick labels are drawn vertically. % Run: matlab -batch "plotcurve" % (Copy of analysis/matlab/variation_plot.m; see make_variation_plot.m.) @@ -14,15 +15,20 @@ if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -days = unique(D.day); % 0-indexed -xd = days + 1; % plot as 1-indexed training day (paper axis) - +days = unique(D.day); +xd = days + 1; % plot as 1-indexed training day (paper axis) red = [0.85 0.10 0.10]; blue = [0.10 0.30 0.85]; -[Ma, Sa, na] = localCurve(D, 1, days); % anodal / treatment (stim = 1) -[Mc, Sc, nc] = localCurve(D, 0, days); % control (stim = 0) +localPlot(D, days, xd, 'count', '# successes', ... + fullfile(here, 'learning_curve.png'), vname, red, blue); +localPlot(D, days, xd, 'rate', 'success rate', ... + fullfile(here, 'learning_curve_rate.png'), vname, red, blue); +% ------------------------------------------------------------------ helpers +function localPlot(D, days, xd, metric, ylab, outFile, vname, red, blue) +[Ma, Sa, na] = localCurve(D, 1, days, metric); % anodal / treatment +[Mc, Sc, nc] = localCurve(D, 0, days, metric); % control fig = figure('Visible', 'off', 'Color', 'w', 'Position', [100 100 560 460]); hold on e1 = errorbar(xd, Ma, Sa, '-o', 'Color', red, 'MarkerFaceColor', red, 'LineWidth', 2); @@ -30,26 +36,30 @@ e2 = errorbar(xd, Mc, Sc, '-o', 'Color', blue, 'MarkerFaceColor', blue, 'LineWid hold off legend([e1 e2], {sprintf('anodal, N = %d', na), sprintf('control, N = %d', nc)}, ... 'Location', 'northwest', 'Box', 'off'); -xlabel('training day'); -ylabel('# successes'); +xlabel('training day'); ylabel(ylab); title(vname, 'Interpreter', 'none'); set(gca, 'XTick', xd, 'FontName', 'Arial', 'FontSize', 13, 'LineWidth', 1.5, 'Box', 'off'); - -outFile = fullfile(here, 'learning_curve.png'); +xtickangle(90); % vertical x-axis tick labels exportgraphics(fig, outFile, 'Resolution', 150); close(fig); -fprintf('%s: wrote learning_curve.png (anodal N=%d, control N=%d)\n', vname, na, nc); +fprintf('%s: wrote %s (anodal N=%d, control N=%d)\n', vname, outFile, na, nc); +end -% ------------------------------------------------------------------ helper -function [M, S, n] = localCurve(D, stimVal, days) -%LOCALCURVE Per-day mean and SEM of successes across the animals in a group. +function [M, S, n] = localCurve(D, stimVal, days, metric) +%LOCALCURVE Per-day mean and SEM across the animals in a group, for a metric. subs = unique(D.subject(D.stim == stimVal)); n = numel(subs); X = nan(numel(days), n); for j = 1:n for i = 1:numel(days) r = D.subject == subs(j) & D.day == days(i); - if any(r); X(i, j) = mean(D.success(r)); end + if ~any(r); continue; end + if strcmp(metric, 'rate') + tot = sum(D.total(r)); + if tot > 0; X(i, j) = sum(D.success(r)) / tot; end + else + X(i, j) = mean(D.success(r)); + end end end M = mean(X, 2, 'omitnan'); diff --git a/analysis/matlab/variations/naive_boxa_d6_10/power_result.txt b/analysis/matlab/variations/naive_boxa_d6_10/power_result.txt index 0da438b..5bc7b7a 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/power_result.txt +++ b/analysis/matlab/variations/naive_boxa_d6_10/power_result.txt @@ -1,11 +1,11 @@ ============================================================================== -POWER SIMULATION -- naive_boxa_d6_10 +POWER SIMULATION -- naive_boxa_d6_10 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + day + stim:day + (1|rat) (success COUNT; day within-window) +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = # successes (count); day within-window) observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 -ground truth: stim:day=-1.62/day, rat SD=11.24, residual SD=10.14, days=5 +ground truth: stim:day=-1.619/day, rat SD=11.24, residual SD=10.14, days=5 - true stim:day interaction = -1.62 (100% of observed) + true stim:day interaction = -1.619 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.05 | 0.08 <- observed @@ -15,7 +15,7 @@ ground truth: stim:day=-1.62/day, rat SD=11.24, residual SD=10.14, days=5 16 | 0.28 | 0.30 24 | 0.41 | 0.45 - true stim:day interaction = -0.81 (50% of observed) + true stim:day interaction = -0.8095 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.03 | 0.08 <- observed @@ -25,5 +25,4 @@ ground truth: stim:day=-1.62/day, rat SD=11.24, residual SD=10.14, days=5 16 | 0.11 | 0.09 24 | 0.12 | 0.13 -Read the per-animal column as the honest power. At the observed N this study -is typically underpowered; per-animal power reaches ~0.8 only at larger N. +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d6_10/power_result_rate.txt b/analysis/matlab/variations/naive_boxa_d6_10/power_result_rate.txt new file mode 100644 index 0000000..b58db33 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d6_10/power_result_rate.txt @@ -0,0 +1,28 @@ +============================================================================== +POWER SIMULATION -- naive_boxa_d6_10 [metric: success RATE] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE; day within-window) +observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 +ground truth: stim:day=-0.02112/day, rat SD=0.06482, residual SD=0.06465, days=5 + + true stim:day interaction = -0.02112 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.10 | 0.23 <- observed + 5 | 0.21 | 0.33 + 8 | 0.47 | 0.53 + 12 | 0.73 | 0.78 + 16 | 0.87 | 0.87 + 24 | 0.93 | 0.93 + + true stim:day interaction = -0.01056 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.03 | 0.10 <- observed + 5 | 0.07 | 0.11 + 8 | 0.16 | 0.21 + 12 | 0.15 | 0.17 + 16 | 0.32 | 0.38 + 24 | 0.33 | 0.35 + +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d6_10/powersim.m b/analysis/matlab/variations/naive_boxa_d6_10/powersim.m index 554d032..ecd9c6c 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/powersim.m +++ b/analysis/matlab/variations/naive_boxa_d6_10/powersim.m @@ -1,45 +1,49 @@ % Variation power simulation -- Monte-Carlo power for the paper's stim x day -% interaction, using THIS folder's data as the ground truth. +% interaction, using THIS folder's data as the ground truth, for BOTH metrics: +% metric = count : behavior = # successes -> power_result.txt +% metric = rate : behavior = success / attempts -> power_result_rate.txt % % Ground truth: fitlme(behavior ~ stim + day + stim:day + (1|rat)) on data.csv -% (success COUNT; day within-window). Its fixed effects, per-rat intercept SD, -% and residual SD generate NREP synthetic datasets at each rats-per-group N and -% each true-effect multiplier (1 = observed slope, 0.5 = half). Each dataset is -% scored at alpha = 0.05 two ways: -% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest -- the -% honest power, matching the random-slope / per-animal inference) +% (day within-window). Its fixed effects, per-rat intercept SD, and residual SD +% generate NREP synthetic datasets at each rats-per-group N and each true-effect +% multiplier (1 = observed, 0.5 = half). Each is scored at alpha=0.05 by: +% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest power) % LME : the fitlme stim:day p (observation-level DF -- optimistic) -% Writes power_result.txt beside this script. Run: matlab -batch "powersim" +% Writes power_result[_rate].txt. Run: matlab -batch "powersim" % (Copy of analysis/matlab/variation_power.m; see make_variation_power.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); + +localPower(D, 'count', here, vname); +localPower(D, 'rate', here, vname); + +% ---------------------------------------------------------------- per metric +function localPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; - -warnState = warning('off', 'all'); -rng(1); - +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); day0 = min(D.day); -tbl0 = table(D.success, D.day - day0, double(D.stim), categorical(D.subject), ... +tbl0 = table(beh, D.day - day0, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); - nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nPOWER SIMULATION -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: %s (success COUNT; day within-window)\n', FORMULA)]; +s = sprintf('%s\nPOWER SIMULATION -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: %s (behavior = %s; day within-window)\n', FORMULA, mlabel)]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for a power simulation (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for a power simulation.\n')]; + localFinish(s, here, suffix); warning(warnState); return end lme = fitlme(tbl0, FORMULA); @@ -49,14 +53,13 @@ bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(lme); sRat = sqrt(psi{1}); sRes = sqrt(lme.MSE); days = (0:max(tbl0.day))'; -s = [s sprintf('ground truth: stim:day=%+.2f/day, rat SD=%.2f, residual SD=%.2f, days=%d\n', ... +s = [s sprintf('ground truth: stim:day=%+.4g/day, rat SD=%.4g, residual SD=%.4g, days=%d\n', ... bInt, sRat, sRes, numel(days))]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:day interaction = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n', 'N/group')]; %#ok - s = [s sprintf(' %s\n', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:day interaction = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -70,20 +73,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end - -s = [s sprintf(['\nRead the per-animal column as the honest power. At the observed N this study\n' ... - 'is typically underpowered; per-animal power reaches ~0.8 only at larger N.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead the per-animal column as the honest power; LME is optimistic (obs-level DF).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'power_result.txt'), 'w'); +fid = fopen(fullfile(here, ['power_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d6_10/result.txt b/analysis/matlab/variations/naive_boxa_d6_10/result.txt index d33c85c..f918c3a 100644 --- a/analysis/matlab/variations/naive_boxa_d6_10/result.txt +++ b/analysis/matlab/variations/naive_boxa_d6_10/result.txt @@ -1,5 +1,5 @@ ============================================================================== -VARIATION: naive_boxa_d6_10 +VARIATION: naive_boxa_d6_10 [metric: success COUNT] ============================================================================== model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT) day = training day within window (0 = first analyzed day) @@ -60,9 +60,9 @@ effect t(df) / F(df1) p (resid) Satterthwaite: p (df) stim x day (interaction) t(46)= -0.73 F(1)= 0.535 p=0.4682 p=0.4687 (df=40) day (learning) t(46)= 4.77 F(1)= 22.779 p=1.877e-05 p=2.436e-05 (df=40) stim (main, window start) t(46)= 2.54 F(1)= 6.429 p=0.01469 p=0.02198 (df=16) -interaction 95% CI: [-6.07, +2.84] +interaction 95% CI: [-6.074, +2.836] HONEST LME (per-animal random slope, day|rat): interaction F(1,10.0)=0.41, p=0.5344 (Satterthwaite DF ~= residual on this random-intercept model; the random-slope model above is the honest learning-rate test -- DF collapses toward the animal count.) -INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.4682, slope diff=-1.62) -Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008. +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.4682, slope diff=-1.619) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d6_10/result_rate.txt b/analysis/matlab/variations/naive_boxa_d6_10/result_rate.txt new file mode 100644 index 0000000..cd9fa18 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d6_10/result_rate.txt @@ -0,0 +1,68 @@ +============================================================================== +VARIATION: naive_boxa_d6_10 [metric: success RATE (success/attempts)] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE (success/attempts)) +day = training day within window (0 = first analyzed day) +treatment (stim=1): Electrode-Box-B2 +control (stim=0): Electrode-Box-A, Electrode-Box-A2, Naive +N = 10 rats, 50 sessions raw day coverage: treat 6..10, control 6..10 +(equal day coverage over this window) + +============================================================================== +FULL MODEL SUMMARY -- fitlme +============================================================================== + +Linear mixed-effects model fit by ML + +Model information: + Number of observations 50 + Fixed effects coefficients 4 + Random effects coefficients 10 + Covariance parameters 2 + +Formula: + behavior ~ 1 + day*stim + (1 | rat) + +Model fit statistics: + AIC BIC LogLikelihood Deviance + -102.02 -90.545 57.008 -114.02 + +Fixed effects coefficients (95% CIs): + Name Estimate SE tStat DF pValue + {'(Intercept)'} 0.4775 0.030961 15.422 46 8.4559e-20 + {'day' } 0.033319 0.0077275 4.3117 46 8.4717e-05 + {'stim' } 0.1661 0.056527 2.9385 46 0.0051416 + {'day:stim' } -0.021119 0.014108 -1.4969 46 0.14124 + + + Lower Upper + 0.41518 0.53982 + 0.017764 0.048873 + 0.052321 0.27989 + -0.049518 0.0072795 + +Random effects covariance parameters (95% CIs): +Group: rat (10 Levels) + Name1 Name2 Type Estimate + {'(Intercept)'} {'(Intercept)'} {'std'} 0.064825 + + + Lower Upper + 0.038261 0.10983 + +Group: Error + Name Estimate Lower Upper + {'Res Std'} 0.064653 0.05193 0.080492 + + +effect t(df) / F(df1) p (resid) Satterthwaite: p (df) +---------------------------------------------------------------------------- +stim x day (interaction) t(46)= -1.50 F(1)= 2.241 p=0.1412 p=0.1423 (df=40) +day (learning) t(46)= 4.31 F(1)= 18.591 p=8.472e-05 p=0.0001028 (df=40) +stim (main, window start) t(46)= 2.94 F(1)= 8.635 p=0.005142 p=0.009071 (df=17) +interaction 95% CI: [-0.04952, +0.00728] +HONEST LME (per-animal random slope, day|rat): interaction F(1,10.0)=1.71, p=0.2198 + (Satterthwaite DF ~= residual on this random-intercept model; the random-slope + model above is the honest learning-rate test -- DF collapses toward the animal count.) +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.1412, slope diff=-0.02112) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d6_13/analyze.m b/analysis/matlab/variations/naive_boxa_d6_13/analyze.m index 82da5ed..0d40402 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/analyze.m +++ b/analysis/matlab/variations/naive_boxa_d6_13/analyze.m @@ -1,30 +1,58 @@ -% Variation analysis -- the paper's linear mixed model on the successful-reach -% COUNT, fit on this folder's curated data subset. +% Variation analysis -- the paper's linear mixed model on this folder's data, +% for BOTH metrics: +% metric = count : behavior = # successes -> result.txt +% metric = rate : behavior = success / attempts -> result_rate.txt +% (rate uses only sessions with attempts > 0) % % model: behavior ~ stim + day + stim:day + (1|rat) -% behavior = successful reaches (count per session) -% stim = 1 for the treatment group(s), 0 for the control group(s) -% day = training day within this window (0 = first analyzed day) -% rat = subject (random intercept) -% -% Self-contained: reads data.csv beside this script and writes result.txt. -% Run headless from this folder with: matlab -batch "analyze" -% (This is a copy of analysis/matlab/variation_analyze.m; see make_variations.m.) +% stim = 1 treatment / 0 control; day = training day within window (0 = +% first analyzed day); rat = subject (random intercept). +% For the interaction we report residual DF, Satterthwaite DF, and the honest +% per-animal random-slope test. Self-contained: reads data.csv beside this +% script. Run headless with: matlab -batch "analyze" +% (Copy of analysis/matlab/variation_analyze.m; see make_variations.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end -vname = regexprep(here, '.*[/\\]', ''); % folder name = variation id +vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -tbl = table(D.success, D.day - min(D.day), double(D.stim), categorical(D.subject), ... +Rc = localAnalyze(D, 'count', here, vname); +Rr = localAnalyze(D, 'rate', here, vname); + +% Machine-readable handoff for SUMMARY.csv (count drives it; rate appended). +VARRESULT = struct('name', vname, 'nRats', Rc.nRats, 'nObs', Rc.nObs, ... + 'interP', Rc.interP, 'interEst', Rc.interEst, ... + 'interPsatt', Rc.interPsatt, 'interPrs', Rc.interPrs, ... + 'stimP', Rc.stimP, 'dayP', Rc.dayP, 'covEqual', Rc.covEqual, ... + 'interPrate', Rr.interP, 'interEstRate', Rr.interEst, 'interPrsRate', Rr.interPrs); + +% ------------------------------------------------------------------ helper +function R = localAnalyze(D, metric, here, vname) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); + beh = D.success ./ D.total; + mlabel = 'success RATE (success/attempts)'; suffix = '_rate'; +else + beh = D.success; + mlabel = 'success COUNT'; suffix = ''; +end +R = struct('interP', NaN, 'interEst', NaN, 'interPsatt', NaN, 'interPrs', NaN, ... + 'stimP', NaN, 'dayP', NaN, 'nRats', numel(unique(D.subject)), ... + 'nObs', height(D), 'covEqual', false); +if numel(unique(D.stim)) < 2 || numel(unique(D.day)) < 2 + localWrite(sprintf('VARIATION: %s [metric: %s]\nInsufficient data for this metric.\n', ... + vname, mlabel), here, suffix); + return +end + +tbl = table(beh, D.day - min(D.day), double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); m = fitlme(tbl, 'behavior ~ stim + day + stim:day + (1|rat)'); C = m.Coefficients; A = anova(m); ci = coefCI(m); -As = anova(m, 'DFMethod', 'satterthwaite'); % Satterthwaite denominator DF +As = anova(m, 'DFMethod', 'satterthwaite'); -% Honest test: refit with a per-animal random SLOPE so the interaction DF -% collapses toward the animal count (guarded -- may not converge in short windows). rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; wst = warning('off', 'all'); try @@ -43,6 +71,7 @@ row = @(nm, t) sprintf('%-26s t(%d)=%6.2f F(%d)=%7.3f p=%.4g p=%.4g (df=%.0f C.DF(gi(t)), C.tStat(gi(t)), A.DF1(ga(t)), A.FStat(ga(t)), C.pValue(gi(t)), ... As.pValue(gs(t)), As.DF2(gs(t))); +ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); maxT = max(D.day(D.stim == 1)); minT = min(D.day(D.stim == 1)); maxC = max(D.day(D.stim == 0)); minC = min(D.day(D.stim == 0)); if abs(maxT - maxC) > 2 @@ -50,20 +79,14 @@ if abs(maxT - maxC) > 2 else cov = '(equal day coverage over this window)'; end - -ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); -if pI >= 0.05 - verdict = 'n.s. -- slopes parallel (no differential learning rate)'; -elseif eI > 0 - verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; -else - verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; -end +if pI >= 0.05; verdict = 'n.s. -- slopes parallel (no differential learning rate)'; +elseif eI > 0; verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; +else; verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; end bar = repmat('=', 1, 78); raw = regexprep(evalc('disp(m)'), '', ''); -s = sprintf('%s\nVARIATION: %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT)\n')]; +s = sprintf('%s\nVARIATION: %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('day = training day within window (0 = first analyzed day)\n')]; s = [s sprintf('treatment (stim=1): %s\n', strjoin(cellstr(unique(D.group(D.stim == 1))), ', '))]; s = [s sprintf('control (stim=0): %s\n', strjoin(cellstr(unique(D.group(D.stim == 0))), ', '))]; @@ -74,7 +97,7 @@ s = [s sprintf('%-26s %-18s %-12s %s\n%s\n', 'effect', 't(df) / F(df1)', 'p (res s = [s row('stim x day (interaction)', 'day:stim')]; s = [s row('day (learning)', 'day')]; s = [s row('stim (main, window start)', 'stim')]; -s = [s sprintf('interaction 95%% CI: [%+.2f, %+.2f]\n', ci(ii, 1), ci(ii, 2))]; +s = [s sprintf('interaction 95%% CI: [%+.4g, %+.4g]\n', ci(ii, 1), ci(ii, 2))]; if rsOk s = [s sprintf('HONEST LME (per-animal random slope, day|rat): interaction F(1,%.1f)=%.2f, p=%.4g\n', rsDf, rsF, rsP)]; else @@ -82,17 +105,18 @@ else end s = [s sprintf([' (Satterthwaite DF ~= residual on this random-intercept model; the random-slope\n' ... ' model above is the honest learning-rate test -- DF collapses toward the animal count.)\n'])]; -s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.2f)\n', verdict, pI, eI)]; -s = [s sprintf('Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.4g)\n', verdict, pI, eI)]; +s = [s sprintf('Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +localWrite(s, here, suffix); + +R = struct('interP', pI, 'interEst', eI, 'interPsatt', As.pValue(gs('day:stim')), ... + 'interPrs', rsP, 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... + 'nRats', numel(unique(D.subject)), 'nObs', height(D), 'covEqual', abs(maxT - maxC) <= 2); +end + +function localWrite(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'result.txt'), 'w'); -fprintf(fid, '%s', s); -fclose(fid); - -% Machine-readable handoff for the summary table (see make_variations.m). -VARRESULT = struct('name', vname, 'nRats', numel(unique(D.subject)), ... - 'nObs', height(D), 'interP', pI, 'interEst', eI, ... - 'interPsatt', As.pValue(gs('day:stim')), 'interPrs', rsP, ... - 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... - 'covEqual', abs(maxT - maxC) <= 2); +fid = fopen(fullfile(here, ['result' suffix '.txt']), 'w'); +fprintf(fid, '%s', s); fclose(fid); +end diff --git a/analysis/matlab/variations/naive_boxa_d6_13/learning_curve.png b/analysis/matlab/variations/naive_boxa_d6_13/learning_curve.png index 01ba17f..480a1ee 100644 Binary files a/analysis/matlab/variations/naive_boxa_d6_13/learning_curve.png and b/analysis/matlab/variations/naive_boxa_d6_13/learning_curve.png differ diff --git a/analysis/matlab/variations/naive_boxa_d6_13/learning_curve_rate.png b/analysis/matlab/variations/naive_boxa_d6_13/learning_curve_rate.png new file mode 100644 index 0000000..7258b0b Binary files /dev/null and b/analysis/matlab/variations/naive_boxa_d6_13/learning_curve_rate.png differ diff --git a/analysis/matlab/variations/naive_boxa_d6_13/logpower_result.txt b/analysis/matlab/variations/naive_boxa_d6_13/logpower_result.txt index 25c42f8..71ba4a8 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/logpower_result.txt +++ b/analysis/matlab/variations/naive_boxa_d6_13/logpower_result.txt @@ -1,7 +1,7 @@ ============================================================================== -LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d6_13 +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d6_13 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT) +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = # successes (count)) log(day) uses 1-indexed training day (our day 0 = paper "Day 1") observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 @@ -10,9 +10,9 @@ stim x log(day) interaction: F(1,69)=0.239 p(resid)=0.6263 p(Satt)=0.6264 (df= honest per-animal random slope (log-day): F(1,9.1)=0.07 p=0.8013 Cohen's f (interaction, partial eta^2=0.001) = 0.034 (small; f: .10 small, .25 medium, .40 large) ---- power simulation (log-day ground truth: stim:logday=+6.70, ratSD=10.78, resSD=11.38) --- +--- power simulation (log-day ground truth: stim:logday=+6.701, ratSD=10.78, resSD=11.38) --- - true stim:log(day) = +6.70 (100% of observed) + true stim:log(day) = +6.701 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.05 | 0.07 <- observed @@ -32,5 +32,4 @@ Cohen's f (interaction, partial eta^2=0.001) = 0.034 (small; f: .10 small, .25 16 | 0.06 | 0.07 24 | 0.12 | 0.11 -Read the per-animal column as the honest power; the LME column matches the -paper's power code (anova interaction p, observation-level DF) and is optimistic. +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d6_13/logpower_result_rate.txt b/analysis/matlab/variations/naive_boxa_d6_13/logpower_result_rate.txt new file mode 100644 index 0000000..98940c6 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d6_13/logpower_result_rate.txt @@ -0,0 +1,35 @@ +============================================================================== +LOG-DAY MODEL + COHEN'S f + POWER -- naive_boxa_d6_13 [metric: success RATE] +============================================================================== +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = success RATE) +log(day) uses 1-indexed training day (our day 0 = paper "Day 1") +observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 + +--- fitted on real data --- +stim x log(day) interaction: F(1,69)=0.000 p(resid)=0.9874 p(Satt)=0.9874 (df=65) + honest per-animal random slope (log-day): F(1,8.6)=0.03 p=0.8653 +Cohen's f (interaction, partial eta^2=0.000) = 0.000 (small; f: .10 small, .25 medium, .40 large) + +--- power simulation (log-day ground truth: stim:logday=+0.001382, ratSD=0.06213, resSD=0.07275) --- + + true stim:log(day) = +0.001382 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.03 | 0.06 <- observed + 5 | 0.08 | 0.07 + 8 | 0.01 | 0.02 + 12 | 0.03 | 0.05 + 16 | 0.05 | 0.07 + 24 | 0.02 | 0.03 + + true stim:log(day) = +0.0006912 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.04 | 0.06 <- observed + 5 | 0.04 | 0.04 + 8 | 0.08 | 0.04 + 12 | 0.05 | 0.04 + 16 | 0.05 | 0.04 + 24 | 0.05 | 0.04 + +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/naive_boxa_d6_13/logpowersim.m b/analysis/matlab/variations/naive_boxa_d6_13/logpowersim.m index e0686ee..5e38720 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/logpowersim.m +++ b/analysis/matlab/variations/naive_boxa_d6_13/logpowersim.m @@ -1,50 +1,52 @@ -% Variation log-day analysis + Cohen's f + power simulation. +% Variation log-day analysis + Cohen's f + power simulation, for BOTH metrics: +% metric = count : behavior = # successes -> logpower_result.txt +% metric = rate : behavior = success / attempts -> logpower_result_rate.txt % -% The paper's power code models behavior against LOG training day, not raw day: +% The paper's power code models behavior against LOG training day: % behavior ~ stim + log(day) + stim:log(day) + (1|rat). -% Their day is 1-indexed (1..10); our data.csv day is 0-indexed (day 0 = paper -% "Day 1"), so log(day + 1) reproduces their transform exactly. -% -% This script (a) refits that log-day model on data.csv, (b) reports the -% interaction (residual DF, Satterthwaite DF, and the honest per-animal -% random-slope test) and Cohen's f -- the partial-eta^2 effect size of the -% interaction, var(fitted_full) - var(fitted_no_interaction) over var(behavior) -% -- and (c) runs the Monte-Carlo power simulation on the log-day model, -% scoring per-animal (cluster-honest) and LME power across N. -% Writes logpower_result.txt. Run: matlab -batch "logpowersim" +% Their day is 1-indexed; our data.csv day is 0-indexed, so log(day + 1) +% reproduces their transform (our day 0 = paper "Day 1"). For each metric this +% refits that model, reports the interaction (residual / Satterthwaite / honest +% per-animal random-slope DF) and Cohen's f (partial-eta^2 effect size), then +% runs the Monte-Carlo power sim (per-animal cluster-honest + LME power). +% Run: matlab -batch "logpowersim" % (Copy of analysis/matlab/variation_logpower.m; see make_variation_logpower.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' column = log(day+1) -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; -warnState = warning('off', 'all'); -rng(1); +localLogPower(D, 'count', here, vname); +localLogPower(D, 'rate', here, vname); -logday = log(D.day + 1); % 0-indexed day -> their log(1-indexed day) -tbl0 = table(D.success, logday, double(D.stim), categorical(D.subject), ... +% ---------------------------------------------------------------- per metric +function localLogPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; +FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' = log(day+1) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); +logday = log(D.day + 1); +tbl0 = table(beh, logday, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT)\n')]; +s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('log(day) uses 1-indexed training day (our day 0 = paper "Day 1")\n')]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for this analysis (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for this analysis.\n')]; + localFinish(s, here, suffix); warning(warnState); return end -% ---- fitted model on the real data ---- full = fitlme(tbl0, FORMULA); An = anova(full); Asatt = anova(full, 'DFMethod', 'satterthwaite'); ii = strcmp(An.Term, 'day:stim'); is = strcmp(Asatt.Term, 'day:stim'); @@ -52,7 +54,6 @@ reduced = fitlme(tbl0, 'behavior ~ stim + day + (1|rat)'); eta2part = max((var(fitted(full)) - var(fitted(reduced))) / var(tbl0.behavior), 0); cohenf = sqrt(eta2part / (1 - eta2part)); -% honest per-animal random-slope interaction rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; try mr = fitlme(tbl0, 'behavior ~ stim + day + stim:day + (day|rat)'); @@ -76,19 +77,18 @@ end s = [s sprintf('Cohen''s f (interaction, partial eta^2=%.3f) = %.3f (%s; f: .10 small, .25 medium, .40 large)\n', ... eta2part, cohenf, mag)]; -% ---- power simulation on the log-day ground truth ---- cn = full.CoefficientNames; be = full.fixedEffects; b0 = be(strcmp(cn, '(Intercept)')); bStim = be(strcmp(cn, 'stim')); bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(full); sRat = sqrt(psi{1}); sRes = sqrt(full.MSE); -days = unique(tbl0.day); % the log(day) grid +days = unique(tbl0.day); -s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.2f, ratSD=%.2f, resSD=%.2f) ---\n', ... +s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.4g, ratSD=%.4g, resSD=%.4g) ---\n', ... bInt, sRat, sRes)]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:log(day) = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:log(day) = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -102,19 +102,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end -s = [s sprintf(['\nRead the per-animal column as the honest power; the LME column matches the\n' ... - 'paper''s power code (anova interaction p, observation-level DF) and is optimistic.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead per-animal as the honest power; LME matches the paper''s power code (optimistic).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'logpower_result.txt'), 'w'); +fid = fopen(fullfile(here, ['logpower_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d6_13/plotcurve.m b/analysis/matlab/variations/naive_boxa_d6_13/plotcurve.m index bfc1386..193a44b 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/plotcurve.m +++ b/analysis/matlab/variations/naive_boxa_d6_13/plotcurve.m @@ -1,11 +1,12 @@ -% Variation learning-curve plot, in the style of the paper: +% Variation learning-curve plots, in the style of the paper: % "Lines indicate mean (and SEM) across animals in the anodal (red) and % control (blue) groups." -% Plots mean +/- SEM successful reaches per training day for the treatment / -% anodal group (stim = 1, red) and the control group (stim = 0, blue), reading -% this folder's data.csv and saving learning_curve.png. The per-group N is read -% from the data (each variation pools different groups), so the legend shows the -% actual counts. Training day is 1-indexed (our day 0 = the paper's "Day 1"). +% Produces TWO figures from this folder's data.csv: +% learning_curve.png # successes (count) per training day +% learning_curve_rate.png success rate (success/attempts) per training day +% anodal / treatment = stim 1 (red); control = stim 0 (blue). Per-group N is +% read from the data. Training day is 1-indexed (our day 0 = paper "Day 1") and +% the x-axis tick labels are drawn vertically. % Run: matlab -batch "plotcurve" % (Copy of analysis/matlab/variation_plot.m; see make_variation_plot.m.) @@ -14,15 +15,20 @@ if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -days = unique(D.day); % 0-indexed -xd = days + 1; % plot as 1-indexed training day (paper axis) - +days = unique(D.day); +xd = days + 1; % plot as 1-indexed training day (paper axis) red = [0.85 0.10 0.10]; blue = [0.10 0.30 0.85]; -[Ma, Sa, na] = localCurve(D, 1, days); % anodal / treatment (stim = 1) -[Mc, Sc, nc] = localCurve(D, 0, days); % control (stim = 0) +localPlot(D, days, xd, 'count', '# successes', ... + fullfile(here, 'learning_curve.png'), vname, red, blue); +localPlot(D, days, xd, 'rate', 'success rate', ... + fullfile(here, 'learning_curve_rate.png'), vname, red, blue); +% ------------------------------------------------------------------ helpers +function localPlot(D, days, xd, metric, ylab, outFile, vname, red, blue) +[Ma, Sa, na] = localCurve(D, 1, days, metric); % anodal / treatment +[Mc, Sc, nc] = localCurve(D, 0, days, metric); % control fig = figure('Visible', 'off', 'Color', 'w', 'Position', [100 100 560 460]); hold on e1 = errorbar(xd, Ma, Sa, '-o', 'Color', red, 'MarkerFaceColor', red, 'LineWidth', 2); @@ -30,26 +36,30 @@ e2 = errorbar(xd, Mc, Sc, '-o', 'Color', blue, 'MarkerFaceColor', blue, 'LineWid hold off legend([e1 e2], {sprintf('anodal, N = %d', na), sprintf('control, N = %d', nc)}, ... 'Location', 'northwest', 'Box', 'off'); -xlabel('training day'); -ylabel('# successes'); +xlabel('training day'); ylabel(ylab); title(vname, 'Interpreter', 'none'); set(gca, 'XTick', xd, 'FontName', 'Arial', 'FontSize', 13, 'LineWidth', 1.5, 'Box', 'off'); - -outFile = fullfile(here, 'learning_curve.png'); +xtickangle(90); % vertical x-axis tick labels exportgraphics(fig, outFile, 'Resolution', 150); close(fig); -fprintf('%s: wrote learning_curve.png (anodal N=%d, control N=%d)\n', vname, na, nc); +fprintf('%s: wrote %s (anodal N=%d, control N=%d)\n', vname, outFile, na, nc); +end -% ------------------------------------------------------------------ helper -function [M, S, n] = localCurve(D, stimVal, days) -%LOCALCURVE Per-day mean and SEM of successes across the animals in a group. +function [M, S, n] = localCurve(D, stimVal, days, metric) +%LOCALCURVE Per-day mean and SEM across the animals in a group, for a metric. subs = unique(D.subject(D.stim == stimVal)); n = numel(subs); X = nan(numel(days), n); for j = 1:n for i = 1:numel(days) r = D.subject == subs(j) & D.day == days(i); - if any(r); X(i, j) = mean(D.success(r)); end + if ~any(r); continue; end + if strcmp(metric, 'rate') + tot = sum(D.total(r)); + if tot > 0; X(i, j) = sum(D.success(r)) / tot; end + else + X(i, j) = mean(D.success(r)); + end end end M = mean(X, 2, 'omitnan'); diff --git a/analysis/matlab/variations/naive_boxa_d6_13/power_result.txt b/analysis/matlab/variations/naive_boxa_d6_13/power_result.txt index f391e92..40d1ec7 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/power_result.txt +++ b/analysis/matlab/variations/naive_boxa_d6_13/power_result.txt @@ -1,11 +1,11 @@ ============================================================================== -POWER SIMULATION -- naive_boxa_d6_13 +POWER SIMULATION -- naive_boxa_d6_13 [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + day + stim:day + (1|rat) (success COUNT; day within-window) +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = # successes (count); day within-window) observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 -ground truth: stim:day=+0.87/day, rat SD=10.75, residual SD=11.53, days=8 +ground truth: stim:day=+0.8737/day, rat SD=10.75, residual SD=11.53, days=8 - true stim:day interaction = +0.87 (100% of observed) + true stim:day interaction = +0.8737 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.07 | 0.11 <- observed @@ -15,7 +15,7 @@ ground truth: stim:day=+0.87/day, rat SD=10.75, residual SD=11.53, days=8 16 | 0.28 | 0.33 24 | 0.38 | 0.41 - true stim:day interaction = +0.44 (50% of observed) + true stim:day interaction = +0.4368 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.06 | 0.10 <- observed @@ -25,5 +25,4 @@ ground truth: stim:day=+0.87/day, rat SD=10.75, residual SD=11.53, days=8 16 | 0.07 | 0.07 24 | 0.16 | 0.15 -Read the per-animal column as the honest power. At the observed N this study -is typically underpowered; per-animal power reaches ~0.8 only at larger N. +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d6_13/power_result_rate.txt b/analysis/matlab/variations/naive_boxa_d6_13/power_result_rate.txt new file mode 100644 index 0000000..d8f2f16 --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d6_13/power_result_rate.txt @@ -0,0 +1,28 @@ +============================================================================== +POWER SIMULATION -- naive_boxa_d6_13 [metric: success RATE] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE; day within-window) +observed groups: stim n=3, control n=7 nrep=120, alpha=0.05 +ground truth: stim:day=+0.001651/day, rat SD=0.06197, residual SD=0.07346, days=8 + + true stim:day interaction = +0.001651 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.03 | 0.04 <- observed + 5 | 0.10 | 0.08 + 8 | 0.05 | 0.03 + 12 | 0.04 | 0.05 + 16 | 0.06 | 0.07 + 24 | 0.03 | 0.06 + + true stim:day interaction = +0.0008253 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.05 | 0.07 <- observed + 5 | 0.04 | 0.06 + 8 | 0.07 | 0.05 + 12 | 0.03 | 0.05 + 16 | 0.03 | 0.05 + 24 | 0.07 | 0.07 + +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/naive_boxa_d6_13/powersim.m b/analysis/matlab/variations/naive_boxa_d6_13/powersim.m index 554d032..ecd9c6c 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/powersim.m +++ b/analysis/matlab/variations/naive_boxa_d6_13/powersim.m @@ -1,45 +1,49 @@ % Variation power simulation -- Monte-Carlo power for the paper's stim x day -% interaction, using THIS folder's data as the ground truth. +% interaction, using THIS folder's data as the ground truth, for BOTH metrics: +% metric = count : behavior = # successes -> power_result.txt +% metric = rate : behavior = success / attempts -> power_result_rate.txt % % Ground truth: fitlme(behavior ~ stim + day + stim:day + (1|rat)) on data.csv -% (success COUNT; day within-window). Its fixed effects, per-rat intercept SD, -% and residual SD generate NREP synthetic datasets at each rats-per-group N and -% each true-effect multiplier (1 = observed slope, 0.5 = half). Each dataset is -% scored at alpha = 0.05 two ways: -% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest -- the -% honest power, matching the random-slope / per-animal inference) +% (day within-window). Its fixed effects, per-rat intercept SD, and residual SD +% generate NREP synthetic datasets at each rats-per-group N and each true-effect +% multiplier (1 = observed, 0.5 = half). Each is scored at alpha=0.05 by: +% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest power) % LME : the fitlme stim:day p (observation-level DF -- optimistic) -% Writes power_result.txt beside this script. Run: matlab -batch "powersim" +% Writes power_result[_rate].txt. Run: matlab -batch "powersim" % (Copy of analysis/matlab/variation_power.m; see make_variation_power.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); + +localPower(D, 'count', here, vname); +localPower(D, 'rate', here, vname); + +% ---------------------------------------------------------------- per metric +function localPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; - -warnState = warning('off', 'all'); -rng(1); - +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); day0 = min(D.day); -tbl0 = table(D.success, D.day - day0, double(D.stim), categorical(D.subject), ... +tbl0 = table(beh, D.day - day0, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); - nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nPOWER SIMULATION -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: %s (success COUNT; day within-window)\n', FORMULA)]; +s = sprintf('%s\nPOWER SIMULATION -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: %s (behavior = %s; day within-window)\n', FORMULA, mlabel)]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for a power simulation (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for a power simulation.\n')]; + localFinish(s, here, suffix); warning(warnState); return end lme = fitlme(tbl0, FORMULA); @@ -49,14 +53,13 @@ bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(lme); sRat = sqrt(psi{1}); sRes = sqrt(lme.MSE); days = (0:max(tbl0.day))'; -s = [s sprintf('ground truth: stim:day=%+.2f/day, rat SD=%.2f, residual SD=%.2f, days=%d\n', ... +s = [s sprintf('ground truth: stim:day=%+.4g/day, rat SD=%.4g, residual SD=%.4g, days=%d\n', ... bInt, sRat, sRes, numel(days))]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:day interaction = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n', 'N/group')]; %#ok - s = [s sprintf(' %s\n', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:day interaction = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -70,20 +73,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end - -s = [s sprintf(['\nRead the per-animal column as the honest power. At the observed N this study\n' ... - 'is typically underpowered; per-animal power reaches ~0.8 only at larger N.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead the per-animal column as the honest power; LME is optimistic (obs-level DF).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'power_result.txt'), 'w'); +fid = fopen(fullfile(here, ['power_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/naive_boxa_d6_13/result.txt b/analysis/matlab/variations/naive_boxa_d6_13/result.txt index 869fbd6..ce8141f 100644 --- a/analysis/matlab/variations/naive_boxa_d6_13/result.txt +++ b/analysis/matlab/variations/naive_boxa_d6_13/result.txt @@ -1,5 +1,5 @@ ============================================================================== -VARIATION: naive_boxa_d6_13 +VARIATION: naive_boxa_d6_13 [metric: success COUNT] ============================================================================== model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT) day = training day within window (0 = first analyzed day) @@ -60,9 +60,9 @@ effect t(df) / F(df1) p (resid) Satterthwaite: p (df) stim x day (interaction) t(69)= 0.63 F(1)= 0.397 p=0.5308 p=0.5309 (df=65) day (learning) t(69)= 4.26 F(1)= 18.109 p=6.446e-05 p=6.931e-05 (df=64) stim (main, window start) t(69)= 2.21 F(1)= 4.886 p=0.03039 p=0.0412 (df=17) -interaction 95% CI: [-1.89, +3.64] +interaction 95% CI: [-1.893, +3.641] HONEST LME (per-animal random slope, day|rat): interaction F(1,8.5)=0.16, p=0.6966 (Satterthwaite DF ~= residual on this random-intercept model; the random-slope model above is the honest learning-rate test -- DF collapses toward the animal count.) -INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.5308, slope diff=+0.87) -Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008. +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.5308, slope diff=+0.8737) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/naive_boxa_d6_13/result_rate.txt b/analysis/matlab/variations/naive_boxa_d6_13/result_rate.txt new file mode 100644 index 0000000..db3fcde --- /dev/null +++ b/analysis/matlab/variations/naive_boxa_d6_13/result_rate.txt @@ -0,0 +1,68 @@ +============================================================================== +VARIATION: naive_boxa_d6_13 [metric: success RATE (success/attempts)] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE (success/attempts)) +day = training day within window (0 = first analyzed day) +treatment (stim=1): Electrode-Box-B2 +control (stim=0): Electrode-Box-A, Electrode-Box-A2, Naive +N = 10 rats, 73 sessions raw day coverage: treat 6..13, control 6..13 +(equal day coverage over this window) + +============================================================================== +FULL MODEL SUMMARY -- fitlme +============================================================================== + +Linear mixed-effects model fit by ML + +Model information: + Number of observations 73 + Fixed effects coefficients 4 + Random effects coefficients 10 + Covariance parameters 2 + +Formula: + behavior ~ 1 + day*stim + (1 | rat) + +Model fit statistics: + AIC BIC LogLikelihood Deviance + -143.92 -130.17 77.958 -155.92 + +Fixed effects coefficients (95% CIs): + Name Estimate SE tStat DF pValue + {'(Intercept)'} 0.50639 0.029658 17.074 69 1.7857e-26 + {'day' } 0.013881 0.0046444 2.9887 69 0.0038784 + {'stim' } 0.13179 0.05426 2.4289 69 0.017756 + {'day:stim' } 0.0016506 0.0088296 0.18693 69 0.85226 + + + Lower Upper + 0.44722 0.56555 + 0.0046157 0.023146 + 0.023544 0.24004 + -0.015964 0.019265 + +Random effects covariance parameters (95% CIs): +Group: rat (10 Levels) + Name1 Name2 Type Estimate + {'(Intercept)'} {'(Intercept)'} {'std'} 0.061972 + + + Lower Upper + 0.036939 0.10397 + +Group: Error + Name Estimate Lower Upper + {'Res Std'} 0.073458 0.061716 0.087434 + + +effect t(df) / F(df1) p (resid) Satterthwaite: p (df) +---------------------------------------------------------------------------- +stim x day (interaction) t(69)= 0.19 F(1)= 0.035 p=0.8523 p=0.8523 (df=65) +day (learning) t(69)= 2.99 F(1)= 8.933 p=0.003878 p=0.003964 (df=64) +stim (main, window start) t(69)= 2.43 F(1)= 5.899 p=0.01776 p=0.02572 (df=18) +interaction 95% CI: [-0.01596, +0.01927] +HONEST LME (per-animal random slope, day|rat): interaction F(1,8.1)=0.00, p=0.9836 + (Satterthwaite DF ~= residual on this random-intercept model; the random-slope + model above is the honest learning-rate test -- DF collapses toward the animal count.) +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.8523, slope diff=+0.001651) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/prev_f_full/analyze.m b/analysis/matlab/variations/prev_f_full/analyze.m index 82da5ed..0d40402 100644 --- a/analysis/matlab/variations/prev_f_full/analyze.m +++ b/analysis/matlab/variations/prev_f_full/analyze.m @@ -1,30 +1,58 @@ -% Variation analysis -- the paper's linear mixed model on the successful-reach -% COUNT, fit on this folder's curated data subset. +% Variation analysis -- the paper's linear mixed model on this folder's data, +% for BOTH metrics: +% metric = count : behavior = # successes -> result.txt +% metric = rate : behavior = success / attempts -> result_rate.txt +% (rate uses only sessions with attempts > 0) % % model: behavior ~ stim + day + stim:day + (1|rat) -% behavior = successful reaches (count per session) -% stim = 1 for the treatment group(s), 0 for the control group(s) -% day = training day within this window (0 = first analyzed day) -% rat = subject (random intercept) -% -% Self-contained: reads data.csv beside this script and writes result.txt. -% Run headless from this folder with: matlab -batch "analyze" -% (This is a copy of analysis/matlab/variation_analyze.m; see make_variations.m.) +% stim = 1 treatment / 0 control; day = training day within window (0 = +% first analyzed day); rat = subject (random intercept). +% For the interaction we report residual DF, Satterthwaite DF, and the honest +% per-animal random-slope test. Self-contained: reads data.csv beside this +% script. Run headless with: matlab -batch "analyze" +% (Copy of analysis/matlab/variation_analyze.m; see make_variations.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end -vname = regexprep(here, '.*[/\\]', ''); % folder name = variation id +vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -tbl = table(D.success, D.day - min(D.day), double(D.stim), categorical(D.subject), ... +Rc = localAnalyze(D, 'count', here, vname); +Rr = localAnalyze(D, 'rate', here, vname); + +% Machine-readable handoff for SUMMARY.csv (count drives it; rate appended). +VARRESULT = struct('name', vname, 'nRats', Rc.nRats, 'nObs', Rc.nObs, ... + 'interP', Rc.interP, 'interEst', Rc.interEst, ... + 'interPsatt', Rc.interPsatt, 'interPrs', Rc.interPrs, ... + 'stimP', Rc.stimP, 'dayP', Rc.dayP, 'covEqual', Rc.covEqual, ... + 'interPrate', Rr.interP, 'interEstRate', Rr.interEst, 'interPrsRate', Rr.interPrs); + +% ------------------------------------------------------------------ helper +function R = localAnalyze(D, metric, here, vname) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); + beh = D.success ./ D.total; + mlabel = 'success RATE (success/attempts)'; suffix = '_rate'; +else + beh = D.success; + mlabel = 'success COUNT'; suffix = ''; +end +R = struct('interP', NaN, 'interEst', NaN, 'interPsatt', NaN, 'interPrs', NaN, ... + 'stimP', NaN, 'dayP', NaN, 'nRats', numel(unique(D.subject)), ... + 'nObs', height(D), 'covEqual', false); +if numel(unique(D.stim)) < 2 || numel(unique(D.day)) < 2 + localWrite(sprintf('VARIATION: %s [metric: %s]\nInsufficient data for this metric.\n', ... + vname, mlabel), here, suffix); + return +end + +tbl = table(beh, D.day - min(D.day), double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); m = fitlme(tbl, 'behavior ~ stim + day + stim:day + (1|rat)'); C = m.Coefficients; A = anova(m); ci = coefCI(m); -As = anova(m, 'DFMethod', 'satterthwaite'); % Satterthwaite denominator DF +As = anova(m, 'DFMethod', 'satterthwaite'); -% Honest test: refit with a per-animal random SLOPE so the interaction DF -% collapses toward the animal count (guarded -- may not converge in short windows). rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; wst = warning('off', 'all'); try @@ -43,6 +71,7 @@ row = @(nm, t) sprintf('%-26s t(%d)=%6.2f F(%d)=%7.3f p=%.4g p=%.4g (df=%.0f C.DF(gi(t)), C.tStat(gi(t)), A.DF1(ga(t)), A.FStat(ga(t)), C.pValue(gi(t)), ... As.pValue(gs(t)), As.DF2(gs(t))); +ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); maxT = max(D.day(D.stim == 1)); minT = min(D.day(D.stim == 1)); maxC = max(D.day(D.stim == 0)); minC = min(D.day(D.stim == 0)); if abs(maxT - maxC) > 2 @@ -50,20 +79,14 @@ if abs(maxT - maxC) > 2 else cov = '(equal day coverage over this window)'; end - -ii = gi('day:stim'); pI = C.pValue(ii); eI = C.Estimate(ii); -if pI >= 0.05 - verdict = 'n.s. -- slopes parallel (no differential learning rate)'; -elseif eI > 0 - verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; -else - verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; -end +if pI >= 0.05; verdict = 'n.s. -- slopes parallel (no differential learning rate)'; +elseif eI > 0; verdict = 'SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates)'; +else; verdict = 'SIGNIFICANT negative -- treatment improves SLOWER (groups converge)'; end bar = repmat('=', 1, 78); raw = regexprep(evalc('disp(m)'), '', ''); -s = sprintf('%s\nVARIATION: %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT)\n')]; +s = sprintf('%s\nVARIATION: %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + day + stim:day + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('day = training day within window (0 = first analyzed day)\n')]; s = [s sprintf('treatment (stim=1): %s\n', strjoin(cellstr(unique(D.group(D.stim == 1))), ', '))]; s = [s sprintf('control (stim=0): %s\n', strjoin(cellstr(unique(D.group(D.stim == 0))), ', '))]; @@ -74,7 +97,7 @@ s = [s sprintf('%-26s %-18s %-12s %s\n%s\n', 'effect', 't(df) / F(df1)', 'p (res s = [s row('stim x day (interaction)', 'day:stim')]; s = [s row('day (learning)', 'day')]; s = [s row('stim (main, window start)', 'stim')]; -s = [s sprintf('interaction 95%% CI: [%+.2f, %+.2f]\n', ci(ii, 1), ci(ii, 2))]; +s = [s sprintf('interaction 95%% CI: [%+.4g, %+.4g]\n', ci(ii, 1), ci(ii, 2))]; if rsOk s = [s sprintf('HONEST LME (per-animal random slope, day|rat): interaction F(1,%.1f)=%.2f, p=%.4g\n', rsDf, rsF, rsP)]; else @@ -82,17 +105,18 @@ else end s = [s sprintf([' (Satterthwaite DF ~= residual on this random-intercept model; the random-slope\n' ... ' model above is the honest learning-rate test -- DF collapses toward the animal count.)\n'])]; -s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.2f)\n', verdict, pI, eI)]; -s = [s sprintf('Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +s = [s sprintf('INTERPRETATION: stim x day interaction %s (p=%.4g, slope diff=%+.4g)\n', verdict, pI, eI)]; +s = [s sprintf('Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.\n')]; +localWrite(s, here, suffix); + +R = struct('interP', pI, 'interEst', eI, 'interPsatt', As.pValue(gs('day:stim')), ... + 'interPrs', rsP, 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... + 'nRats', numel(unique(D.subject)), 'nObs', height(D), 'covEqual', abs(maxT - maxC) <= 2); +end + +function localWrite(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'result.txt'), 'w'); -fprintf(fid, '%s', s); -fclose(fid); - -% Machine-readable handoff for the summary table (see make_variations.m). -VARRESULT = struct('name', vname, 'nRats', numel(unique(D.subject)), ... - 'nObs', height(D), 'interP', pI, 'interEst', eI, ... - 'interPsatt', As.pValue(gs('day:stim')), 'interPrs', rsP, ... - 'stimP', C.pValue(gi('stim')), 'dayP', C.pValue(gi('day')), ... - 'covEqual', abs(maxT - maxC) <= 2); +fid = fopen(fullfile(here, ['result' suffix '.txt']), 'w'); +fprintf(fid, '%s', s); fclose(fid); +end diff --git a/analysis/matlab/variations/prev_f_full/learning_curve.png b/analysis/matlab/variations/prev_f_full/learning_curve.png index 5b7419f..7971adb 100644 Binary files a/analysis/matlab/variations/prev_f_full/learning_curve.png and b/analysis/matlab/variations/prev_f_full/learning_curve.png differ diff --git a/analysis/matlab/variations/prev_f_full/learning_curve_rate.png b/analysis/matlab/variations/prev_f_full/learning_curve_rate.png new file mode 100644 index 0000000..8a6ddf1 Binary files /dev/null and b/analysis/matlab/variations/prev_f_full/learning_curve_rate.png differ diff --git a/analysis/matlab/variations/prev_f_full/logpower_result.txt b/analysis/matlab/variations/prev_f_full/logpower_result.txt index 2cac7ea..cc0652b 100644 --- a/analysis/matlab/variations/prev_f_full/logpower_result.txt +++ b/analysis/matlab/variations/prev_f_full/logpower_result.txt @@ -1,7 +1,7 @@ ============================================================================== -LOG-DAY MODEL + COHEN'S f + POWER -- prev_f_full +LOG-DAY MODEL + COHEN'S f + POWER -- prev_f_full [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT) +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = # successes (count)) log(day) uses 1-indexed training day (our day 0 = paper "Day 1") observed groups: stim n=12, control n=12 nrep=120, alpha=0.05 @@ -10,9 +10,9 @@ stim x log(day) interaction: F(1,227)=8.356 p(resid)=0.004218 p(Satt)=0.004253 honest per-animal random slope (log-day): F(1,24.3)=3.32 p=0.08079 Cohen's f (interaction, partial eta^2=0.012) = 0.112 (small-medium; f: .10 small, .25 medium, .40 large) ---- power simulation (log-day ground truth: stim:logday=+2.39, ratSD=4.33, resSD=4.34) --- +--- power simulation (log-day ground truth: stim:logday=+2.389, ratSD=4.328, resSD=4.338) --- - true stim:log(day) = +2.39 (100% of observed) + true stim:log(day) = +2.389 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.15 | 0.28 @@ -22,7 +22,7 @@ Cohen's f (interaction, partial eta^2=0.012) = 0.112 (small-medium; f: .10 smal 16 | 0.92 | 0.94 24 | 0.99 | 0.98 - true stim:log(day) = +1.19 (50% of observed) + true stim:log(day) = +1.195 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.05 | 0.09 @@ -32,5 +32,4 @@ Cohen's f (interaction, partial eta^2=0.012) = 0.112 (small-medium; f: .10 smal 16 | 0.46 | 0.49 24 | 0.46 | 0.49 -Read the per-animal column as the honest power; the LME column matches the -paper's power code (anova interaction p, observation-level DF) and is optimistic. +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/prev_f_full/logpower_result_rate.txt b/analysis/matlab/variations/prev_f_full/logpower_result_rate.txt new file mode 100644 index 0000000..9156a86 --- /dev/null +++ b/analysis/matlab/variations/prev_f_full/logpower_result_rate.txt @@ -0,0 +1,35 @@ +============================================================================== +LOG-DAY MODEL + COHEN'S f + POWER -- prev_f_full [metric: success RATE] +============================================================================== +model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = success RATE) +log(day) uses 1-indexed training day (our day 0 = paper "Day 1") +observed groups: stim n=12, control n=12 nrep=120, alpha=0.05 + +--- fitted on real data --- +stim x log(day) interaction: F(1,227)=2.888 p(resid)=0.0906 p(Satt)=0.09072 (df=208) + honest per-animal random slope (log-day): F(1,23.7)=2.05 p=0.1655 +Cohen's f (interaction, partial eta^2=0.006) = 0.079 (small; f: .10 small, .25 medium, .40 large) + +--- power simulation (log-day ground truth: stim:logday=+0.02595, ratSD=0.06838, resSD=0.08017) --- + + true stim:log(day) = +0.02595 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.06 | 0.07 + 5 | 0.17 | 0.18 + 8 | 0.33 | 0.38 + 12 | 0.38 | 0.47 <- observed + 16 | 0.55 | 0.62 + 24 | 0.64 | 0.60 + + true stim:log(day) = +0.01298 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.04 | 0.04 + 5 | 0.07 | 0.11 + 8 | 0.10 | 0.12 + 12 | 0.17 | 0.17 <- observed + 16 | 0.10 | 0.15 + 24 | 0.15 | 0.17 + +Read per-animal as the honest power; LME matches the paper's power code (optimistic). diff --git a/analysis/matlab/variations/prev_f_full/logpowersim.m b/analysis/matlab/variations/prev_f_full/logpowersim.m index e0686ee..5e38720 100644 --- a/analysis/matlab/variations/prev_f_full/logpowersim.m +++ b/analysis/matlab/variations/prev_f_full/logpowersim.m @@ -1,50 +1,52 @@ -% Variation log-day analysis + Cohen's f + power simulation. +% Variation log-day analysis + Cohen's f + power simulation, for BOTH metrics: +% metric = count : behavior = # successes -> logpower_result.txt +% metric = rate : behavior = success / attempts -> logpower_result_rate.txt % -% The paper's power code models behavior against LOG training day, not raw day: +% The paper's power code models behavior against LOG training day: % behavior ~ stim + log(day) + stim:log(day) + (1|rat). -% Their day is 1-indexed (1..10); our data.csv day is 0-indexed (day 0 = paper -% "Day 1"), so log(day + 1) reproduces their transform exactly. -% -% This script (a) refits that log-day model on data.csv, (b) reports the -% interaction (residual DF, Satterthwaite DF, and the honest per-animal -% random-slope test) and Cohen's f -- the partial-eta^2 effect size of the -% interaction, var(fitted_full) - var(fitted_no_interaction) over var(behavior) -% -- and (c) runs the Monte-Carlo power simulation on the log-day model, -% scoring per-animal (cluster-honest) and LME power across N. -% Writes logpower_result.txt. Run: matlab -batch "logpowersim" +% Their day is 1-indexed; our data.csv day is 0-indexed, so log(day + 1) +% reproduces their transform (our day 0 = paper "Day 1"). For each metric this +% refits that model, reports the interaction (residual / Satterthwaite / honest +% per-animal random-slope DF) and Cohen's f (partial-eta^2 effect size), then +% runs the Monte-Carlo power sim (per-animal cluster-honest + LME power). +% Run: matlab -batch "logpowersim" % (Copy of analysis/matlab/variation_logpower.m; see make_variation_logpower.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' column = log(day+1) -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; -warnState = warning('off', 'all'); -rng(1); +localLogPower(D, 'count', here, vname); +localLogPower(D, 'rate', here, vname); -logday = log(D.day + 1); % 0-indexed day -> their log(1-indexed day) -tbl0 = table(D.success, logday, double(D.stim), categorical(D.subject), ... +% ---------------------------------------------------------------- per metric +function localLogPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; +FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; % 'day' = log(day+1) +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); +logday = log(D.day + 1); +tbl0 = table(beh, logday, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (success COUNT)\n')]; +s = sprintf('%s\nLOG-DAY MODEL + COHEN''S f + POWER -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: behavior ~ stim + log(day) + stim:log(day) + (1|rat) (behavior = %s)\n', mlabel)]; s = [s sprintf('log(day) uses 1-indexed training day (our day 0 = paper "Day 1")\n')]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for this analysis (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for this analysis.\n')]; + localFinish(s, here, suffix); warning(warnState); return end -% ---- fitted model on the real data ---- full = fitlme(tbl0, FORMULA); An = anova(full); Asatt = anova(full, 'DFMethod', 'satterthwaite'); ii = strcmp(An.Term, 'day:stim'); is = strcmp(Asatt.Term, 'day:stim'); @@ -52,7 +54,6 @@ reduced = fitlme(tbl0, 'behavior ~ stim + day + (1|rat)'); eta2part = max((var(fitted(full)) - var(fitted(reduced))) / var(tbl0.behavior), 0); cohenf = sqrt(eta2part / (1 - eta2part)); -% honest per-animal random-slope interaction rsP = NaN; rsDf = NaN; rsF = NaN; rsOk = false; try mr = fitlme(tbl0, 'behavior ~ stim + day + stim:day + (day|rat)'); @@ -76,19 +77,18 @@ end s = [s sprintf('Cohen''s f (interaction, partial eta^2=%.3f) = %.3f (%s; f: .10 small, .25 medium, .40 large)\n', ... eta2part, cohenf, mag)]; -% ---- power simulation on the log-day ground truth ---- cn = full.CoefficientNames; be = full.fixedEffects; b0 = be(strcmp(cn, '(Intercept)')); bStim = be(strcmp(cn, 'stim')); bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(full); sRat = sqrt(psi{1}); sRes = sqrt(full.MSE); -days = unique(tbl0.day); % the log(day) grid +days = unique(tbl0.day); -s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.2f, ratSD=%.2f, resSD=%.2f) ---\n', ... +s = [s sprintf('\n--- power simulation (log-day ground truth: stim:logday=%+.4g, ratSD=%.4g, resSD=%.4g) ---\n', ... bInt, sRat, sRes)]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:log(day) = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:log(day) = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -102,19 +102,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end -s = [s sprintf(['\nRead the per-animal column as the honest power; the LME column matches the\n' ... - 'paper''s power code (anova interaction p, observation-level DF) and is optimistic.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead per-animal as the honest power; LME matches the paper''s power code (optimistic).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'logpower_result.txt'), 'w'); +fid = fopen(fullfile(here, ['logpower_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/prev_f_full/plotcurve.m b/analysis/matlab/variations/prev_f_full/plotcurve.m index bfc1386..193a44b 100644 --- a/analysis/matlab/variations/prev_f_full/plotcurve.m +++ b/analysis/matlab/variations/prev_f_full/plotcurve.m @@ -1,11 +1,12 @@ -% Variation learning-curve plot, in the style of the paper: +% Variation learning-curve plots, in the style of the paper: % "Lines indicate mean (and SEM) across animals in the anodal (red) and % control (blue) groups." -% Plots mean +/- SEM successful reaches per training day for the treatment / -% anodal group (stim = 1, red) and the control group (stim = 0, blue), reading -% this folder's data.csv and saving learning_curve.png. The per-group N is read -% from the data (each variation pools different groups), so the legend shows the -% actual counts. Training day is 1-indexed (our day 0 = the paper's "Day 1"). +% Produces TWO figures from this folder's data.csv: +% learning_curve.png # successes (count) per training day +% learning_curve_rate.png success rate (success/attempts) per training day +% anodal / treatment = stim 1 (red); control = stim 0 (blue). Per-group N is +% read from the data. Training day is 1-indexed (our day 0 = paper "Day 1") and +% the x-axis tick labels are drawn vertically. % Run: matlab -batch "plotcurve" % (Copy of analysis/matlab/variation_plot.m; see make_variation_plot.m.) @@ -14,15 +15,20 @@ if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); -days = unique(D.day); % 0-indexed -xd = days + 1; % plot as 1-indexed training day (paper axis) - +days = unique(D.day); +xd = days + 1; % plot as 1-indexed training day (paper axis) red = [0.85 0.10 0.10]; blue = [0.10 0.30 0.85]; -[Ma, Sa, na] = localCurve(D, 1, days); % anodal / treatment (stim = 1) -[Mc, Sc, nc] = localCurve(D, 0, days); % control (stim = 0) +localPlot(D, days, xd, 'count', '# successes', ... + fullfile(here, 'learning_curve.png'), vname, red, blue); +localPlot(D, days, xd, 'rate', 'success rate', ... + fullfile(here, 'learning_curve_rate.png'), vname, red, blue); +% ------------------------------------------------------------------ helpers +function localPlot(D, days, xd, metric, ylab, outFile, vname, red, blue) +[Ma, Sa, na] = localCurve(D, 1, days, metric); % anodal / treatment +[Mc, Sc, nc] = localCurve(D, 0, days, metric); % control fig = figure('Visible', 'off', 'Color', 'w', 'Position', [100 100 560 460]); hold on e1 = errorbar(xd, Ma, Sa, '-o', 'Color', red, 'MarkerFaceColor', red, 'LineWidth', 2); @@ -30,26 +36,30 @@ e2 = errorbar(xd, Mc, Sc, '-o', 'Color', blue, 'MarkerFaceColor', blue, 'LineWid hold off legend([e1 e2], {sprintf('anodal, N = %d', na), sprintf('control, N = %d', nc)}, ... 'Location', 'northwest', 'Box', 'off'); -xlabel('training day'); -ylabel('# successes'); +xlabel('training day'); ylabel(ylab); title(vname, 'Interpreter', 'none'); set(gca, 'XTick', xd, 'FontName', 'Arial', 'FontSize', 13, 'LineWidth', 1.5, 'Box', 'off'); - -outFile = fullfile(here, 'learning_curve.png'); +xtickangle(90); % vertical x-axis tick labels exportgraphics(fig, outFile, 'Resolution', 150); close(fig); -fprintf('%s: wrote learning_curve.png (anodal N=%d, control N=%d)\n', vname, na, nc); +fprintf('%s: wrote %s (anodal N=%d, control N=%d)\n', vname, outFile, na, nc); +end -% ------------------------------------------------------------------ helper -function [M, S, n] = localCurve(D, stimVal, days) -%LOCALCURVE Per-day mean and SEM of successes across the animals in a group. +function [M, S, n] = localCurve(D, stimVal, days, metric) +%LOCALCURVE Per-day mean and SEM across the animals in a group, for a metric. subs = unique(D.subject(D.stim == stimVal)); n = numel(subs); X = nan(numel(days), n); for j = 1:n for i = 1:numel(days) r = D.subject == subs(j) & D.day == days(i); - if any(r); X(i, j) = mean(D.success(r)); end + if ~any(r); continue; end + if strcmp(metric, 'rate') + tot = sum(D.total(r)); + if tot > 0; X(i, j) = sum(D.success(r)) / tot; end + else + X(i, j) = mean(D.success(r)); + end end end M = mean(X, 2, 'omitnan'); diff --git a/analysis/matlab/variations/prev_f_full/power_result.txt b/analysis/matlab/variations/prev_f_full/power_result.txt index 9368d88..91f2c44 100644 --- a/analysis/matlab/variations/prev_f_full/power_result.txt +++ b/analysis/matlab/variations/prev_f_full/power_result.txt @@ -1,11 +1,11 @@ ============================================================================== -POWER SIMULATION -- prev_f_full +POWER SIMULATION -- prev_f_full [metric: # successes (count)] ============================================================================== -model: behavior ~ stim + day + stim:day + (1|rat) (success COUNT; day within-window) +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = # successes (count); day within-window) observed groups: stim n=12, control n=12 nrep=120, alpha=0.05 -ground truth: stim:day=+0.56/day, rat SD=4.32, residual SD=4.49, days=10 +ground truth: stim:day=+0.5602/day, rat SD=4.316, residual SD=4.489, days=10 - true stim:day interaction = +0.56 (100% of observed) + true stim:day interaction = +0.5602 (100% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.13 | 0.27 @@ -15,7 +15,7 @@ ground truth: stim:day=+0.56/day, rat SD=4.32, residual SD=4.49, days=10 16 | 0.91 | 0.93 24 | 0.98 | 0.98 - true stim:day interaction = +0.28 (50% of observed) + true stim:day interaction = +0.2801 (50% of observed) N/group | per-animal power | LME power ------------------------------------------ 3 | 0.04 | 0.07 @@ -25,5 +25,4 @@ ground truth: stim:day=+0.56/day, rat SD=4.32, residual SD=4.49, days=10 16 | 0.33 | 0.37 24 | 0.45 | 0.47 -Read the per-animal column as the honest power. At the observed N this study -is typically underpowered; per-animal power reaches ~0.8 only at larger N. +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/prev_f_full/power_result_rate.txt b/analysis/matlab/variations/prev_f_full/power_result_rate.txt new file mode 100644 index 0000000..8970c48 --- /dev/null +++ b/analysis/matlab/variations/prev_f_full/power_result_rate.txt @@ -0,0 +1,28 @@ +============================================================================== +POWER SIMULATION -- prev_f_full [metric: success RATE] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE; day within-window) +observed groups: stim n=12, control n=12 nrep=120, alpha=0.05 +ground truth: stim:day=+0.006292/day, rat SD=0.06824, residual SD=0.08152, days=10 + + true stim:day interaction = +0.006292 (100% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.06 | 0.09 + 5 | 0.17 | 0.16 + 8 | 0.28 | 0.35 + 12 | 0.37 | 0.41 <- observed + 16 | 0.55 | 0.62 + 24 | 0.62 | 0.62 + + true stim:day interaction = +0.003146 (50% of observed) + N/group | per-animal power | LME power + ------------------------------------------ + 3 | 0.03 | 0.04 + 5 | 0.07 | 0.05 + 8 | 0.12 | 0.15 + 12 | 0.17 | 0.18 <- observed + 16 | 0.12 | 0.17 + 24 | 0.15 | 0.13 + +Read the per-animal column as the honest power; LME is optimistic (obs-level DF). diff --git a/analysis/matlab/variations/prev_f_full/powersim.m b/analysis/matlab/variations/prev_f_full/powersim.m index 554d032..ecd9c6c 100644 --- a/analysis/matlab/variations/prev_f_full/powersim.m +++ b/analysis/matlab/variations/prev_f_full/powersim.m @@ -1,45 +1,49 @@ % Variation power simulation -- Monte-Carlo power for the paper's stim x day -% interaction, using THIS folder's data as the ground truth. +% interaction, using THIS folder's data as the ground truth, for BOTH metrics: +% metric = count : behavior = # successes -> power_result.txt +% metric = rate : behavior = success / attempts -> power_result_rate.txt % % Ground truth: fitlme(behavior ~ stim + day + stim:day + (1|rat)) on data.csv -% (success COUNT; day within-window). Its fixed effects, per-rat intercept SD, -% and residual SD generate NREP synthetic datasets at each rats-per-group N and -% each true-effect multiplier (1 = observed slope, 0.5 = half). Each dataset is -% scored at alpha = 0.05 two ways: -% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest -- the -% honest power, matching the random-slope / per-animal inference) +% (day within-window). Its fixed effects, per-rat intercept SD, and residual SD +% generate NREP synthetic datasets at each rats-per-group N and each true-effect +% multiplier (1 = observed, 0.5 = half). Each is scored at alpha=0.05 by: +% per-animal : Welch t on per-rat behavior~day slopes (cluster-honest power) % LME : the fitlme stim:day p (observation-level DF -- optimistic) -% Writes power_result.txt beside this script. Run: matlab -batch "powersim" +% Writes power_result[_rate].txt. Run: matlab -batch "powersim" % (Copy of analysis/matlab/variation_power.m; see make_variation_power.m.) here = fileparts(mfilename('fullpath')); if isempty(here); here = pwd; end vname = regexprep(here, '.*[/\\]', ''); - D = readtable(fullfile(here, 'data.csv'), 'TextType', 'string'); + +localPower(D, 'count', here, vname); +localPower(D, 'rate', here, vname); + +% ---------------------------------------------------------------- per metric +function localPower(D, metric, here, vname) +NS = [3 5 8 12 16 24]; EFFMULS = [1 0.5]; NREP = 120; FORMULA = 'behavior ~ stim + day + stim:day + (1|rat)'; -NS = [3 5 8 12 16 24]; -EFFMULS = [1 0.5]; -NREP = 120; - -warnState = warning('off', 'all'); -rng(1); - +if strcmp(metric, 'rate') + D = D(D.total > 0, :); beh = D.success ./ D.total; mlabel = 'success RATE'; suffix = '_rate'; +else + beh = D.success; mlabel = '# successes (count)'; suffix = ''; +end +warnState = warning('off', 'all'); rng(1); day0 = min(D.day); -tbl0 = table(D.success, D.day - day0, double(D.stim), categorical(D.subject), ... +tbl0 = table(beh, D.day - day0, double(D.stim), categorical(D.subject), ... 'VariableNames', {'behavior', 'day', 'stim', 'rat'}); - nStim = numel(unique(D.subject(D.stim == 1))); nCtrl = numel(unique(D.subject(D.stim == 0))); bar = repmat('=', 1, 78); -s = sprintf('%s\nPOWER SIMULATION -- %s\n%s\n', bar, vname, bar); -s = [s sprintf('model: %s (success COUNT; day within-window)\n', FORMULA)]; +s = sprintf('%s\nPOWER SIMULATION -- %s [metric: %s]\n%s\n', bar, vname, mlabel, bar); +s = [s sprintf('model: %s (behavior = %s; day within-window)\n', FORMULA, mlabel)]; s = [s sprintf('observed groups: stim n=%d, control n=%d nrep=%d, alpha=0.05\n', nStim, nCtrl, NREP)]; if nStim < 2 || nCtrl < 2 || numel(unique(tbl0.day)) < 2 - s = [s sprintf('\nInsufficient data for a power simulation (need >=2 animals/group and >=2 days).\n')]; - localFinish(s, here); warning(warnState); return + s = [s sprintf('\nInsufficient data for a power simulation.\n')]; + localFinish(s, here, suffix); warning(warnState); return end lme = fitlme(tbl0, FORMULA); @@ -49,14 +53,13 @@ bDay = be(strcmp(cn, 'day')); bInt = be(strcmp(cn, 'day:stim')); psi = covarianceParameters(lme); sRat = sqrt(psi{1}); sRes = sqrt(lme.MSE); days = (0:max(tbl0.day))'; -s = [s sprintf('ground truth: stim:day=%+.2f/day, rat SD=%.2f, residual SD=%.2f, days=%d\n', ... +s = [s sprintf('ground truth: stim:day=%+.4g/day, rat SD=%.4g, residual SD=%.4g, days=%d\n', ... bInt, sRat, sRes, numel(days))]; for eMul = EFFMULS bI = bInt * eMul; - s = [s sprintf('\n true stim:day interaction = %+.2f (%.0f%% of observed)\n', bI, eMul * 100)]; %#ok - s = [s sprintf(' %-8s | per-animal power | LME power\n', 'N/group')]; %#ok - s = [s sprintf(' %s\n', repmat('-', 1, 42))]; %#ok + s = [s sprintf('\n true stim:day interaction = %+.4g (%.0f%% of observed)\n', bI, eMul * 100)]; + s = [s sprintf(' %-8s | per-animal power | LME power\n %s\n', 'N/group', repmat('-', 1, 42))]; for N = NS sigPA = 0; sigL = 0; for r = 1:NREP @@ -70,20 +73,18 @@ for eMul = EFFMULS end star = ''; if N == nStim || N == nCtrl; star = ' <- observed'; end - s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; %#ok + s = [s sprintf(' %-8d | %5.2f | %5.2f%s\n', N, sigPA / NREP, sigL / NREP, star)]; end end - -s = [s sprintf(['\nRead the per-animal column as the honest power. At the observed N this study\n' ... - 'is typically underpowered; per-animal power reaches ~0.8 only at larger N.\n'])]; - -localFinish(s, here); +s = [s sprintf('\nRead the per-animal column as the honest power; LME is optimistic (obs-level DF).\n')]; +localFinish(s, here, suffix); warning(warnState); +end % ---------------------------------------------------------------- helpers -function localFinish(s, here) +function localFinish(s, here, suffix) fprintf('%s', s); -fid = fopen(fullfile(here, 'power_result.txt'), 'w'); +fid = fopen(fullfile(here, ['power_result' suffix '.txt']), 'w'); fprintf(fid, '%s', s); fclose(fid); end diff --git a/analysis/matlab/variations/prev_f_full/result.txt b/analysis/matlab/variations/prev_f_full/result.txt index 0383c95..3b0d61b 100644 --- a/analysis/matlab/variations/prev_f_full/result.txt +++ b/analysis/matlab/variations/prev_f_full/result.txt @@ -1,5 +1,5 @@ ============================================================================== -VARIATION: prev_f_full +VARIATION: prev_f_full [metric: success COUNT] ============================================================================== model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success COUNT) day = training day within window (0 = first analyzed day) @@ -60,9 +60,9 @@ effect t(df) / F(df1) p (resid) Satterthwaite: p (df) stim x day (interaction) t(227)= 2.66 F(1)= 7.088 p=0.008315 p=0.008365 (df=208) day (learning) t(227)= 8.94 F(1)= 79.838 p=1.43e-16 p=2.16e-16 (df=209) stim (main, window start) t(227)= 0.92 F(1)= 0.839 p=0.3606 p=0.3656 (df=37) -interaction 95% CI: [+0.15, +0.97] +interaction 95% CI: [+0.1456, +0.9749] HONEST LME (per-animal random slope, day|rat): interaction F(1,24.0)=3.01, p=0.09541 (Satterthwaite DF ~= residual on this random-intercept model; the random-slope model above is the honest learning-rate test -- DF collapses toward the animal count.) -INTERPRETATION: stim x day interaction SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates) (p=0.008315, slope diff=+0.56) -Paper (N=24): interaction t(227)=2.68, F(1)=7.12, p=0.008. +INTERPRETATION: stim x day interaction SIGNIFICANT positive -- treatment improves FASTER (benefit accumulates) (p=0.008315, slope diff=+0.5602) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008. diff --git a/analysis/matlab/variations/prev_f_full/result_rate.txt b/analysis/matlab/variations/prev_f_full/result_rate.txt new file mode 100644 index 0000000..1ef97a5 --- /dev/null +++ b/analysis/matlab/variations/prev_f_full/result_rate.txt @@ -0,0 +1,68 @@ +============================================================================== +VARIATION: prev_f_full [metric: success RATE (success/attempts)] +============================================================================== +model: behavior ~ stim + day + stim:day + (1|rat) (behavior = success RATE (success/attempts)) +day = training day within window (0 = first analyzed day) +treatment (stim=1): b2_f +control (stim=0): a2_f +N = 24 rats, 231 sessions raw day coverage: treat 0..9, control 0..9 +(equal day coverage over this window) + +============================================================================== +FULL MODEL SUMMARY -- fitlme +============================================================================== + +Linear mixed-effects model fit by ML + +Model information: + Number of observations 231 + Fixed effects coefficients 4 + Random effects coefficients 24 + Covariance parameters 2 + +Formula: + behavior ~ 1 + day*stim + (1 | rat) + +Model fit statistics: + AIC BIC LogLikelihood Deviance + -441.62 -420.96 226.81 -453.62 + +Fixed effects coefficients (95% CIs): + Name Estimate SE tStat DF pValue + {'(Intercept)'} 0.25306 0.024155 10.477 227 3.3213e-21 + {'day' } 0.017111 0.0027396 6.246 227 2.0536e-09 + {'stim' } 0.040889 0.034129 1.1981 227 0.23214 + {'day:stim' } 0.006292 0.0038198 1.6472 227 0.1009 + + + Lower Upper + 0.20547 0.30066 + 0.011713 0.02251 + -0.026361 0.10814 + -0.0012348 0.013819 + +Random effects covariance parameters (95% CIs): +Group: rat (24 Levels) + Name1 Name2 Type Estimate + {'(Intercept)'} {'(Intercept)'} {'std'} 0.06824 + + + Lower Upper + 0.049364 0.094332 + +Group: Error + Name Estimate Lower Upper + {'Res Std'} 0.081522 0.07404 0.08976 + + +effect t(df) / F(df1) p (resid) Satterthwaite: p (df) +---------------------------------------------------------------------------- +stim x day (interaction) t(227)= 1.65 F(1)= 2.713 p=0.1009 p=0.101 (df=209) +day (learning) t(227)= 6.25 F(1)= 39.012 p=2.054e-09 p=2.316e-09 (df=209) +stim (main, window start) t(227)= 1.20 F(1)= 1.435 p=0.2321 p=0.2378 (df=41) +interaction 95% CI: [-0.001235, +0.01382] +HONEST LME (per-animal random slope, day|rat): interaction F(1,22.9)=1.78, p=0.1956 + (Satterthwaite DF ~= residual on this random-intercept model; the random-slope + model above is the honest learning-rate test -- DF collapses toward the animal count.) +INTERPRETATION: stim x day interaction n.s. -- slopes parallel (no differential learning rate) (p=0.1009, slope diff=+0.006292) +Paper (N=24, count): interaction t(227)=2.68, F(1)=7.12, p=0.008.