BonamentePythonCodesData.tar

10points.py0000664000175000017500000000346214114424555011255 0ustar  maxmaxexec(open('imports.py').read())

# use the 10 measurements for chi2 and F tests
x=np.array([10,12,15,11,13,16,12,10,18,13])

print('Problem 9.5')
x1=x[0:2]
x2=x[2:10]
m1=np.mean(x1)
m2=np.mean(x2)
s21=np.var(x1,ddof=1)
s22=np.var(x2,ddof=1)
F=s22/s21
N1=2
N2=8
print('means of 0-1 and 2-10 segments: %3.3f, %3.3f, variances: %3.3f, %3.3f, S2: %3.3f %3.3f, F: %3.3f'
        %(m1,m2,s21,s22,s21*(N1-1),s22*(N2-1),F))
fCrit=f.ppf(0.95,N1-1,N2-1)
fPValue=f.sf(F,N1-1,N2-1)
print('F: %3.3f (crit value 0.90: %3.3f)'%(F,fCrit))
print('F: p-value %3.3f'%fPValue)

quit()
####################################################
# This is used for the examples in Chapter 9
x1=x[0:5]
x2=x[5:10]
mu=13
yMod=mu*np.ones(5)
sigma=2**0.5
yErr=sigma*np.ones(5)
chi21=chisquared(x1,yErr,yMod)
chi22=chisquared(x2,yErr,yMod)
chi2Crit=chi2.ppf(0.95,5)
N=5
print(x1,x2)
print('chi2: %3.3f, %3.3f (crit value 0.95: %3.3f'%(chi21,chi22,chi2Crit))
F=chi22/chi21
fCrit=f.ppf(0.95,N,N)
fPValue=f.sf(F,N,N)
print('F: %3.3f (crit value 0.95: %3.3f)'%(F,fCrit))
print('F: p-value %3.3f'%fPValue)
# now with sample means
m1=np.mean(x1)
m2=np.mean(x2)
s21=np.var(x1,ddof=1)
s22=np.var(x2,ddof=1)
F=s22/s21
# ==== z-score of first five measurements
N=5
z1=(m1-mu)/(sigma/N**0.5)
p1=norm.sf(abs(z1),loc=0,scale=1)*2
print('zscore of 5 measurements: %3.3f, pvalue: %3.3f'%(z1,p1))
# ==== t stat for the 5 measurements
tStat1=(m1-mu)/(s21/N)**0.5
tPvalue=t.sf(abs(tStat1),N-1) # one-sided prob.
tPvalue=tPvalue*2
print('t statistic: %3.3f, 2-sided p-value: %3.3f'%(tStat1,tPvalue))
# ==== t stat for the 10 measurements
s2=(s21*(N-1)+s22*(N-1))/(N+N-2)
tStatComp=(m1-m2)/s2**0.5/(1/N+1/N)**0.5
print('s2= %3.3f, t stat for comparison of two sample means: %3.3f'%(s2,tStatComp))
tPValueComp=2*t.sf(abs(tStatComp),2*(N-1))
print('p-value for the comparison: %3.3f'%tPValueComp)

5points.py0000664000175000017500000002411114116666717011205 0ustar  maxmaxexec(open('imports.py').read())

dataTypes=[('number','I'),('x','f'),('y','f')]
# 5 point data for prob11.2 
data = np.genfromtxt('5points.dat',skip_header=1,dtype=dataTypes)
print(data)
x=data['x']
y=data['y']
yerr=y**0.5

# ==================
print('Problem 11.2')
popt, pcov = curve_fit(linear, x,y, sigma=yerr,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f, sigma2_ab=%3.3f, r=%3.3f'
        %(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5,pcov[1][0],pcov[1][0]/(pcov[0][0]*pcov[1][1])**0.5))

ymodel=linear(x,popt[0],popt[1])
#print(ymodel)
chimin=chisquared(y,yerr,ymodel)
pValue=chi2.sf(chimin,3)
print('chi^2=%3.2f (p value: %3.3f)'%(chimin,pValue))
print('critical value for chi2(3) at 0.9 and 0.99: %f, %f'%(chi2.ppf(0.9,3),chi2.ppf(0.99,3)))
print(y,ymodel,yerr)
# =================
#### intrinsic scatter for problem 17.5
print('Problem 17.5')
sigmaint=intrinsicScatter(y,ymodel,yerr,2)
print('intrinsic scatter: %3.3f'%sigmaint)

xmin=-1
xmax=5

xline=[xmin,xmax]
yline=np.zeros(2)
yline[0]=linear(xline[0],popt[0],popt[1])
yline[1]=linear(xline[1],popt[0],popt[1])

fig,ax=plt.subplots(figsize=(8,6))
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
ax.yaxis.set_major_locator(MultipleLocator(10))
ax.yaxis.set_minor_locator(MultipleLocator(5))

plt.errorbar(x,y,linewidth=2,yerr=yerr,linestyle='',fmt='o',capsize=4, fillstyle='none',label='Data',color='black')
plt.plot(xline,yline,linewidth=2,linestyle='--',label='Best-fit model',color='black')
plt.xlabel('X')
plt.ylabel('Y')
plt.legend(prop={'size': 12})
ax.set_xlim(xmin,xmax)
ax.set_ylim(10,100)

for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())


ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
ax.yaxis.set_major_locator(MultipleLocator(10))
ax.yaxis.set_minor_locator(MultipleLocator(5))
plt.grid(which='both')
plt.savefig('5points.pdf')
# ===========================================

# ===========================================
# 3. Now onto 2-D confidence contours 
# also problem 12.3
fig,ax=plt.subplots(figsize=(8,6))

Na=100
Nb=100
# search a range +-n sigma from the best-fit
n=3
amin=popt[0]-n*pcov[0][0]**0.5
amax=popt[0]+n*pcov[0][0]**0.5
bmin=popt[1]-n*pcov[1][1]**0.5
bmax=popt[1]+n*pcov[1][1]**0.5
a=np.linspace(amin,amax,Na)
b=np.linspace(bmin,bmax,Nb)
chi2Surface=np.zeros((Na,Nb))

for i in range(Na):
    for j in range(Nb):
        ymodelCurrent=linear(x,a[i],b[j])
        chi2Surface[i][j]=chisquared(y,yerr,ymodelCurrent)-chimin

#print(chi2Surface)
levels=[1.0,2.3,2.7,4.6]
CS=ax.contour(a,b,chi2Surface,levels,linewidth=2,colors=['red','green','blue','black'])
ax.clabel(CS,inline=True,fontsize=12,fmt='+%1.1f')
ax.set_xlabel('Parameter a')
ax.set_ylabel('Parameter b')
# adjust the view area to make room for label
plt.gcf().subplots_adjust(left=0.20)

ax.xaxis.set_major_locator(MultipleLocator(10))
ax.xaxis.set_minor_locator(MultipleLocator(2))
ax.yaxis.set_major_locator(MultipleLocator(2))
ax.yaxis.set_minor_locator(MultipleLocator(1))


plt.plot(popt[0],popt[1],linewidth=2,linestyle='',marker='+',color='black',label='Point of minimum $\chi^2$')
plt.grid(which='both')
plt.legend(prop={'size' : 12},loc=1)
ax.set_xlim(12,38)
ax.set_ylim(7,18)
plt.savefig('5pointsContours.pdf')
# ================================================

# Additional 2-d Contours with illustration of interesting parameters
fig,ax=plt.subplots(figsize=(8,6))
levels=[1.0,2.3,2.7,4.6]
CS=ax.contour(a,b,chi2Surface,levels,linewidth=2,colors=['red','green','blue','black'])
ax.clabel(CS,inline=True,fontsize=12,fmt='+%1.1f')
ax.set_xlabel('Parameter a')
ax.set_ylabel('Parameter b')
# adjust the view area to make room for label
plt.gcf().subplots_adjust(left=0.20)
print('68.3 pct crit value for 2 dof: %f'%(chi2.ppf(0.683,2)))
ax.xaxis.set_major_locator(MultipleLocator(10))
ax.xaxis.set_minor_locator(MultipleLocator(2))
ax.yaxis.set_major_locator(MultipleLocator(2))
ax.yaxis.set_minor_locator(MultipleLocator(1))
plt.plot(popt[0],popt[1],linewidth=2,linestyle='',marker='+',color='black')
plt.grid(which='both')
# vertical line to illustrate marginalization over b parameter
aValue=27.4
plt.vlines(aValue,7,18,color='black',linestyle='--')
ax.set_xlim(12,38)
ax.set_ylim(7,18)

plt.vlines(popt[0]-pcov[0][0]**0.5,7,18,color='red',linestyle='-.')
plt.vlines(popt[0]+pcov[0][0]**0.5,7,18,color='red',linestyle='-.')
plt.vlines(aValue,7,18,color='black',linestyle='--')
ax.axvspan(popt[0]-pcov[0][0]**0.5,popt[0]+pcov[0][0]**0.5, alpha=0.5, color='red',
        label='68.3% confidence interval  on a')

#ax.axhspan(popt[1]-pcov[1][1]**0.5,popt[1]+pcov[1][1]**0.5, alpha=0.5, color='red',
#        label='p=0.68 confidenceinterval  on b')


plt.legend(prop={'size' : 12},loc=1)
plt.savefig('5pointsContours2.pdf')


######################

# F-test for the additional linear component (example 19.2)
print('Fit to a constant model')
popt, pcov = curve_fit(constant, x,y, sigma=yerr,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f'%(popt[0],pcov[0][0]**0.5))
ymodelC=constant(x,popt[0])
print(ymodelC)
chiminC=chisquared(y,yerr,ymodelC)
print('chi^2 = %3.3f'%chiminC)

# now use only the middle three measurements
#iStart=1
#iEnd=4 # these are for example 19.2

# problem 19.3
print('Problem 19.3')

iStart=2
iEnd=5 # these are for prob19.3

xM=x[iStart:iEnd]
yM=y[iStart:iEnd]
yerrM=yerr[iStart:iEnd]
print('Using the  three measurements')
print(xM,yM,yerrM)
# linear model for middle three datapoints
poptM, pcovM = curve_fit(linear, xM,yM, sigma=yerrM,absolute_sigma=True)
print('Linear fit ',poptM,pcovM)
print('a=%3.4f +- %3.4f, b=%3.4f +- %3.4f'%(poptM[0],pcovM[0][0]**0.5,poptM[1],pcovM[1][1]**0.5))
ymodelM=linear(xM,poptM[0],poptM[1])
print(ymodelM)
chiminM=chisquared(yM,yerrM,ymodelM)
chiminMPval=chi2.sf(chiminM,1)
print('chi^2 = %3.3f (p-value %3.3f)'%(chiminM,chiminMPval))

# constant model for middle three datapoints
print('Constant fit')
poptMC, pcovMC = curve_fit(constant, xM,yM, sigma=yerrM,absolute_sigma=True)
print('curve_fit M',poptMC,pcovMC)
print('a=%3.4f +- %3.4f'%(poptMC[0],pcovMC[0][0]**0.5))
ymodelMC=constant(xM,poptMC[0])
print(ymodelMC)
chiminMC=chisquared(yM,yerrM,ymodelMC)
chiminMCPval=chi2.sf(chiminMC,2)
print('chi^2 = %3.3f (p-value %3.3f)'%(chiminMC,chiminMCPval))

# ftest for addition of slope to middle three points, for 1,1 d.o.f
print('F test for addition of slope parameter in three datapoints')
nu1=1
nu2=3-2
DeltaChiM=chiminMC-chiminM
print('Delta chi2 = %3.3f'%DeltaChiM)
FM=(DeltaChiM/nu1)/(chiminM/nu2)
FPval=f.sf(FM,nu1,nu2)
print('F=%3.3f, p-value: %3.3f'%(FM,FPval))

#################
#### bootstrap, prob20.6 (code from Hubble)
print('Problem 20.6, bootstrap of 5-point dataset')
NBoot=1000
N=len(x)
aBoot=np.zeros(NBoot)
bBoot=np.zeros(NBoot)
indices=np.arange(N)
print('Indices',indices,x,x[indices])
for i in range(NBoot):
    indicesChoice=choices(indices,k=N)
    # if all are the same index, need to re-draw b/c no fit is possible
    if(indicesChoice.count(indicesChoice[0]) == N):
        indicesChoice=choices(indices,k=N) # odds of two consecutive bad draws is ignored
        print('***')
    #print('%d: '%i,indicesChoice)
    bBoot[i],aBoot[i],r_value, p_value, std_err = stats.linregress(x[indicesChoice],y[indicesChoice])
    #print('%3.2f %3.2f'%(aBoot[i],bBoot[i]))
fig,ax=plt.subplots(figsize=(8,6))
aMin=0
aMax=40
bMin=0
bMax=18
binsa=np.linspace(aMin,aMax,200)
binsb=np.linspace(bMin,bMax,90)
#print('bins:',binsa,binsb)
plt.hist(aBoot,bins=binsa,linewidth=2,density=True,histtype='step',color='black',label='Parameter a')
plt.hist(bBoot,bins=binsb,linewidth=2,density=True,histtype='step',color='blue',label='Parameter b')
plt.xlabel('Parameter value')
plt.ylabel('Distribution')
plt.legend(prop={'size': 12})
plt.xlim((bMin,aMax))
plt.ylim((0,0.5))
ax.xaxis.set_major_locator(MultipleLocator(4))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(.05))
plt.grid(which='both')

# mean, median and percentiles
xaxis=np.linspace(bMin,aMax,2000) # shared by two distrib.
p=0.683
alpha=(1-p)/2
beta=(1+p)/2
print('Using percentiles %3.2f and %3.2f'%(alpha,beta))
#print('aBoot',aBoot)
aBootMean=np.mean(aBoot)
aBootMedian=np.median(aBoot)
erraBoot=np.std(aBoot)
alphaa=np.percentile(aBoot,alpha*100)
# do the best--fit gaussians from median and 68% C.I.
betaa=np.percentile(aBoot,beta*100)
yaxis=norm.pdf(xaxis,loc=aBootMedian,scale=(betaa-alphaa)/2)
plt.plot(xaxis,yaxis,linewidth=1,linestyle='--',color='black')

bBootMean=np.mean(bBoot)
bBootMedian=np.median(bBoot)
errbBoot=np.std(bBoot)
alphab=np.percentile(bBoot,alpha*100)
betab=np.percentile(bBoot,beta*100)
yaxis=norm.pdf(xaxis,loc=bBootMedian,scale=(betab-alphab)/2)
plt.plot(xaxis,yaxis,linewidth=1,linestyle='--',color='blue')
print('Bootstrap estimates a=%3.3f+-%3.3f, b=%3.3f+-%3.3f'
    %(aBootMean,erraBoot,bBootMean,errbBoot))
print('Bootstrap median and p=%3.3f central intervals:'%p)
print('a=%3.3f+-%3.3f (%3.3f-%3.3f), b=%3.3f+-%3.3f (%3.3f-%3.3f)'
    %(aBootMedian,(betaa-alphaa)/2,alphaa,betaa,
        bBootMedian,(betab-alphab)/2,alphab,betab))
plt.savefig('bootstrap5Point.pdf')

###### jackknife, prob20.7
print('Problem 20.7, jackknife on Hubble data')
aJackj=np.zeros(N)
bJackj=np.zeros(N)
aJackjStar=np.zeros(N)
bJackjStar=np.zeros(N)
xj=np.zeros((N,N-1)) # these are the resampled datasets, N sets of size N-1
yj=np.zeros((N,N-1))
# start with the estimates from the original data
bhat,ahat,r_value, p_value, std_err = stats.linregress(x,y)
for j in range(N):
    xj[j,:]=np.delete(x,j)
    yj[j,:]=np.delete(y,j)
    # these are the theta_j estimates from Zj
    bJackj[j],aJackj[j],r_value, p_value, std_err = stats.linregress(xj[j,:],yj[j,:])
    print(j, xj[j,:],yj[j,:],aJackj[j],bJackj[j])
    aJackjStar[j]=N*ahat-(N-1)*aJackj[j]
    bJackjStar[j]=N*bhat-(N-1)*bJackj[j]
meanaJackStar=np.mean(aJackjStar)
erraJackStar=np.std(aJackjStar,ddof=1)/N**0.5 # this is error on mean
meanbJackStar=np.mean(bJackjStar)
errbJackStar=np.std(bJackjStar,ddof=1)/N**0.5

print('Jackknife estimates: a=%3.3f +-%3.3f, b=%3.3f+-%3.3f'%(meanaJackStar,erraJackStar,meanbJackStar,errbJackStar))

6points.py0000644000175000017500000001352614115646523011204 0ustar  maxmaxexec(open('imports.py').read())

dataTypes=[('number','I'),('x','f'),('y','f')]
data = np.genfromtxt('6points.dat',skip_header=1,dtype=dataTypes)
print(data)
x=data['x']
y=data['y']
yerr=np.ones(6) # this is to day they all have same error

popt, pcov = curve_fit(linear, x,y, sigma=yerr,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f, sigma2_ab=%3.3f, r=%3.3f'
        %(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5,pcov[1][0],pcov[1][0]/(pcov[0][0]*pcov[1][1])**0.5))

ymodel=linear(x,popt[0],popt[1])
#print(ymodel)
chimin=chisquared(y,yerr,ymodel)
pValue=chi2.sf(chimin,3)
print('chi^2=%3.2f (p value: %3.3f)'%(chimin,pValue))
print('critical value for chi2(3) at 0.9 and 0.99: %f, %f'%(chi2.ppf(0.9,3),chi2.ppf(0.99,3)))

xmin=-1
xmax=5

xline=[xmin,xmax]
yline=np.zeros(2)
yline[0]=linear(xline[0],popt[0],popt[1])
yline[1]=linear(xline[1],popt[0],popt[1])

fig,ax=plt.subplots(figsize=(8,6))
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
ax.yaxis.set_major_locator(MultipleLocator(10))
ax.yaxis.set_minor_locator(MultipleLocator(5))

plt.errorbar(x,y,linewidth=2,yerr=yerr,linestyle='',fmt='o',capsize=4, fillstyle='none',label='Data',color='black')
plt.plot(xline,yline,linewidth=2,linestyle='--',label='Best-fit model',color='black')
plt.xlabel('X')
plt.ylabel('Y')
plt.legend(prop={'size': 12})
ax.set_xlim(xmin,xmax)
ax.set_ylim(10,100)

for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())


ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
ax.yaxis.set_major_locator(MultipleLocator(10))
ax.yaxis.set_minor_locator(MultipleLocator(5))
plt.grid(which='both')
plt.savefig('6points.pdf')


fig,ax=plt.subplots(figsize=(8,6))
quit()
# below is unused as of now =============================
# 3. Now onto 2-D confidence contours 
Na=100
Nb=100
# search a range +-n sigma from the best-fit
n=3
amin=popt[0]-n*pcov[0][0]**0.5
amax=popt[0]+n*pcov[0][0]**0.5
bmin=popt[1]-n*pcov[1][1]**0.5
bmax=popt[1]+n*pcov[1][1]**0.5
a=np.linspace(amin,amax,Na)
b=np.linspace(bmin,bmax,Nb)
chi2Surface=np.zeros((Na,Nb))

for i in range(Na):
    for j in range(Nb):
        ymodelCurrent=linear(x,a[i],b[j])
        chi2Surface[i][j]=chisquared(y,yerr,ymodelCurrent)-chimin

#print(chi2Surface)
levels=[1.0,2.7,4.6]
CS=ax.contour(a,b,chi2Surface,levels,linewidth=2,colors=['red','blue','black'])
ax.clabel(CS,inline=True,fontsize=12,fmt='+%1.1f')
ax.set_xlabel('Parameter a')
ax.set_ylabel('Parameter b')
# adjust the view area to make room for label
plt.gcf().subplots_adjust(left=0.20)

ax.xaxis.set_major_locator(MultipleLocator(10))
ax.xaxis.set_minor_locator(MultipleLocator(2))
ax.yaxis.set_major_locator(MultipleLocator(2))
ax.yaxis.set_minor_locator(MultipleLocator(1))


plt.plot(popt[0],popt[1],linewidth=2,linestyle='',marker='+',color='black',label='Point of minimum $\chi^2$')
plt.grid(which='both')
plt.legend(prop={'size' : 12},loc=1)
ax.set_xlim(12,38)
ax.set_ylim(7,18)
plt.savefig('5pointsContours.pdf')

# Additional 2-d Contours with illustration of interesting parameters
fig,ax=plt.subplots(figsize=(8,6))
levels=[1.0,2.3,2.7,4.6]
CS=ax.contour(a,b,chi2Surface,levels,linewidth=2,colors=['red','green','blue','black'])
ax.clabel(CS,inline=True,fontsize=12,fmt='+%1.1f')
ax.set_xlabel('Parameter a')
ax.set_ylabel('Parameter b')
# adjust the view area to make room for label
plt.gcf().subplots_adjust(left=0.20)
print('68.3 pct crit value for 2 dof: %f'%(chi2.ppf(0.683,2)))
ax.xaxis.set_major_locator(MultipleLocator(10))
ax.xaxis.set_minor_locator(MultipleLocator(2))
ax.yaxis.set_major_locator(MultipleLocator(2))
ax.yaxis.set_minor_locator(MultipleLocator(1))
plt.plot(popt[0],popt[1],linewidth=2,linestyle='',marker='+',color='black')
plt.grid(which='both')
# vertical line to illustrate marginalization over b parameter
aValue=27.4
plt.vlines(aValue,7,18,color='black',linestyle='--')
ax.set_xlim(12,38)
ax.set_ylim(7,18)

plt.vlines(popt[0]-pcov[0][0]**0.5,7,18,color='red',linestyle='-.')
plt.vlines(popt[0]+pcov[0][0]**0.5,7,18,color='red',linestyle='-.')
plt.vlines(aValue,7,18,color='black',linestyle='--')
ax.axvspan(popt[0]-pcov[0][0]**0.5,popt[0]+pcov[0][0]**0.5, alpha=0.5, color='red',
        label='68.3% confidence interval  on a')

#ax.axhspan(popt[1]-pcov[1][1]**0.5,popt[1]+pcov[1][1]**0.5, alpha=0.5, color='red',
#        label='p=0.68 confidenceinterval  on b')


plt.legend(prop={'size' : 12},loc=1)
plt.savefig('5pointsContours2.pdf')

# F-test for the additional linear component
print('Fit to a constant model')
popt, pcov = curve_fit(constant, x,y, sigma=yerr,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f'%(popt[0],pcov[0][0]**0.5))
ymodelC=constant(x,popt[0])
print(ymodelC)
chiminC=chisquared(y,yerr,ymodelC)
print('chi^2 = %3.3f'%chiminC)

# now use only the middle three measurements
xM=x[1:4]
yM=y[1:4]
yerrM=yerr[1:4]
print(xM,yM,yerrM)
# linear model for middle three datapoints
poptM, pcovM = curve_fit(linear, xM,yM, sigma=yerrM,absolute_sigma=True)
print('curve_fit M',poptM,pcovM)
print('a=%3.4f +- %3.4f'%(poptM[0],pcovM[0][0]**0.5))
ymodelM=linear(xM,poptM[0],poptM[1])
print(ymodelM)
chiminM=chisquared(yM,yerrM,ymodelM)
chiminMPval=chi2.sf(chiminM,1)
print('chi^2 = %3.3f (p-value %3.3f)'%(chiminM,chiminMPval))

# constant model for middle three datapoints
poptMC, pcovMC = curve_fit(constant, xM,yM, sigma=yerrM,absolute_sigma=True)
print('curve_fit M',poptMC,pcovMC)
print('a=%3.4f +- %3.4f'%(poptMC[0],pcovMC[0][0]**0.5))
ymodelMC=constant(xM,poptMC[0])
print(ymodelMC)
chiminMC=chisquared(yM,yerrM,ymodelMC)
chiminMCPval=chi2.sf(chiminMC,2)
print('chi^2 = %3.3f (p-value %3.3f)'%(chiminMC,chiminMCPval))

# ftest, for 1,1 d.o.f
nu1=1
nu2=3-2
DeltaChiM=chiminMC-chiminM
print('Delta chi2 = %3.3f'%DeltaChiM)
FM=(DeltaChiM/nu1)/(chiminM/nu2)
FPval=f.sf(FM,nu1,nu2)
print('F=%3.3f, p-value: %3.3f'%(FM,FPval))
CBias.py0000644000175000017500000000457114104316102010544 0ustar  maxmaxexec(open('imports.py').read())
# Numerical calculation of the bias using chi^2 as a function of Poisson mean
# when fitting data with a constant model


#mu=[0.01,0.03,0.05,0.1,0.3,0.5,1,3,5,10,30,50,100]
# WARNING: can't use datasets with 0 counts, since the estimated variance would be 0.

mu=np.logspace(1,3,30) # Values of parent Poisson means to try
nSim=1000 # Number of simulated datasets for a given value of mu
N=100   # Number of datapoints in the dataset

sampleP=np.zeros(nSim)
a=np.zeros(nSim)    # for best-fit a
aB=np.zeros(nSim)   # for best-fit a from chi^2 fit

aBMean=np.zeros(len(mu))
aBStd=np.zeros(len(mu))
aMean=np.zeros(len(mu))
aStd=np.zeros(len(mu))
Bias=np.zeros(len(mu))

for i in range(len(mu)):
    for j in range(nSim):
        sampleP=poisson.rvs(mu[i],size=N) # random values

        a[j]=np.average(sampleP)                    # this is the sample mean
        aB[j]=np.average(sampleP,weights=sampleP**(-2.0))# this is the weighted mean
    aMean[i]=np.mean(a)
    aStd[i]=np.std(a)
    aBMean[i]=np.nanmean(aB)
    aBStd[i]=np.nanstd(aB)
    Bias[i]=100*(aBMean[i]-mu[i])/mu[i]
    print('mu=%3.3f: a=%3.3f+-%3.3f, aB=%3.3f+-%3.3f, fract. bias: %3.3f'%(mu[i],aMean[i],aStd[i],aBMean[i],aBStd[i],Bias[i]))

# ============================================
# Now summarize the simulation into a plot
fig,ax=plt.subplots(figsize=(8,6))

ax.errorbar(mu,aMean,yerr=aStd,label='Poisson fit (sample average)')
ax.errorbar(mu,aBMean,yerr=aBStd,label='$\chi^2$ fit (weighted mean)')

ax.set_ylim(10,300)
ax.set_yscale('log')
ax.set_xscale('log')
ax.set_xlim(10,300)
ax.set_xlabel('Parent Poisson mean')
ax.set_ylabel('Measured Value')

# formatting log axis with linear labels
for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())

plt.grid(which='both')
plt.legend(loc=4,prop={'size': 12})
plt.savefig('CBias.eps')


# ============================================
# Plot the bias
fig,ax=plt.subplots(figsize=(8,6))
ax.plot(mu,Bias,color='black',linewidth=2)
ax.set_ylim(-30,6)
ax.set_yscale('linear')
ax.set_xscale('log')
ax.set_xlim(10,1000)
ax.set_xlabel('Parent Poisson mean')
ax.set_ylabel('Bias (Percentage of Poisson mean)')
ax.hlines(0,10,1000,linestyle='--',linewidth=2,color='blue')
# formatting log axis with linear labels
for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())

plt.grid(which='both')

plt.savefig('CBias2.pdf')



ch10.py0000644000175000017500000000210113770644065010325 0ustar  maxmaxexec(open('imports.py').read())

# Make a plot with the Gaussians to illustrate fitting


ymin=0
ymax=5.5
6
xline=np.array([0,10])
#yline=np.array([0,0])
# assume this line model
a=1.0
b=0.4
yline=a+xline*b


fig,ax=plt.subplots(figsize=(8,6))

plt.plot(xline,yline,linewidth=2,color='black')



# add Gaussians on the other axis
nPoints=4
# add a fictional data set
xi=np.array([0.7,3,5,7.5])
yi=np.array([1.7,1.8,3.4,3.7])
sigmai=np.array([0.3,0.3,0.2,0.5])
# plot the data with error bars
plt.errorbar(xi,yi,yerr=sigmai,color='blue',capsize=4,fmt='o',capthick=2,label='Gaussian data')

x=np.linspace(-10,10,1000)
mu=np.zeros(nPoints)
# mean of the Gaussians
mu=a+xi*b
x=np.linspace(ymin,ymax,1000)
for i in range(nPoints):
    y=norm.pdf(x,loc=mu[i],scale=sigmai[i]) # distribution of i-th data point
    ax.plot(y+xi[i],x,color='grey',linewidth=2)
    ax.vlines(xi[i],ymin,ymax,linestyle=':',linewidth=2,color='black')

ax.scatter(xi,mu,marker='_',linewidth=4,color='black')
ax.set_xlabel('X')
ax.set_ylabel('Y')
ax.set_ylim(ymin,ymax)
ax.set_xlim(0,10)
plt.savefig('gaussianFit.pdf')
ch2Example.py0000644000175000017500000000541714104213111011547 0ustar  maxmaxexec(open('imports.py').read())

# Simulate a normal distribution
fig,ax=plt.subplots(figsize=(8,6))

# Simulate N sample from a Gaussian distribution
N=1000
mu=5
sigma=1.
sample=norm.rvs(size=N,loc=mu,scale=sigma)
bins=np.linspace(0,10,51)
ax.hist(sample,bins=bins,histtype='bar',hatch='///',edgecolor='black',color='white',cumulative=False,density=True,linewidth=2,alpha=0.75,label='Sample distribution')
# the expected distribution for large N is a Gaussian  ---------------------------
ax.plot(bins,norm.pdf(bins,loc=mu,scale=sigma),color='red',linewidth=2,label='Parent distribution')
print(bins)
# --------------------------------------------------------------------------------
# Overplot it using number instead of pdf
ax.set_xlabel('Variable X')
ax.set_ylabel('Probability of occurrence')
ax2=ax.twinx()
ax2.hist(sample,bins=bins,histtype='bar',alpha=0)
ax2.set_ylabel('Number of occurrence')
ax.legend(loc=2,prop={'size': 12})
#plt.show()
plt.savefig('ch2Example.pdf')

# -------------------------------------------
# also plot an exponential function
x=np.linspace(0,10,100)
Lambda=0.5
fig,ax=plt.subplots(figsize=(8,6))
ax.plot(x,expon.pdf(x,scale=1/Lambda),color='black',linewidth=2,label='Exponential PDF')
ax.plot(x,expon.cdf(x,scale=1/Lambda),color='black',linestyle='--',linewidth=2,label='Exponential CDF')
ax.set_xlabel('Variable X')
ax.set_ylabel('Distributions f(x) and F(x)')
ax.legend(loc=2,prop={'size': 12})
#plt.show()
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
ax.set_ylim(0,1)
ax.set_xlim(0,10)

plt.grid()
plt.savefig('ch2Exponential.pdf')

# -------------------------------------------
# For Chapter 7, use the figure for confidence intervals

x=np.linspace(0,10,100)
Lambda=0.5
fig,ax=plt.subplots(figsize=(8,6))
ax.plot(x,expon.pdf(x,scale=1/Lambda),color='black',linewidth=2,label='Exponential PDF')
ax.plot(x,expon.cdf(x,scale=1/Lambda),color='black',linestyle='--',linewidth=2,label='Exponential CDF')
ax.set_xlabel('Variable X')
ax.set_ylabel('Distributions f(x) and F(x)')
#plt.show()
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
ax.set_ylim(0,1)
ax.set_xlim(0,10)

# add the 90% confidence intervals
pp5=expon.ppf(0.05,scale=1/Lambda)
pp95=expon.ppf(0.95,scale=1/Lambda)

ax.vlines(pp5,0,1,color='gray')
ax.vlines(pp95,0,1,color='gray')
ax.hlines(0.05,0,10,color='red',linestyle='--')
ax.hlines(0.95,0,10,color='red',linestyle='--')
ax.axvspan(pp5,pp95,color='gray',alpha=0.5,label='90% Confidence Interval')
plt.grid()
ax.legend(loc=5,prop={'size': 12})
plt.savefig('ch7Exponential.pdf')




ch3Distributions.py0000644000175000017500000003604414115645263013042 0ustar  maxmaxexec(open('imports.py').read())

# Simulate a normal distribution
fig,ax=plt.subplots(figsize=(8,6))

#############################################################
# Binomial distribution
fig,ax=plt.subplots(figsize=(8,6))
xmax=20
N,p=10,0.5
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=binom.pmf(x,N,p)
plt.plot(x,y,linestyle='--',marker='o',color='black',label='$N$=10, $p$=0.5')
# ----------- overplot a different p
N,p=10,0.10
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=binom.pmf(x,N,p)
plt.plot(x,y,linestyle='--',marker='o',color='blue',label='$N$=10, $p$=0.1')
# --------------------------------------------------
plt.legend()
plt.grid(which='both')
plt.xlim(0,10)
plt.xlabel('$n$')
#ax.xaxis.set_major_locator(MultipleLocator(1))
#ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
# linear tick mark labels
ax.xaxis.set_major_formatter(ScalarFormatter())
plt.ylabel('Probability $P_N(n)$')
plt.savefig('binomial.pdf')

# ============================================
# --------------------------------------------
#  Test the significance of the Astra-Zeneca and Pfizer tests
choice=3 #0-2 are AstraZeneca, 3 is Pfizer
choiceName=['AstraSD-SD','AstraAll','AstraLD-SD','Pfizer']
b=[27, 30, 3,8]     # no. of covid-19 cases for vaccinated
d=[71, 101, 30,162]    # no. of covid-10 cases for unvaccinated
n1=[4440, 5807, 1367,21720] # total vaccinated
n2=[4455, 5829, 1374,21728] #total un-vaccinated
mu=b[choice]+d[choice]
# vaccine efficacy formula, assuming same number in two samples
# --------------------------------------------
def ve(b,d,n1,n2):
	return 1.-(b/n1)/(d/n2) # if 2:1 ratio
# ------------------------------------
VE=ve(b[choice],d[choice],n1[choice],n2[choice])
print('Vaccine Efficacy (%s): %3.4f'%(choiceName[choice],VE))

pConf=0.95 # confidence probability, 95%
# 1. ---------------------------------
# endpoints of the range that contains 95% probability
# This is for the "observed" ratio, not the posterior on parent mean
# The observed ratio is b/mu2, sick and vaccinated
interval=binom.interval(pConf,mu,b[choice]/mu)
print('Interval containing 95pct. range of b=%d (%s);'%(b[choice],choiceName[choice]),interval)
VEInterval=np.zeros(2)
for i in range(2):
    VEInterval[i]=ve(interval[i],mu-interval[i],n1[choice],n2[choice])
print('corresponding VE range (from obs. ratio)',VEInterval)

# ======================================================
# 2. -----------------------------------
# Now also look at the posterior on parent value of beta/delta
# Lower and upper limit on b/mu, assuming mu is fixed

fig,ax=plt.subplots(figsize=(8,6))
ax.yaxis.set_major_locator(MultipleLocator(0.2))
ax.yaxis.set_minor_locator(MultipleLocator(0.1))
ax.xaxis.set_major_locator(MultipleLocator(0.10))
ax.xaxis.set_minor_locator(MultipleLocator(0.025))

betaMu=np.linspace(0,1,100)
# cumulative distr. function for fixed b, d and the parent value of beta
cdfB=binom.cdf(b[choice],mu,betaMu)
sfB=binom.sf(b[choice],mu,betaMu)
ax.plot(betaMu,cdfB,color='black',linewidth=2,label='P($>$ %d out of %d)'%(b[choice],mu))
ax.plot(betaMu,sfB,color='blue',linewidth=2,label='P($\leq$ %d out of %d)'%(b[choice],mu))
ax.set_xlabel(r"Parent $\beta$/m")
ax.set_ylabel('Probability')
ax.grid(which='both')
plt.legend(loc=6)
plt.hlines(0.025,0,1,linestyle=':')
plt.hlines(0.975,0,1,linestyle=':')

#plt.hlines(0.0015,0,1,linestyle='--')
#plt.hlines(0.9985,0,1,linestyle='--')
ax.set_xlim(0,0.2)
# also convert beta/mu into vaccine efficacy
# the efficiency can become -infty, so be careful
#ax2=twiny()
#VE=ve(betaMu[0:50],1-betaMu[0:50])
#print(VE)

# look at the solution in the plot BY EYE and calculate VE
betaMu1= [0.20,0.17, 0.03,0.022] 
betaMu2= [0.375,0.31, 0.24,0.09] 
print('Posterior beta/m2: %3.3f-%3.3f, VE p=0.95 conf. interval: %3.3f-%3.3f'%(betaMu1[choice],betaMu2[choice],ve(betaMu1[choice],1-betaMu1[choice],n1[choice],n2[choice]),ve(betaMu2[choice],1-betaMu2[choice],n1[choice],n2[choice])))

plt.savefig('%s.pdf'%choiceName[choice])

# ==================================================
# --- add a check on whether two sub-samples
# --- are consistent with the general sample
m2=b[1]+d[1]
rho=b[1]/m2 # parent value of b/m2 

# Can the following two datasets come from a binomial with pi,
# assuming a fixed value of events from the marginals?
b1=b[0]
m21=b[0]+d[0]
b2=b[2]
m22=b[2]+d[2]
p=0.95
# this is done with the 95% intervals for m21 and m22
print('================================')
print('Using model from All: b=%d, m2=%d, rho=%3.3f'%(b[1],m2,rho))
print('Interval with rho=%3.3f and m=%d (observed: %d)'%(rho,m21,b1),binom.interval(p,m21,rho))
print('Interval with rho=%3.3f and m=%d (observed: %d)'%(rho,m22,b2),binom.interval(p,m22,rho))

# --- check the smaller subsample using the larger subsample
rho=b1/m21
print('================================')
print('Using model from SD/SD: b=%d, m2=%d, rho=%3.3f'%(b1,m21,rho))
print('Interval with rho=%3.3f and m=%d (observed: %d)'%(rho,m22,b2),binom.interval(0.95,m22,rho))



# ================================================
# ================================================
# Gaussian distribution
fig,ax=plt.subplots(figsize=(8,6))
xmin=-20
xmax=20
mu,sigma=5,(10*0.5*0.5)**0.5 # to match the first binomial
loc,scale=mu,sigma
# ------------------------------------------
x=np.linspace(xmin,xmax,10000,endpoint=True)
y=norm.pdf(x,loc,scale)
plt.plot(x,y,linestyle='-',linewidth=2,color='black',label='$\mu$=%2.1f, $\sigma^2$=%2.2f'%(mu,sigma))
# add vertical lines)
ax.vlines(mu,0,norm.pdf(mu,loc,scale),linestyle='-.',color='grey')
ax.vlines(mu+sigma,0,norm.pdf(mu+sigma,loc,scale),linestyle='--',color='grey')
ax.vlines(mu-sigma,0,norm.pdf(mu-sigma,loc,scale),linestyle='--',color='grey')

# ----------- overplot a different p
mu,sigma=0,1
loc,scale=mu,sigma
x=np.linspace(xmin,xmax,10000,endpoint=True)
y=norm.pdf(x,loc,scale)
plt.plot(x,y,linestyle='-',linewidth=2,color='blue',label='$\mu$=%1.0f, $\sigma^2$=%1.0f'%(mu,sigma))
# --------------------------------------------------
plt.legend()
plt.grid(which='both')
plt.xlim(-5,10)
ymin=-0.02
ymax=0.42
plt.ylim(ymin,ymax)
plt.xlabel('$x$')
ax.xaxis.set_major_locator(MultipleLocator(2))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.025))
# add vertical lines)
ax.vlines(0,0,norm.pdf(0,loc,scale),linestyle='-.',color='grey')
ax.vlines(1,0,norm.pdf(1,loc,scale),linestyle='--',color='grey')
ax.vlines(-1,0,norm.pdf(-1,loc,scale),linestyle='--',color='grey')
plt.ylabel('Distribution function $f(x)$')
plt.savefig('normal.pdf')

#calculate  FWHM ========================================
HWHM = (2*np.log(2))**0.5

peak=norm.pdf(0)
print('HWHM = %2.3f, FWHM= %2.3f'%(HWHM,2*HWHM))
print('Prob. in FWHM = %3.4f'%(1-2*norm.sf(HWHM)))
for i in range(5):
    sigma=i+1
    print('prob between +- %d: %5.6f'%(sigma,1-2*norm.sf(sigma)))
# ======================================================

# Add another Figure of gaussian for Appendix
fig,ax=plt.subplots(figsize=(8,6))
mu,sigma=0,1
loc,scale=mu,sigma
x=np.linspace(xmin,xmax,10000,endpoint=True)
y=norm.pdf(x,loc,scale)/norm.pdf(0,loc,scale) # normalized to peak value at x=0
plt.plot(x,y,linestyle='-',linewidth=2,color='black',label='Normalized distr. function')
y=norm.cdf(x,loc,scale)
plt.plot(x,y,linestyle='-',linewidth=2,color='red',label='Cumulative distr. function')
# --------------------------------------------------
plt.legend(loc=2,prop={'size': 12})
#plt.grid(which='both')
plt.grid(which='major',color='black')
plt.grid(which='minor')
plt.xlim(-5,5)
plt.ylim((0,1.0))
plt.xlabel('$z$')
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.025))
plt.ylabel('Distributions')
plt.savefig('normalAppendix.pdf')

# ==========================================================================

#================================================
# === Poisson distribution
fig,ax=plt.subplots(figsize=(8,6))
xmax=20
mu=5
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=poisson.pmf(x,mu)
plt.plot(x,y,linestyle='--',marker='o',color='black',label='$\mu$=%2.1f'%mu)
# ----------- overplot a different p
mu=1
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=poisson.pmf(x,mu)
plt.plot(x,y,linestyle='--',marker='o',color='blue',label='$\mu$=%2.1f'%mu)
# --------------------------------------------------
plt.legend()
plt.grid(which='both')
plt.xlim(0,10)
plt.xlabel('$n$')
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.ylabel('Probability $P(n)$')
plt.savefig('poisson.pdf')

# ===============================================
# Comparison of the three distributions
# Binomial distribution
fig,ax=plt.subplots(figsize=(8,6))
xmax=20
# -------- binomial ---------------------------------
# --------------------------------------------------
N,p=10,0.5 # mean of 5
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=binom.pmf(x,N,p)
plt.plot(x,y,linestyle='--',marker='o',color='black',label='Binom(%d,%2.1f)'%(N,p))
print('binom: mean=%3.2f, variance=%3.2f'%(N*p,N*p*(1-p)))
# ----------- overplot a different p
N,p=10,0.10 # mean of 1
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=binom.pmf(x,N,p)
plt.plot(x,y,linestyle='--',marker='o',color='blue',label='Binom(%d,%2.1f)'%(N,p))
print('binom: mean=%3.2f, variance=%3.2f'%(N*p,N*p*(1-p)))
# --------------------------------------------------
# ------------------------------------------------
# ---------- gaussian ----------------------------
xmin=-20
xmax=20
mu,sigma=5,(10*0.5*0.5)**0.5 # to match the first binomial
loc,scale=mu,sigma
x=np.linspace(xmin,xmax,10000,endpoint=True)
y=norm.pdf(x,loc,scale)
plt.plot(x,y,linestyle='-',linewidth=2,color='black',label='N(%2.1f,%2.1f)'%(mu,sigma**2))
# ----------- overplot a different gaussian
mu,sigma=1,1
loc,scale=mu,sigma
x=np.linspace(xmin,xmax,10000,endpoint=True)
y=norm.pdf(x,loc,scale)
plt.plot(x,y,linestyle='-',linewidth=2,color='blue',label='N(%2.1f,%2.1f)'%(mu,sigma**2))
# --------------------------------------------------
# --------------------------------------------------
# -------------- Poisson ---------------------------
xmax=20
mu=5
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=poisson.pmf(x,mu)
plt.plot(x,y,linestyle=':',marker='o',fillstyle='none',color='black',label='Poiss(%2.1f)'%mu)
# ----------- overplot a different mu
mu=1
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
y=poisson.pmf(x,mu)
plt.plot(x,y,linestyle=':',marker='o',fillstyle='none',color='blue',label='Poiss(%2.1f)'%mu)
# ------------------------------------------------

plt.legend(loc=1,prop={'size': 12})
plt.grid(which='both')
plt.xlim(-2,12)
plt.xlabel('$x$ or $n$')
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.ylabel('Probability function')
plt.savefig('comparison.pdf')

# ============================================================
# Quantify difference between Poisson and Gaussian of same mean
fig,ax=plt.subplots(figsize=(8,6))
N=100
mu=np.arange(1,N+1)
avgFracDiff=np.zeros((3,N))
for m in range(N):
    # find integer closest to mean, to use for Poisson distribution
    G=norm.pdf(m,loc=mu[m],scale=mu[m]**0.5)
    P=poisson.pmf(m,mu[m])
    print('ind=',m, 'mu[m]=',mu[m], 'G=',G,'P=', P)#, 'P ,',P, 'G ,',G)
    print('for mu=%3.3f, G=%3.3f, for ind=%d, P=%3.3f'%(mu[m],G,m,P))
    avgFracDiff[0,m]=np.mean((P-G)/G) # percent, using one point
    # now another comparison, by averaging for values within 3 sigma
#    if (m>=3):
#        rangem=np.arange(m-3,m+3,7) # if m>=3
#        G=norm.pdf(rangem,loc=mu[m],scale=mu[m]**0.5)
#        P=poisson.pmf(rangem,mu[m])
#        avgFracDiff[1,m]=np.mean((P-G)/G) # percent
    # now another comparison, by averaging all values up to 3 sigma
    #indMin=max(0,mu[m]-3*mu[m]**0.5)
    #indMax=mu[m]+3*mu[m]**0.5
    #print(int(indMin),int(indMax)) # this should cover approx the +-3 sigma range
    #murangePlus=np.arange(int(indMin),int(indMax))
    #G=norm.pdf(murangePlus,loc=mu[m],scale=mu[m]**0.5)
    #P=poisson.pmf(murangePlus,mu[m])
    #avgFracDiff[2,m]=100*np.mean((G-P)/P) # percent
    # or maybe calculate 90% confidence intervals from the two distributions

# these are the 5th and 95th percentiles; the medians too
# They are normalized by the mean - is it needed?
G50=norm.ppf(0.50,loc=mu,scale=mu**0.5) # median
G95=norm.ppf(0.95,loc=mu,scale=mu**0.5) #- G50   # upper error
G05=norm.ppf(0.05,loc=mu,scale=mu**0.5) #- G50   # lower error

P50=poisson.ppf(0.5,mu) # median
P95=poisson.ppf(0.95,mu)# - P50 # upper error
P05=poisson.ppf(0.05,mu)# - P50 # lower error

# print confidence intervals
for i in range(100):
    #i=2
    print("%d: mu=%2.1f, G: %3.3f - %3.3f - %3.3f; P: %3.3f - %3.3f - %3.3f"%
            (i,mu[i],G05[i],G50[i],G95[i],P05[i],P50[i],P95[i]))

P50Ratio=(P50-mu)/mu
P95Ratio=(P95-mu)/mu
P05Ratio=(P05-mu)/mu

G50Ratio=(G50-mu)/mu
G95Ratio=(G95-mu)/mu
G05Ratio=(G05-mu)/mu

ax.yaxis.set_major_locator(MultipleLocator(0.20))
ax.yaxis.set_minor_locator(MultipleLocator(0.10))
ax.xaxis.set_major_locator(LogLocator())
ax.xaxis.set_minor_locator(LogLocator())

ax2=ax.twinx()
#ax.plot(mu[3:N],avgFracDiff[1,3:N],linewidth=2,color='blue',label='for x=$\mu - 3, \ldots, \mu+3$')
#ax.plot(mu,avgFracDiff[2],linewidth=2,color='red',label='for $x=\mu - 3 \sigma, \ldots, \mu+3 \sigma$')
print((P05-G05)/G05,P05,G05)
ax2.xaxis.set_major_locator(LogLocator())
ax2.xaxis.set_minor_locator(LogLocator())
ax2.yaxis.set_major_locator(MultipleLocator(0.20))
ax2.yaxis.set_minor_locator(MultipleLocator(0.10))

#ax2.plot(mu,P50Ratio,linewidth=2,color='red',label='test')
ax2.plot(mu,P95Ratio,linewidth=1,color='black')
ax2.plot(mu,P05Ratio,linewidth=1,color='black')

#ax2.plot(mu,G50Ratio,linewidth=2,color='black',linestyle='--')
ax2.plot(mu,G95Ratio,linewidth=2,color='black',linestyle='--',label='N($\mu$,$\mu^2$) Confidence Interval')
ax2.plot(mu,G05Ratio,linewidth=2,color='black',linestyle='--')
#ax.xaxis.set_major_locator(MultipleLocator(1))
#ax.xaxis.set_minor_locator(MultipleLocator(1))
ax2.fill_between(mu,P95Ratio,P05Ratio,facecolor='red',alpha=1.0,label='Poiss($\mu$) Confidence Interval')

ax.plot(mu,avgFracDiff[0],linewidth=3,color='black',label='Peak probability ratio')
ax.hlines(0,1,100,linestyle=':',linewidth=2,color='black')
ax.legend(loc=4,prop={'size': 12})
ax.set_xscale('log')
ax.xaxis.set_major_formatter(ScalarFormatter())
ax.set_xlabel('$\mu$')
ax.set_ylabel('Poiss($\mu$)/N($\mu$,$\mu^2$)- 1')

ax2.set_ylabel('Normalized Confidence Interval')
l=ax2.legend(loc=1,prop={'size': 12})
#l.set_zorder(10)
ax.set_ylim(-1.10,1.30)
ax2.set_ylim(-1.10,1.30)
ax.set_xlim(1,100)
plt.tight_layout()
#ax.grid(axis='x',which='both')
ax.set_zorder(ax2.get_zorder()+1)
ax.patch.set_visible(False)
plt.savefig('difference.pdf')

# compare Stirling's approximation to exact factorial
for i in np.arange(30):
     print('n=%d, (S-n!)/n! = %3.4f'%(i,(Stirling(i)-math.factorial(i))/math.factorial(i)))

ch5.py0000644000175000017500000000233314104226577010254 0ustar  maxmaxexec(open('imports.py').read())

# Simulate a normal distribution
fig,ax=plt.subplots(figsize=(8,6))

# exponential distribution

x=np.linspace(0,10,100)
Lambda=0.5
y=expon.pdf(x,scale=1/Lambda)
yCDF=expon.cdf(x,scale=1/Lambda)
ax.xaxis.set_major_locator(MultipleLocator(2))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.20))
ax.yaxis.set_minor_locator(MultipleLocator(0.10))


ax.plot(x,y,color='black',label='Probability Distribution f(x)',linewidth=2)
# same curve again. but labeled as quantile function
ax.plot(x,yCDF,color='red',linewidth=2,linestyle='-',label='Quantile Function $F^{-1}(p)$')
ax.plot(x,yCDF,color='black',linewidth=2,linestyle='--',label='Cumulative Distribution F(x)')

ax.set_xlim(0,10)
ax.set_ylim(0,1)
ax.set_xlabel('x')
ax.set_ylabel('Distribution')
plt.grid(which='both')
ax.legend(loc=5,prop={'size': 12})
ax2=ax.twinx()
ax2.set_ylabel('Quantile $p$')
ax2.yaxis.label.set_color('red')
ax2.tick_params(axis='y', colors='red')


ax3=ax.twiny()
ax3.set_xlim(0,10)
ax3.xaxis.label.set_color('red')
ax3.tick_params(axis='x', colors='red')
ax3.set_xlabel('Quantile Function $x=F^{-1}(p)$',color='red')

#plt.legend(loc=2,prop={'size': 12})
plt.savefig('exponential.pdf')

ch6ML.py0000644000175000017500000000143313763161345010507 0ustar  maxmaxexec(open('imports.py').read())

# 1. use the example with gaussian and Poisson treatment
# number of counts and times 
x=np.array([100,180,33])
t=np.array([10,20,3])

cr=x/t

print("count rate",cr)

wAvg=np.average(x,weights=t**(-0.5))
print("weighted average of counts",wAvg)
# assuming Gaussian errors
crErr=x**0.5/t
print('Gaussian errors',crErr)


# 2. do weighted average of the count rates, which
# is foolish because we can add number together, but it's an example

wAvgcr=np.average(cr,weights=1/crErr**2)
print('Weighted average of count rates',wAvgcr)
wAvgcrErr=(1/sum(1/crErr**2))**0.5
print('Std. dev. of weighted average',wAvgcrErr)

# 3. Combine the measurements with a Direct average

crD=sum(x)/sum(t)
crDErr=sum(x)**0.5/sum(t)
print("Poisson average and std. dev:",crD,crDErr)
ch8Example.py0000644000175000017500000000071513766460370011601 0ustar  maxmaxexec(open('imports.py').read())


x=np.array([1.2,0.8])
xE=np.array([0.24,0.16])

xWAvg=np.average(x,weights=1/xE**2)
print('Weighted average: %3.4f'%xWAvg)


log10X=np.log10(x)
log10XE=xE/x/np.log(10)
lnX=np.log(x)
lnXE=xE/x
print('log10:',log10X)
print('ln:',lnX)
print(log10XE,lnXE)

lnXAvg=np.mean(lnX)
log10XAvg=np.mean(log10X)
print('average of log10: %3.4f (%3.4f), average of ln: %3.4f (%3.4f)'
        %(log10XAvg,10**log10XAvg,lnXAvg,np.exp(lnXAvg)))
ch9Hypothesis.py0000644000175000017500000002702014114422321012322 0ustar  maxmaxexec(open('imports.py').read())

# Simulate a normal distribution
fig,ax=plt.subplots(figsize=(8,6))

xObs=61
xmin=50
xmax=100
mu=13
sigma=2**0.5
N=5
x=np.linspace(xmin,xmax,1000)
y=norm.pdf(x,N*mu,sigma*N**0.5)

ax.plot(x,y,color='black',linewidth=2,label='N($N\mu,N\sigma^2$)')
plt.grid(which='both')
ax.set_xlim(53,77)
ax.yaxis.set_major_locator(MultipleLocator(0.02))
ax.yaxis.set_minor_locator(MultipleLocator(0.01))
ax.xaxis.set_major_locator(MultipleLocator(5))
ax.xaxis.set_minor_locator(MultipleLocator(1.0))
ax.set_xlabel('Statistic Y')
ax.set_ylabel('Probability Distribution')

# add the 95% confidence limits
x1=N*mu-1.96*(sigma*N**0.5)
x2=N*mu+1.96*(sigma*N**0.5)
x1range=[i for i in np.arange(1000) if x[i]<=x1]
x2range=[i for i in np.arange(1000) if x[i]>=x2]
ax.fill_between(x[x1range], y[x1range],facecolor='none',hatch='XXXX',edgecolor="b",label='95% rejection region')
ax.fill_between(x[x2range], y[x2range],facecolor='none',hatch='XXXX',edgecolor="b")
plt.legend(loc=1,prop={'size':12})
# repeat the same for 68.3% rejection region
x1=N*mu-1*(sigma*N**0.5)
x2=N*mu+1*(sigma*N**0.5)
x1range=[i for i in np.arange(1000) if x[i]<=x1]
x2range=[i for i in np.arange(1000) if x[i]>=x2]
ax.fill_between(x[x1range], y[x1range],facecolor='none',hatch='//',edgecolor="gray",label='68.3% rejection region')
ax.fill_between(x[x2range], y[x2range],facecolor='none',hatch='//',edgecolor="gray")
# also report the measurement, needed for the p-value
ax.vlines(xObs,0,norm.pdf(xObs,N*mu,sigma*N**0.5),color='r',linewidth=2,label='Measurement')
ax.vlines(N*mu+abs(N*mu-xObs),0,norm.pdf(N*mu+abs(N*mu-xObs),N*mu,sigma*N**0.5),color='r',linewidth=2,linestyle='--')
plt.legend(loc=1,prop={'size':12})
ax.set_ylim(0.,0.13)
plt.savefig('hypothesisTesting.pdf')


# ====== pdf of chi^2
xmin=0
xmax=20
dof=5
x=np.linspace(xmin,xmax,1000)
y=chi2.pdf(x,dof)
fig,ax=plt.subplots(figsize=(8,6))

ax.plot(x,y,linewidth=2,color='black',label='$\chi^2$(N)')
ax.set_xlabel('$\chi^2$')
ax.set_ylabel('Probability Distribution')
ax.yaxis.set_major_locator(MultipleLocator(0.02))
ax.yaxis.set_minor_locator(MultipleLocator(0.01))
ax.xaxis.set_major_locator(MultipleLocator(5))
ax.xaxis.set_minor_locator(MultipleLocator(1))
plt.grid(which='both')
ax.set_xlim(0,20)
ax.set_ylim(0,0.16)
chi2Crit95=chi2.ppf(0.95,dof)
chi2Crit68=chi2.ppf(0.683,dof)
x1range=[i for i in np.arange(1000) if x[i]>=chi2Crit95]
x2range=[i for i in np.arange(1000) if x[i]>=chi2Crit68]
ax.fill_between(x[x2range], y[x2range],facecolor='none',hatch='//',edgecolor="gray",label='68.3% rejection region')
ax.fill_between(x[x1range], y[x1range],facecolor='none',hatch='XXXX',edgecolor="b",label='95% rejection region')

plt.legend(loc=1,prop={'size':12})
plt.savefig('chi2.pdf')

# example in section 9.3
# also calculate critical value for N=5 dof
p=[0.683,0.95]
chi2Crit68=chi2.ppf(p,5)
print('chi2 crit: ',chi2Crit68)

# p value
pVal=chi2.sf(9,5)
print('p-value',pVal)

# new table with Delta chi^2 values
print('New table in Ch. 12')
p=[0.683,0.90,0.99]
dof=[1,2,3,4,5]
for i in range(5):
    chi2Crit=chi2.ppf(p,dof[i])
    print('chi2 crit: ',chi2Crit)

# another example for the sampling distr. of variance
x=np.array([10, 12, 15, 11, 13, 16, 12, 10, 18, 13])
N=len(x)
xbar=np.mean(x)
s2=np.var(x,ddof=1)
S2=s2*(N-1)
print('xbar=%3.3f, S2=%3.3f'%(xbar,S2))
pVal=chi2.sf(32,9)
print('p-value',pVal)

# ==============================================
# ---- another chi^2 plot for the appendix

fig,ax=plt.subplots(figsize=(8,6))
colors=['darkgrey','gray','dimgray']
dof=[5,20,100]
xmin=0
for i in range(len(dof)):
	xmax=dof[i]+5*(2*dof[i])**0.5 # range of chi^2 distr. 
	x=np.linspace(xmin,xmax,1000)
	xOverf=(x-dof[i])/(2*dof[i])**0.5
	yNorm=chi2.pdf(x,dof[i])/max(chi2.pdf(x,dof[i])) # normalized chi^2 distr
	yNormCDF=chi2.cdf(x,dof[i])
	plt.plot(xOverf,yNorm,linewidth=2,color=colors[i],label='$\chi^2$(%d)'%dof[i])
	plt.plot(xOverf,yNormCDF,linewidth=2,color=colors[i])
x=np.linspace(-5,5,1000)
loc=0
scale=1
yGauss=norm.pdf(x,loc,scale)/norm.pdf(0,loc,scale)
plt.plot(x,yGauss,label='N(0,1)',color='black',linewidth=2,linestyle='-')
yGaussCDF=norm.cdf(x,loc,scale)
plt.plot(x,yGaussCDF,color='black',linewidth=2,linestyle='-')

ax.yaxis.set_major_locator(MultipleLocator(0.2))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.2))
plt.grid(which='both')
plt.xlim((-3,3))
plt.ylim((0,1))
plt.xlabel('Standardized x')
plt.ylabel('Normalized distributions')
plt.legend(loc=2,prop={'size':12})
plt.savefig('chi2Appendix.pdf')

# ==============================================
# F distribution
xmin=0
xmax=10
dfn=5   # degrees of freedom of numerator
dfd=5   # degrees of freedom of denominator
fig,ax=plt.subplots(figsize=(8,6))

x=np.linspace(xmin,xmax,1000)
y=f.pdf(x,dfn,dfd)
plt.plot(x,y,color='black',linewidth=2,label='F distr. ($f_1=5$, $f_2$=5)')
ax.set_xlabel('F')
ax.set_ylabel('Probability Distribution')
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
ax.xaxis.set_major_locator(MultipleLocator(2))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
plt.grid(which='both')
ax.set_xlim(0,8)
ax.set_ylim(0,0.68)

fCrit68=f.ppf(0.683,dfn,dfd)
fCrit95=f.ppf(0.95,dfn,dfd)
x1range=[i for i in np.arange(1000) if x[i]>=fCrit95]
x2range=[i for i in np.arange(1000) if x[i]>=fCrit68]
ax.fill_between(x[x2range], y[x2range],facecolor='none',hatch='//',edgecolor="gray",label='68.3% rejection region')
ax.fill_between(x[x1range], y[x1range],facecolor='none',hatch='XXXX',edgecolor="b",label='95% rejection region')

# add also another distribution with 1 fewer dof
dfn=4
dfd=4
y=f.pdf(x,dfn,dfd)
plt.plot(x,y,color='black',linewidth=2,linestyle='--',label='F distr. ($f_1=4$, $f_2$=4)')

# rearrange the labels
handles,labels = ax.get_legend_handles_labels()

handles = [handles[0], handles[2], handles[3],handles[1]]
labels = [labels[0], labels[2], labels[3],labels[1]]
plt.legend(handles,labels,loc=1,prop={'size':12})


plt.savefig('f.pdf')

# Another F distribution for the appendix ========
# F distribution
xmin=0
xmax=10

dfn=[5,10,20,50,100]   # degrees of freedom of numerator
dfd=dfn   # degrees of freedom of denominator
fig,ax=plt.subplots(figsize=(8,6))
colors=['silver','darkgrey','gray','dimgray','black']
x=np.linspace(xmin,xmax,1000)
for i in range(len(dfd)):
    y=f.pdf(x,dfn[i],dfd[i])/max(f.pdf(x,dfn[i],dfd[i]))
    plt.plot(x,y,linewidth=2,color=colors[i],label='F distr. ($f_1=%d$, $f_2$=%d)'%(dfd[i],dfn[i]))
ax.set_xlabel('F')
ax.set_ylabel('Normalized Prob. Distr.')
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.025))
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.1))
plt.grid(which='both')
ax.set_xlim(0,5)
ax.set_ylim(0,1)
plt.legend(loc=1,prop={'size':12})
plt.savefig('fAppendix.pdf')

# ============================================
'''
# use the 10 measurements for chi2 and F tests
print('Problem 9.5')

x=np.array([10,12,15,11,13,16,12,10,18,13])

x1=x[0:5]
x2=x[5:10]
mu=13
yMod=mu*np.ones(5)
sigma=2**0.5
yErr=sigma*np.ones(5)
chi21=chisquared(x1,yErr,yMod)
chi22=chisquared(x2,yErr,yMod)
chi2Crit=chi2.ppf(0.95,5)
N=5
print(x1,x2)
print('chi2: %3.3f, %3.3f (crit value 0.95: %3.3f'%(chi21,chi22,chi2Crit))
F=chi22/chi21
fCrit=f.ppf(0.95,N,N)
fPValue=f.sf(F,N,N)
print('F: %3.3f (crit value 0.95: %3.3f)'%(F,fCrit))
print('F: p-value %3.3f'%fPValue)
# now with sample means
m1=np.mean(x1)
m2=np.mean(x2)
s21=np.var(x1,ddof=1)
s22=np.var(x2,ddof=1)
F=s22/s21
print('means of two 5-point segments: %3.3f, %3.3f, variances: %3.3f, %3.3f, S2: %3.3f %3.3f, F: %3.3f'
        %(m1,m2,s21,s22,s21*(N-1),s22*(N-1),F))
fCrit=f.ppf(0.95,N-1,N-1)
fPValue=f.sf(F,N-1,N-1)
print('F: %3.3f (crit value 0.95: %3.3f)'%(F,fCrit))
print('F: p-value %3.3f'%fPValue)
# ==== z-score of first five measurements
N=5
z1=(m1-mu)/(sigma/N**0.5)
p1=norm.sf(abs(z1),loc=0,scale=1)*2
print('zscore of 5 measurements: %3.3f, pvalue: %3.3f'%(z1,p1))
# ==== t stat for the 5 measurements
tStat1=(m1-mu)/(s21/N)**0.5
tPvalue=t.sf(abs(tStat1),N-1) # one-sided prob.
tPvalue=tPvalue*2
print('t statistic: %3.3f, 2-sided p-value: %3.3f'%(tStat1,tPvalue))
# ==== t stat for the 10 measurements
s2=(s21*(N-1)+s22*(N-1))/(N+N-2)
tStatComp=(m1-m2)/s2**0.5/(1/N+1/N)**0.5
print('s2= %3.3f, t stat for comparison of two sample means: %3.3f'%(s2,tStatComp))
tPValueComp=2*t.sf(abs(tStatComp),2*(N-1))
print('p-value for the comparison: %3.3f'%tPValueComp)

'''
quit()
#=================================================
# ==== Student t distribution

xmin=-5
xmax=5
N=5
f=N-1
x=np.linspace(xmin,xmax,1000)
y=t.pdf(x,f)

fig,ax=plt.subplots(figsize=(8,6))

ax.plot(x,y,linewidth=2,color='black',label='t distr. (f=4)')
ax.set_xlabel('t')
ax.set_ylabel('Probability Distribution')
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.025))
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.25))
plt.grid(which='both')
ax.set_xlim(-4,4.2)
ax.set_ylim(0,0.42)

# add the 95% confidence limits
p1=0.95 # probability
l1=p1+0.5*(1-p1) # upper bound of SF 
l2=0.5*(1-p1)   # lower bound of SF
x1=t.ppf(l1,f)
x2=t.ppf(l2,f)
x1range=[i for i in np.arange(1000) if x[i]>=x1]
x2range=[i for i in np.arange(1000) if x[i]<=x2]
ax.fill_between(x[x1range], y[x1range],facecolor='none',hatch='XXXX',edgecolor="b",label='95% rejection region')
ax.fill_between(x[x2range], y[x2range],facecolor='none',hatch='XXXX',edgecolor="b")
# repeat the same for 68.3% rejection region
p1=0.683 # probability
l1=p1+0.5*(1-p1) # upper bound of SF
l2=0.5*(1-p1)   # lower bound of SF
x1=t.ppf(l1,f)
x2=t.ppf(l2,f)
x1range=[i for i in np.arange(1000) if x[i]>=x1]
x2range=[i for i in np.arange(1000) if x[i]<=x2]
ax.fill_between(x[x1range], y[x1range],facecolor='none',hatch='//',edgecolor="gray",label='68.3% rejection region')
ax.fill_between(x[x2range], y[x2range],facecolor='none',hatch='//',edgecolor="gray")

# add a N(0,1) for comparison
y=norm.pdf(x)
ax.plot(x,y,linewidth=2,linestyle='--',color='black',label='N(0,1)')

# add t(1), also known as Cauchy distribution
y=t.pdf(x,1)
ax.plot(x,y,color='red',linewidth=2,label='t distr. (f=1) - Cauchy')

handles,labels = ax.get_legend_handles_labels()
handles = [handles[0], handles[3], handles[4],handles[1],handles[2]]
labels = [labels[0], labels[3], labels[4],labels[1],labels[2]]
plt.legend(handles,labels,loc=1,prop={'size':12})
plt.savefig('t.pdf')

# -- Add another table of Tcrit, with $p$ level fixed
pLev=[0.683,0.9,0.95,0.99] # this needs to be a central CI
fN=[1,2,5,10,20,50,100] # number of degrees of freedom
for i in range(len(fN)):
    print('%d'%fN[i])
    for j in range(len(pLev)):
        tcrit=t.ppf((1+pLev[j])/2,fN[i])
        normcrit=norm.ppf((1+pLev[j])/2)
        print("%3.3f %3.3f"%(tcrit, normcrit))

# ===== Add another t distribution for the appendix

xmin=-5
xmax=5
f=[1,5,10,30,100]
colors=['red','darkgrey','gray','dimgray','black']
x=np.linspace(xmin,xmax,1000)

fig,ax=plt.subplots(figsize=(8,6))
for i in range(len(f)):
    y=t.pdf(x,f[i])#/max(t.pdf(x,f[i]))
    ax.plot(x,y,linewidth=2,color=colors[i],label='t distr. (f=%d)'%f[i])
y=norm.pdf(x)#/max(norm.pdf(x))
ax.plot(x,y,linewidth=2,linestyle='--',color='black',label='N(0,1)')
ax.set_xlabel('t')
ax.set_ylabel('Probability Distribution')
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.025))
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.25))
plt.grid(which='both')
plt.grid(which='major',color='black')
ax.set_xlim(-4,4.2)
ax.set_ylim(0,0.4)
plt.legend(loc=1,prop={'size':12})
plt.savefig('tAppendix.pdf')
CiApprox.py0000644000175000017500000000715514116214212011313 0ustar  maxmax# These are the analytical function from Bonamente20
# Taken from cstat.py in ./pg1116

exec(open('imports.py').read())

# Simulate a normal distribution
fig,ax=plt.subplots(figsize=(8,6))

#def funcECmin(x, a,b,c,d,e,beta,f,alpha):
#    return (a+b*x+c*(x-d)**2.0)*np.exp(-alpha*x)+e*np.exp(-beta*x)+f
#def funcVarC(x, a,b,c,d,e,f,g,h,alpha,beta,i):
#    return  (a+b*x**2+c*(x-d)**2.0)*np.exp(-alpha*x) +  (e+f*x+g*(x-h)**2.0)*np.exp(-beta*x) +i

#iimport numpy as np
#from matplotlib import pyplot as plt
#import math


# Parameter values as reported in Bonamente20
parNamesE=['a','b','c','d','e','beta','f','alpha']
parNamesVar=['a','b','c','d','e','f','g','h','alpha','beta','i']

# These are the best-fit values for E[Ci] on full and narrow range, 
# followed by Var[Ci] for full and narrow range
A=[0.065672, -0.56709, -2.4637, -3.1971]
B=[-6.9461,-2.7336,1.5109,1.5118]
C=[-8.0124,-2.3603,-1.5109,-1.5118]
D=[0.40165,0.52816,0.60509,0.79384]
E=[0.261037,0.33133,1.4761,1.9294]
F=[1.00512,1.0174,18.358,6.1740]
G=[0,0,0.87316e-3,22.360e-3]
H=[0,0,-0.08592,-7.2981]
I=[0,0,2.02343,2.08378]
alphaB=[5.5178,3.9375,0.62652,0.750315]
betaB=[0.34817,0.48446,7.8187,4.49654]

colors=['black','red','gray','orange']
descriptor=['E[$C_i$] (range $\mu$=0.01-100)','E[$C_i$] (range $\mu$=0.1-10)','Var($C_i$) (range $\mu$=0.01-100)','Var($C_i$) (range $\mu$=0.1-10)'] 
# ==============================================

# Plot the four functions
size=100
mu1=np.logspace(-2,2,num=size)
mu2=mu=np.logspace(-1,1,num=size)
yaxis=np.zeros((4,size))

for j in range(2):
    if j==1: 
        mu=mu2
    if j==0:
        mu=mu1
    for i in range(size):
            yaxis[j][i]=funcECmin(mu[i],A[j],B[j],C[j],D[j],E[j],betaB[j],F[j],alphaB[j])
    print(yaxis[j][i])
    plt.semilogx(mu,yaxis[j],linewidth=2,color=colors[j],label=descriptor[j])
for j in range(2,4,1):
    if j==2:
        mu=mu1
    if j==3:
        mu=mu2
    for i in range(size):
            yaxis[j][i]=funcVarC(mu[i],A[j],B[j],C[j],D[j],E[j],F[j],G[j],H[j],alphaB[j],betaB[j],I[j])
    print(yaxis[j][i])
    plt.semilogx(mu,yaxis[j],linewidth=2,color=colors[j],label=descriptor[j])

for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())
# Add horizontal and vertical lines
plt.hlines(1,mu1[0],mu1[size-1],linewidth=2,linestyle='--')
plt.hlines(2,mu1[0],mu1[size-1],linewidth=2,linestyle='--')
ax.set_xlabel('Poisson mean $\mu$')
ax.set_ylabel('Appr. Expectation and Variance')
plt.vlines(0.1,0.0,2.5,linestyle='--',color='black',linewidth=1)
plt.vlines(10,0.0,2.5,linestyle='--',color='black',linewidth=1)
ax.yaxis.set_major_locator(MultipleLocator(0.5))
ax.yaxis.set_minor_locator(MultipleLocator(0.1))

plt.grid(which='both')
plt.xlim(0.01,100)
plt.ylim(0,2.5)
plt.legend(loc=2,prop={'size': 12})
plt.savefig('CiMeanVariance.pdf')

# ============
#mean=0.1
#  problem 15.2
print('Problem 15.2')
mean=0.2 
j=1
Expect=funcECmin(mean,A[j],B[j],C[j],D[j],E[j],betaB[j],F[j],alphaB[j])
j=3
Variance=funcVarC(mean,A[j],B[j],C[j],D[j],E[j],F[j],G[j],H[j],alphaB[j],betaB[j],I[j])
print('for mu=%3.3f: E=%3.3f, Var=%3.3f'%(mean,Expect,Variance))
N=100
EC=Expect*N
VarC=N*Variance
print('C=%3.3f \pm %3.3f'%(EC,VarC**0.5))
# For N=100, Gaussian approx for C applies.
q=1.3
print('C crit for N=%d: %3.3f'%(N,EC+q*(VarC)**0.5))
# ====================
# problem 15.3
print('Problem 15.3')
mean=3.0 # 
j=1
Expect=funcECmin(mean,A[j],B[j],C[j],D[j],E[j],betaB[j],F[j],alphaB[j])
j=3
Variance=funcVarC(mean,A[j],B[j],C[j],D[j],E[j],F[j],G[j],H[j],alphaB[j],betaB[j],I[j])
print('for mu=%3.3f: E=%3.3f, Var=%3.3f'%(mean,Expect,Variance))
N=100
EC=Expect*N
VarC=N*Variance
print('C=%3.3f \pm %3.3f'%(EC,VarC**0.5))



CiAsympt.py0000664000175000017500000000123714116212562011322 0ustar  maxmaxexec(open('imports.py').read())

# Simulate a normal distribution
fig,ax=plt.subplots(figsize=(8,6))

def function(x):
    result=x*np.log(x)
    return result

N=100
mu=np.arange(N)+1
ENorm=np.zeros(N)
for i in range(len(mu)):
    P=poisson(mu[i])
    ENorm[i]=P.dist.expect(lambda x: function(x), P.args,lb=1,ub=1000)-function(mu[i])
    print(mu[i], ENorm[i])

plt.plot(mu,ENorm,linewidth=2,color='black',label='E[$y_i \ln y_i$]-$\mu_i \ln \mu_i$' )
plt.hlines(0.5,0,100,color='red',linestyle='--',linewidth=2,label='Asymptotic value')
plt.xlabel('Parent mean $\mu_i$')
plt.ylabel('Expectation')
plt.legend()
plt.xlim(0,100)
plt.grid()
plt.savefig('CiAsymptotic.pdf')
CiCritical.py0000644000175000017500000000665514104233413011601 0ustar  maxmaxexec(open('imports.py').read())
# Numerical calculation of the critical values for Ci
# In the low--count and low--N limit where no approximation is available

def CContribwTwo(Ni,mu):
    if (Ni==0):
        return 2*mu
    if (Ni>0):
        return 2*(mu - Ni +Ni*np.log(Ni/mu)) # factor of 2 included here

# =========================================
# Simple calculations in the case of large N, where C~Normal
N=100
chiCrit=chi2.ppf(0.9,N)
mu=N
sigma=(2*N)**0.5
NormCrit=norm.ppf(0.9,N,sigma)
print('Critical value of chi2(100): %3.3f, N(100,200):%3.3f'%(chiCrit,NormCrit))

# -------------
yi=2
mui=0.3
CTest=CContribwTwo(yi,mui)
print('Ci=%3.3f for yi=%d and mui=%3.3f'%(CTest,yi,mui))

# =========================================
# Under the null hypothesis that Mu is the parent mean,
# Draw random Poisson samples from Ni~Poiss(mu) and calculate
# Critical values at various confidence levels

# This should have been done in the Bonamente20 paper, but wasn't

nProb=5
p = [0.683,0.90,0.95,0.99,0.999]

# Choose whether to have many values or just one
#size=100
#mu=np.logspace(-2,2,num=size)
size=8
mu=[0.1,0.3,0.5,1.0,3,5,10.,100]
colors=['blue','red','green','magenta','brown','orange','violet','yellow']
Ciquantile=np.zeros((size,nProb)) # quantiles of Ci, approximate
chi2quantile=np.zeros((size,nProb)) # this is done for comparison

sizeSample=100000
sampleC=np.zeros((size,sizeSample))
for i in range(size):
    # 1. generate random values
    sampleP=poisson.rvs(mu[i],size=sizeSample) # random values
    # 2. Now sample values of Ci
    for k in range(sizeSample):
        sampleC[i,k]=CContribwTwo(sampleP[k],mu[i])
    # 3. find the p quantile
    print('Ci quantiles for mu=%4.3f'%mu[i])
    for j in range(nProb):
        Ciquantile[i,j]=np.quantile(sampleC[i],p[j])
        chi2quantile[i,j]=chi2.ppf(p[j],1) # this remains the same as a function ofi mu, 
        # so it should not be re-calculated
    print('Quantiles of Ci     ',Ciquantile[i,:],'\nQuantiles of chi2(1)',chi2quantile[i,:])

# ==================================================================
# Print the unique values of C_i for mu=0.1
CiUnique=np.unique(sampleC[0,:])
print('Unique value of Ci for mu=0.1',CiUnique)

# Check that these correspond to n=0,1,2,3 counts
A=np.zeros(4)
muTest=0.1
for i in range(4):
    A[i]=CContribwTwo(i,muTest)
print(A)

# ===================================================================
# Plot a few specific cases
PLOT=1
bins=np.linspace(0.,10,1000)

fig,ax=plt.subplots(figsize=(8,6))

for i in range(size*PLOT):
    n,bins,_=ax.hist(sampleC[i],bins=bins,cumulative=True,histtype='step',log=False,density=True,linewidth=2,alpha=0.5,color=colors[i],label='mean $\mu_i$=%2.2f'%mu[i])
    # overplot the chi2(1) distribution using the bins in the histogram plot
    binsCenter=(bins[1:] + bins[:-1])/2 # arithmetic mean
#    binsCenter=np.sqrt(bins[1:] * bins[:-1]) # geometric mean

ax.plot(binsCenter,chi2.cdf(binsCenter,1),linewidth=2,color='black',linestyle='--',label='$\chi^2(1)$')
ax.set_xlabel('Values of $C_i$')
ax.set_ylabel('Sample Cumulative Distribution')
ax.set_yticks(np.linspace(0,1,11,dtype=float))
ax.set_xticks(np.arange(0,11,step=1.0))
ax.set_ylim(0,1)
ax.set_yscale('linear')
ax.set_xscale('log')
ax.set_xlim(0.07,10)

# formatting log axis with linear labels
for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())


plt.grid(which='both')
plt.legend(loc=4,prop={'size': 12})
plt.savefig('CiCritical.pdf')
#plt.show()


clinicalTestingContingency.py0000644000175000017500000003630114115644462015113 0ustar  maxmaxexec(open('imports.py').read())
from math import factorial as fact

# describe the positive predictive value (posterior)
# as a function of sensitivity (SE), specificity (SP) and prevalence

# positive and negative likelihood ratios
def LR12n(SE, SP, n):
    result=(SE/(1-SP))**n
    return result

def LR21n(SE, SP, n):
    result=((1-SE)/SP)**n
    return result
# ---------------------------------------

def PPV(SE, SP, PR):
    result=SE*PR/(SE*PR+(1-SP)*(1-PR))
    return result

# let's try PPV after two positive tests
def PPV2(SE, SP, PR):
    #result = SE**2*PR/(SE**2*PR + (1-SP)*(1-PR)*(SE*PR+(1-SP)*(1-PR)))
    PPV1=PPV(SE, SP,PR) # this is PPV after one test
    result=SE*PPV1/(SE*PPV1+(1-SP)*(1-PPV1))
    return result

# Let's generalize the PPV after n repeated tests
def PPVn(SE, SP, PR, n):
    ppv=np.zeros(n+1)
    ppv[0]=PR # is this right?
    for i in range(1,n+1):
        ppv[i]=PPV(SE, SP,ppv[i-1])
    return ppv[n]


# test that this is equivalent to the formua in the Zhou book
def PPVnLR(SE, SP, PR, n):
    lr=LR12n(SE, SP, n) # first the likelihood ratio
    result=1/(1+(1-PR)/(PR*lr))
    return result

def NPV(SE, SP, PR):
    result=SP*(1-PR)/(SP*(1-PR)+(1-SE)*PR) 
    return result

def NPVn(SE, SP, PR, n):
    npv=np.zeros(n+1)
    pr=np.zeros(n+1) # prevalence, updated by successive negative tests
    pr[0]=PR # is this right? Same as the PPVn function
    for i in range(1,n+1):
        npv[i]=NPV(SE, SP,pr[i-1])
        pr[i]=1-npv[i] # this is the new prevalence
    return npv[n]

def NPVnLR(SE, SP, PR, n):
    lr=LR21n(SE, SP, n) # likelihood ratio
    result=1/(1+PR*lr/(1-PR)) # notice lr at different location than for PPVnLR
    return result


# make the contingency table by specifying N
# question is: how to ensure that PP, SE and SP are consistent with one another??
# I think that prevalence is needed too
'''
def makeTable(SE, SP, PP, PR, N):
# USING PRevalence; beware that both SE and SP can't be used cnsistently
#    n1=PP*N
#    n2=N-n1 # n1 and n2 will always be positive.
#    m1=N*PR
#    m2=N*(1-PR)
# NOT using PRevalence
    n1=PP*N
    n2=N-n1
    m1=N*(1-PP-SP)/(1-SE-sp)
    m2=N-m1
    a=m1*SE
    b=n1-a
    c=n2-m2*SP
    d=n2-c
    prevalence=m1/N
    return(a,b,c,d,n1,n2, m1,m2,prevalence)


def makeTablePR(SE, SP, PP,PR,N): 
    # Let's analyze in detail the ORDER in which quantities are detemined.
    n1=PP*N
    n2=N-n1
    a=n1*PPV(SE,SP,PR)
    b=n1-a
    # m1=PR*N  # was this also for the calibration test?
    #m2=N-m1
    # a=m1*SE was this for the calibration test?
    d=n2*NPV(SE,SP,PR)
    c=N-a-b-d
    m1=a+c
    m2=b+d
    estimatedPrevalence=m1/N # is this even meaningful for these data?!
    # I don't think one can estimate prevalence from a sample, only
    # from a calibration set with a Golden Standard 
    return(a,b,c,d,n1,n2, m1,m2,estimatedPrevalence)

# What is the point of "making" a contingency table from a test?
# We don't have the Golden Standard in the sample data.
# We have assumed a prevalence, and we don't get that prevalence back
# if we use the PPV() and NPV() functions, using Bayes' theorem

SP=0.55
SE=0.65
PP=0.55
PR=0.1

# test calculations of PPV
print('PPV for SPecificity=%3.2f, SEnsitivity=%3.2f and PRevalence=%3.2f: %f'%(SP,SE,PR,PPV(SE,SP,PP)))

# test my PPV formula  vs. the one in the Zhou book
# looks like they are equivalent - but not fully tested
test1=PPVn(SE, SP, PR, 3)
test2=PPVnLR(SE, SP, PR, 3)
print('PPV=%3.4f PPVnLR=%3.4f'%(test1,test2))
test1=NPVn(SE, SP, PR, 3)
test2=NPVnLR(SE, SP, PR, 3)
print('NPV=%3.4f NPVnLR=%3.4f'%(test1,test2))

N=100
#print('PP+SP=%3.4f, check PP<SE and PP+SP>1 to have m1>0 (PP=%3.2f, SE=%3.2f)'%(PP+SP,PP, SE))

print(makeTablePR(SE, SP, PP, PR,N))
'''

#problem 10.8 - use a golden-standard test to calibrate a diagnostic test
print('Problem 10.8')
a=120 # true positives
b=20  # false positives
c=40 # true negatives
d=100  # false negatives

N=a+b+c+d
m1=a+c # people with disease
n1=a+b # people who test positive
m2=N-m1
n2=N-n1
SE=a/m1
SP=d/m2
print('Using calibration table %d, %d, %d, %d'%(a,b,c,d))
print('N=%d, SE=%2.3f, SP=%2.3f, PLR=%3.3f, NLR=%3.3f'%(N, SE, SP, SE/(1-SP),(1-SE)/SP))

#problem 10.9 - Now use a diagnostic test with same SE and SP as prob10.8
# 
print('Problem 10.9')
PR1=0.1 # 10 % prevalence
ppv1=PPV(SE,SP,PR1)
PR2=0.01 # 10 % prevalence
ppv2=PPV(SE,SP,PR2)
print('PPV= %3.2f (PR=%2.3f), %3.2f (PR=%3.3f)'%(ppv1,PR1,ppv2,PR2))
npv1=NPV(SE,SP,PR1)
print('NPV = %3.3f (PR=%3.3f)'%(npv1,PR1))

#problem 10.10 - flying in the age of covid-19
print('Problem 10.10')
SE=0.7
SP=0.99
PR=0.10
npv1=NPV(SE,SP,PR) # prob. of being healthy, after 1 test
npv3=NPVn(SE,SP,PR,3) # after 3 tests
print('For PR=%3.2f 1 test NPV: %3.3f (therefore %3.3f), 3 tests %3.3f (therefore %3.3f)'%(PR,npv1,1-npv1,npv3,1-npv3))
# what if PR=0.5??
PR=0.50
npv1=NPV(SE,SP,PR) # prob. of being healthy, after 1 test
npv3=NPVn(SE,SP,PR,3) # after 3 tests
print('For PR=%3.2f 1 test NPV: %3.3f (therefore %3.3f), 3 tests %3.3f (therefore %3.3f)'%(PR,npv1,1-npv1,npv3,1-npv3))
print('problem 10.11')
# Now see if a positive test should deny boarding on a plane
PR=0.01
ppv=PPV(SE,SP,PR)
print('For PR=%3.2f 1 test PPV: %3.3f'%(PR,ppv))
quit()

# Below here to reproduce figures in the textbook 
# ==========================================
# 2. Now plot  PPV and NPV as a function of its parameters
#pr is prevalence
NPoints=11
se=np.linspace(0.0,1.0,NPoints)#0.99999,NPoints)
sp=np.linspace(0.0,1.0,NPoints)#0.99999,NPoints)
pr=np.linspace(0.0,1.0,NPoints*10) # more points for prevalence

PPVFixedSE=np.zeros((NPoints*10,NPoints))
PPVFixedSP=np.zeros((NPoints*10,NPoints))
NPVFixedSE=np.zeros((NPoints*10,NPoints))
NPVFixedSP=np.zeros((NPoints*10,NPoints))
# 2.1 for a fixed sensitivity, pr vs. PPV 
fixedSE=0.5
fixedSP=0.5


for j in range(NPoints): # j: specificity or sensitivity
    for i in range(NPoints*10):    # i: prevalence
        PPVFixedSE[i,j]=PPV(fixedSE,sp[j],pr[i])
        NPVFixedSE[i,j]=NPV(fixedSE,sp[j],pr[i])
        PPVFixedSP[i,j]=PPV(se[j],fixedSP,pr[i])
        NPVFixedSP[i,j]=NPV(se[j],fixedSP,pr[i])


# plot PPV versus prevalence, as a function of specificity
fig,ax=plt.subplots(figsize=(8,6))
for j in range(NPoints): # j: specificity or sensitivity, alpha=1 for SP=1 or SE=1
    plt.plot(pr,PPVFixedSE[:,j],linewidth=2+0.1*j,alpha=0.5+0.05*j,color='black')
    plt.plot(pr,PPVFixedSP[:,j],linewidth=2+0.1*j,alpha=0.5+0.05*j,linestyle='--',color='black')

# plot the last curves for the legend on them
plt.plot(pr,PPVFixedSE[:,NPoints-1],linewidth=2+0.1*j,alpha=0.5+0.05*j,color='black',label='Fixed SE=%2.1f, SP=%2.1f-%2.1f'%(fixedSE,sp[0],sp[NPoints-1]))
plt.plot(pr,PPVFixedSP[:,NPoints-1],linewidth=2+0.1*j,alpha=0.5+0.05*j,linestyle='--',color='black',label='Fixed SP=%2.1f, SE=%2.1f-%2.1f'%(fixedSP,se[0],se[NPoints-1]))
# --------------------------------------------
plt.legend(loc=4,prop={'size': 12})
plt.xlabel('Prevalence')
plt.ylabel('Positive Predictive Value')
ax.xaxis.set_major_locator(MultipleLocator(0.1))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.grid(which='both')
eps=0.01
plt.xlim(0-eps,1+eps)
plt.ylim(0-eps,1+eps)
plt.savefig('ppv.pdf')

# plot PPV versus prevalence, as a function of specificity
fig,ax=plt.subplots(figsize=(8,6))
for j in range(NPoints): # j: specificity or sensitivity
    plt.plot(pr,NPVFixedSE[:,j],linewidth=2+0.1*j,alpha=0.5+0.05*j,color='black')
    plt.plot(pr,NPVFixedSP[:,j],linewidth=2+0.1*j,alpha=0.5+0.05*j,linestyle='--',color='black')
#plt.legend(loc=4,prop={'size': 10})
# plot the last curves for the legend on them
plt.plot(pr,NPVFixedSE[:,NPoints-1],linewidth=2+0.1*j,alpha=0.5+0.05*j,color='black',label='Fixed SE=%2.1f, SP=%2.1f-%2.1f'%(fixedSE,sp[0],sp[NPoints-1]))
plt.plot(pr,NPVFixedSP[:,NPoints-1],linewidth=2+0.1*j,alpha=0.5+0.05*j,linestyle='--',color='black',label='Fixed SP=%2.1f, SE=%2.1f-%2.1f'%(fixedSP,se[0],se[NPoints-1]))
# -------------------------------------------
plt.xlabel('Prevalence')
plt.ylabel('Negative Predictive Value')
ax.xaxis.set_major_locator(MultipleLocator(0.1))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.grid(which='both')
plt.legend(loc=3,prop={'size': 12})
plt.xlim(0-eps,1+eps)
plt.ylim(0-eps,1+eps)

plt.savefig('npv.pdf')

# ======== Now one more figure for a typical COVID-19 test
# case a: high sensitivity, medium specificity
PPV19a=np.zeros(NPoints*10)
NPV19a=np.zeros(NPoints*10)
fixedSE19a=0.95
fixedSP19a=0.5
# case b: medium sensitivity, high specificity
PPV19b=np.zeros(NPoints*10)
NPV19b=np.zeros(NPoints*10)
fixedSE19b=0.5
fixedSP19b=0.95
# case c: a useless test with 50% sens. and spec.
PPV19c=np.zeros(NPoints*10)
NPV19c=np.zeros(NPoints*10)
# ----------------------------------------------
for i in range(NPoints*10):
    PPV19a[i]=PPV(fixedSE19a,fixedSP19a,pr[i])
    NPV19a[i]=NPV(fixedSE19a,fixedSP19a,pr[i])
    PPV19b[i]=PPV(fixedSE19b,fixedSP19b,pr[i])
    NPV19b[i]=NPV(fixedSE19b,fixedSP19b,pr[i])
# also for 50% SP and 50% SE
    PPV19c[i]=PPV(fixedSE19b,fixedSP19a,pr[i])
    NPV19c[i]=NPV(fixedSE19b,fixedSP19a,pr[i])
fig,ax=plt.subplots(figsize=(8,6))
plt.plot(pr,PPV19a,color='blue',linewidth=2,label='PPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19a,fixedSP19a))
plt.plot(pr,NPV19a,color='blue',linestyle='--',linewidth=2,label='NPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19a,fixedSP19a))
plt.plot(pr,PPV19b,color='red',linewidth=2,label='PPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19b,fixedSP19b))
plt.plot(pr,NPV19b,color='red',linewidth=2,linestyle='--',label='NPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19b,fixedSP19b))
plt.plot(pr,PPV19c,color='black',alpha=0.5,linewidth=2,linestyle='-',label='PPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19b,fixedSP19a))
plt.plot(pr,NPV19c,color='black',alpha=0.5,linewidth=2,linestyle='--',label='NPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19b,fixedSP19a))

ax.xaxis.set_major_locator(MultipleLocator(0.1))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.xlabel('Prevalence')
plt.ylabel('Predictive Value')
plt.grid(which='both')
plt.legend(loc=8,prop={'size': 10})
plt.xlim(0-eps,1+eps)
plt.ylim(0-eps,1+eps)
plt.savefig('ppvnpv19.pdf')

# ==================================================
# ==================================================
# Compare the PPV from one and 2 positive tests, 
#fixedSE19a=0.95
#fixedSP19a=0.5

# This is a test with a small PPV 
fixedSE19a=0.95
fixedSP19a=0.6


PPV219a=np.zeros(NPoints*10)
NPV219a=np.zeros(NPoints*10)
PPV319a=np.zeros(NPoints*10)
NPV319a=np.zeros(NPoints*10)

for i in range(NPoints*10):
    PPV19a[i]=PPV(fixedSE19a,fixedSP19a,pr[i])
#    PPV219a[i]=PPV2(fixedSE19a,fixedSP19a,pr[i])
    PPV219a[i]=PPVnLR(fixedSE19a,fixedSP19a,pr[i],2)
    PPV319a[i]=PPVnLR(fixedSE19a,fixedSP19a,pr[i],3)

    NPV19a[i]=NPV(fixedSE19a,fixedSP19a,pr[i])
#    NPV219a[i]=NPV2(fixedSE19a,fixedSP19a,pr[i])
    NPV219a[i]=NPVnLR(fixedSE19a,fixedSP19a,pr[i],2)
    NPV319a[i]=NPVnLR(fixedSE19a,fixedSP19a,pr[i],3)

fig,ax=plt.subplots(figsize=(8,6))
plt.plot(pr,PPV19a,color='darkblue',linewidth=2,label='PPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19a,fixedSP19a))
plt.plot(pr,PPV219a,color='blue',linewidth=2,label='PPV(2)')
plt.plot(pr,PPV319a,color='cornflowerblue',linewidth=2,label='PPV(3)')

plt.plot(pr,NPV19a,color='red',linewidth=2,label='NPV')
plt.plot(pr,NPV219a,color='darkorange',linewidth=2,label='NPV(2)')
plt.plot(pr,NPV319a,color='orange',linewidth=2,label='NPV(3)')


plt.xlabel('Prevalence')
plt.ylabel('Predictive Value')
ax.xaxis.set_major_locator(MultipleLocator(0.1))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.grid(which='both')
plt.legend(loc=4,prop={'size': 9})
plt.xlim(0-eps,1+eps)
plt.ylim(0-eps,1+eps)

PR=0.25 # this is the prevalence, or initial prevalence


# we need to illustrate this plot with an example
# start at PR prevalence
PPV1=PPV(fixedSE19a,fixedSP19a,PR) # PPV from one positive test
PPV2=PPVn(fixedSE19a,fixedSP19a,PR,2)#PPV1)
PPV3=PPVn(fixedSE19a,fixedSP19a,PR,3)
PPV4=PPVn(fixedSE19a,fixedSP19a,PR,4)
plt.vlines(PR,0,1,linestyle='--',color='black')
plt.hlines(PPV1,0,1,linestyle='--',color='black')
plt.vlines(PPV1,0,1,linestyle='--',color='black')
plt.vlines(PPV2,0,1,linestyle='--',color='black')
plt.hlines(PPV2,0,1,linestyle='--',color='black')
plt.hlines(PPV3,0,1,linestyle='--',color='black')
#plt.hlines(PPV4,0,1,linestyle='--',color='black')
plt.annotate('A',xy=(PR,PPV1),xytext=(-15,2),textcoords='offset points')
plt.annotate('B',(PR,PPV2),xytext=(-15,2),textcoords='offset points')
plt.annotate('B\'',(PPV1,PPV2),xytext=(-19,2),textcoords='offset points')
plt.annotate('C',(PR,PPV3),xytext=(-15,2),textcoords='offset points')
plt.annotate('C\'',(PPV2,PPV3),xytext=(-19,2),textcoords='offset points')

plt.savefig('ppv2-19.pdf')

# ----------------------------------------------------------
# now also for two negative tests, see how the NPV increases

# This is a test with a small NPV; see what additional
# negative tests will do to improve the NPV

fixedSE19a=0.6
fixedSP19a=0.95

for i in range(NPoints*10):
    PPV19a[i]=PPV(fixedSE19a,fixedSP19a,pr[i])
#    PPV219a[i]=PPV2(fixedSE19a,fixedSP19a,pr[i])
    PPV219a[i]=PPVn(fixedSE19a,fixedSP19a,pr[i],2)
    PPV319a[i]=PPVn(fixedSE19a,fixedSP19a,pr[i],3)

    NPV19a[i]=NPV(fixedSE19a,fixedSP19a,pr[i])
#    NPV219a[i]=NPV2(fixedSE19a,fixedSP19a,pr[i])
    NPV219a[i]=NPVn(fixedSE19a,fixedSP19a,pr[i],2)
    NPV319a[i]=NPVn(fixedSE19a,fixedSP19a,pr[i],3)


fig,ax=plt.subplots(figsize=(8,6))
plt.plot(pr,PPV19a,color='darkblue',linewidth=2,label='PPV (SE=%2.2f, SP=%2.2f)'%(fixedSE19a,fixedSP19a))
plt.plot(pr,PPV219a,color='blue',linewidth=2,label='PPV(2)')
plt.plot(pr,PPV319a,color='cornflowerblue',linewidth=2,label='PPV(3)')

plt.plot(pr,NPV19a,color='red',linewidth=2,label='NPV')
plt.plot(pr,NPV219a,color='darkorange',linewidth=2,label='NPV(2)')
plt.plot(pr,NPV319a,color='orange',linewidth=2,label='NPV(3)')


plt.xlabel('Prevalence')
plt.ylabel('Predictive Value')
ax.xaxis.set_major_locator(MultipleLocator(0.1))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
ax.yaxis.set_major_locator(MultipleLocator(0.1))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.grid(which='both')
plt.legend(loc=3,prop={'size': 9})
plt.xlim(0-eps,1+eps)
plt.ylim(0-eps,1+eps)

PR=0.75 # this is the prevalence, or initial prevalence

# we need to illustrate this plot with an example
# start at PR prevalence
NPV1=NPV(fixedSE19a,fixedSP19a,PR) # NPV from one positive test
NPV2=NPVn(fixedSE19a,fixedSP19a,PR,2)
NPV3=NPVn(fixedSE19a,fixedSP19a,PR,3)
plt.vlines(PR,0,1,linestyle='--',color='black')
plt.hlines(NPV1,0,1,linestyle='--',color='black')
plt.vlines(1-NPV1,0,1,linestyle='--',color='black')
plt.vlines(1-NPV2,0,1,linestyle='--',color='black')
plt.hlines(NPV2,0,1,linestyle='--',color='black')
plt.hlines(NPV3,0,1,linestyle='--',color='black')

plt.annotate('A',(PR,NPV1),xytext=(1,2),textcoords='offset points')
plt.annotate('B',(PR,NPV2),xytext=(1,2),textcoords='offset points')
plt.annotate('B\'',(1-NPV1,NPV2),xytext=(1,2),textcoords='offset points')
plt.annotate('C',(PR,NPV3),xytext=(1,2),textcoords='offset points')
plt.annotate('C\'',(1-NPV2,NPV3),xytext=(1,2),textcoords='offset points')

plt.savefig('npv2-19.pdf')

ContingencyTables.py0000644000175000017500000001702414115424661013207 0ustar  maxmaxexec(open('imports.py').read())
from math import factorial as fact
from scipy.stats import fisher_exact,chi2
# ==================================================
# The chi^2 evaluated for a 2x2 contingency table,
# assuming the measured margins, and approximating the
# variance of p1-p2 (Grenwood and Yule 1915)
def chi2Yule(a,b,c,d,N,m1,m2,n1,n2):
    return (a*d-b*c)**2*N/(m1*m2*n1*n2)
# ==================================================

# ==================================================
# Also chi^2 following the original definition 
# by Pearson, summing over 4 terms

def chi2Yates(a,b,c,d,N,m1,m2,n1,n2):
    return (np.absolute(a*d-b*c)-N/2)**2*N/(m1*m2*n1*n2)

def chi2Table(a,b,c,d,N,m1,m2,n1,n2):
    # calculate the expected values
    m1=a+c
    n1=a+b
    m2=b+d
    n2=c+d
    N=n1+n2

    E=[m1*n1/N,m2*n1/N,m1*n2/N,m2*n2/N] # expected values, from proportions
    m=[a,b,c,d] # measured values
    # Now the chi^2 is the sum over 4 terms, carried out explicitly
    result=0
    chiIndividual=np.zeros(4) # to keep track of each deviation
    for i in range(4):
        chiIndividual[i]=(m[i]-E[i])**2/E[i]
        result+=chiIndividual[i]
    return result,chiIndividual,E

# assuming a fixed value of p, from binomial distribution
def chi2TableFixedP(a,b,c,d,N,p):
	m1=a+c
	n1=a+b
	m2=b+d
	n2=c+d
	N=n1+n2

	E=[p*(1-p)*N,(1-p)*(1-p)*N,p*p*N,p*(1-p)*N] # expected values, from p
	m=[a,b,c,d] # measured values
	chiIndividual=np.zeros(4)
	result=0
	for i in range(4):
		chiIndividual[i]=(m[i]-E[i])**2/E[i]
		result+=chiIndividual[i]
	return result,chiIndividual,E
# this is Fisher's exact test, still assuming the observed margins
# it assumes n1<=n2 and m1<=m2 
def fisherExact(a,b,c,d,N,m1,m2,n1,n2):
    result = fact(m1)*fact(m2)*fact(n1)*fact(n2)/(fact(a)*fact(b)*fact(c)*fact(d)*fact(N))
    return result

def barnardExact(a,b,c,d,N,m1,m2,n1,n2,p):
    result = fact(n1)*fact(n2)/(fact(a)*fact(b)*fact(c)*fact(d))*(p**m1*(1-p)**m2)
    return result

# Calculate the probability of having a_i>=a_observed.
# Careful: the total number N and m1, and m2 are held fixed at observe values
# but changing a_observed->a_i means that also b and c must be changed
#   (a) a->a_i (a_i is the trial value of the distribution)
#   (b) b->b_i=n1-a_i 
#   (c) c->c_i=m1-a_i
#   (d) d->d_i=m2-b_i (or n2-c_i, should check that this is the same)


# ========================================================
# This is done already by scipy.stats.fisher_exact
# I am trying to understand its algorithm/logic
# Think of stepping a up to n1; accordingly
#   b steps down
#   c steps down
#   d steps up to keep n1, n2, m1 and m2 all fixed
def fisherExactCumulative(a,b,c,d,N,m1,m2,n1,n2):
    # First, find our how many values of "a" we have
    # Yates says there are min(m1,n1)+1 values of "a".
    # But I can't see how, if observed margins must be held fixed.
    # Line below is OK for the case of Table VIII, given the observed numbers
    #rangeA=np.arange(a-d,n1+1,dtype=int)

    # this is the general criterion, I think
    if n1<=m1:
        amax=n1
    if n1>m1:
        amax=m1
    rangeA=np.arange(0,amax+1,dtype=int)


    #index=np.arange(0,(n1+1)-a,dtype=int)
    probObserved=fisherExact(a,b,c,d,N,m1,m2,n1,n2)
    cumA=0 # cumulative probability 
    for ai in rangeA:
        bi=n1-ai    #b-index[i]
        ci=m1-ai    #c-index[i]
        di=m2-bi    #d+index[i]
        cumContrib=fisherExact(ai,bi,ci,di,N,m1,m2,n1,n2)
        pi=0.75 # THIS IS A TEST FOR NOW
        cumContribB=barnardExact(ai,bi,ci,di,N,m1,m2,n1,n2,pi)
# ========================================================================
# Note that the cumulative probability extends over all values of "a"
# that give more "extreme" probability than observed, not a>=a_observed.
# This seems to be the consensus in the literature (and scipy),
# Although not what Yates1984 says
        #if(cumContrib<=1): #probObserved): # testing that it is properly normalized
        if(cumContrib<=probObserved):
            cumA+=cumContrib
            print('[%d,%d,%d,%d], %e (Barnard: %e) %f ***'%(ai,bi, ci,di,cumContrib,cumContribB,cumContribB/cumContrib))
        else:
            print('[%d,%d,%d,%d], %e (Barnard: %e)'%(ai,bi, ci,di,cumContrib,cumContribB))
# ==========================================================================    

    return cumA # right now it should return 1
# =======================================================

# =================================================
# Python codes for the study of 2x2 contingency tables
# For the new material in the 3rd edition of the textbook
# Notice the requirement that m1<=m2 and n1<=n2 so that
# we can count all possibile tables properly

#problem 10.1
#Simple N=10 table, to illustrate
'''
a=2
b=3 
c=1
d=4
tabLabel='Problem 10.1'
'''
# ==========================================
'''
# problem 10.3 and Example 10.4 of textbook
a=1
b=4
c=5
d=0
tabLabel='Problem 10.3 and Example 10.4'
# Invert columns to satisfy m1<m2, equivalent table
a=0
b=5
c=4
d=1
# ==========================================
'''
# Table 2 of Greenwood and Yule (1915), also reported 
# by Fisher (1925, but inverted columns)
#tabLabel='Yule Table II'
#a=6759
#b=56
#c=11396
#d=272
# Table VIII, with smaller counts
#tabLabel='Yule Table VIII'
#a=105
#b=5
#c=88
#d=11

# ========================================
# Table XV of Greenwood and Yule
# Rearrange so that n1<n2 and m1<m2
tabLabel='Yule Table XV'
a=54
b=5
c=46
d=15

a=5
b=54
c=15
d=46
# ========================================
#tabLabel='Mendel'
#a=101
#b=32
#c=315
#d=108

#tabLabel='prob10.3'
#a=4
#b=1
#c=0
#d=5

M=[[a,b],[c,d]] # this is the measured table
# ===================================
# Evaluate margins of the 2x2 table
N=a+b+c+d
n1=a+b
n2=c+d
p1=a/n1
p2=b/n2
m1=a+c
m2=b+d

print('N=%d, m1=%d, m2=%d, n1=%d, n2=%d'%(N,m1,m2,n1,n2))
print('Assumed n1/N (fixed margins): %3.3f, m1/N: %3.3f'%(n1/N,m1/N))
print('Measured ratios: a/n1: %3.3f b/n1: %3.3f, c/n2=%3.3f'%(a/n1,b/n1,c/n2))
print('Expectation of a: %3.3f'%(n1/N * m1/N * N))
# =================================
# Report also the probability to exceed measured chi^2
# based on the "wrong" chi^2(3) and the correct chi^2(1)
CHI2YULE=chi2Yule(a,b,c,d,N,m1,m2,n1,n2)
print('Yule\'s chi^2 for %s = %4.4f (P=%3.2e (3) or P=%3.2e (1))'%(tabLabel,CHI2YULE,chi2.sf(CHI2YULE,3),chi2.sf(CHI2YULE,1)))

chiPearson,chiIndividual,E=chi2Table(a,b,c,d,N,m1,m2,n1,n2)
CHI2YATES=chi2Yates(a,b,c,d,N,m1,m2,n1,n2)

# example 10.5 with a fixed p=0.75
p=0.75 # parent binomial probability
CHI2P,CHI2PIndiv,EP=chi2TableFixedP(a,b,c,d,N,p)


print('Yates\' chi^2 for %s = %4.4f (P=%3.2e (3) or P=%3.2e (1))'%(tabLabel,CHI2YATES,chi2.sf(CHI2YATES,3),chi2.sf(CHI2YATES,1)))
print('Pearson\'s chi^2 for %s = %4.4f'%(tabLabel,chiPearson))
print('Measured data',M)
print('Expectations based on fixed margins',E)
print('Individual chi^2 contributions',chiIndividual)

CHI2PPval=chi2.sf(CHI2P,3)
print('(Chi2 with p=%3.2f: %3.3f, p-value: %3.3f)'%(p,CHI2P,CHI2PPval))
print('(Expected values for p=%3.2f)'%p,EP)

# ==== EXACT TEST ================
EXACT=1
if EXACT==1:

# For the exact test, need to find probability that a >= a_observed
# 1. find how many values of "a" there are, a=0,...,m1 if m1<=n1
# and a=0,...,n1 if m1>n1
    numberA=min(m1,n1)+1 # this should take care of it

    print('Fisher\'s exact probability for %s that a=%d: %f'%(tabLabel,a,fisherExact(a,b,c,d,N,m1,m2,n1,n2)))
    print('Fisher\'s exact cumulative probability for %s that p(a)<=p(%d): %e'%(tabLabel,a,fisherExactCumulative(a,b,c,d,N,m1,m2,n1,n2)))
    print(fisher_exact(M))


# ===============================================
# Also do testing on Mendel's Table 1.3


convergencemcmc.py0000644000175000017500000001314614045745632012741 0ustar  maxmaxexec(open('imports.py').read())
# Read in data from the MCMC
DeltaaAux=[1,10,50]
DeltabAux=DeltaaAux
color=['red','black','blue']
ZGType=['Full Chain','Thinned Chain','Batch Means']
data = np.genfromtxt('mcmc5points.txt',skip_header=0)

a1=data[:,0]
b1=data[:,1]
a2=data[:,2]
b2=data[:,3]
a3=data[:,4]
b3=data[:,5]
N=len(a1)	# number of links in the chain
# put the chains in a compact form
a=[a1,a2,a3]
b=[b1,b2,b3]
print(len(a[0]))

# this is saved for later use 
out=open('a1.txt','w')
for i in range(N):
	out.write('%3.4f\n'%a1[i])
out.close()

out=open('a2.txt','w')
for i in range(N):
        out.write('%3.4f\n'%a2[i])
out.close()

out=open('a3.txt','w')
for i in range(N):
	out.write('%3.4f\n'%a3[i])
out.close()

out=open('b1.txt','w')
for i in range(N):
	out.write('%3.4f\n'%b1[i])
out.close()

out=open('b2.txt','w')
for i in range(N):
	out.write('%3.4f\n'%b2[i])
out.close()

out=open('b3.txt','w')
for i in range(N):
	out.write('%3.4f\n'%b3[i])
out.close()


thin=10	# thinning of the chain

NA=int(0.1*N)
NB=int(0.5*N)
print('Making Geweke Z scores for %d and %d segments'%(NA,NB))

NSteps=int((N-NA-NB)/thin)
NAthin=int(NA/thin)
NBthin=int(NB/thin)
print('Preparing %d ZG scores'%NSteps)
# setup means and variances
aMeanA=np.zeros((3,NSteps))
aMeanB=np.zeros((3,NSteps))
aVarA=np.zeros((3,NSteps))
aVarB=np.zeros((3,NSteps))
aVarABM=np.zeros((3,NSteps))
aVarBBM=np.zeros((3,NSteps))
aVarAThin=np.zeros((3,NSteps))
aVarBThin=np.zeros((3,NSteps))
bMeanA=np.zeros((3,NSteps))
bMeanB=np.zeros((3,NSteps))
bVarA=np.zeros((3,NSteps))
bVarB=np.zeros((3,NSteps))
bVarABM=np.zeros((3,NSteps))
bVarBBM=np.zeros((3,NSteps))
bVarAThin=np.zeros((3,NSteps))
bVarBThin=np.zeros((3,NSteps))
aZG=np.zeros((3,NSteps))
aZGThin=np.zeros((3,NSteps))
aZGBM=np.zeros((3,NSteps))

bZG=np.zeros((3,NSteps))
bZGThin=np.zeros((3,NSteps))
bZGBM=np.zeros((3,NSteps))

# Test the batch-means method to estimate variance
aVar=np.zeros(3)
aVarBM=np.zeros(3)
aVarThin=np.zeros(3)
iThin=np.arange(0,N,thin,dtype=int)
for j in range(3):
	aVar[j]=np.var(a[j][:],ddof=1) # sample variance
	aVarThin[j]=np.var(a[j][iThin],ddof=1) # thinned sample variance
	aVarBM[j]=VarBM(a[j])	# Batch Mean estimator for variance
	print('Variance of a[%d]: %3.3f, thinned a: %3.3f,  BM Variance: %3.3f'
		%(j,aVar[j],aVarThin[j],aVarBM[j]))

# ==================================================

fig,ax=plt.subplots(2,1,figsize=(8,6))
xaxis=np.arange(0,N-NB-NA,thin,dtype=int)
for i in range(NSteps):
	istart=i*thin
	inA=np.arange(istart,istart+NA,1,dtype=int)
	inAThin=np.arange(istart,istart+NA,thin,dtype=int)	# this moves forward
	inB=np.arange(N-NB,N,1,dtype=int)
	inBThin=np.arange(N-NB,N,thin,dtype=int)		# this is constant
	for j in range(3):	# for the 3 chains with different proposals
		# estimates of the mean in intervals A and B
		aMeanA[j,i]=np.mean(a[j][inA])
		aMeanB[j,i]=np.mean(a[j][inB])
		bMeanA[j,i]=np.mean(b[j][inA])
		bMeanB[j,i]=np.mean(b[j][inB])

		# 3 types of variance estimators: straight, thinned and BM
		# for segment A
		aVarA[j,i]=np.var(a[j][inA],ddof=1)/NA
		aVarAThin[j,i]=np.var(a[j][inAThin],ddof=1)/NAthin # variance of sample mean
		aVarABM[j,i]=VarBM(a[j][inA])/NA # Batch Mean variance
		aVarB[j,i]=np.var(a[j][inB],ddof=1)/NB
		aVarBThin[j,i]=np.var(a[j][inBThin],ddof=1)/NBthin
		aVarBBM[j,i]=VarBM(a[j][inB])/NB #    "      "
		# for segment B
		bVarA[j,i]=np.var(b[j][inA],ddof=1)/NA
		bVarAThin[j,i]=np.var(b[j][inAThin],ddof=1)/NAthin
		bVarABM[j,i]=VarBM(b[j][inA])/NA # Batch Mean variance
		bVarB[j,i]=np.var(b[j][inB],ddof=1)/NB 
		bVarBThin[j,i]=np.var(b[j][inBThin],ddof=1)/NBthin
		bVarBBM[j,i]=VarBM(b[j][inB])/NB #    "      "

		aZG[j,i]=(aMeanA[j,i]-aMeanB[j,i])/((aVarA[j,i]+aVarB[j,i])**0.5)
		aZGThin[j,i]=(aMeanA[j,i]-aMeanB[j,i])/((aVarAThin[j,i]+aVarBThin[j,i])**0.5)
		aZGBM[j,i]=(aMeanA[j,i]-aMeanB[j,i])/((aVarABM[j,i]+aVarBBM[j,i])**0.5)

		bZG[j,i]=(bMeanA[j,i]-bMeanB[j,i])/((bVarA[j,i]+bVarB[j,i])**0.5)
		bZGThin[j,i]=(bMeanA[j,i]-bMeanB[j,i])/((bVarAThin[j,i]+bVarBThin[j,i])**0.5)
		bZGBM[j,i]=(bMeanA[j,i]-bMeanB[j,i])/((bVarABM[j,i]+bVarBBM[j,i])**0.5)

		print('%d: ZG scores A: %2.1f B: %2.1f'%(i,aZG[j,i],bZG[j,i]))
# Now make plots
for j in range(1,2,1):
	ax[0].plot(xaxis,aZG[j,:],color=color[j],label='$\Delta a$=%2.1f %s'%(DeltaaAux[j],ZGType[0]))
	ax[0].plot(xaxis,aZGThin[j,:],color=color[j],linestyle=':',label='$\Delta a$=%2.1f %s'%(DeltaaAux[j],ZGType[1]))
	ax[0].plot(xaxis,aZGBM[j,:],color=color[j],linestyle='--',label='$\Delta a$=%2.1f %s'%(DeltaaAux[j],ZGType[2]))
	ax[1].plot(xaxis,bZG[j,:],color=color[j])#,label='$\Delta b$=%2.1f'%DeltabAux[j])
	ax[1].plot(xaxis,bZGThin[j,:],color=color[j],linestyle=':')
	ax[1].plot(xaxis,bZGBM[j,:],color=color[j],linestyle='--')
ax[0].grid(which='both')
ax[1].grid(which='both')
ax[0].set_ylabel('$Z_G$ Parameter $a$')
ax[1].set_xlabel('Start of segment A')
ax[1].set_ylabel('$Z_G$ Parameter $b$')
for axs in ax.flat:
        axs.label_outer()
ax[0].legend(prop={'size': 12})
#ax[1].legend(prop={'size': 12})
ax[0].set_xlim(xaxis[0],xaxis[-1])
ax[1].set_xlim(xaxis[0],xaxis[-1])
ax[0].hlines(0,xaxis[0],xaxis[-1],linestyle='--',color='black')
ax[0].hlines(3,xaxis[0],xaxis[-1],linestyle=':',color='black')
ax[0].hlines(-3,xaxis[0],xaxis[-1],linestyle=':',color='black')
ax[1].hlines(0,xaxis[0],xaxis[-1],linestyle='--',color='black')
ax[1].hlines(3,xaxis[0],xaxis[-1],linestyle=':',color='black')
ax[1].hlines(-3,xaxis[0],xaxis[-1],linestyle=':',color='black')
ax[1].yaxis.set_major_locator(MultipleLocator(2))
ax[1].yaxis.set_minor_locator(MultipleLocator(0.5))
ax[0].yaxis.set_major_locator(MultipleLocator(2))
ax[0].yaxis.set_minor_locator(MultipleLocator(0.5))

ax[0].set_ylim(-5,5)
ax[1].set_ylim(-5,5)
plt.savefig('mcmcZg.pdf')
covidContingency.py0000644000175000017500000000316113745254651013106 0ustar  maxmaxexec(open('imports.py').read())
from math import factorial as fact
from scipy.stats import fisher_exact,chi2

# Analyze the OpenSAFELY data using contingency tables

data = np.genfromtxt('openSafely.dat')
print(data)

r=6     # 6 rows
c=2     # 2 columns

totals=data[:,1]
deaths=data[:,2]
survivals=totals-deaths
print(deaths,survivals)
N=sum(totals)

# re-arrange the table with survivals, deaths only as 6x2 table
M=np.zeros((r,c))   # this is the table with measured values
M[:,0]=deaths
M[:,1]=survivals

print(M)

#   ===================================================
# Now find the margins
p=deaths/survivals  # measured probabilities
n=totals
m=np.zeros(c)
m[0]=np.sum(deaths)
m[1]=np.sum(survivals)
print('Margins n:', n)
print('Margins m:',m)
print('Measured probabilities',p)
print('m1/N',m[0]/N)

# ===================================================
# Calculate the expected rates, assuming independence
# Also calculate chi^2

E=np.zeros((r,c)) # this is the table with expectations, from independence
for i in range(r):
    for j in range(c):
        E[i,j]=m[j]*n[i]/N
        print('%f (%f) '%(M[i][j],E[i][j]), end=' ')
    print(' ')

# All chi2 values can be obtained as 
Chi2Table=(E-M)**2/E
print('Table with chi2 values:',Chi2Table)
chi2Measured=np.sum(Chi2Table)
print('Total chi2=%f'%chi2Measured)
dof=(r-1)*(c-1) # degrees of freedom for thic chi2
# Use numpy's inverse survival function for critical value, at 90% level
print('Critical value for parent distrib. chi2(%d)=%f'%(dof,chi2.isf(0.1,dof)))

print('Sums of deaths (%d), survivals (%d), combined (%d) and expectations (%f)'%(m[0],m[1],m[0]+m[1],np.sum(E[:,0])))
cstatChi.py0000664000175000017500000000310214116231626011326 0ustar  maxmaxexec(open('imports.py').read())

# sample data for problems for chapters 15 and 16
N=9
#problem 15.4
print('Problem 15.4')
scale=2
y=scale*np.asarray([2,4,6,2,4,6,2,4,6]) # sample dataset
print(y,len(y))
ymodel=4*scale**np.ones(N) # fixed model - no fitting

# statistics
Cstat=cstat(y,ymodel)
chi=chisquared(y,y**0.5,ymodel)

# critical value of chi2
p=0.9
chiCrit=chi2.ppf(p,N) 
# critical value of cstat
mean=4*scale #  prob15.4
j=1
Expect=funcECmin(mean,A[j],B[j],C[j],D[j],E[j],betaB[j],F[j],alphaB[j])
j=3
Variance=funcVarC(mean,A[j],B[j],C[j],D[j],E[j],F[j],G[j],H[j],alphaB[j],betaB[j],I[j])
print('for mu=%3.3f: E=%3.3f, Var=%3.3f'%(mean,Expect,Variance))
EC=Expect*N
VarC=N*Variance
print('Expected C=%3.3f \pm %3.3f'%(EC,VarC**0.5))
# For N=100, Gaussian approx for C applies.
q=1.3
CCrit=EC+q*(VarC)**0.5
print('C crit for N=%d: %3.3f'%(N,CCrit))


print('chi2=%3.3f (%3.2f), cstat=%3.3f (%3.2f)'%(chi,chiCrit,Cstat,CCrit))

######################
#problem 16.8 
print('Problem 16.8')
scale=2
y=scale*np.asarray([2,4,6,2,4,6,2,4,6]) # sample dataset

xaxis=np.ones(len(y)) # this is irrelevant since it's constant model
popt, pcov = curve_fit(constant,xaxis , y, sigma=y**0.5,absolute_sigma=True)
print('chi2 curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f '%(popt[0],pcov[0][0]**0.5))

######## now cstat
ahat=np.mean(y) # this is the known solution
ymod=np.ones(len(y))*ahat
cmin=cstat(y,ymod)
print(' cmin=%3.2f for ahat=%3.2f'%(cmin,ahat))
a=np.linspace(6,10,40)
c=np.zeros(40)
for i in range(len(a)):
    c[i]=cstat(y,a[i]*np.ones(len(y)))
    #print('a=%3.2f cstat=%3.2f'%(a[i],c[i]))

DeltachiF.py0000644000175000017500000000266214015057245011420 0ustar  maxmaxexec(open('imports.py').read())

# Make a simulation of Delta chi^2 for the detection of a line
# Assume that there are 100 bins with a constant model
# and that the line lies in one specific bin, say the first.

alpha=100 # parent value of the continuum
NBins=100
NSim=1000	# number of simulations
DeltaChi2=np.zeros(NSim)
b=np.zeros(NSim)
x=np.arange(NBins)	# x-axis of data, irrelevant
for i in range(NSim):
	data = poisson.rvs(alpha,size=NBins)
	error=data**0.5
	poptC, pcovC = curve_fit(constant, x,data, sigma=error,absolute_sigma=True)	
	modelC=constant(x,poptC[0])
	chiminC=chisquared(data,error,modelC)
	poptCP, pcovCP = curve_fit(constantPlusOne, x,data, sigma=error,absolute_sigma=True,bounds=([-100,-100],[100,0]))
	modelCP=constantPlusOne(x,poptCP[0],poptCP[1])
	b[i]=poptCP[1] # best-fit of parameter "b" of line
	chiminCP=chisquared(data,error,modelCP)
	DeltaChi2[i]=chiminC-chiminCP
	print(DeltaChi2[i],b[i])

# now plot distribution of DeltaChi2
fig,ax=plt.subplots(figsize=(8,6))
xaxis=np.linspace(0,10,100)
yaxis=chi2.pdf(xaxis,1)
plt.hist(DeltaChi2,bins=100,density=True,color='black')
plt.plot(xaxis,yaxis)
plt.xlabel('$\Delta chi^2$')
plt.ylabel('Prob. Distribution')
plt.xlim(0,6)
plt.savefig('Deltachi2F.pdf')

# plot distribution of b values
fig,ax=plt.subplots(figsize=(8,6))
plt.hist(b,bins=100,density=True,color='red')
plt.xlim(-30,30)
plt.xlabel('$b$ parameter')
plt.ylabel('Prob. Distribution')
plt.savefig('Deltachi2F-b.pdf')
DeltaC.py0000644000175000017500000000446214104315757010735 0ustar  maxmaxexec(open('imports.py').read())
# Numerical calculation of Delta C
# In the low--count and low--N limit where no approximation is available
# For the constant model

# Choose a number of measurements and a parent mean
N=10    # number of measurements in teh dataset
mu=0.1  # parent Poisson mean

# Choose a sample size for the simulation
nSize=10000
sampleP=np.zeros(nSize)
sampleCtrue=np.zeros(nSize)
sampleCmin=np.zeros(nSize)
DeltaC=np.zeros(nSize)
Ctrue=np.zeros(nSize)
Cmin=np.zeros(nSize)

# Simulate the distribution of Delta C
for i in range(nSize):
    sampleP=poisson.rvs(mu,size=N) # random values
    a=np.average(sampleP) # this is the best--fit parameter a
    for j in range(N):
#        Calculate the contributions to Ctrue and Cmin
        sampleCtrue[j]=CContribwTwo(sampleP[j],mu)
        sampleCmin[j]=CContribwTwo(sampleP[j],a)
    # Now calculate Cmin and C=C_true(mu)
    Ctrue[i]=np.sum(sampleCtrue)
    Cmin[i]=np.sum(sampleCmin)
    DeltaC[i]=Ctrue[i]-Cmin[i]
    #print(sampleP,end='')
    #print('a=%3.3f, mu=%3.3f '%(a,mu),end='')
    print('%3.3f-%3.3f=%3.3f'%(Ctrue[i],Cmin[i],DeltaC[i]))

fig,ax=plt.subplots(figsize=(8,6))

bins=np.linspace(0.0,10,1000)
n,bins,_=ax.hist(DeltaC,bins=bins,cumulative=True,histtype='step',log=False,density=True,linewidth=2,alpha=1.0,color='black',label='$\Delta C$ for N=%d, $\mu$=%2.2f'%(N,mu))

binsCenter=(bins[1:] + bins[:-1])/2 # arithmetic mean

ax.set_ylim(0,1)
ax.set_yscale('linear')
ax.set_xscale('log')
ax.set_xlim(0.5,10)
plt.grid(which='both')
# formatting log axis with linear labels
for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())

# also overplot horizontal lines for notable critical values
hlines=[0.683,0.90,0.95]
for  i in range(3):
    plt.hlines(hlines[i],0.1,10,color='blue',linewidth=2,linestyle='--')


# overplot the chi^2(1) distribution, for comparison
ax.plot(binsCenter,chi2.cdf(binsCenter,1),linewidth=2,color='red',linestyle='--',label='$\chi^2(1)$')

plt.legend(loc=4,prop={'size': 12})
ax.set_xlabel('Values of $\Delta C$')
ax.set_ylabel('Sample Cumulative Distribution')

#plt.show()
plt.savefig('DeltaC.pdf')

# ===============================
# add calculation of binomial probabilities to obtain datasets

n=[0,1,2,3]
for i in range(len(n)):
    print('Binomial prob for n=%d: %4.3f'%(n[i],binom.pmf(n[i],N,mu)))



fatherMother.py0000644000175000017500000002360014116113360012211 0ustar  maxmaxexec(open('imports.py').read())

# ==================================================================
# Define ad-hoc functions for variance and covariance using binned data
# These will be extended to non-integer data, as in Pearson's biometric data

# x: x coordinate of the binned array, n-dim. array/list
# multX: number of occurrences for the x value, n-dim. array/list
def varBinned(x,multX):
    xMean=np.average(x,weights=multX) 
    N=sum(multX) # total number of counts
    result=0
    for i in range(len(x)): #using the binned data
        result+=multX[i]*(x[i]-xMean)**2
    return result/(N-1)

# for the covariance, 
# x: x coordinate of the binned array, n-dim. array/list
# y: y coordinate, m-dim.
# ( grid: nxm matrix with (x,y) coordinates of the 2-D grid, using meshgrid )
# mult[i,j] is an nxm matrix
# n: # of bins for x variable
# m: # of bins for y variable
def covBinned(x,y,mult):
    # first need multX and multY, the multiplicities at fixed X or Y coordinate
    n=len(x) # 18
    m=len(y) # 19
    N=sum(mult.flatten()) # it needs to be flattened before summing over
    print('N=%f'%N)
    multX=np.zeros(n)
    multY=np.zeros(m)
    print('n=%d, m=%d, x[0]=%f, y[0]=%3.2f'%(n,m,x[0],y[0]))
    for i in range(n):
        multX[i]=sum(mult[:,i]) #multX[j]=sum(data[:,j])
    for j in range(m):
        multY[j]=sum(mult[j,:])

    xMean=np.average(x,weights=multX)
    yMean=np.average(y,weights=multY)
    print('Average father height: %3.4f, Average mother height: %3.4f, N=%d'%(xMean,yMean,N))
    result=0
    for i in range(n):
        for j in range(m):
            # x is mean of father's hight
            currentElement=(x[i]-xMean)*(y[j]-yMean)*mult[j,i]
            #print('i=%d,j=%d,mult[%d,%d]=%f'%(i,j,j,i,mult[j,i]))
            result+=currentElement# i,j]
    result/=(N-1) # check the N-1
    print('result:',result)
    return result

# Read data and debug ================================================
data = np.genfromtxt('fatherMotherStature.dat')
print('Dimensions of data:',len(data))
#print(data)
# mother[i,:] are rows of data for a fixed mother's height
motherRows=19 # this is y axis, m
fatherCols=18 # this is x axis, n
n=fatherCols
m=motherRows
print('Last data point: data[%d,%d]=%f'%(m,n,data[m-1,n-1]))
motherMin=52
motherMax=70
fatherMin=58
fatherMax=75
xaxisFather=np.linspace(fatherMin,fatherMax,num=fatherCols)+0.5
yaxisMother=np.linspace(motherMin,motherMax,num=motherRows)+0.5
print(xaxisFather,yaxisMother)

# -------------------------------------
multX=np.zeros(fatherCols) # 
multY=np.zeros(motherRows)
avgMotherHeightY=np.zeros(fatherCols)
avgFatherHeightX=np.zeros(motherRows)
# also sum of occurrence of that bin
sumNumberMotherHeight=np.zeros(motherRows)
sumNumberFatherHeight=np.zeros(fatherCols)

# Print rows and columns, for debugging
for i in range(motherRows):
    #print('row %d'%i,end=' ')
    #print(data[i,:])
    multY[i]=sum(data[i,:])
    # also add the average X=father's height for that Y
    avgFatherHeightX[i]=sum(data[i,:]*xaxisFather[:])/sum(data[i,:])
    # below is for the marginal distribution
    sumNumberMotherHeight[i]=sum(data[i,:])
print('Fathers height',avgFatherHeightX)
for j in range(fatherCols):
    #print('col %d'%j,end=' ');
    #print(data[:,j])
    multX[j]=sum(data[:,j])
    # also add the average Y=mother's height for that X
    avgMotherHeightY[j]=sum(data[:,j]*yaxisMother[:])/sum(data[:,j])
    sumNumberFatherHeight[j]=sum(data[:,j])
print('Mothers height',avgMotherHeightY)

# test the variance and covariance routines
x=xaxisFather
y=yaxisMother
meanx=np.average(x,weights=multX)
meany=np.average(y,weights=multY)
varx=varBinned(x,multX)
vary=varBinned(y,multY)
covxy=covBinned(x,y,data)
N=sum(data.flatten()) # Number of counts
print('N=%d, E[X]=%3.3f, E[Y]=%3.3f'%(N,meanx,meany))
print('var x: %3.2f, var y: %3.2f, cov: %3.2f'%(varx,vary,covxy))

# ======================================================================

# 1.1 Plot the data to reproduce the figure in the 2nd edition
#plt.imshow(data)
#plt.show()

fig,ax=plt.subplots(figsize=(8,8))
# 1.2 Now try again with a flattened array
dataFlat=data.flatten(order='C')
for i in range(motherRows):
    for j in range(fatherCols):
        # rescale the size of the datapoints
        counts=dataFlat[j+i*fatherCols]
#        counts=dataFlat[i+j*fatherCols] 
        markerSize=4*((counts>0)+counts)
        plt.scatter(xaxisFather[j],yaxisMother[i],s=markerSize,facecolors='none',color='black',marker='o')
        if (counts>0):
            plt.text(xaxisFather[j]-0.45,yaxisMother[i]-0.45,"%2.2f"%counts,fontsize=4)

# Set the tick mark locations
ax.xaxis.set_major_locator(MultipleLocator(5))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(5))
ax.yaxis.set_minor_locator(MultipleLocator(1))

plt.xlabel('X (Father\'s Height)')
plt.ylabel('Y (Mother\'s Height)')
plt.xlim(fatherMin,fatherMax+2) # to preserve the aspect ratio
plt.ylim(motherMin,motherMax+1)
plt.grid(which='both')
#plt.savefig('pearsonScatter2D.pdf')


# ==============================================================
# Now do the two linear regressions
# Use variances and covariance calculated above
# 2.1 X= father's height; Y= mother's height
bYgX=covxy/varx
aYgX=meany-bYgX*meanx
print('Regression Y given X: a=%3.3f, b=%3.3f'%(aYgX,bYgX))
xPlot=np.zeros(2)
xPlot=[fatherMin,fatherMax+1]
yPlot=np.zeros(2)
for i in range(2):
    yPlot[i]=(bYgX*xPlot[i])+aYgX
plt.plot(xPlot,yPlot,linewidth=2,color='blue',label='Regression of Y on X')

# 2.2 X = mother's height; Y= father's height

bXgY=vary/covxy
aXgY=meany-bXgY*meanx
print('Regression X given Y: a=%3.3f, b=%3.3f'%(aXgY,bXgY))
for i in range(2):
    yPlot[i]=(bXgY*xPlot[i])+aXgY
plt.plot(xPlot,yPlot,linewidth=2,color='red',label='Regression of X on Y')
plt.legend(loc=4,prop={'size': 12})
plt.savefig('pearsonScatter2D.pdf')


# problem 14.2 (r statistic)
# Also calculate r^2
r=covxy/(varx*vary)**0.5
print('Problem 14.2')
print('Correlation coefficient r=%3.3f, r**2=%3.3f'%(r,r**2))
print('b*b\'=%3.3f'%(bYgX/bXgY))


# ======================================================================================
# Evaluate the critical values based on symmetric beta r-distribution 
# and the standard beta distr. for r^2
f=N-2
print('Number of degrees of freedom for r: %d'%f)
# Double--sided critical value
# The two--sided probability is accounted for by 0.99 -> 0.995 for the Critical Value
# NOTE: PYTHON HAS NUMERICAL PROBLEMS WITH THIS P-VALUE 
pValuer=rdist.sf(abs(r), f) # this is one-sided survival probability
pValuer=pValuer*2 # add the other tail
critr=rdist.ppf(0.995,f)
print('critical value of r: %3.4f (for r^2=%3.5f), p value of r: %3.3e'%(critr,critr**2,pValuer))
# Single--sided critical value:
pValuer2=beta.sf(r**2, 1/2,f/2)
critr2=beta.ppf(0.99,1/2,f/2)
print('(One--sided) critical value of r2: %3.5f, p value of r2: %3.3e'%(critr2,pValuer2))
# =======================================================================
# Check an alternative way to do regression by using X vs <Y> values
# 3.1 Using X= Father's height, get average <Y> = Mother's height
# this is multX[j]
# problem 14.1
print('Problem 14.1')
fig,ax=plt.subplots(figsize=(8,8))
ax.xaxis.set_major_locator(MultipleLocator(5))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(5))
ax.yaxis.set_minor_locator(MultipleLocator(1))

print('Now analyzing father columns')
for j in range(fatherCols):
    
    print('%d: X=%2.2f, sum <Y>=%2.2f, <Y>=%2.2f'%(j,xaxisFather[j],multX[j],avgMotherHeightY[j]))

plt.plot(xaxisFather,avgMotherHeightY,color='blue',linestyle='',marker='o',label='(X, <Y>) data ')

slope, intercept, r_value, p_value, std_er=linregress(xaxisFather,avgMotherHeightY)
print('Fit of Avg Y given X:', slope,intercept)
for i in range(2):
    yPlot[i]=(slope*xPlot[i])+intercept
plt.plot(xPlot,yPlot,linewidth=2,color='blue',linestyle='--',label='Regression of <Y> on X')
# repeat the plot from the contingency table
for i in range(2):
    yPlot[i]=(bYgX*xPlot[i])+aYgX
plt.plot(xPlot,yPlot,linewidth=2,color='blue',label='Regression of Y on X')

#3.2, usinx Y=Mother's Height, and <X> = Father's height
print('Now analyzing mother rows')
for i in range(motherRows):

    print('%d: X=%2.2f, sum <Y>=%2.2f, <Y>=%2.2f'%(i,yaxisMother[i],multY[i],avgFatherHeightX[i]))

plt.plot(avgFatherHeightX,yaxisMother,color='red',linestyle='',marker='o',label='(<X>,Y) data')
slope, intercept, r_value, p_value, std_er=linregress(avgFatherHeightX,yaxisMother)
print('Fit of Avg. X given Y',slope,intercept)
for i in range(2):
    yPlot[i]=(slope*xPlot[i])+intercept
plt.plot(xPlot,yPlot,linewidth=2,color='red',linestyle='--',label='Regression of <X> on Y')
# repeat the plot from the contingency table
for i in range(2):
    yPlot[i]=(bXgY*xPlot[i])+aXgY
plt.plot(xPlot,yPlot,linewidth=2,color='red',label='Regression of X on Y')

plt.legend(loc=4,prop={'size': 12})
plt.xlabel('X (Father\'s Height)')
plt.ylabel('Y (Mother\'s Height)')
plt.xlim(fatherMin,fatherMax+2) # To preserve the aspect ratio
plt.ylim(motherMin,motherMax+1)
plt.grid(which='both')
plt.savefig('pearsonXvsavgY.pdf')

# ==============================================================
# Plot another figure for Chapter 2, including marginals
# Use the sums by row and column
fig,ax=plt.subplots(figsize=(8,6))
ax.plot(xaxisFather,sumNumberFatherHeight,marker='o',linewidth=2,color='black',label='Father\'s Height')
ax.plot(yaxisMother,sumNumberMotherHeight,marker='o',linewidth=2,color='blue',label='Mother\'s Height')
ax.set_xlabel('Height')
ax.set_ylabel('Number of occurrence')
plt.ylim(0,200)
plt.legend(loc=2,prop={'size': 12})
ax.xaxis.set_major_locator(MultipleLocator(5))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(20))
ax.yaxis.set_minor_locator(MultipleLocator(5))
plt.grid(which='both')

plt.savefig('pearsonMarginals.pdf')
# Check the sample means of father's and mother's heights
meanFHeight=np.average(xaxisFather,weights=sumNumberFatherHeight)
print('Mean height of father: %3.2f'%meanFHeight)
gehrels.py0000644000175000017500000000353314114402041011210 0ustar  maxmaxexec(open('imports.py').read())


# Make a plot to replace Fig. 5.2, to illustrate the confidence
# intervals on a Poisson variable

xmax=10
x=np.linspace(0,xmax,xmax+1,endpoint=True,dtype=int)
n=3 # this is the example of a number of detected counts
mu1=0.83
mu2=7.8
p1 = poisson.pmf(x,mu1)
p2= poisson.pmf(x,mu2)

fig,ax=plt.subplots(figsize=(8,6))

ax.plot(x,p1,color='black',linewidth=2,linestyle=':')#,label='mean $\mu_{lo}$')#=%d'%mu1)
ax.plot(x,p1,color='black',marker='o',linestyle='')
ax.plot(x,p2,color='blue',linewidth=2,linestyle=':')#,label='mean $\mu_{up}$')#=%d'%mu2)
ax.plot(x,p2,color='blue',marker='o',linestyle='')
ax.fill_between(x[3:10],p1[3:10],color='grey',alpha=0.5,label='prob. 1-$p$')
ax.fill_between(x[0:4],p2[0:4],color='blue',alpha=0.5,label='prob. 1-$p$')

plt.xlabel('X')
plt.ylabel('Poisson Distribution')
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(1))
ax.yaxis.set_major_locator(MultipleLocator(0.10))
ax.yaxis.set_minor_locator(MultipleLocator(0.10))
ax.vlines(3,0.0,1.0,linestyle='--',color='red',linewidth=2,label='measurement')
plt.hlines(0.05,0.,10.,color='grey')
plt.hlines(0.95,0.,10.,color='grey')
plt.grid(which='both')
ax.set_xlim(0,8)
ax.set_ylim(0,1.0)

# let's add the cumulative distributions
P1 = poisson.cdf(x,mu1)
P2= poisson.cdf(x,mu2)
ax.plot(x,P1,color='black',linewidth=2,label='mean $\mu_{lo}$')
ax.plot(x,P2,color='blue',linewidth=2,label='mean $\mu_{up}$')



plt.legend(loc=1,prop={'size': 12})
plt.savefig('gehrels.pdf')
print(x)


# Check that the S parameters correspond to the p-quantiles
p=[0.9,0.95,0.99,0.841,0.977,0.999]
S=norm.ppf(p)
print(p,S)

# =======================================
# Also check numerical solution of prob7.2
print('problem 7.2 (c)')
def ULfunc(x,p):
	return np.exp(-x)*(1+x)-(1-p)
p=0.84
muup=fsolve(ULfunc,3,args=(p))
print('Upper limit',muup)

gelmanRubin.py0000644000175000017500000000524014072051333012026 0ustar  maxmaxexec(open('imports.py').read())
# Read in data from the MCMC
# Test with the same chains as for Geweke Z scores
DeltaaAux=[1,10,50]
DeltabAux=DeltaaAux
color=['red','black','blue']
ZGType=['Full Chain','Thinned Chain','Batch Means']
data = np.genfromtxt('mcmc5points.txt',skip_header=0)

a1=data[:,0]
b1=data[:,1]
a2=data[:,2]
b2=data[:,3]
a3=data[:,4]
b3=data[:,5]
print(a1)
# put the chains in a compact form
aMCMC=[a1,a2,a3]
bMCMC=[b1,b2,b3]
quit()
# Set up the RHat score
m=3

n=len(a1)       # number of links in the chain
b=100 # length of each batch
N=int(n/b) # number of batches
thetaMeanI=np.zeros((m,N)) # by stepping the window forward
thetaMean=np.zeros(N) # mean of means
B=np.zeros(N)
W=np.zeros(N)
VHat=np.zeros(N)
RHat=np.zeros(N)
s2i=np.zeros((m,N)) # sample variance of  i-th chain
Ni=np.zeros(N) # number of chain elements considered)
for i in range(N):
	Ni[i]= b*(i+1) # N-i # number of elements of chain considered
	indexRange=np.arange(0,b*(i+1),1) 
	print('from %d to %d'%(min(indexRange),max(indexRange)))
	for j in range(m):  #j-th chain
		thetaMeanI[j,i]=np.mean(aMCMC[j][indexRange]) # from i to N
		s2i[j,i]=np.var(aMCMC[j][indexRange],ddof=1)
	thetaMean[i]=np.mean(thetaMeanI[:,i]) # mean of m chains
	print('thetaMean=%3.2f'%thetaMean[i])
	B[i]=Ni[i]*np.var(thetaMeanI[:,i],ddof=1) 
	W[i]=np.mean(s2i[:,i]) # mean of sample variances
	VHat[i]=((Ni[i]-1)/Ni[i])*W[i] +B[i]/Ni[i] +B[i]/(m*Ni[i])
	RHat[i]=VHat[i]/W[i]
	print('%d N=%d thetaMean=%3.3f, B=%3.3f, B/N=%3.3f, W=%3.3f, VHat=%3.3f, RHat=%3.3f'
		%(i,Ni[i],thetaMean[i], B[i],B[i]/Ni[i],W[i],VHat[i],RHat[i]))
xaxis=np.arange(b,(N+1)*b,b)
# exclude the last few RHat scores because not meaningful
fig,ax=plt.subplots(1,1,figsize=(8,6))
plt.plot(xaxis[0:N],RHat[0:N]**0.5,color='black',linewidth=2,label='$\sqrt{\hat{R}}$ statistic')
plt.plot(xaxis[0:N],VHat[0:N]**0.5,color='blue',linewidth=2,label='$\sqrt{\hat{V}}$ (variance)')
plt.plot(xaxis[0:N],W[0:N]**0.5,color='red',linewidth=2,label='$\sqrt{W}$ (within-chain variance)')
plt.plot(xaxis[0:N],(B[0:N]/Ni)**0.5,color='green',linewidth=2,label='$\sqrt{B/N}$ (between-chain variance)')
plt.plot(xaxis[0:N],thetaMean/10,color='grey',linewidth=2,label='Mean of ${a}/{10}$')
plt.legend(prop={'size': 12})
plt.xlabel('Number of Iterations')
plt.ylabel('Statistic')
plt.grid(which='both')
ax.yaxis.set_major_locator(MultipleLocator(1))
ax.yaxis.set_minor_locator(MultipleLocator(0.2))
ax.xaxis.set_major_locator(MultipleLocator(2000))
ax.xaxis.set_minor_locator(MultipleLocator(500))
plt.hlines(25.44/10,xaxis[0],xaxis[-1],linestyle='--',color='grey')
plt.hlines(4.26,xaxis[0],xaxis[-1],linestyle='--',color='blue')
plt.ylim((0,8))
plt.xlim((0,10000))
plt.savefig('gelmanRubin.pdf')
hubble.py0000664000175000017500000001345014116656673011051 0ustar  maxmaxexec(open('imports.py').read())

# Python code to do statistics from Hubble data
# Bonamente 2020

dataTypes=[('name','S8'),('v','f'),('N','I'),('m','f')]

data = np.genfromtxt('hubbleHumasonData.dat',skip_header=1,dtype=dataTypes)
print(data)

v=data['v']
m=data['m']

logv=np.log10(v)

# Do a simple linear regression (with no errors) prob11.1

slope, intercept, r_value, p_value, std_err = stats.linregress(m,logv)
print("Problem 11.1")
print("slope: %f    intercept: %f" % (slope, intercept))
print('Problem 14.3')
print("r=%3.4f, R-squared: %f" % (r_value,r_value**2))
print("============")
# check that these formulas are the same as using variance and covariance
N=len(m)
covMatrix=np.cov(m,logv,ddof=1)
covXY=covMatrix[0,1]
varX=covMatrix[0,0]
varY=covMatrix[1,1]
EX=np.mean(m)
EY=np.mean(logv)
b=covXY/varX
a=EY-b*EX
print('var(X)=%3.3f, Cov(X,Y)=%3.3f, a=%3.3f, b=%3.3f'%(varX,covXY,a,b))
# Now also repeat the fit but X/Y, and then converting back to a and b
bXY=varY/covXY
aXY=EY-bXY*EX
print(' aXY=%3.3f, bXY=%3.3f'%(aXY,bXY))

fig,ax=plt.subplots(figsize=(8,6))

xmin=11
xmax=20
xline=[xmin,xmax]
# regression of Y on X
yline=[intercept+slope*xline[0],intercept+slope*xline[1]]
# regression of X on Y
ylineXY=[aXY+bXY*xline[0],aXY+bXY*xline[1]]

plt.plot(m,logv,linestyle='',marker='o',color='black', label='Hubble data (no errors)')
plt.plot(xline,yline,color='black',linewidth=2,label='Linear regression of Y on X')
plt.plot(xline,ylineXY,color='blue',linewidth=2,linestyle='--',label='Linear regression of X on Y')
plt.xlabel('Magnitude m')
plt.ylabel('Log velocity')
ax.yaxis.set_major_locator(MultipleLocator(0.5))
ax.yaxis.set_minor_locator(MultipleLocator(0.25))
ax.xaxis.set_major_locator(MultipleLocator(2))
ax.xaxis.set_minor_locator(MultipleLocator(1.0))
plt.grid(which='both')


# Now get the model variance and then repeat the fit - prob12.1
print('Problem 12.1')
ymodel=linear(m,intercept,slope)
dof=2# number of free parameters (notice the misnomer)
mVar=modelVariance(logv,ymodel,dof)
print('Model variance is %f (sqrt: %f)'%(mVar,mVar**0.5))

modelError=np.ones(len(m))*(mVar**0.5)
popt,pcov = curve_fit(linear,m,logv,sigma=modelError)
print("a=%4.3f +- %4.3f, b=%4.3f +- %4.3f"%(popt[0],pcov[0][0]**0.5,popt[1],pcov[1][1]**0.5))
print("Cov(a,b)=%3.3f, or r=%3.3f between a and b"%(pcov[0][1],pcov[0][1]/(pcov[0][0]*pcov[1][1])**0.5))
chi2min=chisquared(logv,modelError,ymodel)
print("chi2min using the model sample variance: %3.3f"%chi2min)
#Add the error bars from model variance to the plot
#plt.errorbar(m,logv,yerr=modelError,marker='.',linestyle='',color='blue',capsize=2,label='model errors')
plt.legend(prop={'size': 12})
plt.savefig('hubbleFit.pdf')


# For problem 17.4, do a fit with sigma_i=0.01 and then find the intrinsic scatter
print('problem 17.4')
sigma=0.01*np.ones(len(logv)) # this is the error assumed
sigmaInt=intrinsicScatter(logv,ymodel,sigma,dof)
print('intrinsic scatter: %4.4f'%sigmaInt)

## ==============================================
# jackknife  on the Hubble data
aJackj=np.zeros(N)
bJackj=np.zeros(N)
aJackjStar=np.zeros(N)
bJackjStar=np.zeros(N)
mj=np.zeros((N,N-1)) # these are the resampled datasets, N sets of size N-1
logvj=np.zeros((N,N-1))
# start with the estimates from the original data
bhat,ahat,r_value, p_value, std_err = stats.linregress(m,logv)
for j in range(N):
	mj[j,:]=np.delete(m,j)
	#print(j, m,mj[0,:])
	logvj[j,:]=np.delete(logv,j)
	# these are the theta_j estimates from Zj
	bJackj[j],aJackj[j],r_value, p_value, std_err = stats.linregress(mj[j,:],logvj[j,:])
	aJackjStar[j]=N*ahat-(N-1)*aJackj[j]
	bJackjStar[j]=N*bhat-(N-1)*bJackj[j]
meanaJackStar=np.mean(aJackjStar)
erraJackStar=np.std(aJackjStar,ddof=1)/N**0.5 # this is error on mean
meanbJackStar=np.mean(bJackjStar)
errbJackStar=np.std(bJackjStar,ddof=1)/N**0.5

print('Jackknife estimates: a=%3.3f +-%3.3f, b=%3.3f+-%3.3f'%(meanaJackStar,erraJackStar,meanbJackStar,errbJackStar))

# ==================================
# bootstrap, also Problem 20.5
print('Problem 20.5, bootstrap of Hubble data')
NBoot=10000
aBoot=np.zeros(NBoot)
bBoot=np.zeros(NBoot)
indices=np.arange(N)
print(indices,m,m[indices])
for i in range(NBoot):
	indicesChoice=choices(indices,k=N)
	bBoot[i],aBoot[i],r_value, p_value, std_err = stats.linregress(m[indicesChoice],logv[indicesChoice])
fig,ax=plt.subplots(figsize=(8,6))
binsa=np.linspace(0,2,60)
binsb=np.linspace(0.12,0.28,60)
print('bins:',binsa,binsb)
plt.hist(aBoot,bins=binsa,linewidth=2,density=True,histtype='step',color='black',label='Parameter a')
plt.hist(bBoot,bins=binsb,linewidth=2,density=True,histtype='step',color='blue',label='Parameter b')
plt.xlabel('Parameter value')
plt.ylabel('Distribution')
plt.legend(prop={'size': 12})
plt.xlim((0.1,1.0))
plt.ylim((0,5))
ax.xaxis.set_major_locator(MultipleLocator(0.1))
ax.xaxis.set_minor_locator(MultipleLocator(0.025))
ax.yaxis.set_major_locator(MultipleLocator(1))
ax.yaxis.set_minor_locator(MultipleLocator(0.2))
plt.grid(which='both')

# mean, median and percentiles
xaxis=np.linspace(0,2,2000)
p=0.683
alpha=(1-p)/2
beta=(1+p)/2
aBootMean=np.mean(aBoot)
aBootMedian=np.median(aBoot)
erraBoot=np.std(aBoot)
alphaa=np.percentile(aBoot,alpha*100)
yaxis=norm.pdf(xaxis,loc=aBootMean,scale=erraBoot)
plt.plot(xaxis,yaxis,linewidth=1,linestyle='--',color='black')
betaa=np.percentile(aBoot,beta*100)
bBootMean=np.mean(bBoot)
bBootMedian=np.median(bBoot)
errbBoot=np.std(bBoot)
alphab=np.percentile(bBoot,alpha*100)
betab=np.percentile(bBoot,beta*100)
yaxis=norm.pdf(xaxis,loc=bBootMean,scale=errbBoot)
plt.plot(xaxis,yaxis,linewidth=1,linestyle='--',color='blue')
print('Bootstrap estimates (mean and std. dev.: a=%3.3f+-%3.3f, b=%3.3f+-%3.3f'
	%(aBootMean,erraBoot,bBootMean,errbBoot))
print('Bootstrap median and p=%3.3f central intervals:'%p)
print('a=%3.3f (%3.3f-%3.3f), b=%3.3f ((%3.3f-%3.3f)'
	%(aBootMedian,alphaa,betaa,bBootMedian,alphab,betab))
plt.savefig('bootstrapHubble.pdf')
imports.py0000644000175000017500000001171314116636357011300 0ustar  maxmaximport math
import numpy as np
import matplotlib.ticker as tick
import scipy.stats as stats
#from kaastra import Ce
from matplotlib import pyplot as plt
from scipy.stats import expon, binom, chi2, norm, poisson, lognorm, f, t, rdist, beta, kstest, ks_2samp, ksone, kstwo, kstwobign, uniform, linregress, pearsonr
from scipy.optimize import curve_fit, fsolve
from matplotlib.ticker import StrMethodFormatter
from matplotlib.ticker import ScalarFormatter
from matplotlib.ticker import FuncFormatter,LogLocator
import matplotlib.ticker as ticker
from random import choices
from scipy.optimize import minimize 
from matplotlib.ticker import (MultipleLocator, FormatStrFormatter,AutoMinorLocator)
from matplotlib import rcParams

plt.rcParams["font.family"] = "Times New Roman"
plt.rcParams.update({'font.size': 18})
plt.rcParams['axes.linewidth'] = 2.0

# xmin and xmax fill the CDF to the min and max values with 0 and 1
def sampleCDF(x,xmin,xmax):
	xOrdered=np.sort(x)
	xUnique,countsUnique=np.unique(xOrdered,return_counts=True)
	nUnique=len(xUnique) # there will be nUnique-1 steps
	tot=np.cumsum(countsUnique)
	totCDF=tot/tot[-1] # normalization
	# now pad the arrays with the min and max values for a nice plot

	xUniquePad=np.zeros(nUnique+2)
	totCDFPad=np.zeros(nUnique+2)
	xUniquePad[0]=xmin
	xUniquePad[1:nUnique+1]=xUnique[0:nUnique]
	xUniquePad[nUnique+1]=xmax
	totCDFPad[0]=0
	totCDFPad[1:nUnique+1]=totCDF[0:nUnique]
	totCDFPad[nUnique+1]=1
	return xUniquePad,totCDFPad	

def CContribwTwo(Ni,mu):
    if (Ni==0):
        return 2*mu
    if (Ni>0):
        return 2*(mu - Ni +Ni*np.log(Ni/mu)) # factor of 2 included here

def linear(x,a,b):
    return a+b*x

def constant(x,a):
	return a+0*x

# constant model with an additional term for the first bin
def constantPlusOne(x,a,b):
	Sizex=len(x)
	model=np.zeros(Sizex)
	# first bin, where the line would be
	model[0] = a+b
	# all other bins
	for j in range(Sizex-1):
		model[j+1]=a
	return model

def chisquared(y,yerr,ymod):
    N=len(y)
    result=0
    for i in range(N):
        result+=((y[i]-ymod[i])/yerr[i])**2
    return result

# This is the bivariate chi2 for linear model
def chisquaredBivLin(x, *args):
    a,b=x
    x,y,xerr,yerr=args
    #print('inside chisquaredBinLin',x,y,xerr,yerr)
    N=len(y)
    #print('Using %d datapoints'%N)
    result=0
    for i in range(N):
        contrib=(y[i]-a-b*x[i])**2/(yerr[i]**2 +(b**2)*xerr[i]**2)
        #print('x=%3.2f+-%3.2f, y=%3.2f+-%3.2f,a=%3.2f, b=%3.2f, %3.3f'%
        #        (x[i],xerr[i],y[i],yerr[i],a,b,contrib))
        result=result+contrib
    return result

def modelVariance(y,ymodel,dof):
    N=len(y)
    result=0
    for i in range(N):
        result+=(y[i]-ymodel[i])**2
    result=result/(N-dof)
    return result

def intrinsicScatter(y,ymodel,yerr,dof):
    result=0
    N=len(y)
    for i in range(N):
            result+=(y[i]-ymodel[i])**2/(N-dof)-yerr[i]**2/N
    return result**0.5

def Stirling(n):
    return (2*np.pi*n)**0.5*(n/np.e)**n

# Implements a batch variance with an=bn=sqrt(n)
# This BM variance needs to be divided by n to have variance of mean
def VarBM(x):
	n=len(x) #hopefully is the square of a number
	an=int(n**0.5)
	#an=20 #(test)
	bn=int(n/an)
	print('an=%d, bn=%d'%(an,bn))
	# In Flegal+20, Ykbar is the mean of the deviations
	ykMean=np.zeros(bn)
	ynMean=np.mean(x) # mean of whole array
	result=0
	for k in range(an):
		# mean of y over the k-th batch
		ykMean[k]=np.mean(x[k*bn:(k+1)*bn])	
	# Now do the sample variance of the batch means
	s2BM=np.var(ykMean,ddof=1)
	# Multiply by bn 
	result=bn*s2BM
	return result

# Ci contribution to Cstat
def CContribwTwo(Ni,mu):
    if (Ni==0):
        return 2*mu
    if (Ni>0):
        return 2*(mu - Ni +Ni*np.log(Ni/mu)) # factor of 2 included here

def cstat(y,ymod):
    N=len(y)
    result=np.zeros(N)
    for i in range(N):
        result[i]=CContribwTwo(y[i],ymod[i]) # these are the Ci's
    return(sum(result))

def funcECmin(x, a,b,c,d,e,beta,f,alpha):
    return (a+b*x+c*(x-d)**2.0)*np.exp(-alpha*x)+e*np.exp(-beta*x)+f
def funcVarC(x, a,b,c,d,e,f,g,h,alpha,beta,i):
    return  (a+b*x**2+c*(x-d)**2.0)*np.exp(-alpha*x) +  (e+f*x+g*(x-h)**2.0)*np.exp(-beta*x) +i

# Parameter values as reported in Bonamente20
parNamesE=['a','b','c','d','e','beta','f','alpha']
parNamesVar=['a','b','c','d','e','f','g','h','alpha','beta','i']

# These are the best-fit values for E[Ci] on full and narrow range,
# followed by Var[Ci] for full and narrow range
A=[0.065672, -0.56709, -2.4637, -3.1971]
B=[-6.9461,-2.7336,1.5109,1.5118]
C=[-8.0124,-2.3603,-1.5109,-1.5118]
D=[0.40165,0.52816,0.60509,0.79384]
E=[0.261037,0.33133,1.4761,1.9294]
F=[1.00512,1.0174,18.358,6.1740]
G=[0,0,0.87316e-3,22.360e-3]
H=[0,0,-0.08592,-7.2981]
I=[0,0,2.02343,2.08378]
alphaB=[5.5178,3.9375,0.62652,0.750315] # note change of name B- Bonamente
betaB=[0.34817,0.48446,7.8187,4.49654]

colors=['black','red','gray','orange']
descriptor=['E[$C_i$] (range $\mu$=0.01-100)','E[$C_i$] (range $\mu$=0.1-10)','Var($C_i$) (range $\mu$=0.01-100)','Var($C_i$) (range $\mu$=0.1-10)']

iris.py0000664000175000017500000001060214116121130010521 0ustar  maxmaxexec(open('imports.py').read())

# Python code to do statistics from Hubble data
# Bonamente 2020

# Import the data from pressureRatio.dat
# ===============================================================
# Character>---->-------Dominant>-------Recessive>------fraction
# SL	SW	PL	PW
dataTypes=[('SL1','f'),('SW1','f'),('PL1','f'),('PW1','f'),('species1','S8'),
        ('SL2','f'),('SW2','f'),('PL2','f'),('PW2','f'),('species2','S8'),
        ('SL3','f'),('SW3','f'),('PL3','f'),('PW3','f'),('species3','S8')]

# 1 - Iris setosa
# 2 - Iris versicolor
# 3 - Iris Virginica
data = np.genfromtxt('irisData.dat',skip_header=1,dtype=dataTypes)
print(data)


# =======================================
# ID: (select one that applies)
#     1 - Iris Setosa 
#     2 - Iris versicolor 
#     3 - Iris Virginica
IDLabel=['Iris Setosa','Iris versicolor','Iris Virginica']
ID=2
# problems  13.1, 13.2, 13.3 and 13.6, 13.7 and 13.8
print('*** NOTE: Selected %s'%IDLabel[ID-1])
y=data['SL%d'%ID]   # sepal length
x1=data['SW%d'%ID]  # sepal width
x2=data['PL%d'%ID]  # petal length
x3=data['PW%d'%ID]  # petal width
m=3
N=len(y)
print('m=%d, N=%d'%(m,N))

# generate the matrix A and the vector beta
X = np.c_[np.ones(N),x1,x2,x3]

# Do this two ways: Eq. (13.5) and solving the system
# 1. Eq. 12.9

A = np.matmul(np.transpose(X),X) 
betaV = np.matmul(np.transpose(X),y)
a= np.matmul(np.linalg.inv(A),betaV)
# this is also problem 13.1, when ID=1 is selected
print('best-fit parameters',a)
print('==============================')

# 2. Using linalg.solve

a = np.linalg.solve(A,betaV)
print('Equivalent solution')
print(a)

# ========================================
# Also get chi^2 assuming sigma^2 is known

# a value of sigma=1 ignores the variance
sigma=1.0
chicrit=chi2.ppf(0.99,46)
ymean=np.mean(y)
yvar=np.var(y,ddof=1) # this is divided by N-1
S2=yvar*(N-1)
print('mean y: %3.3f'%ymean)

# Residual variance ===================
yhat=np.zeros(N)
for i in range(N):
    yhat[i]=(a[0]+a[1]*x1[i]+a[2]*x2[i]+a[3]*x3[i])
print('yhat',yhat)
Sr2=((y-yhat)/sigma)**2
# Explained variance ===================
Se2=((yhat-ymean)/sigma)**2
print('S2=%3.3f, Sr2=%3.3f, Se2=%3.3f, S2=Sr2+Se2=%3.3f, critical value 0.99:%3.3f'%
	(S2,sum(Sr2),sum(Se2),sum(Sr2)+sum(Se2),chicrit))
R2=sum(Se2)/(sum(Se2)+sum(Sr2))
print('Coefficient of determination R^2: %3.3f'%R2)
# problem 13.2

FStat=(sum(Se2)/m)/(sum(Sr2)/(N-m-1))
pF=f.sf(FStat,m,N-m-1)
print('F=%3.3f (p-value %3.2e)'%(FStat,pF))
# estimate the model sample variance for Iris Setosa
s2hat = sum((y-yhat)**2)/(N-m-1)
print('model sample variance: %3.3f, std. dev: %3.3f'%(s2hat,s2hat**0.5))

# Now get the parameter errors using \hat{s}^2
sigma=s2hat**0.5
# error or covariance matrix
e= s2hat*np.linalg.inv(A)
print(e)
aErr=np.zeros(m+1)
aErr=[ e[i,i]**0.5 for i in range(m+1)]
tStat=a/aErr #t-statistic scores
# 2-sided p-value
pValue=2*t.sf(abs(tStat),N-m-1)
print(pValue)
for i in range(m+1):
	print('Parameter %d: %3.3f error: %3.3f, tscore: %3.3f , pvalue=%3.3e'%(i,a[i],aErr[i],tStat[i],pValue[i])) 

# ==================================================
# ==================================================
# now fit with just one predictor 
# this is the usual linear regression
# x1; sepal width
# x2: petal length
# problem 14.4
#xReg=x1 #using sepal width as predictor variable, for problem 14.4
xReg=x2 # using petal length as predictor variable, for problem 14.5
results=stats.linregress(xReg,y)
b=results.slope
a=results.intercept
r=results.rvalue
p=results.pvalue
berr=results.stderr
aerr=results.intercept_stderr
print('====================================================')
print("Simple linear regression with one predictor variable")
print("a: %f+-%f    b: %f+-%f" % (a,aerr, b,berr))
print("r=%3.4f, R-squared: %f (p value: %3.3e)" % (r,r**2,p))
# best-fit model
for i in range(N):
    yhat[i]=(a+b*xReg[i])
print('yhat',yhat)

sigma=1.0 # there is no error reported for the data
m=1
Sr2=((y-yhat)/sigma)**2
# Explained variance ===================
Se2=((yhat-ymean)/sigma)**2
print('S2=%3.3f, Sr2=%3.3f, Se2=%3.3f, Sr2+Se2=%3.3f'%
        (S2,sum(Sr2),sum(Se2),sum(Sr2)+sum(Se2)))
FStat=(sum(Se2)/m)/(sum(Sr2)/(N-m-1))
pF=f.sf(FStat,m,N-m-1)
print('F=%3.3f (p-value %3.2e)'%(FStat,pF))
f=N-2
pr2=beta.sf(r**2,1/2,f/2)
print('r2=%3.3f (p-value %3.2e)'%(r**2,pr2))
# estimate the model sample variance for Iris Setosa
s2hat = sum((y-yhat)**2)/(N-m-1)
print('model sample variance: %3.3f, std. dev: %3.3f'%(s2hat,s2hat**0.5))

kaastra.py0000650000175000017500000000317113724661410011215 0ustar  maxmax# Expectations for C and its variance from Kaastra 2017
def Ce(mu):
        Ce=0
        if (0 <= mu <= 0.5):
                Ce= -0.25*mu**3 +1.38*mu**2-2.0*mu*np.log(mu)
        if (0.5 < mu <= 2):
                Ce=-0.00335*mu**5+0.04259*mu**4 -0.27331*mu**3+1.381*mu**2-2*mu*np.log(mu)
        if (2 < mu <= 5):
                Ce=1.019275+0.1345*mu**(0.461-0.9*np.log(mu))
        if (5 < mu <= 10):
                Ce=1.00624+0.604*mu**(-1.68)
        if (mu>10):
                Ce=1.0+0.1649*mu**(-1)+0.226*mu**(-2)
        return Ce

def Cv(mu):
        Cv=0
        if (0 <= mu <= 0.1):
                Cv=0
                print("Warning this needs to be updated")
        if (0.1 < mu <= 0.2):
                Cv=-262*mu**4+195*mu**3-51.24*mu**2+4.34*mu+0.77005
        if (0.2 < mu <= 0.3):
                Cv=4.23*mu**2-2.8254*mu+1.12522
        if (0.3 < mu <= 0.5):
                Cv=-3.7*mu**3+7.328*mu**2-3.6926*mu+1.20641
        if (0.5 < mu <= 1):
                Cv=1.28*mu**4-5.191*mu**3+7.666*mu**2-3.5446*mu+1.15431
        if (1 < mu <= 2):
                Cv=0.1225*mu**4-0.641*mu**3+0.859*mu**2+1.091*mu-0.05748
        if (2 < mu <= 3):
                Cv=0.089*mu**3-0.872*mu**2+2.8422*mu-0.67539
        if (3 < mu <= 5):
                Cv=2.12366+0.012202*mu**(5.717-2.6*np.log(mu))
        if (5 < mu <= 10):
                Cv=2.05159+0.331*mu**(1.343-np.log(mu))
        if (10 < mu):
                Cv=12.0*mu**(-3)+0.79*mu**(-2)+0.6747*mu**(-1)+2.0
        return Cv

# define Gaussian function
def gaussian(x, mu, sig):
    return 1./(np.sqrt(2.*np.pi)*sig)*np.exp(-np.power(x - mu, 2.) / (2 * np.power(sig, 2.)))

ks.py0000644000175000017500000000044214015475000010175 0ustar  maxmaxexec(open('imports.py').read())

# critical values of KS test in asymptotic regime
N=100
pVal=[0.5,0.6,0.7,0.8,0.9,0.95,0.99]
NpVal=len(pVal)
for i in np.arange(NpVal):
	Cvalbign=kstwobign.ppf(pVal[i])
	Cval=kstwo.ppf(pVal[i],N)
	print('%3.3f %3.3f  %3.3f'%(pVal[i],Cvalbign,Cval))
#kstest
linearCash.py0000664000175000017500000000112214116222405011630 0ustar  maxmaxexec(open('imports.py').read())

from scipy.optimize import fsolve
import numpy as np
X=np.array([0,1,2,3,4]) 
# choose the Y value for the following problems
#Y=np.array([0,0,1,0,1]) # prob16.2 without solution
Y=np.array([1,0,1,0,1]) # prob16.1
N=len(X)
def equations(p):
    a, b = p
    x=X
    y=Y
    N=len(x)
    
    return (sum(y/(a+b*x))-N,sum(y*x/(a+b*x)) - sum(x))

a, b =  fsolve(equations, (1, 0))

print(equations((a, b)))
print('Solution:', (a,b))

# report cstat
ymod=np.zeros(N)
for i in range(N):
    ymod[i]=a+X[i]*b

Cstat=cstat(Y,ymod)
print('best-fit cstat=%3.3f'%Cstat)
lognormal.py0000644000175000017500000000256714104227516011572 0ustar  maxmaxexec(open('imports.py').read())


# Gaussian distribution
fig,ax=plt.subplots(figsize=(8,6))
xmin=-10
xmax=10

x=np.linspace(xmin,xmax,10000,endpoint=True)

sigma=1.0
S=1 # np.e # Euler's number
mu = np.log(S)
loc,scale=mu,sigma

yLog=lognorm.pdf(x,s=sigma,scale=S)
yNorm=norm.pdf(x,loc,scale)

# Now also check directly that the lognormal distribution
# is the same thing as a normal distr for z=log X
xLog=[np.log(xi) for xi in x if xi>0]
#print(yLog)
yNormxLog=norm.pdf(xLog,loc,scale)





# ------------------------------------------

ax.plot(x,yNorm,color='black',linewidth=2,label='Normal Variable Y=ln X~N($\mu$,$\sigma$)')
ax.plot(x,yLog,color='blue',linewidth=2,label='Lognormal Variable X~lognorm($\mu$,$\sigma$)')

#ax.plot(xLog,yNormxLog,color='red',linestyle='--',label='Normal with log(x) at x axis')

ax.set_xlim(-2,5)
ax.set_ylim(0,0.7)
ax.set_xlabel('x')
ax.set_ylabel('Probability Distribution')
ax.xaxis.set_major_locator(MultipleLocator(1))
ax.xaxis.set_minor_locator(MultipleLocator(0.25))
ax.yaxis.set_major_locator(MultipleLocator(0.20))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.grid(which='both')
#ax2=ax.twiny()
#ax2.xaxis.label.set_color('red')
#ax2.tick_params(axis='x', colors='red')
#ax2.plot(xLog,yNormxLog,color='red',label='Normal Variable')
#ax2.set_xlabel('ln X')
#ax2.set_xlim(-2,5)
ax.legend(prop={'size' : 12},loc=1)
plt.savefig('lognormal.pdf')


mcmc5points.py0000644000175000017500000001074114116672524012040 0ustar  maxmaxexec(open('imports.py').read())

dataTypes=[('number','I'),('x','f'),('y','f')]
# Do MCMC on the 5-point dataset
# ============================
data = np.genfromtxt('5points.dat',skip_header=1,dtype=dataTypes)
print(data)
x=data['x']
y=data['y']
yerr=y**0.5
n=len(x)

popt, pcov = curve_fit(linear, x,y, sigma=yerr,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'%(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5))

# Do three chains: with widths 1, 10 and 50
NMCMC=1000 # 1000000
# proposal distribution widths
DeltaaAux=[1,10,50]
DeltabAux=DeltaaAux
nAux=len(DeltaaAux)	# number of chains with difference proposal distr.
# ============================
aMCMC=np.zeros((nAux,NMCMC))
bMCMC=np.zeros((nAux,NMCMC))
aPrime=np.zeros(nAux)
bPrime=np.zeros(nAux)
# Initial values =============
aMCMC[:,0]=np.ones(nAux)*12
bMCMC[:,0]=np.ones(nAux)*6
# ============================
yMCurrent=np.zeros((nAux,n))
yMPrevious=np.zeros((nAux,n))
chi2Current=np.zeros(nAux)
chi2Previous=np.zeros(nAux)
alpha=np.zeros(nAux)
u=np.zeros(nAux)
NAccept=np.zeros(nAux)
fout=open('mcmc5pointsLong.txt','w')
# --- initial chi^2 -----------------------------------
for j in range(nAux):
	yMPrevious[j]=linear(x,aMCMC[j,0],bMCMC[j,0])
	chi2Previous[j]=chisquared(y,yerr,yMPrevious[j])
	fout.write('%3.5f  %3.5f  '%(aMCMC[j,0],bMCMC[j,0]))
fout.write('\n')
# -----------------------------------------------------

for i in range(1,NMCMC):
	for j in range(nAux):
		aPrime[j]=uniform.rvs(loc=aMCMC[j,i-1]-DeltaaAux[j]/2,scale=DeltaaAux[j],size=1) 
		bPrime[j]=uniform.rvs(loc=bMCMC[j,i-1]-DeltabAux[j]/2,scale=DeltabAux[j],size=1)
        # calculate the likelihood, aka chi^2
		yMCurrent[j]=linear(x,aPrime[j],bPrime[j])
		chi2Current[j]=chisquared(y,yerr,yMCurrent[j])
		alpha[j]=min(1,np.exp((chi2Previous[j]-chi2Current[j])/2))
		print('a=%3.3f, b=%3.3f, chi2Current=%3.4f, chi2Previous=%3.4f, alpha=%3.4f'
		%(aPrime[j],bPrime[j],chi2Previous[j],chi2Current[j],alpha[j]))
		u[j]=uniform.rvs(size=1)
        	# Decide whether to accept or reject
		if (alpha[j]>=u[j]):  # accept
			aMCMC[j,i]=aPrime[j]
			bMCMC[j,i]=bPrime[j]
			chi2Previous[j]=chi2Current[j]# this becomes the previous chi2
			print('Accepted')
			NAccept[j]=NAccept[j]+1
		if (alpha[j]<u[j]):
			aMCMC[j,i]=aMCMC[j,i-1]
			bMCMC[j,i]=bMCMC[j,i-1]
			print('Rejected')
		fout.write('%3.5f  %3.5f  '%(aMCMC[j,i],bMCMC[j,i]))
	fout.write('\n')
fout.close()
print('MCMC acceptance:',NAccept/NMCMC)
fig,ax=plt.subplots(1,2,figsize=(8,6))
color=['black','red','blue']
for j in range(nAux):
	ax[0].hist(aMCMC[j],bins=40,linewidth=2,histtype='step',color=color[j],
	label='$\Delta a$=%2.1f'%DeltaaAux[j])
	ax[1].hist(bMCMC[j],bins=40,linewidth=2,histtype='step',color=color[j],
	label='$\Delta b$=%2.1f'%DeltabAux[j])
ax[0].set_xlabel('Parameter a')
ax[0].xaxis.set_major_locator(MultipleLocator(10))
ax[0].xaxis.set_minor_locator(MultipleLocator(2))
ax[0].yaxis.set_major_locator(MultipleLocator(NMCMC*0.01))
ax[0].yaxis.set_minor_locator(MultipleLocator(NMCMC*0.005))
ax[1].set_xlabel('Parameter b')
ax[1].xaxis.set_major_locator(MultipleLocator(5))
ax[1].xaxis.set_minor_locator(MultipleLocator(1))
ax[1].yaxis.set_major_locator(MultipleLocator(NMCMC*0.01))
ax[1].yaxis.set_minor_locator(MultipleLocator(NMCMC*0.005))
ax[0].set_ylabel('Number of occurrence')
ax[0].set_ylim((0,0.09*NMCMC))
ax[1].set_ylim((0,0.09*NMCMC))
ax[0].grid(which='both')
ax[1].grid(which='both')
ax[0].legend(prop={'size': 10})
ax[1].legend(prop={'size': 10})
for axs in ax.flat:
    axs.label_outer()
plt.savefig('5PointMCMCHist.pdf')

xaxis=np.arange(NMCMC)
fig,ax=plt.subplots(2,1,figsize=(8,6))
for j in range(nAux):
	ax[0].scatter(xaxis,aMCMC[j],s=3,color='none',edgecolor=color[j],marker='o',
	label='$\Delta a$=%2.1f'%DeltaaAux[j])
	ax[1].scatter(xaxis,bMCMC[j],s=5,color='none',edgecolor=color[j],marker='o',
	label='$\Delta b$=%2.1f'%DeltabAux[j])
ax[0].set_ylabel('Parameter a')
ax[1].set_xlabel('Iteration Number')
ax[1].set_ylabel('Parameter b')
ax[0].set_xlim(0,NMCMC)
ax[1].set_xlim(0,NMCMC)
ax[0].yaxis.set_major_locator(MultipleLocator(10))
ax[0].yaxis.set_minor_locator(MultipleLocator(2))
ax[1].yaxis.set_major_locator(MultipleLocator(5))
ax[1].yaxis.set_minor_locator(MultipleLocator(1))
ax[1].xaxis.set_major_locator(MultipleLocator(NMCMC*0.100))
ax[0].xaxis.set_major_locator(MultipleLocator(NMCMC*0.100))
ax[0].grid(which='both')
ax[1].grid(which='both')
ax[0].legend(prop={'size': 10})
ax[1].legend(prop={'size': 10})
for axs in ax.flat:
	axs.label_outer()
plt.savefig('5PointMCMCTime.pdf')

mcmcInference.py0000644000175000017500000000322714116673351012335 0ustar  maxmax# Make inferences on longer MCMC for 5-point data
# ===============================================
exec(open('imports.py').read())
# Read in data from the MCMC
DeltaaAux=[1,10,50]
DeltabAux=DeltaaAux
color=['red','black','blue']
ZGType=['Full Chain','Thinned Chain','Batch Means']
# This MCMC must be generated with mcmc5points.py, and the Long suffix
# can be replaced with any other string 
data = np.genfromtxt('mcmc5pointsLong.txt',skip_header=0)

a1=data[:,0]
b1=data[:,1]
a2=data[:,2]
b2=data[:,3]
a3=data[:,4]
b3=data[:,5]
N=len(a1)       # number of links in the chain
# put the chains in a compact form
a=[a1,a2,a3]
b=[b1,b2,b3]
N=len(a[0])
print('Chains of length %d'%N)

# Choose a burn--in period. For a long chain maybe 10,000, 
# but smaller for a shorter chain
#nburn=10000 # to be excised
nburn=100

Mean=np.zeros((2,3))
Stdev=np.zeros((2,3))
StdevThin=np.zeros((2,3))
nThin=10
Median=np.zeros((2,3))
LL=np.zeros((2,3))
UL=np.zeros((2,3))
lowerp=0.16
upperp=0.84
for i in range(3):		# for the 3 choices of proposals
    for j in range(2): 	# for a and b
   # === mean and std.dev
        Mean[j,i]=np.mean(data[nburn:N,2*i+j])
        Median[j,i]=np.median(data[nburn:N,2*i+j])
        Stdev[j,i]=np.std(data[nburn:N,2*i+j])  
        StdevThin[j,i]=np.std(data[nburn:N:nThin,2*i+j])
        LL[j,i]=np.quantile(data[nburn:N,2*i+j],lowerp)
        UL[j,i]=np.quantile(data[nburn:N,2*i+j],upperp)
        print("E[X]=%3.3f  median=%3.3f  stdev=%3.3f (thinned: %3.3f) LL(%2.1f)=%3.3f UL(%2.1f)=%3.3f"%(Mean[j,i],Median[j,i],Stdev[j,i],StdevThin[j,i],lowerp,LL[j,i],upperp,UL[j,i]))
        print("Using C.I.: %3.3f +- %3.3f"%(0.5*(LL[j,i]+UL[j,i]),0.5*(UL[j,i]-LL[j,i])))
mcmc.py0000644000175000017500000000772414024667046010526 0ustar  maxmaxexec(open('imports.py').read())

# Do an MCMC on Hubble's data

dataTypes=[('name','S8'),('v','f'),('N','I'),('m','f')]
data = np.genfromtxt('hubbleHumasonData.dat',skip_header=1,dtype=dataTypes)
print(data)

v=data['v']
m=data['m']
logv=np.log10(v)
n=len(m) # number of datapoints
# Set up priors - uniform
aMin=0.2
aMax=0.9
bMin=0.15
bMax=0.25

# Set up proposals - uniform
Deltaa=0.2
Deltab=0.04
# Set up the chain
NMCMC=1000
aMCMC=np.zeros(NMCMC)
bMCMC=np.zeros(NMCMC)
logvMCurrent=np.zeros(n)
logvMPrevious=np.zeros(n)

fout=open('hubbleMCMC.txt','w')
# Start at an arbitrary initial value
aMCMC[0]=0.5*(aMax+aMin)
bMCMC[0]=0.5*(bMax+bMin)
logvMPrevious=linear(m,aMCMC[0],bMCMC[0])	# current model
fout.write('%3.5f  %3.5f\n'%(aMCMC[0],bMCMC[0]))
# variances of the data were not provided - estimate via model sample variance
# fit without errors
slope, intercept, r_value, p_value, std_err = stats.linregress(m,logv)
# best-fit model
ymodel=linear(m,intercept,slope)
# sample model variance
dof=2 # number of free parameters
mVar=modelVariance(logv,ymodel,dof)
print('Model variance is %f (sqrt: %f)'%(mVar,mVar**0.5))
logvErr=np.ones(len(m))*(mVar**0.5)
chi2Previous=chisquared(logv,logvErr,logvMPrevious)

# Now run the MCMC
NAccept=0
for i in range(1,NMCMC):	# start at i=1
	# Draw random values for candidates in the neighborhood of previous parameter
	aPrime=uniform.rvs(loc=aMCMC[i-1]-Deltaa/2,scale=Deltaa,size=1)
	bPrime=uniform.rvs(loc=bMCMC[i-1]-Deltab/2,scale=Deltab,size=1)
	# calculate the likelihood, aka chi^2
	logvMCurrent=linear(m,aPrime,bPrime)
	chi2Current=chisquared(logv,logvErr,logvMCurrent)
	# Calculate the acceptance probability
	alpha=min(1,np.exp((chi2Previous-chi2Current)/2))
	print('a=%3.3f, b=%3.3f, chi2Current=%3.4f, chi2Previous=%3.4f, alpha=%3.4f'
		%(aPrime,bPrime,chi2Previous,chi2Current,alpha))
	u=uniform.rvs(size=1)
	# Decide whether to accept or reject
	if (alpha>=u):	# accept
		aMCMC[i]=aPrime
		bMCMC[i]=bPrime
		chi2Previous=chi2Current# this becomes the previous chi2
		print('Accepted')
		NAccept=NAccept+1
	if (alpha<u):
		aMCMC[i]=aMCMC[i-1]
		bMCMC[i]=bMCMC[i-1]
		print('Rejected')
	fout.write('%3.5f  %3.5f\n'%(aMCMC[i],bMCMC[i]))
fout.close()
print('Acceptance rate of MCMC = %d of %d (%3.4f)'%(NAccept,NMCMC,NAccept/NMCMC))
# Plot the parameter distribution
fig,ax=plt.subplots(1,2,figsize=(8,6))
ax[0].hist(aMCMC,bins=40,linewidth=2,histtype='step',color='black')
ax[0].set_xlabel('Parameter a')
ax[0].xaxis.set_major_locator(MultipleLocator(0.2))
ax[0].xaxis.set_minor_locator(MultipleLocator(0.05))
ax[0].yaxis.set_major_locator(MultipleLocator(NMCMC*0.01))
ax[0].yaxis.set_minor_locator(MultipleLocator(NMCMC*0.005))
ax[1].hist(bMCMC,bins=40,linewidth=2,histtype='step',color='black')
ax[1].set_xlabel('Parameter b')
ax[1].xaxis.set_major_locator(MultipleLocator(0.02))
ax[1].xaxis.set_minor_locator(MultipleLocator(0.005))
ax[1].yaxis.set_major_locator(MultipleLocator(NMCMC*0.01))
ax[1].yaxis.set_minor_locator(MultipleLocator(NMCMC*0.005))
ax[0].set_ylabel('Number of occurrence')
ax[0].set_ylim((0,0.09*NMCMC))
ax[1].set_ylim((0,0.09*NMCMC))
ax[0].grid(which='both')
ax[1].grid(which='both')
for axs in ax.flat:
    axs.label_outer()
plt.savefig('hubbleMCMCHist.pdf')

fig,ax=plt.subplots(2,1,figsize=(8,6))
xaxis=np.arange(NMCMC)
ax[0].scatter(xaxis,aMCMC,s=30,color='none',edgecolor='black',marker='o')
ax[1].scatter(xaxis,bMCMC,s=30,color='none',edgecolor='black',marker='o')
ax[0].grid(which='both')
ax[1].grid(which='both')
ax[0].set_ylabel('Parameter a')
ax[1].set_xlabel('Iteration Number')
ax[1].set_ylabel('Parameter b')
for axs in ax.flat:
	axs.label_outer()
ax[0].set_xlim(0,NMCMC)
ax[1].set_xlim(0,NMCMC)
ax[0].yaxis.set_major_locator(MultipleLocator(0.2))
ax[0].yaxis.set_minor_locator(MultipleLocator(0.05))
ax[1].yaxis.set_major_locator(MultipleLocator(0.02))
ax[1].yaxis.set_minor_locator(MultipleLocator(0.005))
ax[1].xaxis.set_major_locator(MultipleLocator(NMCMC*0.100))
ax[0].xaxis.set_major_locator(MultipleLocator(NMCMC*0.100))
plt.savefig('hubbleMCMCTime.pdf')
mendel.py0000644000175000017500000000677314114433144011044 0ustar  maxmax# Python code to do statistics from Mendel data
# Bonamente 2020

exec(open('imports.py').read())
# Import the data from mendel.dat
# ===============================================================
# Character		Dominant	Recessive	fraction
dataTypes=[('character','S20'),('dominant','I'),('recessive','I'),('fraction','f')]

data = np.genfromtxt('mendel.dat',skip_header=1,dtype=dataTypes)

character = data['character']
dominant = data['dominant']
recessive = data['recessive']
fraction = data['fraction']

#print(dominant)
#print(recessive)
#print(dominant/(recessive+dominant))
print(character)
print('Fraction of dominants')
print(fraction)

# ======================================================
# 2. Calculate standard deviations of number of dominant
# problem 6.2
total=dominant+recessive
print('Problem 6.2')
p=0.75   # parent value for fraction of dominant
q=0.25  # parent value for fraction of recessive

stdDominant = np.sqrt(total*p*q)
print('Standard deviation of dominants')
print(stdDominant)
stdDominantFraction = stdDominant/total
print('Standard deviation of fraction of dominants')
print(stdDominantFraction)
# also sample variance of fraction of dominants

# =======================================================
# 3.8 Simple calculation of the sample variance of binomial

# 7th line has the long vs short stem
print('Problem 9.7')
index=6
N7=total[index]
D7=dominant[index]
s27=N7*p*q
E7=N7*p
print('Measurement of dominants: %d'%D7)
print('Expectation of dominant  (long vs. short stem): %3.3f'%E7)
print('Variance of binomial (long vs. short stem): %3.3f, N=%d'%(s27,N7))
# Also estimate p7 from the data
p7=dominant[index]/N7
s2p7=N7*p7*(1-p7)
print('"Sample" variance of binomial (long vs. short stem): %3.3f, N=%d'%(s2p7,N7))
zScore=(D7-E7)/s27**0.5
print('z score for number of dominants (long vs. short stem):%3.3f'%zScore)



#problem 3.9, similar to 3.8
print('Problem 3.9')
N=np.sum(total)
print('Moments of distribution for fraction: mean=%3.2f, variance=%3.2e'%(p,p*q/N))
pStDev=(p*q/N)**0.5
print('Standard deviation: %3.2e'%(pStDev))

pData=sum(dominant)/N
print('Measurement of the fraction of dominants: %3.3f'%pData)

zScore=(pData-p)/pStDev
zScorepValue=norm.cdf(abs(zScore))-norm.cdf(-abs(zScore))
print('Approximate zscore and associated probability: %3.2f (%3.3f)'%(zScore,zScorepValue))
print('======================')
# ======================================================
# 3.1 Weighted average of the seven dominants
# problem 9.8
print('Problem 9.8')
weightedAvgDominantFraction=np.average(fraction,weights=1./stdDominantFraction**2)
# also calculate the error on the weighted mean
weightedAvgDominantFractionError=np.sqrt(1./np.sum(1/stdDominantFraction**2))
print("Weighted average of dominant fraction: %4.5f+-%4.5f"%(weightedAvgDominantFraction,weightedAvgDominantFractionError))

# 3.2 this can be checked by a direct sum of all dominants
fractionSum=np.sum(dominant)/np.sum(total)
fractionSumError = np.sqrt(np.sum(total)*p*q)/np.sum(total)
print("Overall dominant fraction: %4.5f+-%4.5f"%(fractionSum,fractionSumError))
# also need the sample variance of fractions
fraction=dominant/total
fractionMean=np.mean(fraction)
fractionStd=np.std(fraction,ddof=1)
print('Mean and Std. dev. of the fractions: %3.4f+-%3.4f %3.3f (%3.3e)'%(fractionMean,fractionStd,p,fractionStd**2))
tStat = (fractionMean-p)/(fractionStd/np.sqrt(7))
# need also the sample variance of the fractions
print('t statistic for dominant characters: %3.4f'%tStat)
tpValue=t.cdf(abs(tStat),7)-t.cdf(-abs(tStat),7)
print('p value for t: %3.2f'%tpValue)
monteCarlo.py0000644000175000017500000000627014116654045011702 0ustar  maxmaxexec(open('imports.py').read())

# integrate 1/4 of a circle
def fCircle(x,r):
	return (r**2-x**2)**0.5

def expFInv(p,lambdaPar):
	value = - np.log(1.-p)/lambdaPar
	return value

def GaussFInv(p1,p2,mu,sigma):
	value = ((-2*np.log(1-p1))**0.5)*np.cos(2*np.pi*p2)*sigma+mu
	return value

# (A) Simple estimate of the integral, also prob20.3
print('problem 20.3')
a=0
b=1
N=100000
#N=1000
unifSample=uniform.rvs(size=N)
fSample=fCircle(unifSample,1)
fSampleMean=np.mean(fSample)
fSampleVar=np.var(fSample)
PiSim=4.0*fSampleMean
PiSimErr=4.0*fSampleVar**0.5/N**0.5 # error according to Eq. 20.6
print('(a) Monte Carlo standard integration')
print('for N=%d expected precision of %3.4f %%'%(N,100*PiSimErr/PiSim))
print('Pi for N=%d: %4.4f +-%4.4f (%4.4f, error=%3.4f or %3.3f %%)'%
		(N,PiSim,PiSimErr,np.pi,PiSim-np.pi,100*(PiSim-np.pi)/np.pi))



# (B) Dart Monte Carlo
# y(-1,1) is related to x(0,1) by y=2x-1 or x=(y+1)/2
print('Problem 20.4')
y1=2*uniform.rvs(size=N)-1 # between -1 and 1
y2=2*uniform.rvs(size=N)-1 # same
NR=0 # number of simulated data points within R
for i in range(N):
	if (y1[i]**2+y2[i]**2)<= 1:
		NR=NR+1

fig,ax=plt.subplots(figsize=(8,8))
# estimated binomial probabilities	
pHat=NR/N
qHat=1-pHat
V=2*2
A=V*NR/N
VarA = (4.0**2)*pHat*qHat/N
plt.plot(y1,y2,linestyle='',marker='x',color='black')
plt.xlabel('x')
plt.ylabel('y')
plt.xlim((-1,1))
plt.ylim((-1,1))
ax.xaxis.set_major_locator(MultipleLocator(0.5))
ax.xaxis.set_minor_locator(MultipleLocator(0.25))
ax.yaxis.set_major_locator(MultipleLocator(0.5))
ax.yaxis.set_minor_locator(MultipleLocator(0.25))
plt.grid(which='both')

circle=plt.Circle((0,0),radius=1,color='blue',linewidth=2,linestyle='--',fill=False)
ax.add_patch(circle)

plt.savefig('dartMC.pdf')
print('(b) Dart method: A=%4.4f+-%4.4f (%d out of %d)'%(A,VarA**0.5,NR,N))
print(' for N=%d expected precision of %3.4f pct'%(N,100*VarA**0.5/np.pi))
# ============================
# ====== simulation of an exponential distribution

N=1000 		# number of samples
lambdaPar=1.0	# parameter of the exponential distribution
uRand=uniform.rvs(size=N)
uRand2=uniform.rvs(size=N)
#print('uRand',uRand)
lambdaRand=expFInv(uRand,lambdaPar)
# also simulate a Gaussian from 2 uniform
mu=5
sigma=1
gaussRand=GaussFInv(uRand,uRand2,mu,sigma)
xaxis=np.linspace(0,10,1000)
# theoretical PDF of an exponential variable
lambdaPDF=expon.pdf(xaxis,scale=1/lambdaPar)
gaussPDF=norm.pdf(xaxis,loc=mu,scale=sigma)

fig,ax=plt.subplots(figsize=(8,6))
ax.xaxis.set_major_locator(MultipleLocator(1.0))
ax.xaxis.set_minor_locator(MultipleLocator(0.5))
ax.yaxis.set_major_locator(MultipleLocator(0.2))
ax.yaxis.set_minor_locator(MultipleLocator(0.1))
plt.grid(which='both')
plt.hist(lambdaRand,bins=50,linewidth=2,histtype='step',density=True,label='Simulated Exponential Sample',color='black')
plt.plot(xaxis,lambdaPDF,linewidth=2,label='Exponential distribution ($\lambda=1$)',color='blue')
plt.hist(gaussRand,bins=50,linewidth=2,histtype='step',density=True,label='Simulated Gaussian Sample',color='grey')
plt.plot(xaxis,gaussPDF,linewidth=2,label='Gaussian distribution ($\mu=5$, $\sigma=1$)',color='red')
plt.legend(prop={'size': 12})
plt.xlim((0,10))
plt.xlabel('x')
plt.ylabel('Distribution')
plt.savefig('exponMonteCarlo.pdf')




pressureRatio.py0000664000175000017500000004054314116642046012447 0ustar  maxmaxexec(open('imports.py').read())

# Install BCES libraries with: pip3 install bces
# For some systems, it may be needed: pip install bces

import bces.bces

# Python code to do statistics from Pressure Ratio data
# Bonamente 2020

# Import the data from pressureRatio.dat
# ===============================================================
# Character>---->-------Dominant>-------Recessive>------fraction
dataTypes=[('num','I'),('Radius','f'),('RadiusEP','f'),('RadiusEM','f'),
        ('Energy1','f'),('Energy1EP','f'),('Energy1EM','f'),
        ('Energy2','f'),('Energy2EP','f'),('Energy2EM','f'),
        ('ratio','f'),('ratioEP','f'),('ratioEM','f')]

data = np.genfromtxt('pressureRatio.dat',skip_header=1,dtype=dataTypes)
#print(data)


# Extract data
radius=data['Radius']
ratio=data['ratio']
Energy1=data['Energy1']
Energy2=data['Energy2']
# Now make symmetrical errors ==================================
Energy1E=0.5*(data['Energy1EP']+data['Energy1EM'])
Energy2E=0.5*(data['Energy2EP']+data['Energy2EM'])
ratioE=0.5*(data['ratioEP']+data['ratioEM'])
radiusE=0.5*(data['RadiusEP']+data['RadiusEM'])
print('Ratio errors',ratioE)
print('Energy 2 errors',Energy2E)


# First, basic statistics such as mean, weighted mean and median
N=len(ratio)
ratioMean=np.mean(ratio)
ratioWMean=np.average(ratio,weights=1/ratioE**2)
ratioMedian=np.median(ratio)
ratioMeanE=(np.var(ratio,ddof=1)/(N-1))**0.5
ratioWMeanE=(1./np.sum(1/ratioE**2))**0.5
ratioMedianE=ratioMeanE*(np.pi/2)**0.5
####################################################
print('Problems 8.1 and 8.2')
print('Mean: %3.2f+-%3.3f, Weigh. Mean: %3.2f+-%3.3f, Median: %3.2f+-%3.3f'
        %(ratioMean,ratioMeanE,ratioWMean,ratioWMeanE,ratioMedian,ratioMedianE))
# add an additional error of 0.1 to ratio
ratioESys=ratioE+0.1
ratioWMeanSys=np.average(ratio,weights=1/ratioESys**2)
print('Weigh. Mean with 0.1 sys. error: %3.2f'%ratioWMeanSys)
print('%%%%%%%%%%%%%%%')
# also do a correlation coefficient between radius and ratio prob18.3
rRR=pearsonr(ratio,radius)
print('Pearson r for ratio vs radius: %3.3f p-value:%3.2e'%(rRR[0],rRR[1]))

# Also do a histogram plot of ratio, and logarithmic average
# This is unused for now
fig,ax=plt.subplots(figsize=(8,6))
plt.hist(ratio)
plt.savefig('histPressureRatio.pdf')
logRatio=np.log(ratio)
#problem 8.5
print('Problem 8.5')
# implement equations  8.7 and 8.8 of 3rd Ed.
meanlogRatio=np.average(logRatio,weights=(ratio/ratioE)**2)
meanlogRatioE=np.sum((ratio/ratioE)**(2))**(-0.5)
# then, to turn the logarithm into the linear value, need to expand
# the confidence interval
meanRatio=np.exp(meanlogRatio)
meanRatioEPlus=np.exp(meanlogRatio+meanlogRatioE)-meanRatio
meanRatioEMinus=meanRatio-np.exp(meanlogRatio-meanlogRatioE)
print('logarithmic average of ratio: %3.3f+-%3.3f , => %3.4f+%3.4f-%3.4f'
        %(meanlogRatio,meanlogRatioE,meanRatio,meanRatioEPlus,meanRatioEMinus))
# also for the ad hoc relative-error weighted mean
meanRatioRelativeErr=np.average(ratio,weights=(ratio/ratioE)**2)

print('relative error weighted average: %3.3f +- %3.3f'%(meanRatioRelativeErr,ratioWMeanE))
# problem 8.4 
# calculate statistics for all other quantities
print('Problem 8.4')
radiusMean=np.mean(radius)
radiusWMean=np.average(radius,weights=1/radiusE**2)
radiusMedian=np.median(radius)
radiusMeanE=(np.var(radius,ddof=1)/(N-1))**0.5
radiusWMeanE=(1./np.sum(1/radiusE**2))**0.5
radiusMedianE=radiusMeanE*(np.pi/2)**0.5
Energy1Mean=np.mean(Energy1)
Energy1WMean=np.average(Energy1,weights=1/Energy1E**2)
Energy1Median=np.median(Energy1)
Energy1MeanE=(np.var(Energy1,ddof=1)/(N-1))**0.5
Energy1WMeanE=(1./np.sum(1/Energy1E**2))**0.5
Energy1MedianE=Energy1MeanE*(np.pi/2)**0.5
Energy2Mean=np.mean(Energy2)
Energy2WMean=np.average(Energy2,weights=1/Energy2E**2)
Energy2Median=np.median(Energy2)
Energy2MeanE=(np.var(Energy2,ddof=1)/(N-1))**0.5
Energy2WMeanE=(1./np.sum(1/Energy2E**2))**0.5
Energy2MedianE=Energy2MeanE*(np.pi/2)**0.5
print('Ratio: Mean: %3.2f+-%3.3f, Weigh. Mean: %3.2f+-%3.3f, Median: %3.2f+-%3.3f'
        %(ratioMean,ratioMeanE,ratioWMean,ratioWMeanE,ratioMedian,ratioMedianE))
print('Radius: Mean: %3.2f+-%3.3f, Weigh. Mean: %3.2f+-%3.3f, Median: %3.2f+-%3.3f'
        %(radiusMean,radiusMeanE,radiusWMean,radiusWMeanE,radiusMedian,radiusMedianE))
print('Energy1: Mean: %3.2f+-%3.3f, Weigh. Mean: %3.2f+-%3.3f, Median: %3.2f+-%3.3f'
        %(Energy1Mean,Energy1MeanE,Energy1WMean,Energy1WMeanE,Energy1Median,Energy1MedianE))
print('Energy2: Mean: %3.2f+-%3.3f, Weigh. Mean: %3.2f+-%3.3f, Median: %3.2f+-%3.3f'
        %(Energy2Mean,Energy2MeanE,Energy2WMean,Energy2WMeanE,Energy2Median,Energy2MedianE))
# ----------------------------------
# problem 11.6
# 1. Fit data radius vs. ratio using symmetrical errors 
# NOTE: absolute_sigma needs to be set, or else errors are re-scaled .... :-(
popt, pcov = curve_fit(linear, radius, ratio, sigma=ratioE,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'%(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5))

ymodel=linear(radius,popt[0],popt[1])
#print(ymodel)
chimin=chisquared(ratio,ratioE,ymodel)
print('chi^2=%3.2f'%chimin)

# 1.5 Also try numpy polyfit
p,cov=np.polyfit(radius, ratio,1, w=1/ratioE,cov='unscaled')
print('polyfit:',p,cov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'%(p[1],cov[1][1]**0.5, p[0],cov[0][0]**0.5))
# This seems to give a result that is consistent with the Delta chi^2 surface

# ======== do fits in small ranges for F test 
iMin=[0,5,21]
iMax=[5,10,26]
chiMinRange=np.zeros(3)
for j in range(3):
	print('Range %d-%d'%(iMin[j],iMax[j]))
	x=radius[iMin[j]:iMax[j]]
	y=ratio[iMin[j]:iMax[j]]
	yerr=ratioE[iMin[j]:iMax[j]]
	#print(x,y,yerr)
	poptRange, pcovRange = curve_fit(linear, x,y, sigma=yerr,absolute_sigma=True)
	#print('curve_fit range',popt,pcov)
	print('a=%3.3f +- %3.3f, b= %3.5f +- %3.5f'%(poptRange[0],pcovRange[0][0]**0.5, poptRange[1],pcovRange[1][1]**0.5))
	ymodelRange=linear(x,poptRange[0],poptRange[1])
#print(ymodel)
	chiMinRange[j]=chisquared(y,yerr,ymodelRange)
	pValue=chi2.sf(chiMinRange[j],3)
	print('chi^2=%3.2f (pValue=%3.3e)'%(chiMinRange[j],pValue))

# add f-test values
F1=chiMinRange[1]/chiMinRange[0]
pValueF=f.sf(F1,3,3)
print('F-stat: %3.3f, p value=%3.3e'%(F1,pValueF))
F1=chiMinRange[0]/chiMinRange[2]
pValueF=f.sf(F1,3,3)
print('F-stat: %3.3f, p value=%3.3e'%(F1,pValueF))

# ===========================================
# 2. plot with best-fit line
xmin=0
xmax=700

xline=[xmin,xmax]
yline=np.zeros(2)
yline[0]=linear(xline[0],popt[0],popt[1])
yline[1]=linear(xline[1],popt[0],popt[1])

fig,ax=plt.subplots(figsize=(8,6))

plt.errorbar(radius, ratio,yerr=ratioE,linestyle='',fmt='o',label='data',color='black')
plt.plot(xline,yline,label='best-fit model',color='blue')
plt.xlabel('Radius')
plt.ylabel('Ratio')
plt.legend(prop={'size': 12})
plt.savefig('ratioPressure.pdf')
# =====================================

# ===================================
# do the intrinsic scatter for radius vs. ratio 
#problem17.1
print('Problem 17.1')
fig,ax=plt.subplots(figsize=(8,6))

sigmaint=intrinsicScatter(ratio,ymodel,ratioE,2)
print('intrinsic scatter radius vs. ratio: %3.3f'%sigmaint)

# also plot the fit with the intrinsic error added
fig,ax=plt.subplots(figsize=(8,6))
ratioENew=(ratioE**2+sigmaint**2)**0.5

# get a new best-fit model with the new error added in quadrature
popt, pcov = curve_fit(linear, radius, ratio, sigma=ratioENew,absolute_sigma=True)
print('curve_fit with sys error added',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'%(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5))
print('------------------------')
xline=[xmin,xmax]
yline=np.zeros(2)
yline[0]=linear(xline[0],popt[0],popt[1])
yline[1]=linear(xline[1],popt[0],popt[1])

plt.errorbar(radius, ratio,yerr=ratioENew,linestyle='',fmt='o',label='data',color='black')
plt.plot(xline,yline,label='best-fit model',color='blue')
plt.xlabel('Radius')
plt.ylabel('Ratio')
plt.legend(prop={'size': 12})
plt.savefig('ratioPressureint.pdf')

# problem 17.2
print('Problem 17.2 - stepping through intrinsic error')
# also additional estimate of sigmaint from deltachi2=1
# do this numerically
sigmaint=np.linspace(0.12,0.22,100)
for i in range(100):
    ratioENew=(ratioE**2+sigmaint[i]**2)**0.5 # new error
    popt, pcov = curve_fit(linear, radius, ratio, sigma=ratioENew,absolute_sigma=True) # new fit
    ymodel=linear(radius,popt[0],popt[1]) # new model
    chimin=chisquared(ratio,ratioENew,ymodel) # new chi2
    print('sigmaint=%3.3f, chimin=%3.3f (red: %3.2f) a=%3.2f, b=%3.5f'
            %(sigmaint[i],chimin,chimin/23,popt[0],popt[1]))

# =================================================
# ========== add the BCES fits to ratio vs. radius
print('BCES fits, Radius vs. Ratio')
AA=np.zeros(len(Energy1)) 
corrFactor=2.0 # for problem 18.3
#corrFactor=1.0 # for problem 18.2
radiusE=radiusE*corrFactor
ratioE=ratioE*corrFactor
b,a,berr,aerr,covab=bces.bces.bces(radius,radiusE,ratio,ratioE,AA)
print('BCES fits radius vs ratio (problem 18.2 or 18.3)',a,b,aerr,berr,covab)
# a,b aerr, berr and covab are for Y/X, X/Y, bisect., orthogonal prob18.2
fitLabel=['Y/X regression','X/Y regression','Bisector','Orthogonal']
fitColor=['blue','red','grey']
fitStyle=['-','-','--']
xmin=0
xmax=800
ymin=0
ymax=2
fig,ax=plt.subplots(figsize=(8,6))
plt.xscale('linear')
plt.yscale('linear')
NPoints=100
xaxis=np.linspace(xmin,xmax,100)
yaxis=np.zeros((4,100))
for i in range(3): # skip the orthogonal fit
        for j in range(NPoints):
                yaxis[i,j]=a[i]+xaxis[j]*b[i]
                yaxis[i,j]=a[i]+xaxis[j]*b[i]
        plt.plot(xaxis,yaxis[i],linestyle=fitStyle[i],linewidth=2,color=fitColor[i],label=fitLabel[i])

for axis in [ax.xaxis, ax.yaxis]:
        axis.set_major_formatter(ScalarFormatter())

plt.errorbar(radius,ratio,yerr=ratioE,xerr=radiusE,capsize=2,linestyle='',color='black')
print(xaxis[0],yaxis[0,0],xaxis[1],yaxis[0,1])
plt.grid(which='both')
plt.xlim(xmin,xmax)
plt.ylim(ymin,ymax)
plt.xlabel('Radius')
plt.ylabel('Ratio')
plt.legend(prop={'size' : 12})
# choose the one that applies
#plt.savefig('BCESRatio.pdf') # problem 18.2
plt.savefig('BCESRatioTwice.pdf') # problem 18.3

#==================
# add a bivariate chi2 minimization prob18.4
print('Problem 18.4 - numerical minimization of bivariate chi^2')
ab=minimize(chisquaredBivLin,x0=(1,0.4),args=(radius,ratio,radiusE,ratioE))
print('Results of bivariate chi2 minimization:',ab)
chi2Biv=chisquaredBivLin(ab.x,radius,ratio,radiusE,ratioE)
print('Results of bivariate chi2 minimization:',chi2Biv)
fig,ax=plt.subplots(figsize=(8,6))
a,b=ab.x
print('Using best-fit values',a,b)
xaxis=[0,800]
yaxis=np.zeros(2)
for i in range(2):
    yaxis[i]=a+xaxis[i]*b
plt.errorbar(radius,ratio,yerr=ratioE,xerr=radiusE,capsize=2,linestyle='',color='black')
plt.plot(xaxis,yaxis,color='red',linewidth=2,linestyle='--')
plt.grid(which='both')
plt.xlim(xmin,xmax)
plt.ylim(ymin,ymax)
plt.xlabel('Radius')
plt.ylabel('Ratio')
plt.savefig('BCESRatioBiv.pdf')

#====================
# 3. Now onto 2-D confidence contours (problem 12.6)
fig,ax=plt.subplots(figsize=(8,6))
# repeat the same fit as for 11.6
# --
popt, pcov = curve_fit(linear, radius, ratio, sigma=ratioE,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'%(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5))
ymodel=linear(radius,popt[0],popt[1])
chimin=chisquared(ratio,ratioE,ymodel)
# --
Na=100
Nb=100
# search a range +-n sigma from the best-fit
amin=popt[0]-2*pcov[0][0]**0.5
amax=popt[0]+2*pcov[0][0]**0.5
bmin=popt[1]-2*pcov[1][1]**0.5
bmax=popt[1]+2*pcov[1][1]**0.5
print('Searching ranges %3.2e-%3.2e, %3.2e-%3.2e'%(amin,amax,bmin,bmax))
a=np.linspace(amin,amax,Na)
b=np.linspace(bmin,bmax,Nb)
chi2Surface=np.zeros((Na,Nb))

for i in range(Na):
    for j in range(Nb):
        ymodelCurrent=linear(radius,a[i],b[j])
        chi2Surface[i][j]=chisquared(ratio,ratioE,ymodelCurrent)-chimin

#print(chi2Surface)
levels=[1.0,2.7,4.6]
CS=ax.contour(a,b,chi2Surface,levels,colors=['red','blue','black'])
ax.clabel(CS,inline=True,fontsize=12,fmt='+%1.1f')
ax.set_xlabel('Parameter a')
ax.set_ylabel('Parameter b')
# adjust the view area to make room for label
plt.gcf().subplots_adjust(left=0.20)

plt.plot(popt[0],popt[1],linestyle='',marker='+',color='black')
plt.grid()
plt.savefig('ratioPressureContours.pdf')

# ======================================
# Do a fit for energy 1 vs. energy 2
popt, pcov = curve_fit(linear, Energy1, Energy2, sigma=Energy2E,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('Energy 1 vs. Energy 2: a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'
	%(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5))

ymodel=linear(Energy1,popt[0],popt[1])
#print(ymodel)
chimin=chisquared(Energy2,Energy2E,ymodel)
chi2p=chi2.sf(chimin,23)
print('Energy 1 vs. Energy 2: chi^2=%3.2f (p-value: %e)'%(chimin,chi2p))

# ------ intrinsic scatter 
# 
sigmaint=intrinsicScatter(Energy2,ymodel,Energy2E,2)
print('E1 vs E2 sigmaint= %3.3f'%sigmaint)
#now also vary the errors to get reduced chi^2=1 
dof=len(Energy2)-2
for i in range(20):
	sigmaint=i/5 
	Energy2ENew=(Energy2E**2+sigmaint**2)**0.5
	popt, pcov = curve_fit(linear, Energy1, Energy2, sigma=Energy2ENew,absolute_sigma=True)
	ymodel=linear(Energy1,popt[0],popt[1])
	chimin=chisquared(Energy2,Energy2ENew,ymodel)
	print('for sigmaint=%3.3f: chi2=%3.3f (dof:%d)'%(sigmaint,chimin,dof))
# ------------------------
xmin=0
xmax=100

xline=[xmin,xmax]
yline=np.zeros(2)
yline[0]=linear(xline[0],popt[0],popt[1])
yline[1]=linear(xline[1],popt[0],popt[1])

ax.xaxis.set_major_locator(MultipleLocator(10))
ax.xaxis.set_minor_locator(MultipleLocator(5))
ax.yaxis.set_major_locator(MultipleLocator(10))
ax.yaxis.set_minor_locator(MultipleLocator(5))

fig,ax=plt.subplots(figsize=(8,6))
plt.errorbar(Energy1, Energy2,linewidth=2,yerr=Energy2E,linestyle='',fmt='o',capsize=4, fillstyle='none',label='Data',color='black')
plt.plot(xline,yline,linewidth=2,linestyle='--',label='Best-fit model',color='black')
plt.xlabel('Energy 1')
plt.ylabel('Energy 2')
plt.legend(prop={'size': 12})
ax.set_xlim(0.6,110)
ax.set_ylim(0.6,110)
ax.set_xscale('log')
ax.set_yscale('log')

for axis in [ax.xaxis, ax.yaxis]:
    axis.set_major_formatter(ScalarFormatter())

plt.grid(which='both')
plt.savefig('Energies.pdf')

# =================================================
# also try energy 2 vs energy 1
popt, pcov = curve_fit(linear, Energy2, Energy1, sigma=Energy1E,absolute_sigma=True)
print('curve_fit',popt,pcov)
print('a=%3.4f +- %3.4f, b= %3.6f +- %3.6f'%(popt[0],pcov[0][0]**0.5, popt[1],pcov[1][1]**0.5))

ymodel=linear(Energy2,popt[0],popt[1])
#print(ymodel)
chimin=chisquared(Energy1,Energy1E,ymodel)
print('Energy 2 vs. Energy 1: chi^2=%3.2f'%chimin)

# ================================================
# =========== Now try BCES from Akritas & Bershady
# =========== as implemented in numpy ============
# pront problem 18.1, BCES fit energy 1 vs energy 2
print('================================================\n',
'The BCES routines (https://github.com/rsnemmen/BCES)\n',
       'return a 4-long list of the following quantities\n',
       'a,b : best-fit parameters a,b of the linear regression such that y = Ax + B.\n',
    'aerr,berr : the standard deviations in a,b\n',
    'covab : the covariance between a and b (e.g. for plotting confidence bands)\n',
    'Each value corresponds to the Y/X, X/Y, bisector and orthogonal (not used) fits\n')

print('BCES fit energy1 vs energy2 (Problem 18.1)')
AA=np.zeros(len(Energy1))
b,a,berr,aerr,covab=bces.bces.bces(Energy1,Energy1E,Energy2,Energy2E,AA)

print('BCES models for the Energy 1 vs Energy 2',a,b,aerr,berr,covab)
# a,b aerr, berr and covab are for Y/X, X/Y, bisect., orthogonal
fitLabel=['Y/X regression','X/Y regression','Bisector','Orthogonal']
fitColor=['blue','red','grey']
fitStyle=['-','-','--']
xmin=0.6
xmax=100
ymin=0.6
ymax=100
fig,ax=plt.subplots(figsize=(8,6))
plt.xscale('log')
plt.yscale('log')
NPoints=100
xaxis=np.logspace(np.log10(xmin),np.log10(xmax),100)
yaxis=np.zeros((4,100))
for i in range(3): # skip the orthogonal fit
	for j in range(NPoints):
		yaxis[i,j]=a[i]+xaxis[j]*b[i]
		yaxis[i,j]=a[i]+xaxis[j]*b[i]
	plt.plot(xaxis,yaxis[i],linestyle=fitStyle[i],linewidth=2,color=fitColor[i],label=fitLabel[i])

for axis in [ax.xaxis, ax.yaxis]:
	axis.set_major_formatter(ScalarFormatter())

plt.errorbar(Energy1,Energy2,yerr=Energy2E,xerr=Energy1E,capsize=2,linestyle='',color='black')
plt.scatter(xaxis[0],yaxis[0,0],marker='o')
plt.scatter(xaxis[1],yaxis[0,1],marker='o')
#print(xaxis[0],yaxis[0,0],xaxis[1],yaxis[0,1])
plt.grid(which='both')
plt.xlim(xmin,xmax)
plt.ylim(ymin,ymax)
plt.xlabel('Energy 1')
plt.ylabel('Energy 2')
plt.legend(prop={'size' : 12})
plt.savefig('BCESE1vsE2.pdf')
prob22.2.py0000664000175000017500000000061314116671436011045 0ustar  maxmaxexec(open('imports.py').read())

A=[10,11,13,11,10]

B=[7,8,1,11,10,8]

meanA=np.mean(A)
varA=np.var(A,ddof=1)
stdevA=np.std(A,ddof=1)
meanB=np.mean(B)
varB=np.var(B,ddof=1)
stdevB=np.std(B,ddof=1)

print('A: %3.3f \pm %3.3f (%3.2f) , B: %3.3f\pm %3.3f (%3.2f)'%
        (meanA,stdevA,varA,meanB,stdevB,varB))

z = (meanA-meanB)/(varA/len(A) + varB/len(B))**0.5
print('Geweke z-score: %3.3f'%z)
prob22.4.py0000664000175000017500000000313314116673635011052 0ustar  maxmaxexec(open('imports.py').read())

# This code is for Problem 22.4 of the textbook
# Set up the RHat score

m=2
# NOTE: these need to be arrays, not lists
a=np.asarray([7,8,10,11,10,8])
b=np.asarray([11,11,8,10,9,12])
MCMC=[a,b]
print(a)
n=len(MCMC[0])       # number of links in the chain
print('Read %d chains of length %d'%(m,n))

indexRange=np.arange(0,3,dtype=int)
print(MCMC[0][indexRange],MCMC[1])

b=3 # length of each batch
N=int(n/b) # number of batches
print('Using %d batches'%N)
thetaMeanI=np.zeros((m,N)) # by stepping the window forward
thetaMean=np.zeros(N) # mean of means
B=np.zeros(N)
W=np.zeros(N)
VHat=np.zeros(N)
RHat=np.zeros(N)
s2i=np.zeros((m,N)) # sample variance of  i-th chain
Ni=np.zeros(N) # number of chain elements considered)
for i in range(N):
    Ni[i]= b*(i+1) # N-i # number of elements of chain considered
    indexRange=np.arange(0,b*(i+1),1,dtype=int)
    print(indexRange)
    print('from %d to %d'%(min(indexRange),max(indexRange)))
    for j in range(m):  #j-th chain
        thetaMeanI[j,i]=np.mean(MCMC[j][indexRange]) # from i to N
        print('mean %3.3f'%thetaMeanI[j,i])
        s2i[j,i]=np.var(MCMC[j][indexRange],ddof=1)
    thetaMean[i]=np.mean(thetaMeanI[:,i]) # mean of m chains
    print('thetaMean=%3.2f'%thetaMean[i])
    B[i]=Ni[i]*np.var(thetaMeanI[:,i],ddof=1)
    W[i]=np.mean(s2i[:,i]) # mean of sample variances
    VHat[i]=((Ni[i]-1)/Ni[i])*W[i] +B[i]/Ni[i] +B[i]/(m*Ni[i])
    RHat[i]=VHat[i]/W[i]
    print('%d N=%d thetaMean=%3.3f, B=%3.3f, B/N=%3.3f, W=%3.3f, VHat=%3.3f, RHatSq=%3.3f'
    %(i,Ni[i],thetaMean[i], B[i],B[i]/Ni[i],W[i],VHat[i],RHat[i]**0.5))

prob9.6.py0000664000175000017500000000065614114433056010775 0ustar  maxmaxexec(open('imports.py').read())

data=[20.3, 20.4, 19.8, 20.4, 19.9, 20.7]
N=len(data)
smean=np.mean(data)
stdev=np.std(data,ddof=1)
print('measurement: %3.3f +- %3.3f'%(smean,stdev))
mu=20
tStat=(smean-mu)/(stdev/N**0.5)
print(' t stat: %3.2f'%tStat)

#  critical value needs to be two--sided
# Use symmetry of t dist around 0, for 90% probability
tCrit=t.ppf(0.95,6)
print('Critical value of t at 0.9 probability: %3.3f'%tCrit)
probVenn.py0000644000175000017500000000461313754247605011376 0ustar  maxmaxexec(open('imports.py').read())

from matplotlib_venn import venn2, venn2_circles
from matplotlib_venn import venn3, venn3_circles

fig,ax=plt.subplots(figsize=(6,8))
#ax = fig.add_subplot(121)
v1= venn2( subsets=(5, 0, 5), set_labels = ('Group A', 'Group B'), alpha = 1.0, set_colors=('blue','red'))
venn2_circles(subsets=(10, 0, 10))
for text in v1.set_labels:
    text.set_fontsize(14)
for text in v1.subset_labels:
    text.set_fontsize(16)
# Hide the labels both set_labels and subset_labels
for idx, subset in enumerate(v1.set_labels):
    v1.set_labels[idx].set_visible(False)
# Change Backgroud
#plt.gca().set_axis_bgcolor('blue')
plt.gca().set_axis_on()
v1.get_label_by_id('10').set_text('A')
v1.get_label_by_id('11').set_text('B')
v1.get_label_by_id('01').set_text('')

 
#ax = fig.add_subplot(122)
#venn3(subsets = (20, 10, 12, 10, 9, 4, 3), set_labels = ('Group A', 'Group B', 'Group C'), alpha = 0.5)

plt.savefig('venn.pdf')

# Left panel =========================================
fig,ax=plt.subplots(figsize=(6,6))
v3=venn3(subsets = (50, 00, 15, 0, 10, 0, 5), set_labels = ('Group A', 'Group B', 'Group C'), alpha = 1.0, set_colors=('none','red','yellow'))
v3.get_label_by_id('100').set_text('$\Omega$')
v3.get_label_by_id('110').set_text('A')
v3.get_patch_by_id('100').set_alpha(0.0) # remove this color
v3.get_label_by_id('100').set_y(0.4)
v3.get_label_by_id('101').set_text('B')
v3.get_label_by_id('111').set_text('A$\cap$B')
for idx, subset in enumerate(v3.set_labels):
    v3.set_labels[idx].set_visible(False)
# add circles around the sets
c = venn3_circles(subsets =  (50, 0, 15, 0, 10, 0, 5), linestyle='solid', linewidth=2, color='black')
plt.savefig('venn1.pdf')

# Right panel ========================================
ig,ax=plt.subplots(figsize=(6,6))
v3=venn3(subsets = (50, 0, 15, 0, 10, 0, 0), set_labels = ('Group A', 'Group B', 'Group C'), alpha = 1.0, set_colors=('none','red','yellow'))
v3.get_label_by_id('100').set_text('$\Omega$')
v3.get_label_by_id('110').set_text('A')
v3.get_patch_by_id('100').set_alpha(0.0) # remove this color
v3.get_label_by_id('100').set_y(0.4)
v3.get_label_by_id('101').set_text('B')
#v3.get_label_by_id('111').set_text('A$\cap$B')
for idx, subset in enumerate(v3.set_labels):
    v3.set_labels[idx].set_visible(False)
# add circles around the sets
c = venn3_circles(subsets =  (50, 00, 15, 0, 10, 0, 0), linestyle='solid', linewidth=2, color='black')
plt.savefig('venn2.pdf')




rDist.py0000644000175000017500000001121114116121226010641 0ustar  maxmaxexec(open('imports.py').read())

# ==============================================
# Plot the rdist for the book.

fig,ax=plt.subplots(figsize=(8,6))

#Note that GAmma(0) is undefined, so f=2 is not meaningful
N=[3, 4,5,7,12,22]  # NUMBER OF data points for r
colors=['black','grey','red','blue','lightgreen','darkgreen']
k=2 # this is simple linear regression with 2 free parameters
x = np.linspace(-1,1,100)
for i in range(len(N)):
    f=N[i]-k
    rdistVar=rdist.var(f)
    plt.plot(x,rdist.pdf(x, f),color=colors[i],linewidth=2,label='f=%d'%f)
    print('f=%d: var=%2.4f, 1/(f+1)=%3.4f'%(f,rdistVar,1/(f+1)))
# overplot the N(0,1) for reference - not useful
#plt.plot(x,norm.pdf(x),color='black',linestyle='--',linewidth=2)

plt.legend(loc=1,prop={'size': 12})
#plt.xlabel('r or r2')
plt.grid(which='both')
plt.xlabel('r')
plt.xlim((-1,1))
plt.ylim((0,1.8))
ax.yaxis.set_major_locator(MultipleLocator(0.2))
ax.yaxis.set_minor_locator(MultipleLocator(0.1))
ax.xaxis.set_major_locator(MultipleLocator(0.5))
ax.xaxis.set_minor_locator(MultipleLocator(0.1))

plt.ylabel('Probability Distribution Function')
plt.savefig('rDist.pdf')

# ===========================================
# Also plot the r^2 distribution
fig,ax=plt.subplots(figsize=(8,6))

k=2 #k=2 or m=1 is for R^2=r^2, simple linear regression
xplus = np.linspace(0,1,100)
for i in range(len(N)):
    a=(k-1)/2.
    b=(N[i]-k)/2.
    f=N[i]-k
    plt.plot(xplus,beta.pdf(xplus,a,b),color=colors[i],linewidth=2,linestyle='-',label='f=%d'%f)
plt.legend(loc=1,prop={'size': 12})
plt.ylim((0,4))
#plt.xlabel('r or r2')
plt.grid(which='both')
plt.xlabel('$r^2$')
plt.xlim((0,1))
ax.yaxis.set_major_locator(MultipleLocator(1.0))
ax.yaxis.set_minor_locator(MultipleLocator(0.2))
ax.xaxis.set_major_locator(MultipleLocator(0.2))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
plt.ylabel('Probability Distribution Function')
plt.savefig('r2Dist.pdf')

# =======================================================
# Also plot the R^2 distribution for multiple regressions
fig,ax=plt.subplots(figsize=(8,6))
N=[4,5,8,13,23]  # NUMBER OF data points for r
k=3 #k=2 or m=1 is for R^2=r^2, simple linear regression
xplus = np.linspace(0,1,100)
for i in range(len(N)):
    a=(k-1)/2.
    b=(N[i]-k)/2.
    f=N[i]-k
    plt.plot(xplus,beta.pdf(xplus,a,b),color=colors[i],linewidth=2,linestyle='-',label='f=%d'%f)
plt.legend(loc=1,prop={'size': 12})
plt.ylim(0,4)
#plt.xlabel('r or r2')
plt.grid(which='both')
plt.xlabel('$R^2$ ($m$=2)')
plt.xlim((0,1))
ax.yaxis.set_major_locator(MultipleLocator(1.0))
ax.yaxis.set_minor_locator(MultipleLocator(0.2))
ax.xaxis.set_major_locator(MultipleLocator(0.2))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
plt.ylabel('Probability Distribution Function')
plt.savefig('R2Dist.pdf')



# Try and calculate r for N=2 data
X=uniform.rvs(size=2)
Y=uniform.rvs(size=2)
b,a,r,p,stdrerr=linregress(X,Y)
print('b=%3.2f, r=%3.3f, p=%3.3f'%(b,r,p))
bPr,aPr,rPr,pPr,stdrerr=linregress(Y,X)
print('bPr=%3.2f, b*bPr=%3.3f, r=%3.3f, p=%3.3f'%(bPr,b*bPr,rPr,pPr))

# ================================================
# Critical value at p significance
N=1079 # Number of data points
k=2 # linear regression
f=N-k# Number of degrees of freedom
# Keep in mind that we look for |r|>rCrit, i.e., a two-sided hypothesis
# so, if we want 99% prob that |r|>rCrit, and since 
# f(r) is symmetrical around 0, we need to do:
p=0.99
q=1-p # 1\%
q=q/2 # 0.5% to the right, and also 0.5% to the left
rCrit=rdist.ppf(1-q,f)
print('N=%d, Two-sided critical value of r at %3.4f significance is %4.3f'%(N,p,rCrit))


# =================================================
# Now also the distribution of r^2, which is a Beta(1,N-1)
# or, for multiple regression Beta(k-1, N-k)
# which makes sense since we are only interested
# in absolute values of r.
a=(k-1)/2
b=(N-k)/2
r2Crit=beta.ppf(p,a,b)
print('N=%d, Critical value of r^2 at %3.4f significance is %4.5f'%(N,p,r2Crit))
print('=> Two-sided Critical value of r at %3.4f significance is %4.3f'%(p,r2Crit**0.5))

#######################################################
#### Solve problem 14.6 using both the standard beta distributions
print('prob14.6')
R2=0.5
N=20
m=5
f=N-m-1 
p=0.90
q=1-p # 10\%
q=q/2 # 5% to the right, and also 5% to the left
# Using rDist, or symmetric beta distr - this is NOT applicable to m>2
rCrit=rdist.ppf(1-q,f)
print('N=%d, m=%d Two-sided critical value of r at %3.4f significance is %4.3f'%(N,m,p,rCrit))
print('Note: this does not apply to R with m>2')
# using R^2, or standard beta distribution
a=m/2
b=(N-m-1)/2
r2Crit=beta.ppf(p,a,b)
print('N=%d, Critical value of R^2 at %3.4f significance is %4.5f'%(N,p,r2Crit))
print('=> Two-sided Critical value of R at %3.4f significance is %4.3f'%(p,r2Crit**0.5))


sumUniforms.py0000644000175000017500000000372314104226101012107 0ustar  maxmaxexec(open('imports.py').read())
# Make figure 4.1 (and problem 4.4)
# This is the sum of 100 uniforms, with a number of samples n=1,000 or 100,000


N=100   # number of uniform distributions
n=1000  # number of samples for each variable

sample=np.zeros((N,n))
sampleSum=np.zeros(n)
# 1. Simulate n samples from N random variables
for i in range(N):
    sample[i,:]=uniform.rvs(size=n)
#2. Sum the values of the N variables, sample by sample
    sampleSum[:]=sum(sample[:,])
#3. Plot the sample distribution of the sum
# =========================================

fig,ax=plt.subplots(figsize=(8,6))

bins=np.linspace(40,60,100)
ax.hist(sampleSum,bins=bins,cumulative=False,histtype='bar',log=False,density=True,linewidth=2,alpha=0.5,color='black',label='Sum of %d uniform variables with %d samples'%(N,n))
ax.hist(sampleSum,bins=bins,cumulative=False,histtype='step',color='black',density=True)
# the expected distribution for large N is a Gaussian  ---------------------------
meanU=0.5 # mean of U(0,1)
varU=1./12. # variance of U(0,1)
mu=N*meanU
var=N*varU
ax.plot(bins,norm.pdf(bins,loc=mu,scale=var**0.5),color='red',linewidth=2,label='Gaussian distribution')
# --------------------------------------------------------------------------------
ax.set_xlabel('y')
ax.set_ylabel('Probability of occurrence')

# -------------------------------------
# ---- overplot with 100,000 samples
n=100000  # number of samples for each variable
sampleSum=np.zeros(n)
sample=np.zeros((N,n))
for i in range(N):
    sample[i,:]=uniform.rvs(size=n)
#2. Sum the values of the N variables, sample by sample
    sampleSum[:]=sum(sample[:,])
bins=np.linspace(40,60,100)
ax.hist(sampleSum,bins=bins,cumulative=False,histtype='bar',log=False,density=True,linewidth=2,alpha=0.5,color='blue',label='Sum of %d uniform variables with %d samples'%(N,n))
ax.hist(sampleSum,bins=bins,cumulative=False,histtype='step',color='blue',density=True)



plt.legend(loc=2,prop={'size': 12})
#plt.show()



plt.savefig('sumUniforms.pdf')
thomson.py0000644000175000017500000002056214116647600011265 0ustar  maxmax# This program calculates means and std. dev. from Thomson data
# Making use of the two tabular datasets
# M. Bonamente, 2020
exec(open('imports.py').read())


# =====================================================
# error propagation formula for m/e(I,W/Q)
# it returns sigma^2/mean^2
def errPropme(I,sigmaI,wq,sigmawq):
    result = 4*(sigmaI/I)**2+(sigmawq/wq)**2
    return result
# error propagation formula for v(I,W/Q); include covariance
def errPropv(I,sigmaI,wq,sigmawq,sigma2Iwq):
    result = (sigmaI/I)**2+(sigmawq/wq)**2 -2*sigma2Iwq/(I*wq)
    return result
# ====================================================


# Read in the two tables =============================
dataTypes=[('gas','S3'),('wq','f'),('I','f'),('me','f'),('me*','f'),('v','f'),('v*','f')]
table1 = np.genfromtxt('thomson1.dat',skip_header=1,dtype=dataTypes)
#print(table1)
table2 = np.genfromtxt('thomson2.dat',skip_header=1,dtype=dataTypes)
#print(table2)

# Generate lists of relevant measurements
# Table1
table1Gas=table1['gas']
table1wq=table1['wq']
table1I=table1['I']
table1me=table1['me']
table1v=table1['v']
#Table2
table2Gas=table2['gas']
table2wq=table2['wq']
table2I=table2['I']
table2me=table2['me']
table2v=table2['v']
# =====================================================

# Do the statistics for the tables
#1. Means and standard deviations ======================
# NOTE: python calculates standard deviations dividing by 1/N, not 1/(N-1)
# the ddof parameters must be specified to calculate the sample standard dev.
table1Meanwq=np.mean(table1wq)
table2Meanwq=np.mean(table2wq)
table1MeanI=np.mean(table1I)
table2MeanI=np.mean(table2I)

table1Stdwq=np.std(table1wq,ddof=1)
table2Stdwq=np.std(table2wq,ddof=1)
table1StdI=np.std(table1I,ddof=1)
table2StdI=np.std(table2I,ddof=1)

# problem 5.2
print('For problem 5.2')
print('================================')
print('Table 1: %d measurements; table 2: %d measurements'%(len(table1wq),len(table2wq)))
print('Table1: w/q = %f +- %f, I = %f +- %f (mean and std. dev.)'%(table1Meanwq,table1Stdwq,table1MeanI,table1StdI))
print('Table2: w/q = %f +- %f, I = %f +- %f (mean and std. dev.)'%(table2Meanwq,table2Stdwq,table2MeanI,table2StdI))

# 2. Covariance and correlations coeff. between w/q and I
table1CovwqI=np.cov(table1wq,table1I,ddof=1)   # this is the 2x2 covariance matrix
table1CorrwqI=np.corrcoef(table1wq,table1I)     # this is Pearson's r coeff.

table2CovwqI=np.cov(table2wq,table2I,ddof=1)   # this is the 2x2 covariance matrix
table2CorrwqI=np.corrcoef(table2wq,table2I)     # this is Pearson's r coeff.

#print(table1CovwqI[0,1],table1CorrwqI[0,1])
print('Table1: Cov(wq,I)=%f, r=%f'%(table1CovwqI[0,1],table1CorrwqI[0,1]))
print('Table2: Cov(wq,I)=%f, r=%f'%(table2CovwqI[0,1],table2CorrwqI[0,1]))
print('================================')

# problem 5.2
# 3. Sample mean and std. dev. for m/e for Tube 1
table1Meanme=np.mean(table1me)
table1Stdme=np.std(table1me,ddof=1)
print('Table 1 m/e',table1me)
print('Table1: m/e = %f +- %f (+- %3.3f for mean), 90 C.I.: %3.3f - %3.3f'%(table1Meanme,table1Stdme,table1Stdme/len(table1me)**0.5,table1Meanme-1.65*table1Stdme,table1Meanme+1.65*table1Stdme))
print('Table1: Std. dev. of m/e from error prop. (assuming no correlation between I and W/Q)')
print('sigma m/e = %f'%(table1Meanme**2*errPropme(table1MeanI,table1StdI,table1Meanwq,table1Stdwq))**0.5)

# also add m/e for Tube 2 (example in Ch. 7 of 3rd Ed.)
table2Meanme=np.mean(table2me)
table2Stdme=np.std(table2me,ddof=1)
# problem 7.5
print('Problem 7.5')
print('Table 2 m/e',table2me)
print('Table2: Mean m/e = %f +- %f (+- %3.3f for mean), 90 C.I.: %3.3f - %3.3f'%(table2Meanme,table2Stdme,table2Stdme/len(table2me)**0.5,table2Meanme-1.65*table2Stdme,table2Meanme+1.65*table2Stdme))

# also add upper limits and lower limits, for both sample and mean
print('Table1: m/e >= %3.4f (%3.4f for mean)'%(table1Meanme-1.28*table1Stdme,table1Meanme-1.28*table1Stdme/len(table1me)**0.5))
# problem 7.5 3rd ed.
print('Table2: m/e >= %3.4f (%3.4f for mean)'%(table2Meanme-1.28*table2Stdme,table2Meanme-1.28*table2Stdme/len(table2me)**0.5))
print('================================')

# problem 5.1
# 4. Sample mean and st. dev. for v for Tube 1
table1Meanv=np.mean(table1v)
table1Stdv=np.std(table1v,ddof=1)
print('For Problem 5.1')
print('Table1: Mean v = %3.2f +- %3.2f'%(table1Meanv,table1Stdv))
print('Table1: Std. dev. of v from error prop. (assuming no correlation between I and W/Q)')
print('sigma v = %f'%(table1Meanv**2*errPropv(table1MeanI,table1StdI,table1Meanwq,table1Stdwq,0))**0.5)
print('Table1: Std. dev. of v from error prop. (using sample covariance)')
print('Sample covariance between W/Q and I',table1CovwqI[0,1])
print('sigma v = %f'%(table1Meanv**2*errPropv(table1MeanI,table1StdI,table1Meanwq,table1Stdwq,table1CovwqI[0,1]))**0.5)

# ==============================================================================
# 4. problem 7.1, calculate confidence intervals and lower/upper limits on v
table1meanv=np.mean(table1v)
table1sigmav=np.std(table1v,ddof=1)
table2meanv=np.mean(table2v)
table2sigmav=np.std(table2v,ddof=1)
print('Problem 7.1')
print('Table 1: v=%4.2f +- %4.2f; Table 2: %4.2f +- %4.2f'%(table1meanv,table1sigmav,table2meanv,table2sigmav))
# ==== 90% confidence intervals are +-1.65 sigma
nSigma=1.65
print('90pct. confidence intervals on v - Table 1: %4.2f - %4.2f; Table 2: %4.2f - %4.2f'%(table1meanv-nSigma*table1sigmav,table1meanv+nSigma*table1sigmav,table2meanv-nSigma*table2sigmav,table2meanv+nSigma*table2sigmav))

# ==== 90% upper and lower limits are +-1.28 sigma
nSigmaL=1.28
print('90pct. confidence upper limit on v - Table 1: %4.2f; Table 2: %4.2f'%(table1meanv+nSigmaL*table1sigmav,table2meanv+nSigmaL*table2sigmav))
print('90pct. confidence lower limit on v - Table 1: %4.2f; Table 2: %4.2f'%(table1meanv-nSigmaL*table1sigmav,table2meanv-nSigmaL*table2sigmav))

# ============================================================================================
# 5. F statistic for measurements in Air for Tables 1 and 2
print('Problem 9.4')
table1Air=table1me[0:7] # first 7 measurements
print('m/e air measurements for table1:',len(table1Air),table1Air)
s12Air = np.var(table1Air,ddof=1)
m1Air= np.mean(table1me[0:6])
# turn table2 data from array to list, for easier concatenation
table2me=np.ndarray.tolist(table2me)
table2meAir=table2me[0:3]+table2me[4:5]+table2me[6:7]+table2me[9:11]
print('m/e air measurements for table2:', len(table2meAir), table2meAir)

s22Air = np.var(table2meAir,ddof=1)
m2Air= np.mean(table2meAir)
print('mean in air for two tables: %4.3f, %4.3f'%(m1Air,m2Air))
print('variances for air measurements of the two tables: %3.4f, %3.4f'%(s12Air,s22Air))
print('mesurements: Air 1 = %3.4f+-%3.4f, Air 2 = %3.4f+-%3.4f'%(m1Air,s12Air**0.5,m2Air,s22Air**0.5))
F=s12Air/s22Air
print('F=%3.2f'%F)
# calculate critical value
fCrit=f.ppf(0.9,6,6)
print('Critical value of f at 0.9 probability: %3.3f'%fCrit)

# ============================================
# ==== K-S test on m/e problem 19.1 and problem 19.2
print('problem 19.1 and 19.2')
print(table1me,table2me)
muM=0.57
sigmaM=0.1
xaxis=np.linspace(0,1,100)
modelG=norm.cdf(xaxis,muM,sigmaM)
fig,ax=plt.subplots(figsize=(8,6))
# check the 'align' so that it is a CDF

# 0 and 1 are the limits along the x-axis, for nice step plot
a1,b1=sampleCDF(table1me,0,1)
a2,b2=sampleCDF(table2me,0,1)
print('a=',a1,'b=',b1)
for i in range(len(a1)):
	plt.vlines(a1[i],0,b1[i],linewidth=0.75,linestyle='--',color='blue')
for i in range(len(a2)):
	plt.vlines(a2[i],0,b2[i],linewidth=0.75,linestyle='--',color='red')
plt.plot(xaxis,modelG,linewidth=2,color='black',label='Model')
plt.step(a1,b1,where='post',linewidth=2,color='blue',label='$m/e$ (Tube 1)')
plt.step(a2,b2,where='post',linewidth=2,color='red',label='$m/e$ (Tube 2)')
# --------------------------------------------------------
plt.legend(prop={'size': 12})
plt.grid(which='both')
plt.xlim(0.2,0.9)
plt.ylim(0,1.05)
ax.xaxis.set_major_locator(MultipleLocator(0.2))
ax.xaxis.set_minor_locator(MultipleLocator(0.05))
ax.yaxis.set_major_locator(MultipleLocator(0.2))
ax.yaxis.set_minor_locator(MultipleLocator(0.05))
plt.xlabel('Variable $m/e$')
plt.ylabel('Cumulative distribution')
plt.savefig('meKS.pdf')
KSV,KSp=kstest(table1me,lambda x: norm.cdf(x, loc=muM, scale=sigmaM))
print('Result of KS test Table 1:',KSV,KSp)
KSV,KSp=kstest(table2me,lambda x: norm.cdf(x, loc=muM, scale=sigmaM))
print('Result of KS test Table 2:',KSV,KSp)

# two-sample KS test
KSV,KSp=ks_2samp(table1me,table2me)
print('Result of 2-sample KS test:',KSV,KSp)
5points.dat0000644000175000017500000000016114064126363011310 0ustar  maxmaxn       X       Y
1       0.0     25
2       1.0     36
3       2.0     64
4       3.0     49
5       4.0     81
6points.dat0000644000175000017500000000020014064136306011301 0ustar  maxmaxn       X       Y
1       0.0     0.0
2       1.0     1.5
3       2.0     1.5
4       3.0     2.5
5       4.0     4.5
6	5.0	5.0
fatherMotherStature.dat0000650000175000017500000000340313655523466013720 0ustar  maxmax0    0    0    0    0    0    0    1     0.5   0     0     0     0     0     0    0    0    0
0    0    0    0    0    0    0    0     0.5   0     0     0     0     0     0    0    0    0
0    0    0    0    0    0    0    0.25  0.25  0     0.5   0     0     0     0    0    0    0
0    0    0    0.5  1    0    0    0.25  0.25  0     0.5   0     0     0     0    0    0    0
0    0    0    0    0.75 1.25 0    1     1.75  1.75  0     0     0     0     0    0    0    0
0    0    0    0.25 1    1.25 1.5  4     3.25  2.5   3     1.25  0.5   0     0    0    0    0
0    0.25 0.75 1.25 1.25 2.75 4    7     5.75  4.5   3.75  1.25  2     0     0    0    0    0
0    1.25 1.25 1    4    4.5  7.75 10    15    16.75 9     5.5   3.25  1.25  1    0.5  0    0
0.25 0.25 0.5  2    4.25 4.5  18   16    24    14.75 23.25 12.75 7.25  5.75  4.25 0.75 0    0
0.25 0.25 0    0    8    8.25 15   17.25 25    20.75 24    14.25 14.25 10    4    0.75 0.5  0
0    0.5  0.5  1.25 4.75 7.75 10   26    21.25 28    28    23    14.25 10.75 4.5  2    1    0.5
0    0    0.25 2    3.5  4.5  9    21    15.75 20.75 19.5  24    22.5  10.75 4    2.25 2.25 0.5
0    0    1.25 0.75 2    6    6.5  9.75  16    18.25 23    16.75 13.75 6.75  4.75 2.25 0.25 1.5
0    0    0    0.25 1.5  1.5  3.25 5.5   9.75  7     15.5  12.75 10.5  6.25  4.25 1.75 0.25 0
0    0    0    0.25 1    0.75 0.5  3.5   5     3     7.25  7.75  7     3.5   2.75 1.5  0.25 0
0    0    0    0    0    0    0    1     2.5   1.5   2.75  3.25  2.75  1.5   1    0.5  0.25 0
0    0    0    0    0    0    0    0     0     1     2.5   1.25  1.25  0.5   1    0.25 0.25 0
0    0    0    0    0    0    0    0     0     0     0     0.25  2.25  0     2    0    0    0
0    0    0    0    0    0    0    0     0     0     0     0     1     0     0.5  0    0    0  

hubbleHumasonData.dat0000650000175000017500000000031313742300302013256 0ustar  maxmaxNebula	v	N	m
Virgo	890	7	12.5
Pegasus	3810	5	15.5
Pisces	4630	4	15.4
Cancer	4820	2	16.0
Perseus	5230	4	16.4
Coma	7500	3	17.0
UMajor	11800	1	18.0
Leo	19600	1	19.0
unnamed	2350	16	13.8
unnamed	630	21	11.6
irisData.dat0000650000175000017500000001056613773650532011464 0ustar  maxmaxSL	SW	PL	PW	species		SL	SW	PL	PW	species		SL	SW	PL	PW	species
5.1	3.5	1.4	0.2	Iris-setosa	7	3.2	4.7	1.4	Iris-versicolor	6.3	3.3	6	2.5	Iris-virginica
4.9	3	1.4	0.2	Iris-setosa	6.4	3.2	4.5	1.5	Iris-versicolor	5.8	2.7	5.1	1.9	Iris-virginica
4.7	3.2	1.3	0.2	Iris-setosa	6.9	3.1	4.9	1.5	Iris-versicolor	7.1	3	5.9	2.1	Iris-virginica
4.6	3.1	1.5	0.2	Iris-setosa	5.5	2.3	4	1.3	Iris-versicolor	6.3	2.9	5.6	1.8	Iris-virginica
5	3.6	1.4	0.2	Iris-setosa	6.5	2.8	4.6	1.5	Iris-versicolor	6.5	3	5.8	2.2	Iris-virginica
5.4	3.9	1.7	0.4	Iris-setosa	5.7	2.8	4.5	1.3	Iris-versicolor	7.6	3	6.6	2.1	Iris-virginica
4.6	3.4	1.4	0.3	Iris-setosa	6.3	3.3	4.7	1.6	Iris-versicolor	4.9	2.5	4.5	1.7	Iris-virginica
5	3.4	1.5	0.2	Iris-setosa	4.9	2.4	3.3	1	Iris-versicolor	7.3	2.9	6.3	1.8	Iris-virginica
4.4	2.9	1.4	0.2	Iris-setosa	6.6	2.9	4.6	1.3	Iris-versicolor	6.7	2.5	5.8	1.8	Iris-virginica
4.9	3.1	1.5	0.1	Iris-setosa	5.2	2.7	3.9	1.4	Iris-versicolor	7.2	3.6	6.1	2.5	Iris-virginica
5.4	3.7	1.5	0.2	Iris-setosa	5	2	3.5	1	Iris-versicolor	6.5	3.2	5.1	2	Iris-virginica
4.8	3.4	1.6	0.2	Iris-setosa	5.9	3	4.2	1.5	Iris-versicolor	6.4	2.7	5.3	1.9	Iris-virginica
4.8	3	1.4	0.1	Iris-setosa	6	2.2	4	1	Iris-versicolor	6.8	3	5.5	2.1	Iris-virginica
4.3	3	1.1	0.1	Iris-setosa	6.1	2.9	4.7	1.4	Iris-versicolor	5.7	2.5	5	2	Iris-virginica
5.8	4	1.2	0.2	Iris-setosa	5.6	2.9	3.6	1.3	Iris-versicolor	5.8	2.8	5.1	2.4	Iris-virginica
5.7	4.4	1.5	0.4	Iris-setosa	6.7	3.1	4.4	1.4	Iris-versicolor	6.4	3.2	5.3	2.3	Iris-virginica
5.4	3.9	1.3	0.4	Iris-setosa	5.6	3	4.5	1.5	Iris-versicolor	6.5	3	5.5	1.8	Iris-virginica
5.1	3.5	1.4	0.3	Iris-setosa	5.8	2.7	4.1	1	Iris-versicolor	7.7	3.8	6.7	2.2	Iris-virginica
5.7	3.8	1.7	0.3	Iris-setosa	6.2	2.2	4.5	1.5	Iris-versicolor	7.7	2.6	6.9	2.3	Iris-virginica
5.1	3.8	1.5	0.3	Iris-setosa	5.6	2.5	3.9	1.1	Iris-versicolor	6	2.2	5	1.5	Iris-virginica
5.4	3.4	1.7	0.2	Iris-setosa	5.9	3.2	4.8	1.8	Iris-versicolor	6.9	3.2	5.7	2.3	Iris-virginica
5.1	3.7	1.5	0.4	Iris-setosa	6.1	2.8	4	1.3	Iris-versicolor	5.6	2.8	4.9	2	Iris-virginica
4.6	3.6	1	0.2	Iris-setosa	6.3	2.5	4.9	1.5	Iris-versicolor	7.7	2.8	6.7	2	Iris-virginica
5.1	3.3	1.7	0.5	Iris-setosa	6.1	2.8	4.7	1.2	Iris-versicolor	6.3	2.7	4.9	1.8	Iris-virginica
4.8	3.4	1.9	0.2	Iris-setosa	6.4	2.9	4.3	1.3	Iris-versicolor	6.7	3.3	5.7	2.1	Iris-virginica
5	3	1.6	0.2	Iris-setosa	6.6	3	4.4	1.4	Iris-versicolor	7.2	3.2	6	1.8	Iris-virginica
5	3.4	1.6	0.4	Iris-setosa	6.8	2.8	4.8	1.4	Iris-versicolor	6.2	2.8	4.8	1.8	Iris-virginica
5.2	3.5	1.5	0.2	Iris-setosa	6.7	3	5	1.7	Iris-versicolor	6.1	3	4.9	1.8	Iris-virginica
5.2	3.4	1.4	0.2	Iris-setosa	6	2.9	4.5	1.5	Iris-versicolor	6.4	2.8	5.6	2.1	Iris-virginica
4.7	3.2	1.6	0.2	Iris-setosa	5.7	2.6	3.5	1	Iris-versicolor	7.2	3	5.8	1.6	Iris-virginica
4.8	3.1	1.6	0.2	Iris-setosa	5.5	2.4	3.8	1.1	Iris-versicolor	7.4	2.8	6.1	1.9	Iris-virginica
5.4	3.4	1.5	0.4	Iris-setosa	5.5	2.4	3.7	1	Iris-versicolor	7.9	3.8	6.4	2	Iris-virginica
5.2	4.1	1.5	0.1	Iris-setosa	5.8	2.7	3.9	1.2	Iris-versicolor	6.4	2.8	5.6	2.2	Iris-virginica
5.5	4.2	1.4	0.2	Iris-setosa	6	2.7	5.1	1.6	Iris-versicolor	6.3	2.8	5.1	1.5	Iris-virginica
4.9	3.1	1.5	0.2	Iris-setosa	5.4	3	4.5	1.5	Iris-versicolor	6.1	2.6	5.6	1.4	Iris-virginica
5	3.2	1.2	0.2	Iris-setosa	6	3.4	4.5	1.6	Iris-versicolor	7.7	3	6.1	2.3	Iris-virginica
5.5	3.5	1.3	0.2	Iris-setosa	6.7	3.1	4.7	1.5	Iris-versicolor	6.3	3.4	5.6	2.4	Iris-virginica
4.9	3.6	1.4	0.1	Iris-setosa	6.3	2.3	4.4	1.3	Iris-versicolor	6.4	3.1	5.5	1.8	Iris-virginica
4.4	3	1.3	0.2	Iris-setosa	5.6	3	4.1	1.3	Iris-versicolor	6	3	4.8	1.8	Iris-virginica
5.1	3.4	1.5	0.2	Iris-setosa	5.5	2.5	4	1.3	Iris-versicolor	6.9	3.1	5.4	2.1	Iris-virginica
5	3.5	1.3	0.3	Iris-setosa	5.5	2.6	4.4	1.2	Iris-versicolor	6.7	3.1	5.6	2.4	Iris-virginica
4.5	2.3	1.3	0.3	Iris-setosa	6.1	3	4.6	1.4	Iris-versicolor	6.9	3.1	5.1	2.3	Iris-virginica
4.4	3.2	1.3	0.2	Iris-setosa	5.8	2.6	4	1.2	Iris-versicolor	5.8	2.7	5.1	1.9	Iris-virginica
5	3.5	1.6	0.6	Iris-setosa	5	2.3	3.3	1	Iris-versicolor	6.8	3.2	5.9	2.3	Iris-virginica
5.1	3.8	1.9	0.4	Iris-setosa	5.6	2.7	4.2	1.3	Iris-versicolor	6.7	3.3	5.7	2.5	Iris-virginica
4.8	3	1.4	0.3	Iris-setosa	5.7	3	4.2	1.2	Iris-versicolor	6.7	3	5.2	2.3	Iris-virginica
5.1	3.8	1.6	0.2	Iris-setosa	5.7	2.9	4.2	1.3	Iris-versicolor	6.3	2.5	5	1.9	Iris-virginica
4.6	3.2	1.4	0.2	Iris-setosa	6.2	2.9	4.3	1.3	Iris-versicolor	6.5	3	5.2	2	Iris-virginica
5.3	3.7	1.5	0.2	Iris-setosa	5.1	2.5	3	1.1	Iris-versicolor	6.2	3.4	5.4	2.3	Iris-virginica
5	3.3	1.4	0.2	Iris-setosa	5.7	2.8	4.1	1.3	Iris-versicolor	5.9	3	5.1	1.8	Iris-virginica
mendel.dat0000650000175000017500000000040314113673423011146 0ustar  maxmaxCharacter		Dominant	Recessive	fraction
Round_seed		5474		1850		0.7474
Yellow_seed		6022		2001		0.7506
Violet_flower		705		224		0.7589
Inflated_pod		882		299		0.7468
Green_yellow_pod	428		152		0.7379
Axial_flower		651		207		0.7587
Long_stem		787		277		0.7397

openSafely.dat0000644000175000017500000000012613745021777012025 0ustar  maxmax1	5914384	54
2	2849984	140
3	3051110	522
4	2392392	1101
5	1938842	2635
6	1131680	6474
pressureRatio.dat0000650000175000017500000000312113655523467012565 0ustar  maxmaxNum	Radius	R_e+	R_e-	E1	E1_e+	E1_e-	E2	E2_e+	E2_e+	ratio	r_e+	r_e-
1	221.1	11	12.3	8.3	0.76	0.88	9.67	1.14	1.12	0.86	0.08	0.07
2	268.5	22.1	20.7	4.92	0.77	0.7	4.19	0.82	0.7	1.17	0.16	0.15
3	138.4	12.7	11.9	3.03	0.53	0.49	2.61	0.59	0.49	1.16	0.2	0.18
4	714.3	23.5	34.5	49.61	3.15	3.19	60.62	4.84	6.13	0.82	0.06	0.05
5	182.3	18.5	15.1	2.75	0.49	0.43	3.3	0.81	0.61	0.83	0.14	0.14
6	72.1	5.5	5.7	1.01	0.23	0.2	0.86	0.14	0.13	1.17	0.24	0.21
7	120.3	8.6	7.5	5.04	0.66	0.57	3.8	0.72	0.57	1.33	0.16	0.15
8	196.2	15.1	15.5	5.18	0.73	0.7	6	1.17	1.11	0.86	0.14	0.11
9	265.7	8.7	8.6	12.17	1.22	1.17	10.56	0.93	0.95	1.14	0.13	0.1
10	200	9.6	10.7	7.74	0.57	0.58	6.26	0.78	0.83	1.24	0.14	0.11
11	78.8	5.6	5.1	1.08	0.16	0.15	0.73	0.11	0.1	1.49	0.26	0.24
12	454.4	20.3	20.3	17.1	2.64	2.03	23.12	2.36	2.32	0.75	0.07	0.06
13	109.4	8.3	8.3	3.31	0.34	0.34	3.06	0.54	0.52	1.09	0.18	0.15
14	156.5	11.5	10.2	2.36	0.61	0.58	2.31	0.36	0.31	1.02	0.26	0.23
15	218	6.6	5.9	14.02	0.75	0.75	21.59	1.82	1.82	0.65	0.04	0.04
16	370.7	7.6	8	31.41	1.56	1.56	29.67	1.56	1.57	1.06	0.06	0.06
17	189.1	16.4	15.4	2.15	0.45	0.39	2.52	0.57	0.51	0.86	0.22	0.18
18	150.5	4.2	4.6	3.39	0.57	0.5	4.75	0.44	0.46	0.72	0.11	0.11
19	326.7	12.1	9.9	15.73	1.43	1.3	18.03	1.54	1.26	0.87	0.06	0.06
20	189.1	9.9	9.1	5.04	0.65	0.55	4.61	0.61	0.5	1.09	0.12	0.12
21	147.7	8	11.1	2.53	0.29	0.3	2.76	0.37	0.48	0.93	0.12	0.1
22	504.6	12.5	11.2	44.97	2.99	2.74	43.93	3.08	2.59	1.02	0.05	0.05
23	170.5	8.6	8.1	3.89	0.3	0.29	3.93	0.49	0.42	0.98	0.1	0.09
24	297.6	13.1	13.6	10.78	1.04	1.02	10.48	1.34	1.22	1.04	0.1	0.11
25	256.2	13.4	14.4	7.27	0.81	0.77	7.37	0.97	0.95	0.99	0.09	0.09
problem10.3.dat0000650000175000017500000000003013655523467011654 0ustar  maxmaxn	x	y
1	0	1
2	1	1
3	2	1
problem16.3.dat0000650000175000017500000000004113655523467011664 0ustar  maxmaxA 10 11 13 11 10
B 7 8 1 11 10 8
problem16.5.dat0000650000175000017500000000004313655523467011670 0ustar  maxmaxA 7 8 1 11 10 8
B 11 11 8 10 9 12

problem8.2.dat0000650000175000017500000000006413655523467011611 0ustar  maxmaxn	x	y
1 	0.0	25
2	1.0	36
3	2.0	47
4	3.0	64
5	4.0	81
problem8.3.dat0000650000175000017500000000010313655523467011604 0ustar  maxmaxn	x	y
1 	0.0	0.0
2	1.0	1.5
3	2.0	1.5
4	3.0	2.5
5	4.0	4.5
6	5.0	5.0
problem8.7.dat0000650000175000017500000000004713655523467011617 0ustar  maxmaxn	x	y
1	-2	-1
2	-1	0
3	0	1
4	1	0
5	2	2
thomson1.dat0000650000175000017500000000057713655523467011502 0ustar  maxmaxGas	w/q	I	m/e	m/e*	v	v*
air	4.6	230	0.57	0.58	4	4.000
air	18	350	0.34	0.34	10	10.286
air	6.1	230	0.43	0.43	5.4	5.304
air	25	400	0.32	0.32	12	12.500
air	5.5	230	0.48	0.48	4.8	4.783
air	10	285	0.4	0.41	7	7.018
air	10	285	0.4	0.41	7	7.018
H*	6	205	0.35	0.35	6	5.854
H	21	460	0.5	0.5	9.2	9.130
CO2	8.4	260	0.4	0.4	7.5	6.462
CO2	14.7	340	0.4	0.39	8.5	8.647
CO2	30	480	0.39	0.38	13	12.500
thomson2.dat0000650000175000017500000000056013755226345011467 0ustar  maxmaxGas	w/q	I	m/e	m/e*	v	v*
air	2.8	175	0.53	0.55	3.3	3.200
air	2.8	175	0.47	0.55	4.1	3.200
air	3.5	181	0.47	0.47	3.8	3.867
H	2.8	175	0.53	0.55	3.3	3.200
air	2.5	160	0.51	0.51	3.1	3.125
CO2	2	148	0.54	0.55	2.5	2.703
air	1.8	151	0.63	0.63	2.3	2.384
H	2.8	175	0.53	0.55	3.3	3.200
H	4.4	201	0.46	0.46	4.4	4.378
Air	2.5	176	0.61	0.62	2.8	2.841
Air	4.2	200	0.48	0.48	4.1	4.200
codesBonamenteSpenceBook.tar0000644000175000017500000036000014116230700014612 0ustar  maxmaxcstatLinear.py0000650000175000017500000014400013730460736012042 0ustar  maxmaxcstatFunctions.py0000650000175000017500000004123213613554567012611 0ustar  maxmax# ========================================================
# python functions that provides the best-fit 
# maximum-linelihood solutions for linear models
# using the Cash statistic
# Author: M. Bonamente, 2019, 2020
# ========================================================

import numpy as np
from scipy.stats import poisson
from scipy.optimize import fsolve, root_scalar

# ====================================================================
def sanityCheck(data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	#1. check that all arrays are of proper size
	Ny=len(y)
	NDeltax=len(Deltax)
	if (N!=Ny) or (N!=NDeltax):
		print('Error: Dimensions of x, y and Deltax not matching')
		print('N=',N,'Ny=',Ny,'NDeltax=',NDeltax)
		quit()
	#2. check that there is no overlap of data points
	L=0
	for i in range(N):
		L+=Deltax[i] # length of i=th bin
		# check that there is no overlap between adjacent bins
		if(i>0) and (x[i]-Deltax[i]/2.)<(x[i-1]+Deltax[i-1]/2.):
			print('Bin',i, 'and',i-1,'overlap',x[i]-Deltax[i]/2,x[i-1]+Deltax[i-1]/2)
			quit()
	if (L>R):
		print('Error: range R not consistent with Deltax')
		print('R=',R,'sum of bin size lengths=',L)			
		quit()

def findGap(data):
	# Modified to handle many gaps
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	gapFound=0
	xa=-1*np.ones(N) # up to N gaps possible
	xb=-1*np.ones(N)
	xG=-1*np.ones(N)
	j=0
	gapFound=0
	RG=np.zeros(N)
	for i in range(N):
		if(i>0) and (x[i]-Deltax[i]/2.)>(x[i-1]+Deltax[i-1]/2.):
		# gap starts at x[i-1]+Deltax[i-1]/2.
		# and ends at x[i]-Deltax[i]/2.
			gapFound+=1
			xa[j]=x[i-1]+Deltax[i-1]/2.
			xb[j]=x[i]-Deltax[i]/2.
			RG[j]=(xb[j]-xa[j])
			xG[j]=0.5*(xb[j]+xa[j])
			print('===>>> Gap in data detected between %3.3f and %3.3f, RG=%3.3f'%(xa[j],xb[j],RG[j]))
			j+=1
	print(gapFound,'gaps found')
	return gapFound,xa,xb,xG,RG

def Rprime(R,RG,xA,xG):
	g=len(RG) # this is how many gaps we have
	SG=0
	RGap=0
	for i in range(g):
		SG+=RG[i]*(xG[i]-xA)	
		RGap+=RG[i] # total gap length
# THIS WAS THE CASE FOR ONE GAP ONLY
#	Rp=(R**2-2*RG*(xG-xA))/(R-RG)
	Rp=(R**2-2*SG)/(R-RGap)
	return Rp

def rangex(data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=x[N-1]-x[0]+Deltax[0]/2.+Deltax[N-1]/2.
	return R

def fAsymptotic(data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	haLim=0.0
	for i in range(N):
		haLim+=-y[i]/(x[i]-xA)
	haLim=haLim/M
	fAsympt=1+R/2.*haLim
	return fAsympt

# ==================================================================
# This function needs to be re-written based on cstatLinear.py
# IT iS OBSOLETE NOW
def findZeros(data):
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	# 0. first some work on the data, find how many unique non-zero points we have
	ind=np.where(y>=1.0)    # find the non-zero indices 
	gaindex=ind[0]
	n=len(gaindex)          # number of unique bins with non-zero counts
	print("Number of unique non-zero bins:%d"%n)
	# if n=1 or 0, there are no solutions =================================
	if (n<=1):
		print('Exiting ...')
		return -1,-1
	# =====================================================================
        # 1. Find point of singularity and zeros of g(a) ======================
	gaSing=np.zeros(n)
	gaZero=np.zeros(n-1)
	for i in range(n):
		gaSing[i]=-1/(x[gaindex[i]]-xA)
		#print("%d-th g(a) singularity at %3.3f (for x=%3.3f)"%(i,gaSing[i],x[gaindex[i]]))
        # (1.5 find zeros of g(a) between singularities) ======================
		E=1.e-10 # this is an Epsilon value
		if i > 0:
			resg=root_scalar(ga,bracket=[gaSing[i-1]+E,gaSing[i]-E],args=data,method="brentq")
			gaZero[i-1]=resg.root
			#print("%d-th g(a) root found at %f (between %f and %f)"%(i,gaZero[i-1],gaSing[i-1],gaSing[i]))
        # 2. Find asymptotic value of F(a) ====================================
#	haLim=0.0
#	for i in range(N):
#		haLim+=-y[i]/(x[i]-xA)
#	haLim=haLim/M
#	fAsympt=1+R/2.*haLim
	fAsympt=fAsymptotic(data)
	print('Asymptotic value of F(a): %3.3e'%fAsympt)
	# This is unnecessary, since there roots are unacceptable ==============
        # 3. Find n-2 roots of F(a) between zeros of g(a)
        # =======================================================================
	FaZero=np.zeros(n-1)
	lambdaZero=np.zeros(n-1)
#	THIS IS COMMENTED OUT BECAUSE THOSE ZEROS ARE UNACCEPTABLE =============
#	for i in range(n-2): # n-2 roots between zeros of g(a)
#		# TRY making smaller EPS value here
#		resF=root_scalar(fa,bracket=[gaZero[i]+E/M,gaZero[i+1]-E/M],args=data,method="brentq")
#		FaZero[i]=resF.root  # best-fit "a" parameter
#		lambdaZero[i]=bestFitLnum(FaZero[i],data) # matching "lambda" parameter
#		print("Unacceptable %d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(i,FaZero[i],gaZero[i],gaZero[i+1],lambdaZero[i]))
#	END OF COMMENT ==========================================================
        # 3.5 now find the remaining zero, ======================================
        # either to left or right of first/last zero of g(a). 
        # NOTE: This is only zero for M=2 =======================================
	INF=np.inf
	# Calculate following values at the extreme of the range ================
	a1=-2./Deltax
	a2=-1./(R-Deltax/2.)
	FLeft=fa(a1,*data)
	FRight=fa(a2,*data)
	print("a1=%3.3f, a2=%3.3f,ga(a1)=%3.3f,ga(a2)=%3.3f,Fa(a1)=%3.2f, Fa(a2)=%3.2f"%(a1,a2,ga(a1,*data),ga(a2,*data),FLeft,FRight))
	FaZero[n-2]=-999 	# reset it
	hasConverged=0
	#print(-2./Deltax, -1./(R-Deltax/2.))
	print("%d-th possible outer solution with fAsympt=%3.3f, FLeft=%3.3f and FRight=%3.3f"%(n-2,fAsympt,FLeft,FRight))
	# Ensure that there is a solution by checking F(-2/Deltax) or F(-1/(R-Deltax/2) 
	if ((fAsympt<0)and(FRight>0)) : # zero is guaranteed to be to the right
		resF=root_scalar(fa,bracket=[gaZero[n-2]+E,+INF],args=data,method="brentq")
		FaZero[n-2]=resF.root
		lambdaZero[n-2]=bestFitLnum(FaZero[n-2],data)
		print("Acceptable %d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(n-2,FaZero[n-2],gaZero[n-2],+INF,lambdaZero[n-2]))
		hasConverged=1 # if code is here, there is an acceptable solution
	if ((fAsympt>0)and(FLeft<0)): # zero is to guaranteed to be to the left
		resF=root_scalar(fa,bracket=[-INF,gaZero[0]-E],args=data,method="brentq")
		FaZero[n-2]=resF.root
		lambdaZero[n-2]=bestFitLnum(FaZero[n-2],data)
		print("Acceptable %d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(n-2,FaZero[n-2],-INF,gaZero[0],lambdaZero[n-2]))	
		hasConverged=1 # if code is here, there is an acceptable solution
# Finally, return the value of the only acceptable solution
	return hasConverged,FaZero[n-2]
# ===========================================================================

def cstat(ymodel,data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	cplot=np.zeros(N)
	for i in range(N):
		if(y[i]>0):
			cplot[i]=2.*(ymodel[i]-y[i]-y[i]*np.log(ymodel[i])+y[i]*np.log(y[i]))
		if(y[i]==0):
			cplot[i]=2.*ymodel[i]
	return cplot

def cminLinear(l,a,data):
	result=0.
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	#R=rangex(data) I don't think that this is needed
	M=sum(y)
	cplot=np.zeros(N) # the contribution to Cmin of each point
	#print('a=%3.2f, lambda=%3.2f, R=%3.2f, xa=%3.2f, Deltax=%3.2f, '%(a,l,R,xA,Deltax))
	ymodel=np.zeros(N)
        # --------------------------------------------
	for i in range(N):
		# Remember the Deltax to convert f(x) to ymodel
		ymodel[i]=flinear(x[i],xA,l,a)*Deltax[i]
		if(y[i]>0):
			cplot[i]=2.*(ymodel[i]-y[i]-y[i]*np.log(ymodel[i])+y[i]*np.log(y[i]))
			result+=cplot[i]
		if(y[i]==0):
			cplot[i]=2*ymodel[i]
			result+=cplot[i]
		#print('%d cplot[i]=%3.3f, y[i]=%d x[i]=%3.2f ymodel[i]=%3.3f'%(i,cplot[i],y[i],x[i],ymodel[i]))
	return result,cplot

def flinear(x,xA,l,a):
# this is the density function f(x), counts/unit length of the bin
        return l*(1.+a*(x-xA))

# ===================================================
# For the extended model with Pivot points at A and B

def flinearPivotA(data):
# this is the model with just one fixed parameter, with analytical solution 
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	bestFitL=2*M/R**2
	
# this is the model for y data, contains already Deltax!
	return bestFitL,bestFitL*(x-xA)*Deltax,bestFitL*(x-xA)

def flinearPivotAGap(data,dataGap):
# this is the model with just one fixed parameter, with analytical solution 
	x,y,xA,Deltax=data # unpack the data
	xa,xb,xG,RG=dataGap
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	g=len(RG)
	SG=0
	for i in range(g):
		SG+=RG[i]*(xG[i]-xA)
		#RGap+=RG[i] # total gap length
# this was for one gap
#	bestFitL=2*M/(R**2-2*RG*(xG-xA))
	bestFitL=2*M/(R**2-2*SG)
# this is the model for y data, contains already Deltax!
	return bestFitL,bestFitL*(x-xA)*Deltax,bestFitL*(x-xA)

def flinearPivotB(data):
# this is the model with just one fixed parameter, with analytical solution 
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	bestFitL=2*M/R
# this is the model for y data, contains already Deltax!
	return bestFitL, bestFitL*(1.-(x-xA)/R)*Deltax,bestFitL*(1.-(x-xA)/R)

def flinearPivotBGap(data,dataGap):
	x,y,xA,Deltax=data # unpack the data
	xa,xb,xG,RG=dataGap
	N=len(x)
	R=rangex(data)
	xB=xA+R
	M=sum(y)
	g=len(RG)
	SB=0
	for i in range(g):
		SB+=(RG[i]/R)*(xB-xG[i])
                #RGap+=RG[i] # total gap length
# this was for just one gap
#	bestFitL=2*M/(R+2*(RG/R)*(xG-xB))
	bestFitL=2*M/(R-2*SB)
# this is the model for y data, contains already Deltax!
	return bestFitL, bestFitL*(1.-(x-xA)/R)*Deltax,bestFitL*(1.-(x-xA)/R)

def flinearConst(data):
# this is the model with just one fixed parameter, the costant
# as shown in the other paper, const = sample mean
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	bestFitL=M/R
	return bestFitL,bestFitL*np.ones(N)*Deltax,bestFitL*np.ones(N)

def flinearConstGap(data,dataGap):
# this is the model with just one fixed parameter, the costant
# as shown in the other paper, const = sample mean
	x,y,xA,Deltax=data # unpack the data
	xa,xb,xG,RG=dataGap
	N=len(x)
	R=rangex(data)
        # when there is a gap, R must account for the gap
	M=sum(y)
	RGap=sum(RG) # sum of all gap lengths
	bestFitL=M/(R-RGap)
	return bestFitL,bestFitL*np.ones(N)*Deltax,bestFitL*np.ones(N)
# =======================================================
# g(a) is the function whose zeros cause a singularity in f(a)
def ga(a, *data):
	x,y,xA,Deltax=data #unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
        # --------------------------------------------
	result=0
	for i in range(len(x)):
		if y[i]==0:
			localga=0
		if y[i]>0:
			localga=y[i]*(x[i]-xA)/(1.+a*(x[i]-xA))
		result+=localga
		#print('%d, localga=%3.3f, y[i]=%3.3f, x[i]=%3.3f, xA=%3.3f,a=%3.3f'%(i,localga,y[i],x[i],xA,a))	
	return result
	
def fa(a, *data):
	x,y,xA,Deltax=data #unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
        # --------------------------------------------
	result=0
	galocal=0
	galocal=ga(a,*data)	
	result=1.+a*R/2.-M*R/(2.*galocal)
	# if the dataset has all 0's, lambda=0, which makes a indeterminate. 
	# for M=0 the equation should return a=-2/R, which is fine, since the
	# best-fit model will be y=0 anyways.
	return result

# also its derivative, which is used for finding the zero of the function
def faprime(a,*data):
	x,y,xA,Deltax=data #unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
        # --------------------------------------------
	result=0
	localSum1=0
	localSum2=0
	for i in range(len(x)):
		localSum1+=y[i]*(x[i]-xA)/(1.+a*(x[i]-xA))
		localSum2+=y[i]*(x[i]-xA)**2*(1.+a*(x[i]-xA))**(-2)
        # result is f(a) such that f(a)=0
	result=R/2.-(M*R/2.0)*localSum1**(-2)*localSum2
        # if the dataset has all 0's, lambda=0, which makes a indeterminate. 
        # for M=0 the equation should return a=-2/R, which is fine, since the
        # best-fit model will be y=0 anyways.
	return result

def bestFitLnum(a, data):
	x,y,xA,Deltax=data #unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	return M/(R*(1.+a*R/2.))

# If there is a gap, there is a different function lambda(a)
def bestFitLnumGap(a,data,dataGap):
	x,y,xA,Deltax=data #unpack the data
	xa,xb,xG,RG=dataGap
	R=rangex(data)
	M=sum(y)
	g=len(RG) # this is how many gaps we have
	RGap=0
	SG=0
	for i in range(g):
		SG+=RG[i]*(xG[i]-xA)
		RGap+=RG[i] # total gap length
# this was for one gap	
#	return M/(R*(1.+a*R/2.) - RG*(1+a*(xG-xA)))
	return M/(R*(1.+a*R/2.) -(RGap+a*SG))
# ===================================================
# ==== FUNCTIONS BELOW ARE UNUSED ===================
# These are the approximations for the small-a case
def bestFitL(data):
# R is range, x2 is final point of range, x is array of N measurements (binned)
	x,y,xA,Deltax=data
	N=len(x)
	R=rangex(data)
	result=0
	for i in range(len(x)):
                # use mid point of bin as x coordinate for the bin
		xbinlocal=x[i]-xA
		result+=y[i]*xbinlocal
	result=result*2./R**2
	return result

def bestFitA(data):
# l is the best-fit value of lambda, using prior function
	x,y,xA,Deltax=data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	l=bestFitL(data)
	return (2./R)*(M/(l*R)-1.)
# =======================================================

def isModelAcceptable(a,data):
	result=0
	x,y,xA,Deltax=data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	if ((a<-2/Deltax[0]) or (a>-1/(R-Deltax[N-1]/2))):
		result=1
	return result
# ==================================================
# These functions come from the earlier paper
def funcECmin(x, a,b,c,d,e,f,alpha,beta):
        return (a+b*x+c*(x-d)**2.0)*np.exp(-alpha*x)+e*np.exp(-beta*x)+f

# function that calculates E[Cmin] for a linear model from n simulations

def funcVarC(x, a,b,c,d,e,f,g,h,i,alpha,beta):
        return  (a+b*x**2+c*(x-d)**2.0)*np.exp(-alpha*x) +  (e+f*x+g*(x-h)**2.0)*np.exp(-beta*x) +i


# =========================================================
# This function does all the work of simulating and finding 
# best-fit value for the linear model  ====================
def  ECminSimLinear(AParent, LambdaParent, dataSim):
	xbin,xA,Deltax,n=dataSim # unpack the data for the simulations
	# n is the number of simulated datasets
	N=len(xbin)
	mu=np.zeros(N)

	for i in range(N):
		mu[i]=flinear(xbin[i],xA,LambdaParent,AParent)*Deltax

	cminSim=np.zeros(n)
	cminSim.fill(np.nan)
	y=np.zeros(N) # simulated dataset
	aMLnum=np.zeros(n)
	aMLnum.fill(np.nan)
	lMLnum=np.zeros(n)
	lMLnum.fill(np.nan)
	cminplot=np.zeros(N)
	counterNan=0
	for j in range(n): # iterate over the n simulations
		for i in range(N):
			y[i]=poisson.rvs(mu[i],size=1) # generate Poisson sample
		data=(xbin,y,xA,Deltax)
		M=sum(y) # to check if there are all 0's
		# solve numerically for best-fit "a" parameter	
		# seems like a dataset with all 0's can't be handled by fsolve
		# the "Aparent" parameter is the best-guess for the solution
		# when there are all 0's, this should be -2/R. Need to work on this []
#		if(M==0):
#			aMLnum[j]=0
#			lMLnum[j]=0
#			cminSim[j]=0
		EPS=0.0
#	===========================================================
		if(M>=0): # this is the proper case
			# this is the function we need to write====
			hasAConverged,res=findZeros(data)
			# ==========================================
			#res=root_scalar(fa,x0=AParent+EPS,fprime=faprime,args=data,method='newton')
			# res=fsolve(fa,AParent,full_output=1,xtol=1.e-8,args=data)
			#hasAConverged=res[2] # this is for fsolve
			#hasAConverged=res.converged # this is for root_scalar
			if (hasAConverged!=1): # not converged: discard this dataset
				counterNan+=1
				#print(res.flag, M)
			if (hasAConverged==1): # converged
				#aMLnum[j]=res[0] # accept the solution of fsolve
				#aMLnum[j]=res.root # accept solution of root_scalar
				aMLnum[j]=res
				lMLnum[j]=bestFitLnum(aMLnum[j],data) # and calculate corresponding Lambda
# (c) calculate cmin for this simulated dataset ---------------
# it is possible that, for low S/N data, y(xi) is negative,
# making it such that Cmin=NaN. Try by resetting them
				cminSim[j],cminplot=cminLinear(lMLnum[j],aMLnum[j],data)
				#print('-')
				#print(j, y,cminSim[j])
				if(np.isnan(cminSim[j])): # there are NaN in Cmin
					aMLnum[j]=float('NaN')
					lMLnum[j]=float('NaN')
					counterNan+=1
#	==========================================================
	EcminSim=np.nanmean(cminSim)
# Standard deviation - should it be devided by sqrt(n) for error of mean?
	EcminStdev=np.nanstd(cminSim)#/n**0.5
# Calculate % of sets that have not converged or have negative best--fit model
	return cminSim,aMLnum,lMLnum,EcminSim,EcminStdev,counterNan
		
cstatLinearFunction.py0000654000175000017500000001747213613554567013576 0ustar  maxmax# ========================================================
# python code that provides the best-fit 
# maximum-linelihood solutions for linear models
# using the Cash statistic
# bestFitLinear(data,hasGap,dataGap) and
# bestFitLinearExtended(data,hasGap,dataGap)
#
# Author: M. Bonamente, 2019, 2020
# This is intended to be a function to find best-fit
# values of the Scargle+13 linear model, if it exists.
# This was taken from cstatLinear.py and adapted as a function.
# It has the option to skip the inner, un-acceptable zeros of
# the function F(a)=0, to save computational time.
# ========================================================
import numpy as np
from scipy.optimize import root_scalar
from cstatFunctions import cminLinear,isModelAcceptable,ga,fa,rangex,Rprime,bestFitLnum,bestFitLnumGap
from cstatFunctions import cstat,flinearPivotA,flinearPivotB,flinearConst
from cstatFunctions import flinearPivotAGap,flinearPivotBGap,flinearConstGap
#
# ========= Function to FIT THE DATA (steps 1-7) ==========
def bestFitLinear(data,hasGap,dataGap):
	# -----optionally skip inner zeros of F(a)---------
	nMin=0 # this is the default, do all zeros
	doInner=0 # this is a flag to skip inner zeros
	# -----------------------------
	xbin,y,xA,Deltax=data # unpack the data
	N=len(xbin)
	R=rangex(data)
	M=sum(y)	
	# Find how many non-zero bins we have =============
	ind=np.where(y>=1.0)    # find the non-zero indices 
	gaindex=ind[0]
	n=len(gaindex) # number of bins with non-zero counts
	if doInner==0: # skip n-2 zeros of F(a)
		nMin=n-2
	if n<=1:
		print("*** No solution available for n<=1")
		return 0,np.NaN,np.NaN,np.NaN
# Define the relevant arrays ==============================
	gaSing=np.zeros(n)
	gaZero=np.zeros(n-1)
	FaZero=np.zeros(n-1)
	lambdaZero=np.zeros(n-1)
	ymodelAcceptable=np.zeros(n-1,dtype=int)
	cmin=np.zeros(n-1)
	cminplot=np.zeros((n-1,N))
# step (1) ===============================================
	if hasGap>0:
		xa,xb,xG,RG=dataGap	
		Rp=Rprime(R,RG,xA,xG)
		R=Rp # overwrites R with R'
	if (M<=1): # case when M=2 but both in same bin, for example
		print("*** No solution available for M=1")
		return 0,np.NaN,np.NaN,np.NaN
# if we are here, M>=2 ====================================

# step (2): Find n points of singularity of g(a) ==========
	for i in range(n):
		gaSing[i]=-1/(xbin[gaindex[i]]-xA)
		#print("%d-th g(a) singularity at %3.3f (for x=%3.3f)"%(i,gaSing[i],xbin[gaindex[i]]))
# step (3): Find n-1 zeros of g(a) between points of singularity
# Only need the first and last zero of g(a), since ==========
# F(a)=0 solutions between zeros of g(a) are unacceptable ===
		E=1.e-10 # this is an Epsilon value
		# if doInner=0, only seeks first and last zero of g(a)
		if ((i>0)and(doInner==1)) or((i==1)or(i==n-1)and(doInner==0)): 
			resg=root_scalar(ga,bracket=[gaSing[i-1]+E,gaSing[i]-E],args=data,method="brentq")
			gaZero[i-1]=resg.root	
			#print("%d-th g(a) root found at %f (between %f and %f)"%(i,gaZero[i-1],gaSing[i-1],gaSing[i]))
# step (4): Find asymptotic value of F(a) ==================
	haLim=0.0
	for i in range(N):
		haLim+=-y[i]/(xbin[i]-xA)
	haLim=haLim/M
	fAsympt=1+R/2.*haLim
	print('Asymptotic value of F(a): %3.e'%fAsympt)
# step (5): Find n-2 roots of F(a) between zeros of g(a) ===
# ---- this can be skipped --------------------------------
	for i in range(nMin,n-2): # n-2 roots between zeros of g(a)
		resF=root_scalar(fa,bracket=[gaZero[i]+E,gaZero[i+1]-E],args=data,method="brentq")
		FaZero[i]=resF.root  # best-fit "a" parameter
		#print("%d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(i,FaZero[i],gaZero[i],gaZero[i+1],lambdaZero[i]))
		if(hasGap==0):
			lambdaZero[i]=bestFitLnum(FaZero[i],data)
		if(hasGap>=1):
			lambdaZero[i]=bestFitLnumGap(FaZero[i],data,dataGap)	
#-------- -------------------------------------------------
# step (6): Find the external zero of F(a) ================
	INF=1e+6
# IMPORTANT NOTE =================================================================
# *** If fAsympt is sufficiently close to 0, the solution may be near infinity and 
# difficult to obtain numerically. Set an arbitrary EPS value 
# ================================================================================
	if fAsympt<0-E: # zero is to the right
		print('Seeking solution between gaZero[n-2]+E=%e and +INF=%e'%(gaZero[n-2]+E,+INF))
		print('f(a)=%e, f(b)=%e'%(fa(gaZero[n-2]+E,*data),fa(+INF,*data)))
		resF=root_scalar(fa,bracket=[gaZero[n-2]+E,+INF],args=data,method="brentq")
		FaZero[n-2]=resF.root
		print("%d-th F(a) root found at %e (between %e and %e)"%(n-2,FaZero[n-2],gaZero[n-2],+INF))
	if fAsympt>0+E: # zero is to the left
		print('Seeking solution between -INF=%e and gaZero[0]-E=%e'%(-INF,gaZero[0]-E))
		print('f(a)=%e, f(b)=%e'%(fa(-INF,*data),fa(gaZero[0]-E,*data)))
		resF=root_scalar(fa,bracket=[-INF,gaZero[0]-E],args=data,method="brentq")
		FaZero[n-2]=resF.root
		print("%d-th F(a) root found at %e (between %e and %e)"%(n-2,FaZero[n-2],-INF,gaZero[0]))
# IMPORTANT NOTE (cont'd) =========================================================
# Need to also take care of the case of -E < fAsympt < E 
	if (fAsympt>=-E)and(fAsympt<=E):
		FaZero[n-2]=INF # set to infinity, and lambda will be 0, thus zeroing out the model
# =================================================================================	
	# after calculating the correct value of a, calculate Lambda(a)
	if(hasGap==0):
		lambdaZero[n-2]=bestFitLnum(FaZero[n-2],data)
	if(hasGap>=1):
		lambdaZero[n-2]=bestFitLnumGap(FaZero[n-2],data,dataGap)
	print("Lambda=%3.4f (n-1=%d, nMin+1=%d)"%(lambdaZero[n-2],n-1,nMin+1))
# step (7): Check that the solution is acceptable =========
# Also calculate C statistic ==============================
	for i in range(nMin,n-1):
	# It is redundant to check for the first n-2 models, but can do it anyways
		ymodelAcceptable[i]=isModelAcceptable(FaZero[i],data)
		print('ymodelAcceptable[i]=',ymodelAcceptable[i])
		if (ymodelAcceptable[i]==1):
			cmin[i],cminplot[i]=cminLinear(lambdaZero[i],FaZero[i],data)
			print('%d-th solution acceptable: Cmin=%3.3f'%(i,cmin[i]))
		if (ymodelAcceptable[i]==0):
			print('%d-th solution not acceptable'%i)	
			cmin[i]=np.NaN
# ==============================================================
	return ymodelAcceptable[n-2],FaZero[n-2],lambdaZero[n-2],cmin[n-2]
# ==============================================================

# ==============================================================
# Also a similar function for the extended model ===============
def bestFitLinearExtended(data,hasGap,dataGap):
# Calculate the pivotA, pivotB and Constant model ======================
	xbin,y,xA,Deltax=data # unpack the data
	N=len(xbin)
	#R=rangex(data)
	#M=sum(y)

	cminplotPivotA=np.zeros(N)
	cminplotPivotB=np.zeros(N)
	cminplotConst=np.zeros(N)
	ymodelPivotA=np.zeros(N)
	ymodelPivotA=np.zeros(N)
	fmodelPivotA=np.zeros(N)
	ymodelPivotB=np.zeros(N)
	fmodelPivotB=np.zeros(N)
	ymodelConst=np.zeros(N)
	fmodelConst=np.zeros(N)
	if hasGap==0:
		bestFitLambdaA,ymodelPivotA,fmodelPivotA=flinearPivotA(data) # this is the model pivoted at point xA
		bestFitLambdaB,ymodelPivotB,fmodelPivotB=flinearPivotB(data) # this is the model pivoted at point xB
		bestFitLambdaC,ymodelConst,fmodelConst=flinearConst(data)
	if hasGap>=1:
		bestFitLambdaA,ymodelPivotA,fmodelPivotA=flinearPivotAGap(data,dataGap) 
		bestFitLambdaB,ymodelPivotB,fmodelPivotB=flinearPivotBGap(data,dataGap) 
		bestFitLambdaC,ymodelConst,fmodelConst=flinearConstGap(data,dataGap)

	cminplotPivotA=cstat(ymodelPivotA,data)
	cminplotPivotB=cstat(ymodelPivotB,data)
	cminplotConst=cstat(ymodelConst,data)
	CstatA=sum(cminplotPivotA) # "pivotA" model
	CstatB=sum(cminplotPivotB) # "pivotB" model
	CstatC=sum(cminplotConst)  # constant model
	# sort the Cstats in ascending order
	cstatExtended=[CstatA,CstatB,CstatC]
	name=[1,2,3] # 1: pivotA; 2: pivotB; 3: constant (0: standard)
	indexSort=np.argsort(cstatExtended)

	return cstatExtended[indexSort[0]],name[indexSort[0]]
plots.py0000654000175000017500000001653713612611665010751 0ustar  maxmaximport math
import numpy as np
import matplotlib.ticker as tick
import scipy.stats as stats

from scipy.stats import poisson
from scipy.stats import uniform
from cstatFunctions import  flinear,cminLinear,fa,faprime,bestFitLnum,bestFitL,bestFitA,ga
from matplotlib import pyplot as plt
from scipy.optimize import curve_fit,root_scalar
from scipy.optimize import fsolve
from matplotlib.ticker import StrMethodFormatter
from matplotlib.ticker import ScalarFormatter
from matplotlib.ticker import FuncFormatter
plt.rcParams["font.family"] = "Times New Roman"
plt.rcParams.update({'font.size': 18})
plt.rcParams['axes.linewidth'] = 2.0
plt.rcParams['figure.figsize'] = 5,5
plt.rcParams.update({'figure.autolayout': True})


# =====================================================
# Figure 1 ============================================
# parameters of the data ==============================
N=10
xA=0. #starting point of x-axis, 1/2 bin to the left of x[0]
xB=N
Deltax=1 # size of each bin, assumed uniform in this example
# xbin has the mid-point of each bin
xbin=np.arange(Deltax/2.,N+Deltax/2.,1)
R=xbin[N-1]-xbin[0]+Deltax # range of independent variable
l=1
a=1/R

ymodel=np.zeros(N)

for i in range(N):
        ymodel[i]=flinear(xbin[i],xA,l,a)

plt.figure(figsize=(8, 5))
print('xA,xB',xA,xB)
plt.tick_params(axis='y', direction='in',which='both', labelleft='false', labelright='true',width=2,length=6)
plt.tick_params(axis='x',width=2,length=8)
plt.xticks([xA,0.5,1.5,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5,xB],('\n$\mathbf{x_A}$','$x_1$','','','$x_i$','$x_{i+1}$','','','','','$x_{N}$','\n$\mathbf{x_B}$'))
plt.yticks([l,l*(1+a*R)],('$\lambda$','$\lambda(1+aR)$'),rotation='vertical',va='center')
plt.plot([xA,xB],[flinear(xA,xA,l,a),flinear(xB,xA,l,a)],color='black',linewidth=2,label='linear model $f(x)=\lambda(1+a(x-x_A))$')
plt.step(xbin,ymodel,where='mid',color='black',linewidth=2,linestyle='--',label='step-wise linear model $y(x_i)$')
# add the two 1/2 missing steps
plt.hlines(ymodel[0],xbin[0]-Deltax/2,xbin[0],linestyle='--',color='black',linewidth=2)
plt.hlines(ymodel[N-1],xbin[N-1],xbin[N-1]+Deltax/2.,linestyle='--',color='black',linewidth=2)
# ==============================
plt.xlabel('')
plt.ylabel('y (dependent variable)')
plt.grid(axis='y')
plt.vlines(N,1,2,color="red",linewidth=2)
plt.vlines(N,0,1,linestyle='--',color="red")
plt.vlines(0,0,1,linestyle='--',color="red")
plt.vlines(3.5,0,ymodel[3],linestyle='--',color="green",linewidth=2)
plt.vlines(4.5,0,ymodel[4],linestyle='--',color="green",linewidth=2)
plt.hlines(1,xA,xB,color='blue',linewidth=2)
plt.legend(loc=2)

#plt.annotate('',xy=(N+0.2,1),xytext=(N+0.2,2),arrowprops=dict(facecolor='red', color='red',arrowstyle='|-|'))
plt.text(N+0.2,1.5,'$\lambda a R$',color='red',rotation=90)
#plt.arrow(N+0.1,1,0,1,shape='full',color='red',linestyle='-',width=0.02,length_includes_head='true',head_width=0.1)
#bbox_props = dict(boxstyle="darrow,pad=0.3", fc="cyan", ec="b", lw=2)
plt.text(5,1.05,"$R$",color='blue')#,bbox=bbox_props)
#plt.annotate('',xy=(3,1.5),xytext=(4,1.5),arrowprops=dict(facecolor='red', color='red',arrowstyle='|-|'))
plt.hlines(1.4,3.,4.,color="green",linewidth=2)
plt.text(3.2,1.45,'$\Delta x_i$',color="green")
plt.xlim(-1,N+1)
plt.ylim(0.5,2.3)
plt.show()
# =========== end Fig. 1 =================================


# Figure 4 ===============================================
# Now plot the intervals of acceptability of "a" solutions
Deltax=1
R=100
xA=0
xB=10
# this is a/(R/2), i.e., a rescaled by R/2
a=np.arange(-2*R/(Deltax),R+1,0.01)
M=1
Lambda=np.zeros(len(a))
for i in range(len(a)):
	Lambda[i]=M/R/(1+a[i])

plt.figure(figsize=(7, 7))
#plt.axis('off')
plt.xlim(-2*R,0.3*R)
plt.ylim(-0.012,0.012)
plt.plot(a,Lambda,color='black',linewidth=3,label='$\lambda(a)/M$')
plt.vlines(-R/Deltax,-0.12,0.12,color='blue',label='$a=-2/\Delta x_1$')
plt.vlines(-1/2,-0.12,0.12,color='red',label='$a=-1/(R-\Delta x_N/2)$')
# scale the x axis as a/(R/2)
plt.xscale('symlog',linthreshx=1.,linscalex=.4)
plt.xticks([-R/Deltax,-1,-0.5,0],('','$-1$','','0'))
plt.yticks([-0.01,0.0,0.01],('-0.01','0','0.01'))
plt.tick_params(axis='y', right=True,labelright=True,which='both',left=True, labelleft=False,labelrotation=90)
plt.tick_params(axis='x',width=2,length=8)
plt.grid(axis='x',linewidth=2, linestyle='--')
plt.hlines(0,min(a),max(a),linewidth=2, linestyle='--')
#plt.vlines(-1,-0.012,0.012,color='blue',linestyle='--',label='Singularity ($a=-2/R$)',linewidth=4)
plt.xlabel('Normalized parameter $a/(R/2)$')
plt.ylabel('Normalized parameter $\lambda(a)/M$')
plt.fill([-R/Deltax,-R/Deltax,-1/2,-1/2],[-1,1,1,-1],facecolor="none",alpha=0.7,hatch='/',edgecolor='black',label='Unacceptable solutions of $F(a)=0$')
plt.fill([-R/Deltax,-R/Deltax,-1/2,-1/2],[-1,1,1,-1],"yellow",alpha=0.5,label='Range of singularities of $F(a)$')
plt.legend(loc=2,prop={'size': 12})
#plt.text(-0.8*R,0.006,"Unacceptable solutions",fontsize=13,bbox=dict(facecolor='red', alpha=0.5))
plt.show()
# ======= end Fig. 4 =====================================

# Fig. 7 ====================================================
# Make a plot of % of datasets with unacceptable solutions, as function of M
# and % of datasets with negative asymptotic value

accept0=np.genfromtxt("acceptRatio-a0.txt")
accept1=np.genfromtxt("acceptRatio-a+0.01.txt")
accept2=np.genfromtxt("acceptRatio-a-0.01.txt")
accept=(accept0,accept1,accept2)

print(accept[0][:,1])
n=len(accept[0][:,1]) # assuming all files of same length ...
m=np.zeros((3,n),dtype=float)
frac=np.zeros((3,n),dtype=float)
fracErr=np.zeros((3,n),dtype=float)
fracPlus=np.zeros((3,n),dtype=float)
fracMinus=np.zeros((3,n),dtype=float)
Finf=np.zeros((3,n),dtype=float)
FinfPlus=np.zeros((3,n),dtype=float)
FinfMinus=np.zeros((3,n),dtype=float)
FinfErr=np.zeros((3,n),dtype=float)
colors=['red','green','blue']
labelText=['uniform','positive slope','negative slope']
for j in range(3):
	m[j]=accept[j][:,1]
	frac[j]=accept[j][:,2]
	Finf[j]=1-accept[j][:,3]
	N=100 # this is the number of simulations
	# also estimate an error for the simulations, var = N p q since it's a binomial
	for i in range(n):
#		if m[i]>20:
#			N=100
		fracErr[j,i]=(N*frac[j,i]*(1.-frac[j,i]))**0.5/(N*frac[j,i])
		FinfErr[j,i]=(N*Finf[j,i]*(1.-Finf[j,i]))**0.5/(N*Finf[j,i])
		fracPlus[j,i]=frac[j,i]+fracErr[j,i]
		FinfPlus[j,i]=Finf[j,i]+FinfErr[j,i]
		fracMinus[j,i]=frac[j,i]-fracErr[j,i]
		FinfMinus[j,i]=Finf[j,i]-FinfErr[j,i]
		if fracPlus[j,i]>1.0:
			fracPlus[j,i]=1.0
		if FinfPlus[j,i]>1.0:
			FinfPlus[j,i]=1.0
		if fracMinus[j,i]<0.0:	
			fracMinus[j,i]=0.0
		if FinfMinus[j,i]<0:
			FinfMinus[j,i]=0
	print(m[j],frac[j],fracErr[j])
	plt.xlim(2,200)
	plt.semilogx(m[j],frac[j],linewidth=2,color='black')
#plt.semilogx(m,fracPlus)
#plt.semilogx(m,fracMinus)
	plt.fill_between(m[j],fracMinus[j],fracPlus[j],color=colors[j],alpha=0.5,label=labelText[j])

plt.xticks([2,5,10,20,50,100,200],['2','5','10','20','50','100','200'])
plt.xlabel('Number of counts M')
plt.ylabel('Fraction with acceptable model')
plt.legend(loc=4,prop={'size': 10})
plt.hlines(1,2,200,linestyle='--',color='black')
plt.show()

plt.xlim(2,200)
plt.xlabel('Number of counts M')
plt.ylabel('Fraction with $F_{\infty}<0$')
plt.hlines(1,2,200,linestyle='--',color='black')
for j in range(3):
	plt.semilogx(m[j],Finf[j],linewidth=2,color='black')
	plt.fill_between(m[j],FinfMinus[j],FinfPlus[j],color=colors[j],alpha=0.5,label=labelText[j])
plt.legend(loc=4,prop={'size': 10})
plt.show()
# =========== end Fig. 7 ========================================
cstatFraction.py0000644000175000017500000000623213613544466012407 0ustar  maxmaximport math
import numpy as np
import matplotlib.ticker as tick
import scipy.stats as stats
import statistics
from scipy.stats import poisson
from scipy.stats import uniform
from cstatLinearFunction import bestFitLinear,bestFitLinearExtended
from cstatFunctions import fAsymptotic

# ==========================================================
# define how many simulations and counts (M) in each simulation
Mvalues=[2,3,4,5,6,7,8,9,10,12,15,20,30,50,70,100,150,200,500,1000]
NSim=100# number of simulated datasets
# ==========================================================

# define the data ==========================================
N=100 	# number of datapoints
xA=0. #starting point of x-axis, 1/2 bin to the left of x[0]
Deltax=1 # size of each bin, assumed uniform in this example
Deltax=Deltax*np.ones(N)
# xbin has the mid-point of each bin
hasGap=0
xbin=np.arange(0.5,N+0.5,1)
# ==========================================================a

res=np.zeros(NSim)
Lambda=np.zeros(NSim)
cstat=np.zeros(NSim)
hasConverged=np.zeros(NSim, dtype=int)
hasPositiveFinf=np.zeros(NSim, dtype=int)
y=np.zeros((NSim,N),dtype=int) # all zeros to begin with

fratio=open("acceptRatio.txt","w")

# ===================================
# decide which model to simulate
modelNumber=3
# 1: constant model
# 2: increasing model
# 3: decreasing model
# ==================================
for k in range(len(Mvalues)):
	M=Mvalues[k]
	print("M=%d"%M)
	simIndex=np.zeros((NSim,M),dtype=int)
	randomFloats=np.zeros((NSim,M),dtype=float)
	nAccept=0
	for i in range(NSim):
                # 1. M random indices between 0 and N-1
		if (modelNumber==1):
			simIndex[i]=np.random.randint(0,N,size=M)
                # ==================================
                # If we want to use a non-uniform distribution, got to change this
                # 2. This is for an increasing f(x)=2x , x in 0,1, distribution
		if (modelNumber==2):
			randomFloats[i]=N*(np.random.uniform(0,1,M)**0.5)
			simIndex[i]=randomFloats[i]
                # 3. This is for a decreasing, 2-2x distribution
		if (modelNumber==3): 
			randomFloats[i]=N*(1-np.random.uniform(0,1,M)**0.5)
			simIndex[i]=randomFloats[i]
                # =====================================================
                #print('Simulated indices',simIndex[i])
                # Need to reset the data
		y[i]=np.zeros(N)
		for j in range(M):
			y[i,simIndex[i,j]]+=1 # possibility of a bin with >1 counts
		data=(xbin,y[i],xA,Deltax)
		# =======================================================================
		# This is the main function "bestFitLinear"
		hasConverged[i],res[i],Lambda[i],cstat[i]=bestFitLinear(data,hasGap,data)
		# =======================================================================
		if hasConverged[i]==1:
			nAccept+=1
                # also check the sign of F_infinity for that data
		Finf=fAsymptotic(data)
		hasPositiveFinf[i]=np.sign(Finf) # +-1 is returned
	acceptRatio=nAccept/NSim
	positiveFinfRatio=sum(map(lambda x : x==1, hasPositiveFinf))/NSim
	fratio.write("%d %d %3.3f %3.3f\n"%(k, M,acceptRatio,positiveFinfRatio))
	print("%d of %d simulations for M=%d had acceptable solution (fraction: %3.3f)"%(nAccept,NSim,M,acceptRatio))
fratio.close()
xbin.dat0000654000175000017500000000075213571311216010651 0ustar  maxmax0.5 1.5 2.5 3.5 4.5 5.5 6.5 7.5 8.5 9.5 10.5 11.5 12.5 13.5 14.5 15.5 16.5 17.5 18.5 19.5 20.5 21.5 22.5 23.5 24.5 25.5 26.5 27.5 28.5 29.5 30.5 31.5 32.5 33.5 34.5 35.5 36.5 37.5 38.5 39.5 40.5 41.5 42.5 43.5 44.5 45.5 46.5 47.5 48.5 49.5 50.5 51.5 52.5 53.5 54.5 55.5 56.5 57.5 58.5 59.5 60.5 61.5 62.5 63.5 64.5 65.5 66.5 67.5 68.5 69.5 70.5 71.5 72.5 73.5 74.5 75.5 76.5 77.5 78.5 79.5 80.5 81.5 82.5 83.5 84.5 85.5 86.5 87.5 88.5 89.5 90.5 91.5 92.5 93.5 94.5 95.5 96.5 97.5 98.5 99.5
data.txt0000650000175000017500000000045413531324432010664 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 2. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 1. 0. 0. 0. 0 
params.dat0000654000175000017500000000001013571310170011155 0ustar  maxmax0 100 1
data2_Figure1.txt0000654000175000017500000000045513557674473012362 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 
data2_Figure2.txt0000654000175000017500000000045513557674473012363 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 
data2_Figure3.txt0000654000175000017500000000045413562317700012342 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
xbinGap.dat0000654000175000017500000000005313573706625011311 0ustar  maxmax0.5 1.5 2.5 6.25 6.75 7.25 7.75 8.25 8.75 
dataGap.dat0000654000175000017500000000002213573706625011256 0ustar  maxmax1 1 1 1 1 1 1 1 1
paramsGap.dat0000654000175000017500000000002213571723571011625 0ustar  maxmax0 9 DeltaxGap.dat
cstatFunctions.py0000650000175000017500000004123213613554567012611 0ustar  maxmax# ========================================================
# python functions that provides the best-fit 
# maximum-linelihood solutions for linear models
# using the Cash statistic
# Author: M. Bonamente, 2019, 2020
# ========================================================

import numpy as np
from scipy.stats import poisson
from scipy.optimize import fsolve, root_scalar

# ====================================================================
def sanityCheck(data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	#1. check that all arrays are of proper size
	Ny=len(y)
	NDeltax=len(Deltax)
	if (N!=Ny) or (N!=NDeltax):
		print('Error: Dimensions of x, y and Deltax not matching')
		print('N=',N,'Ny=',Ny,'NDeltax=',NDeltax)
		quit()
	#2. check that there is no overlap of data points
	L=0
	for i in range(N):
		L+=Deltax[i] # length of i=th bin
		# check that there is no overlap between adjacent bins
		if(i>0) and (x[i]-Deltax[i]/2.)<(x[i-1]+Deltax[i-1]/2.):
			print('Bin',i, 'and',i-1,'overlap',x[i]-Deltax[i]/2,x[i-1]+Deltax[i-1]/2)
			quit()
	if (L>R):
		print('Error: range R not consistent with Deltax')
		print('R=',R,'sum of bin size lengths=',L)			
		quit()

def findGap(data):
	# Modified to handle many gaps
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	gapFound=0
	xa=-1*np.ones(N) # up to N gaps possible
	xb=-1*np.ones(N)
	xG=-1*np.ones(N)
	j=0
	gapFound=0
	RG=np.zeros(N)
	for i in range(N):
		if(i>0) and (x[i]-Deltax[i]/2.)>(x[i-1]+Deltax[i-1]/2.):
		# gap starts at x[i-1]+Deltax[i-1]/2.
		# and ends at x[i]-Deltax[i]/2.
			gapFound+=1
			xa[j]=x[i-1]+Deltax[i-1]/2.
			xb[j]=x[i]-Deltax[i]/2.
			RG[j]=(xb[j]-xa[j])
			xG[j]=0.5*(xb[j]+xa[j])
			print('===>>> Gap in data detected between %3.3f and %3.3f, RG=%3.3f'%(xa[j],xb[j],RG[j]))
			j+=1
	print(gapFound,'gaps found')
	return gapFound,xa,xb,xG,RG

def Rprime(R,RG,xA,xG):
	g=len(RG) # this is how many gaps we have
	SG=0
	RGap=0
	for i in range(g):
		SG+=RG[i]*(xG[i]-xA)	
		RGap+=RG[i] # total gap length
# THIS WAS THE CASE FOR ONE GAP ONLY
#	Rp=(R**2-2*RG*(xG-xA))/(R-RG)
	Rp=(R**2-2*SG)/(R-RGap)
	return Rp

def rangex(data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=x[N-1]-x[0]+Deltax[0]/2.+Deltax[N-1]/2.
	return R

def fAsymptotic(data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	haLim=0.0
	for i in range(N):
		haLim+=-y[i]/(x[i]-xA)
	haLim=haLim/M
	fAsympt=1+R/2.*haLim
	return fAsympt

# ==================================================================
# This function needs to be re-written based on cstatLinear.py
# IT iS OBSOLETE NOW
def findZeros(data):
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	# 0. first some work on the data, find how many unique non-zero points we have
	ind=np.where(y>=1.0)    # find the non-zero indices 
	gaindex=ind[0]
	n=len(gaindex)          # number of unique bins with non-zero counts
	print("Number of unique non-zero bins:%d"%n)
	# if n=1 or 0, there are no solutions =================================
	if (n<=1):
		print('Exiting ...')
		return -1,-1
	# =====================================================================
        # 1. Find point of singularity and zeros of g(a) ======================
	gaSing=np.zeros(n)
	gaZero=np.zeros(n-1)
	for i in range(n):
		gaSing[i]=-1/(x[gaindex[i]]-xA)
		#print("%d-th g(a) singularity at %3.3f (for x=%3.3f)"%(i,gaSing[i],x[gaindex[i]]))
        # (1.5 find zeros of g(a) between singularities) ======================
		E=1.e-10 # this is an Epsilon value
		if i > 0:
			resg=root_scalar(ga,bracket=[gaSing[i-1]+E,gaSing[i]-E],args=data,method="brentq")
			gaZero[i-1]=resg.root
			#print("%d-th g(a) root found at %f (between %f and %f)"%(i,gaZero[i-1],gaSing[i-1],gaSing[i]))
        # 2. Find asymptotic value of F(a) ====================================
#	haLim=0.0
#	for i in range(N):
#		haLim+=-y[i]/(x[i]-xA)
#	haLim=haLim/M
#	fAsympt=1+R/2.*haLim
	fAsympt=fAsymptotic(data)
	print('Asymptotic value of F(a): %3.3e'%fAsympt)
	# This is unnecessary, since there roots are unacceptable ==============
        # 3. Find n-2 roots of F(a) between zeros of g(a)
        # =======================================================================
	FaZero=np.zeros(n-1)
	lambdaZero=np.zeros(n-1)
#	THIS IS COMMENTED OUT BECAUSE THOSE ZEROS ARE UNACCEPTABLE =============
#	for i in range(n-2): # n-2 roots between zeros of g(a)
#		# TRY making smaller EPS value here
#		resF=root_scalar(fa,bracket=[gaZero[i]+E/M,gaZero[i+1]-E/M],args=data,method="brentq")
#		FaZero[i]=resF.root  # best-fit "a" parameter
#		lambdaZero[i]=bestFitLnum(FaZero[i],data) # matching "lambda" parameter
#		print("Unacceptable %d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(i,FaZero[i],gaZero[i],gaZero[i+1],lambdaZero[i]))
#	END OF COMMENT ==========================================================
        # 3.5 now find the remaining zero, ======================================
        # either to left or right of first/last zero of g(a). 
        # NOTE: This is only zero for M=2 =======================================
	INF=np.inf
	# Calculate following values at the extreme of the range ================
	a1=-2./Deltax
	a2=-1./(R-Deltax/2.)
	FLeft=fa(a1,*data)
	FRight=fa(a2,*data)
	print("a1=%3.3f, a2=%3.3f,ga(a1)=%3.3f,ga(a2)=%3.3f,Fa(a1)=%3.2f, Fa(a2)=%3.2f"%(a1,a2,ga(a1,*data),ga(a2,*data),FLeft,FRight))
	FaZero[n-2]=-999 	# reset it
	hasConverged=0
	#print(-2./Deltax, -1./(R-Deltax/2.))
	print("%d-th possible outer solution with fAsympt=%3.3f, FLeft=%3.3f and FRight=%3.3f"%(n-2,fAsympt,FLeft,FRight))
	# Ensure that there is a solution by checking F(-2/Deltax) or F(-1/(R-Deltax/2) 
	if ((fAsympt<0)and(FRight>0)) : # zero is guaranteed to be to the right
		resF=root_scalar(fa,bracket=[gaZero[n-2]+E,+INF],args=data,method="brentq")
		FaZero[n-2]=resF.root
		lambdaZero[n-2]=bestFitLnum(FaZero[n-2],data)
		print("Acceptable %d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(n-2,FaZero[n-2],gaZero[n-2],+INF,lambdaZero[n-2]))
		hasConverged=1 # if code is here, there is an acceptable solution
	if ((fAsympt>0)and(FLeft<0)): # zero is to guaranteed to be to the left
		resF=root_scalar(fa,bracket=[-INF,gaZero[0]-E],args=data,method="brentq")
		FaZero[n-2]=resF.root
		lambdaZero[n-2]=bestFitLnum(FaZero[n-2],data)
		print("Acceptable %d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(n-2,FaZero[n-2],-INF,gaZero[0],lambdaZero[n-2]))	
		hasConverged=1 # if code is here, there is an acceptable solution
# Finally, return the value of the only acceptable solution
	return hasConverged,FaZero[n-2]
# ===========================================================================

def cstat(ymodel,data):
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	cplot=np.zeros(N)
	for i in range(N):
		if(y[i]>0):
			cplot[i]=2.*(ymodel[i]-y[i]-y[i]*np.log(ymodel[i])+y[i]*np.log(y[i]))
		if(y[i]==0):
			cplot[i]=2.*ymodel[i]
	return cplot

def cminLinear(l,a,data):
	result=0.
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	#R=rangex(data) I don't think that this is needed
	M=sum(y)
	cplot=np.zeros(N) # the contribution to Cmin of each point
	#print('a=%3.2f, lambda=%3.2f, R=%3.2f, xa=%3.2f, Deltax=%3.2f, '%(a,l,R,xA,Deltax))
	ymodel=np.zeros(N)
        # --------------------------------------------
	for i in range(N):
		# Remember the Deltax to convert f(x) to ymodel
		ymodel[i]=flinear(x[i],xA,l,a)*Deltax[i]
		if(y[i]>0):
			cplot[i]=2.*(ymodel[i]-y[i]-y[i]*np.log(ymodel[i])+y[i]*np.log(y[i]))
			result+=cplot[i]
		if(y[i]==0):
			cplot[i]=2*ymodel[i]
			result+=cplot[i]
		#print('%d cplot[i]=%3.3f, y[i]=%d x[i]=%3.2f ymodel[i]=%3.3f'%(i,cplot[i],y[i],x[i],ymodel[i]))
	return result,cplot

def flinear(x,xA,l,a):
# this is the density function f(x), counts/unit length of the bin
        return l*(1.+a*(x-xA))

# ===================================================
# For the extended model with Pivot points at A and B

def flinearPivotA(data):
# this is the model with just one fixed parameter, with analytical solution 
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	bestFitL=2*M/R**2
	
# this is the model for y data, contains already Deltax!
	return bestFitL,bestFitL*(x-xA)*Deltax,bestFitL*(x-xA)

def flinearPivotAGap(data,dataGap):
# this is the model with just one fixed parameter, with analytical solution 
	x,y,xA,Deltax=data # unpack the data
	xa,xb,xG,RG=dataGap
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	g=len(RG)
	SG=0
	for i in range(g):
		SG+=RG[i]*(xG[i]-xA)
		#RGap+=RG[i] # total gap length
# this was for one gap
#	bestFitL=2*M/(R**2-2*RG*(xG-xA))
	bestFitL=2*M/(R**2-2*SG)
# this is the model for y data, contains already Deltax!
	return bestFitL,bestFitL*(x-xA)*Deltax,bestFitL*(x-xA)

def flinearPivotB(data):
# this is the model with just one fixed parameter, with analytical solution 
	x,y,xA,Deltax=data # unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
	bestFitL=2*M/R
# this is the model for y data, contains already Deltax!
	return bestFitL, bestFitL*(1.-(x-xA)/R)*Deltax,bestFitL*(1.-(x-xA)/R)

def flinearPivotBGap(data,dataGap):
	x,y,xA,Deltax=data # unpack the data
	xa,xb,xG,RG=dataGap
	N=len(x)
	R=rangex(data)
	xB=xA+R
	M=sum(y)
	g=len(RG)
	SB=0
	for i in range(g):
		SB+=(RG[i]/R)*(xB-xG[i])
                #RGap+=RG[i] # total gap length
# this was for just one gap
#	bestFitL=2*M/(R+2*(RG/R)*(xG-xB))
	bestFitL=2*M/(R-2*SB)
# this is the model for y data, contains already Deltax!
	return bestFitL, bestFitL*(1.-(x-xA)/R)*Deltax,bestFitL*(1.-(x-xA)/R)

def flinearConst(data):
# this is the model with just one fixed parameter, the costant
# as shown in the other paper, const = sample mean
	x,y,xA,Deltax=data # unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	bestFitL=M/R
	return bestFitL,bestFitL*np.ones(N)*Deltax,bestFitL*np.ones(N)

def flinearConstGap(data,dataGap):
# this is the model with just one fixed parameter, the costant
# as shown in the other paper, const = sample mean
	x,y,xA,Deltax=data # unpack the data
	xa,xb,xG,RG=dataGap
	N=len(x)
	R=rangex(data)
        # when there is a gap, R must account for the gap
	M=sum(y)
	RGap=sum(RG) # sum of all gap lengths
	bestFitL=M/(R-RGap)
	return bestFitL,bestFitL*np.ones(N)*Deltax,bestFitL*np.ones(N)
# =======================================================
# g(a) is the function whose zeros cause a singularity in f(a)
def ga(a, *data):
	x,y,xA,Deltax=data #unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
        # --------------------------------------------
	result=0
	for i in range(len(x)):
		if y[i]==0:
			localga=0
		if y[i]>0:
			localga=y[i]*(x[i]-xA)/(1.+a*(x[i]-xA))
		result+=localga
		#print('%d, localga=%3.3f, y[i]=%3.3f, x[i]=%3.3f, xA=%3.3f,a=%3.3f'%(i,localga,y[i],x[i],xA,a))	
	return result
	
def fa(a, *data):
	x,y,xA,Deltax=data #unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
        # --------------------------------------------
	result=0
	galocal=0
	galocal=ga(a,*data)	
	result=1.+a*R/2.-M*R/(2.*galocal)
	# if the dataset has all 0's, lambda=0, which makes a indeterminate. 
	# for M=0 the equation should return a=-2/R, which is fine, since the
	# best-fit model will be y=0 anyways.
	return result

# also its derivative, which is used for finding the zero of the function
def faprime(a,*data):
	x,y,xA,Deltax=data #unpack the data
        # derive the parameters needed for the formula
	N=len(x)
	R=rangex(data)
	M=sum(y)
        # --------------------------------------------
	result=0
	localSum1=0
	localSum2=0
	for i in range(len(x)):
		localSum1+=y[i]*(x[i]-xA)/(1.+a*(x[i]-xA))
		localSum2+=y[i]*(x[i]-xA)**2*(1.+a*(x[i]-xA))**(-2)
        # result is f(a) such that f(a)=0
	result=R/2.-(M*R/2.0)*localSum1**(-2)*localSum2
        # if the dataset has all 0's, lambda=0, which makes a indeterminate. 
        # for M=0 the equation should return a=-2/R, which is fine, since the
        # best-fit model will be y=0 anyways.
	return result

def bestFitLnum(a, data):
	x,y,xA,Deltax=data #unpack the data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	return M/(R*(1.+a*R/2.))

# If there is a gap, there is a different function lambda(a)
def bestFitLnumGap(a,data,dataGap):
	x,y,xA,Deltax=data #unpack the data
	xa,xb,xG,RG=dataGap
	R=rangex(data)
	M=sum(y)
	g=len(RG) # this is how many gaps we have
	RGap=0
	SG=0
	for i in range(g):
		SG+=RG[i]*(xG[i]-xA)
		RGap+=RG[i] # total gap length
# this was for one gap	
#	return M/(R*(1.+a*R/2.) - RG*(1+a*(xG-xA)))
	return M/(R*(1.+a*R/2.) -(RGap+a*SG))
# ===================================================
# ==== FUNCTIONS BELOW ARE UNUSED ===================
# These are the approximations for the small-a case
def bestFitL(data):
# R is range, x2 is final point of range, x is array of N measurements (binned)
	x,y,xA,Deltax=data
	N=len(x)
	R=rangex(data)
	result=0
	for i in range(len(x)):
                # use mid point of bin as x coordinate for the bin
		xbinlocal=x[i]-xA
		result+=y[i]*xbinlocal
	result=result*2./R**2
	return result

def bestFitA(data):
# l is the best-fit value of lambda, using prior function
	x,y,xA,Deltax=data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	l=bestFitL(data)
	return (2./R)*(M/(l*R)-1.)
# =======================================================

def isModelAcceptable(a,data):
	result=0
	x,y,xA,Deltax=data
	N=len(x)
	R=rangex(data)
	M=sum(y)
	if ((a<-2/Deltax[0]) or (a>-1/(R-Deltax[N-1]/2))):
		result=1
	return result
# ==================================================
# These functions come from the earlier paper
def funcECmin(x, a,b,c,d,e,f,alpha,beta):
        return (a+b*x+c*(x-d)**2.0)*np.exp(-alpha*x)+e*np.exp(-beta*x)+f

# function that calculates E[Cmin] for a linear model from n simulations

def funcVarC(x, a,b,c,d,e,f,g,h,i,alpha,beta):
        return  (a+b*x**2+c*(x-d)**2.0)*np.exp(-alpha*x) +  (e+f*x+g*(x-h)**2.0)*np.exp(-beta*x) +i


# =========================================================
# This function does all the work of simulating and finding 
# best-fit value for the linear model  ====================
def  ECminSimLinear(AParent, LambdaParent, dataSim):
	xbin,xA,Deltax,n=dataSim # unpack the data for the simulations
	# n is the number of simulated datasets
	N=len(xbin)
	mu=np.zeros(N)

	for i in range(N):
		mu[i]=flinear(xbin[i],xA,LambdaParent,AParent)*Deltax

	cminSim=np.zeros(n)
	cminSim.fill(np.nan)
	y=np.zeros(N) # simulated dataset
	aMLnum=np.zeros(n)
	aMLnum.fill(np.nan)
	lMLnum=np.zeros(n)
	lMLnum.fill(np.nan)
	cminplot=np.zeros(N)
	counterNan=0
	for j in range(n): # iterate over the n simulations
		for i in range(N):
			y[i]=poisson.rvs(mu[i],size=1) # generate Poisson sample
		data=(xbin,y,xA,Deltax)
		M=sum(y) # to check if there are all 0's
		# solve numerically for best-fit "a" parameter	
		# seems like a dataset with all 0's can't be handled by fsolve
		# the "Aparent" parameter is the best-guess for the solution
		# when there are all 0's, this should be -2/R. Need to work on this []
#		if(M==0):
#			aMLnum[j]=0
#			lMLnum[j]=0
#			cminSim[j]=0
		EPS=0.0
#	===========================================================
		if(M>=0): # this is the proper case
			# this is the function we need to write====
			hasAConverged,res=findZeros(data)
			# ==========================================
			#res=root_scalar(fa,x0=AParent+EPS,fprime=faprime,args=data,method='newton')
			# res=fsolve(fa,AParent,full_output=1,xtol=1.e-8,args=data)
			#hasAConverged=res[2] # this is for fsolve
			#hasAConverged=res.converged # this is for root_scalar
			if (hasAConverged!=1): # not converged: discard this dataset
				counterNan+=1
				#print(res.flag, M)
			if (hasAConverged==1): # converged
				#aMLnum[j]=res[0] # accept the solution of fsolve
				#aMLnum[j]=res.root # accept solution of root_scalar
				aMLnum[j]=res
				lMLnum[j]=bestFitLnum(aMLnum[j],data) # and calculate corresponding Lambda
# (c) calculate cmin for this simulated dataset ---------------
# it is possible that, for low S/N data, y(xi) is negative,
# making it such that Cmin=NaN. Try by resetting them
				cminSim[j],cminplot=cminLinear(lMLnum[j],aMLnum[j],data)
				#print('-')
				#print(j, y,cminSim[j])
				if(np.isnan(cminSim[j])): # there are NaN in Cmin
					aMLnum[j]=float('NaN')
					lMLnum[j]=float('NaN')
					counterNan+=1
#	==========================================================
	EcminSim=np.nanmean(cminSim)
# Standard deviation - should it be devided by sqrt(n) for error of mean?
	EcminStdev=np.nanstd(cminSim)#/n**0.5
# Calculate % of sets that have not converged or have negative best--fit model
	return cminSim,aMLnum,lMLnum,EcminSim,EcminStdev,counterNan
		
cstatLinearFunction.py0000654000175000017500000001747213613554567013576 0ustar  maxmax# ========================================================
# python code that provides the best-fit 
# maximum-linelihood solutions for linear models
# using the Cash statistic
# bestFitLinear(data,hasGap,dataGap) and
# bestFitLinearExtended(data,hasGap,dataGap)
#
# Author: M. Bonamente, 2019, 2020
# This is intended to be a function to find best-fit
# values of the Scargle+13 linear model, if it exists.
# This was taken from cstatLinear.py and adapted as a function.
# It has the option to skip the inner, un-acceptable zeros of
# the function F(a)=0, to save computational time.
# ========================================================
import numpy as np
from scipy.optimize import root_scalar
from cstatFunctions import cminLinear,isModelAcceptable,ga,fa,rangex,Rprime,bestFitLnum,bestFitLnumGap
from cstatFunctions import cstat,flinearPivotA,flinearPivotB,flinearConst
from cstatFunctions import flinearPivotAGap,flinearPivotBGap,flinearConstGap
#
# ========= Function to FIT THE DATA (steps 1-7) ==========
def bestFitLinear(data,hasGap,dataGap):
	# -----optionally skip inner zeros of F(a)---------
	nMin=0 # this is the default, do all zeros
	doInner=0 # this is a flag to skip inner zeros
	# -----------------------------
	xbin,y,xA,Deltax=data # unpack the data
	N=len(xbin)
	R=rangex(data)
	M=sum(y)	
	# Find how many non-zero bins we have =============
	ind=np.where(y>=1.0)    # find the non-zero indices 
	gaindex=ind[0]
	n=len(gaindex) # number of bins with non-zero counts
	if doInner==0: # skip n-2 zeros of F(a)
		nMin=n-2
	if n<=1:
		print("*** No solution available for n<=1")
		return 0,np.NaN,np.NaN,np.NaN
# Define the relevant arrays ==============================
	gaSing=np.zeros(n)
	gaZero=np.zeros(n-1)
	FaZero=np.zeros(n-1)
	lambdaZero=np.zeros(n-1)
	ymodelAcceptable=np.zeros(n-1,dtype=int)
	cmin=np.zeros(n-1)
	cminplot=np.zeros((n-1,N))
# step (1) ===============================================
	if hasGap>0:
		xa,xb,xG,RG=dataGap	
		Rp=Rprime(R,RG,xA,xG)
		R=Rp # overwrites R with R'
	if (M<=1): # case when M=2 but both in same bin, for example
		print("*** No solution available for M=1")
		return 0,np.NaN,np.NaN,np.NaN
# if we are here, M>=2 ====================================

# step (2): Find n points of singularity of g(a) ==========
	for i in range(n):
		gaSing[i]=-1/(xbin[gaindex[i]]-xA)
		#print("%d-th g(a) singularity at %3.3f (for x=%3.3f)"%(i,gaSing[i],xbin[gaindex[i]]))
# step (3): Find n-1 zeros of g(a) between points of singularity
# Only need the first and last zero of g(a), since ==========
# F(a)=0 solutions between zeros of g(a) are unacceptable ===
		E=1.e-10 # this is an Epsilon value
		# if doInner=0, only seeks first and last zero of g(a)
		if ((i>0)and(doInner==1)) or((i==1)or(i==n-1)and(doInner==0)): 
			resg=root_scalar(ga,bracket=[gaSing[i-1]+E,gaSing[i]-E],args=data,method="brentq")
			gaZero[i-1]=resg.root	
			#print("%d-th g(a) root found at %f (between %f and %f)"%(i,gaZero[i-1],gaSing[i-1],gaSing[i]))
# step (4): Find asymptotic value of F(a) ==================
	haLim=0.0
	for i in range(N):
		haLim+=-y[i]/(xbin[i]-xA)
	haLim=haLim/M
	fAsympt=1+R/2.*haLim
	print('Asymptotic value of F(a): %3.e'%fAsympt)
# step (5): Find n-2 roots of F(a) between zeros of g(a) ===
# ---- this can be skipped --------------------------------
	for i in range(nMin,n-2): # n-2 roots between zeros of g(a)
		resF=root_scalar(fa,bracket=[gaZero[i]+E,gaZero[i+1]-E],args=data,method="brentq")
		FaZero[i]=resF.root  # best-fit "a" parameter
		#print("%d-th F(a) root found at %f (between %f and %f), lambda=%3.4f"%(i,FaZero[i],gaZero[i],gaZero[i+1],lambdaZero[i]))
		if(hasGap==0):
			lambdaZero[i]=bestFitLnum(FaZero[i],data)
		if(hasGap>=1):
			lambdaZero[i]=bestFitLnumGap(FaZero[i],data,dataGap)	
#-------- -------------------------------------------------
# step (6): Find the external zero of F(a) ================
	INF=1e+6
# IMPORTANT NOTE =================================================================
# *** If fAsympt is sufficiently close to 0, the solution may be near infinity and 
# difficult to obtain numerically. Set an arbitrary EPS value 
# ================================================================================
	if fAsympt<0-E: # zero is to the right
		print('Seeking solution between gaZero[n-2]+E=%e and +INF=%e'%(gaZero[n-2]+E,+INF))
		print('f(a)=%e, f(b)=%e'%(fa(gaZero[n-2]+E,*data),fa(+INF,*data)))
		resF=root_scalar(fa,bracket=[gaZero[n-2]+E,+INF],args=data,method="brentq")
		FaZero[n-2]=resF.root
		print("%d-th F(a) root found at %e (between %e and %e)"%(n-2,FaZero[n-2],gaZero[n-2],+INF))
	if fAsympt>0+E: # zero is to the left
		print('Seeking solution between -INF=%e and gaZero[0]-E=%e'%(-INF,gaZero[0]-E))
		print('f(a)=%e, f(b)=%e'%(fa(-INF,*data),fa(gaZero[0]-E,*data)))
		resF=root_scalar(fa,bracket=[-INF,gaZero[0]-E],args=data,method="brentq")
		FaZero[n-2]=resF.root
		print("%d-th F(a) root found at %e (between %e and %e)"%(n-2,FaZero[n-2],-INF,gaZero[0]))
# IMPORTANT NOTE (cont'd) =========================================================
# Need to also take care of the case of -E < fAsympt < E 
	if (fAsympt>=-E)and(fAsympt<=E):
		FaZero[n-2]=INF # set to infinity, and lambda will be 0, thus zeroing out the model
# =================================================================================	
	# after calculating the correct value of a, calculate Lambda(a)
	if(hasGap==0):
		lambdaZero[n-2]=bestFitLnum(FaZero[n-2],data)
	if(hasGap>=1):
		lambdaZero[n-2]=bestFitLnumGap(FaZero[n-2],data,dataGap)
	print("Lambda=%3.4f (n-1=%d, nMin+1=%d)"%(lambdaZero[n-2],n-1,nMin+1))
# step (7): Check that the solution is acceptable =========
# Also calculate C statistic ==============================
	for i in range(nMin,n-1):
	# It is redundant to check for the first n-2 models, but can do it anyways
		ymodelAcceptable[i]=isModelAcceptable(FaZero[i],data)
		print('ymodelAcceptable[i]=',ymodelAcceptable[i])
		if (ymodelAcceptable[i]==1):
			cmin[i],cminplot[i]=cminLinear(lambdaZero[i],FaZero[i],data)
			print('%d-th solution acceptable: Cmin=%3.3f'%(i,cmin[i]))
		if (ymodelAcceptable[i]==0):
			print('%d-th solution not acceptable'%i)	
			cmin[i]=np.NaN
# ==============================================================
	return ymodelAcceptable[n-2],FaZero[n-2],lambdaZero[n-2],cmin[n-2]
# ==============================================================

# ==============================================================
# Also a similar function for the extended model ===============
def bestFitLinearExtended(data,hasGap,dataGap):
# Calculate the pivotA, pivotB and Constant model ======================
	xbin,y,xA,Deltax=data # unpack the data
	N=len(xbin)
	#R=rangex(data)
	#M=sum(y)

	cminplotPivotA=np.zeros(N)
	cminplotPivotB=np.zeros(N)
	cminplotConst=np.zeros(N)
	ymodelPivotA=np.zeros(N)
	ymodelPivotA=np.zeros(N)
	fmodelPivotA=np.zeros(N)
	ymodelPivotB=np.zeros(N)
	fmodelPivotB=np.zeros(N)
	ymodelConst=np.zeros(N)
	fmodelConst=np.zeros(N)
	if hasGap==0:
		bestFitLambdaA,ymodelPivotA,fmodelPivotA=flinearPivotA(data) # this is the model pivoted at point xA
		bestFitLambdaB,ymodelPivotB,fmodelPivotB=flinearPivotB(data) # this is the model pivoted at point xB
		bestFitLambdaC,ymodelConst,fmodelConst=flinearConst(data)
	if hasGap>=1:
		bestFitLambdaA,ymodelPivotA,fmodelPivotA=flinearPivotAGap(data,dataGap) 
		bestFitLambdaB,ymodelPivotB,fmodelPivotB=flinearPivotBGap(data,dataGap) 
		bestFitLambdaC,ymodelConst,fmodelConst=flinearConstGap(data,dataGap)

	cminplotPivotA=cstat(ymodelPivotA,data)
	cminplotPivotB=cstat(ymodelPivotB,data)
	cminplotConst=cstat(ymodelConst,data)
	CstatA=sum(cminplotPivotA) # "pivotA" model
	CstatB=sum(cminplotPivotB) # "pivotB" model
	CstatC=sum(cminplotConst)  # constant model
	# sort the Cstats in ascending order
	cstatExtended=[CstatA,CstatB,CstatC]
	name=[1,2,3] # 1: pivotA; 2: pivotB; 3: constant (0: standard)
	indexSort=np.argsort(cstatExtended)

	return cstatExtended[indexSort[0]],name[indexSort[0]]
plots.py0000654000175000017500000001653713612611665010751 0ustar  maxmaximport math
import numpy as np
import matplotlib.ticker as tick
import scipy.stats as stats

from scipy.stats import poisson
from scipy.stats import uniform
from cstatFunctions import  flinear,cminLinear,fa,faprime,bestFitLnum,bestFitL,bestFitA,ga
from matplotlib import pyplot as plt
from scipy.optimize import curve_fit,root_scalar
from scipy.optimize import fsolve
from matplotlib.ticker import StrMethodFormatter
from matplotlib.ticker import ScalarFormatter
from matplotlib.ticker import FuncFormatter
plt.rcParams["font.family"] = "Times New Roman"
plt.rcParams.update({'font.size': 18})
plt.rcParams['axes.linewidth'] = 2.0
plt.rcParams['figure.figsize'] = 5,5
plt.rcParams.update({'figure.autolayout': True})


# =====================================================
# Figure 1 ============================================
# parameters of the data ==============================
N=10
xA=0. #starting point of x-axis, 1/2 bin to the left of x[0]
xB=N
Deltax=1 # size of each bin, assumed uniform in this example
# xbin has the mid-point of each bin
xbin=np.arange(Deltax/2.,N+Deltax/2.,1)
R=xbin[N-1]-xbin[0]+Deltax # range of independent variable
l=1
a=1/R

ymodel=np.zeros(N)

for i in range(N):
        ymodel[i]=flinear(xbin[i],xA,l,a)

plt.figure(figsize=(8, 5))
print('xA,xB',xA,xB)
plt.tick_params(axis='y', direction='in',which='both', labelleft='false', labelright='true',width=2,length=6)
plt.tick_params(axis='x',width=2,length=8)
plt.xticks([xA,0.5,1.5,2.5,3.5,4.5,5.5,6.5,7.5,8.5,9.5,xB],('\n$\mathbf{x_A}$','$x_1$','','','$x_i$','$x_{i+1}$','','','','','$x_{N}$','\n$\mathbf{x_B}$'))
plt.yticks([l,l*(1+a*R)],('$\lambda$','$\lambda(1+aR)$'),rotation='vertical',va='center')
plt.plot([xA,xB],[flinear(xA,xA,l,a),flinear(xB,xA,l,a)],color='black',linewidth=2,label='linear model $f(x)=\lambda(1+a(x-x_A))$')
plt.step(xbin,ymodel,where='mid',color='black',linewidth=2,linestyle='--',label='step-wise linear model $y(x_i)$')
# add the two 1/2 missing steps
plt.hlines(ymodel[0],xbin[0]-Deltax/2,xbin[0],linestyle='--',color='black',linewidth=2)
plt.hlines(ymodel[N-1],xbin[N-1],xbin[N-1]+Deltax/2.,linestyle='--',color='black',linewidth=2)
# ==============================
plt.xlabel('')
plt.ylabel('y (dependent variable)')
plt.grid(axis='y')
plt.vlines(N,1,2,color="red",linewidth=2)
plt.vlines(N,0,1,linestyle='--',color="red")
plt.vlines(0,0,1,linestyle='--',color="red")
plt.vlines(3.5,0,ymodel[3],linestyle='--',color="green",linewidth=2)
plt.vlines(4.5,0,ymodel[4],linestyle='--',color="green",linewidth=2)
plt.hlines(1,xA,xB,color='blue',linewidth=2)
plt.legend(loc=2)

#plt.annotate('',xy=(N+0.2,1),xytext=(N+0.2,2),arrowprops=dict(facecolor='red', color='red',arrowstyle='|-|'))
plt.text(N+0.2,1.5,'$\lambda a R$',color='red',rotation=90)
#plt.arrow(N+0.1,1,0,1,shape='full',color='red',linestyle='-',width=0.02,length_includes_head='true',head_width=0.1)
#bbox_props = dict(boxstyle="darrow,pad=0.3", fc="cyan", ec="b", lw=2)
plt.text(5,1.05,"$R$",color='blue')#,bbox=bbox_props)
#plt.annotate('',xy=(3,1.5),xytext=(4,1.5),arrowprops=dict(facecolor='red', color='red',arrowstyle='|-|'))
plt.hlines(1.4,3.,4.,color="green",linewidth=2)
plt.text(3.2,1.45,'$\Delta x_i$',color="green")
plt.xlim(-1,N+1)
plt.ylim(0.5,2.3)
plt.show()
# =========== end Fig. 1 =================================


# Figure 4 ===============================================
# Now plot the intervals of acceptability of "a" solutions
Deltax=1
R=100
xA=0
xB=10
# this is a/(R/2), i.e., a rescaled by R/2
a=np.arange(-2*R/(Deltax),R+1,0.01)
M=1
Lambda=np.zeros(len(a))
for i in range(len(a)):
	Lambda[i]=M/R/(1+a[i])

plt.figure(figsize=(7, 7))
#plt.axis('off')
plt.xlim(-2*R,0.3*R)
plt.ylim(-0.012,0.012)
plt.plot(a,Lambda,color='black',linewidth=3,label='$\lambda(a)/M$')
plt.vlines(-R/Deltax,-0.12,0.12,color='blue',label='$a=-2/\Delta x_1$')
plt.vlines(-1/2,-0.12,0.12,color='red',label='$a=-1/(R-\Delta x_N/2)$')
# scale the x axis as a/(R/2)
plt.xscale('symlog',linthreshx=1.,linscalex=.4)
plt.xticks([-R/Deltax,-1,-0.5,0],('','$-1$','','0'))
plt.yticks([-0.01,0.0,0.01],('-0.01','0','0.01'))
plt.tick_params(axis='y', right=True,labelright=True,which='both',left=True, labelleft=False,labelrotation=90)
plt.tick_params(axis='x',width=2,length=8)
plt.grid(axis='x',linewidth=2, linestyle='--')
plt.hlines(0,min(a),max(a),linewidth=2, linestyle='--')
#plt.vlines(-1,-0.012,0.012,color='blue',linestyle='--',label='Singularity ($a=-2/R$)',linewidth=4)
plt.xlabel('Normalized parameter $a/(R/2)$')
plt.ylabel('Normalized parameter $\lambda(a)/M$')
plt.fill([-R/Deltax,-R/Deltax,-1/2,-1/2],[-1,1,1,-1],facecolor="none",alpha=0.7,hatch='/',edgecolor='black',label='Unacceptable solutions of $F(a)=0$')
plt.fill([-R/Deltax,-R/Deltax,-1/2,-1/2],[-1,1,1,-1],"yellow",alpha=0.5,label='Range of singularities of $F(a)$')
plt.legend(loc=2,prop={'size': 12})
#plt.text(-0.8*R,0.006,"Unacceptable solutions",fontsize=13,bbox=dict(facecolor='red', alpha=0.5))
plt.show()
# ======= end Fig. 4 =====================================

# Fig. 7 ====================================================
# Make a plot of % of datasets with unacceptable solutions, as function of M
# and % of datasets with negative asymptotic value

accept0=np.genfromtxt("acceptRatio-a0.txt")
accept1=np.genfromtxt("acceptRatio-a+0.01.txt")
accept2=np.genfromtxt("acceptRatio-a-0.01.txt")
accept=(accept0,accept1,accept2)

print(accept[0][:,1])
n=len(accept[0][:,1]) # assuming all files of same length ...
m=np.zeros((3,n),dtype=float)
frac=np.zeros((3,n),dtype=float)
fracErr=np.zeros((3,n),dtype=float)
fracPlus=np.zeros((3,n),dtype=float)
fracMinus=np.zeros((3,n),dtype=float)
Finf=np.zeros((3,n),dtype=float)
FinfPlus=np.zeros((3,n),dtype=float)
FinfMinus=np.zeros((3,n),dtype=float)
FinfErr=np.zeros((3,n),dtype=float)
colors=['red','green','blue']
labelText=['uniform','positive slope','negative slope']
for j in range(3):
	m[j]=accept[j][:,1]
	frac[j]=accept[j][:,2]
	Finf[j]=1-accept[j][:,3]
	N=100 # this is the number of simulations
	# also estimate an error for the simulations, var = N p q since it's a binomial
	for i in range(n):
#		if m[i]>20:
#			N=100
		fracErr[j,i]=(N*frac[j,i]*(1.-frac[j,i]))**0.5/(N*frac[j,i])
		FinfErr[j,i]=(N*Finf[j,i]*(1.-Finf[j,i]))**0.5/(N*Finf[j,i])
		fracPlus[j,i]=frac[j,i]+fracErr[j,i]
		FinfPlus[j,i]=Finf[j,i]+FinfErr[j,i]
		fracMinus[j,i]=frac[j,i]-fracErr[j,i]
		FinfMinus[j,i]=Finf[j,i]-FinfErr[j,i]
		if fracPlus[j,i]>1.0:
			fracPlus[j,i]=1.0
		if FinfPlus[j,i]>1.0:
			FinfPlus[j,i]=1.0
		if fracMinus[j,i]<0.0:	
			fracMinus[j,i]=0.0
		if FinfMinus[j,i]<0:
			FinfMinus[j,i]=0
	print(m[j],frac[j],fracErr[j])
	plt.xlim(2,200)
	plt.semilogx(m[j],frac[j],linewidth=2,color='black')
#plt.semilogx(m,fracPlus)
#plt.semilogx(m,fracMinus)
	plt.fill_between(m[j],fracMinus[j],fracPlus[j],color=colors[j],alpha=0.5,label=labelText[j])

plt.xticks([2,5,10,20,50,100,200],['2','5','10','20','50','100','200'])
plt.xlabel('Number of counts M')
plt.ylabel('Fraction with acceptable model')
plt.legend(loc=4,prop={'size': 10})
plt.hlines(1,2,200,linestyle='--',color='black')
plt.show()

plt.xlim(2,200)
plt.xlabel('Number of counts M')
plt.ylabel('Fraction with $F_{\infty}<0$')
plt.hlines(1,2,200,linestyle='--',color='black')
for j in range(3):
	plt.semilogx(m[j],Finf[j],linewidth=2,color='black')
	plt.fill_between(m[j],FinfMinus[j],FinfPlus[j],color=colors[j],alpha=0.5,label=labelText[j])
plt.legend(loc=4,prop={'size': 10})
plt.show()
# =========== end Fig. 7 ========================================
cstatFraction.py0000644000175000017500000000623213613544466012407 0ustar  maxmaximport math
import numpy as np
import matplotlib.ticker as tick
import scipy.stats as stats
import statistics
from scipy.stats import poisson
from scipy.stats import uniform
from cstatLinearFunction import bestFitLinear,bestFitLinearExtended
from cstatFunctions import fAsymptotic

# ==========================================================
# define how many simulations and counts (M) in each simulation
Mvalues=[2,3,4,5,6,7,8,9,10,12,15,20,30,50,70,100,150,200,500,1000]
NSim=100# number of simulated datasets
# ==========================================================

# define the data ==========================================
N=100 	# number of datapoints
xA=0. #starting point of x-axis, 1/2 bin to the left of x[0]
Deltax=1 # size of each bin, assumed uniform in this example
Deltax=Deltax*np.ones(N)
# xbin has the mid-point of each bin
hasGap=0
xbin=np.arange(0.5,N+0.5,1)
# ==========================================================a

res=np.zeros(NSim)
Lambda=np.zeros(NSim)
cstat=np.zeros(NSim)
hasConverged=np.zeros(NSim, dtype=int)
hasPositiveFinf=np.zeros(NSim, dtype=int)
y=np.zeros((NSim,N),dtype=int) # all zeros to begin with

fratio=open("acceptRatio.txt","w")

# ===================================
# decide which model to simulate
modelNumber=3
# 1: constant model
# 2: increasing model
# 3: decreasing model
# ==================================
for k in range(len(Mvalues)):
	M=Mvalues[k]
	print("M=%d"%M)
	simIndex=np.zeros((NSim,M),dtype=int)
	randomFloats=np.zeros((NSim,M),dtype=float)
	nAccept=0
	for i in range(NSim):
                # 1. M random indices between 0 and N-1
		if (modelNumber==1):
			simIndex[i]=np.random.randint(0,N,size=M)
                # ==================================
                # If we want to use a non-uniform distribution, got to change this
                # 2. This is for an increasing f(x)=2x , x in 0,1, distribution
		if (modelNumber==2):
			randomFloats[i]=N*(np.random.uniform(0,1,M)**0.5)
			simIndex[i]=randomFloats[i]
                # 3. This is for a decreasing, 2-2x distribution
		if (modelNumber==3): 
			randomFloats[i]=N*(1-np.random.uniform(0,1,M)**0.5)
			simIndex[i]=randomFloats[i]
                # =====================================================
                #print('Simulated indices',simIndex[i])
                # Need to reset the data
		y[i]=np.zeros(N)
		for j in range(M):
			y[i,simIndex[i,j]]+=1 # possibility of a bin with >1 counts
		data=(xbin,y[i],xA,Deltax)
		# =======================================================================
		# This is the main function "bestFitLinear"
		hasConverged[i],res[i],Lambda[i],cstat[i]=bestFitLinear(data,hasGap,data)
		# =======================================================================
		if hasConverged[i]==1:
			nAccept+=1
                # also check the sign of F_infinity for that data
		Finf=fAsymptotic(data)
		hasPositiveFinf[i]=np.sign(Finf) # +-1 is returned
	acceptRatio=nAccept/NSim
	positiveFinfRatio=sum(map(lambda x : x==1, hasPositiveFinf))/NSim
	fratio.write("%d %d %3.3f %3.3f\n"%(k, M,acceptRatio,positiveFinfRatio))
	print("%d of %d simulations for M=%d had acceptable solution (fraction: %3.3f)"%(nAccept,NSim,M,acceptRatio))
fratio.close()
xbin.dat0000654000175000017500000000075213571311216010651 0ustar  maxmax0.5 1.5 2.5 3.5 4.5 5.5 6.5 7.5 8.5 9.5 10.5 11.5 12.5 13.5 14.5 15.5 16.5 17.5 18.5 19.5 20.5 21.5 22.5 23.5 24.5 25.5 26.5 27.5 28.5 29.5 30.5 31.5 32.5 33.5 34.5 35.5 36.5 37.5 38.5 39.5 40.5 41.5 42.5 43.5 44.5 45.5 46.5 47.5 48.5 49.5 50.5 51.5 52.5 53.5 54.5 55.5 56.5 57.5 58.5 59.5 60.5 61.5 62.5 63.5 64.5 65.5 66.5 67.5 68.5 69.5 70.5 71.5 72.5 73.5 74.5 75.5 76.5 77.5 78.5 79.5 80.5 81.5 82.5 83.5 84.5 85.5 86.5 87.5 88.5 89.5 90.5 91.5 92.5 93.5 94.5 95.5 96.5 97.5 98.5 99.5
data.txt0000650000175000017500000000045413531324432010664 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 1. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 2. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 1. 0. 0. 0. 0 
params.dat0000654000175000017500000000001013571310170011155 0ustar  maxmax0 100 1
data2_Figure1.txt0000654000175000017500000000045513557674473012362 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 
data2_Figure2.txt0000654000175000017500000000045513557674473012363 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 
data2_Figure3.txt0000654000175000017500000000045413562317700012342 0ustar  maxmax0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
xbinGap.dat0000654000175000017500000000005313573706625011311 0ustar  maxmax0.5 1.5 2.5 6.25 6.75 7.25 7.75 8.25 8.75 
dataGap.dat0000654000175000017500000000002213573706625011256 0ustar  maxmax1 1 1 1 1 1 1 1 1
paramsGap.dat0000654000175000017500000000002213571723571011625 0ustar  maxmax0 9 DeltaxGap.dat
README-book0000664000175000017500000000031514066252476011037 0ustar  maxmaxAdapt one of the examples from the paper for a 
simple example in the book

python3 cstatLinear.py xbin.dat data2_Figure1.txt params.dat

python3 cstatLinear.py xbin-book.dat data-book.txt params-book.dat
README0000644000175000017500000001065213643067676010120 0ustar  maxmaxThis README file accompanies the paper
 `A semi–analytical solution to the maximum–likelihood 
fit of Poisson data to a linear model using the Cash statistic' 
by M. Bonamente and D. Spence, 2020.
It contains instructions to reproduce all Figures and significant results
presented in the paper. The following python codes are also provided:

cstatLinear.py (main code to fit data and plot Figures 2,3)
cstatFunctions.py (supporting functions)
cstatLinearFunction.py (supporting functions containing main fitting routins)
plots.py (to plot Figures 1, 4, and 7)
cstatFraction.py (to simulate data)

The code cstatLinear.py needs supporting files:
 - xbin.dat	A list of N numbers, the centers of the bins
 - data.txt	A list of N numbers, the counts in each bin
 - params.dat	A parameter file, with either 
		(a) xA xB Deltax; or
		(b) xA xB Deltax.dat
 - Deltax.dat 	(optional) A list of N numbers, the width of the bins
Note: If Deltax is provided as a single number, the software uses constant
error bars for all N bins.

Case studies:

1: Dataset with 2 counts,  used to generate Fig. 2, Fig. 8 (left panel)  : 

python3 cstatLinear.py xbin.dat data2_Figure1.txt params.dat

These data are also used for Example 5.3, although a figure
for that example was not reported.

For Fig. 8, the plot parameters must be set at the following values:

rescale=1               # rescales the models for visibility
plotUnacceptable=1      # plot the negative (unacceptable) models too
plotExtended=1          # plot also the extended models (pivoted and constant)
plotallticks=0          # write all ticks (for data with few bins)
plotStepModel=0         # plots the step-wise Poisson mean model

2. Dataset with 3 counts, used to generate Fig. 3 and Fig. 8 (right panel):

python3 cstatLinear.py xbin.dat data2_Figure2.txt params.dat

For Fig. 8, the plot parameters must be set at the following values:

rescale=1               # rescales the models for visibility
plotUnacceptable=1      # plot the negative (unacceptable) models too
plotExtended=1          # plot also the extended models (pivoted and constant)
plotallticks=0          # write all ticks (for data with few bins)
plotStepModel=0         # plots the step-wise Poisson mean model


3. Dataset with 5 counts, used to generate Fig. 5 and Fig. 6:

python3 cstatLinear.py xbin.dat data2_Figure3.txt params.dat

For Fig. 5, the shaded area is plotted using

plotRangeUnacceptable=1 # shaded area showing range of unacceptable solutions

These data are also discussed in Example 5.9.
Note that the plot parameters must be set at the following values:

rescale=1               # rescales the models for visibility
plotUnacceptable=1      # plot the negative (unacceptable) models too
plotExtended=0          # plot also the extended models (pivoted and constant)
plotallticks=0          # write all ticks (for data with few bins)
plotStepModel=0         # plots the step-wise Poisson mean model

4. Datasets with gaps, used to generate Fig. 9:

python3 cstatLinear.py xbinGap.dat dataGap.dat paramsGap.dat

Note that the plot parameters must be set at the following values:

rescale=0               # rescales the models for visibility
plotUnacceptable=0      # plot the negative (unacceptable) models too
plotExtended=1          # plot also the extended models (pivoted and constant)
plotallticks=1          # write all ticks (for data with few bins)
plotStepModel=1         # plots the step-wise Poisson mean model


5. To generate Figs. 1, 4, and 7

python3 plots.py 

Figs. 1 and 4 do not need any additional data. 
For Fig. 7, the following data are provided:

accept0=np.genfromtxt("acceptRatio-a0.txt")
accept1=np.genfromtxt("acceptRatio-a+0.01.txt")
accept2=np.genfromtxt("acceptRatio-a-0.01.txt")

The three ASCII files: 
acceptRatio-a0.txt, acceptRatio-a+0.01.txt and acceptRatio-a-0.01.txt
can be generated using the script cstatFraction.py:

python3 cstatFraction.py

Notice that the parameter "modelNumber" in the script

# ===================================
# decide which model to simulate
modelNumber=3
# 1: constant model
# 2: increasing model
# 3: decreasing model
# ==================================

must be set according to the model that needs to be simulated. Also notice
that each run of the simulation will yield slightly different results, so that
the Figure in the paper will not be reproduced exactly.
The script cstatFraction.py generates a file named "acceptRatio.txt"
that needs to be re-named after each run, before generating Fig. 7.

README-book0000664000175000017500000000031514066252476011037 0ustar  maxmaxAdapt one of the examples from the paper for a 
simple example in the book

python3 cstatLinear.py xbin.dat data2_Figure1.txt params.dat

python3 cstatLinear.py xbin-book.dat data-book.txt params-book.dat
README0000644000175000017500000001065213643067676010120 0ustar  maxmaxThis README file accompanies the paper
 `A semi–analytical solution to the maximum–likelihood 
fit of Poisson data to a linear model using the Cash statistic' 
by M. Bonamente and D. Spence, 2020.
It contains instructions to reproduce all Figures and significant results
presented in the paper. The following python codes are also provided:

cstatLinear.py (main code to fit data and plot Figures 2,3)
cstatFunctions.py (supporting functions)
cstatLinearFunction.py (supporting functions containing main fitting routins)
plots.py (to plot Figures 1, 4, and 7)
cstatFraction.py (to simulate data)

The code cstatLinear.py needs supporting files:
 - xbin.dat	A list of N numbers, the centers of the bins
 - data.txt	A list of N numbers, the counts in each bin
 - params.dat	A parameter file, with either 
		(a) xA xB Deltax; or
		(b) xA xB Deltax.dat
 - Deltax.dat 	(optional) A list of N numbers, the width of the bins
Note: If Deltax is provided as a single number, the software uses constant
error bars for all N bins.

Case studies:

1: Dataset with 2 counts,  used to generate Fig. 2, Fig. 8 (left panel)  : 

python3 cstatLinear.py xbin.dat data2_Figure1.txt params.dat

These data are also used for Example 5.3, although a figure
for that example was not reported.

For Fig. 8, the plot parameters must be set at the following values:

rescale=1               # rescales the models for visibility
plotUnacceptable=1      # plot the negative (unacceptable) models too
plotExtended=1          # plot also the extended models (pivoted and constant)
plotallticks=0          # write all ticks (for data with few bins)
plotStepModel=0         # plots the step-wise Poisson mean model

2. Dataset with 3 counts, used to generate Fig. 3 and Fig. 8 (right panel):

python3 cstatLinear.py xbin.dat data2_Figure2.txt params.dat

For Fig. 8, the plot parameters must be set at the following values:

rescale=1               # rescales the models for visibility
plotUnacceptable=1      # plot the negative (unacceptable) models too
plotExtended=1          # plot also the extended models (pivoted and constant)
plotallticks=0          # write all ticks (for data with few bins)
plotStepModel=0         # plots the step-wise Poisson mean model


3. Dataset with 5 counts, used to generate Fig. 5 and Fig. 6:

python3 cstatLinear.py xbin.dat data2_Figure3.txt params.dat

For Fig. 5, the shaded area is plotted using

plotRangeUnacceptable=1 # shaded area showing range of unacceptable solutions

These data are also discussed in Example 5.9.
Note that the plot parameters must be set at the following values:

rescale=1               # rescales the models for visibility
plotUnacceptable=1      # plot the negative (unacceptable) models too
plotExtended=0          # plot also the extended models (pivoted and constant)
plotallticks=0          # write all ticks (for data with few bins)
plotStepModel=0         # plots the step-wise Poisson mean model

4. Datasets with gaps, used to generate Fig. 9:

python3 cstatLinear.py xbinGap.dat dataGap.dat paramsGap.dat

Note that the plot parameters must be set at the following values:

rescale=0               # rescales the models for visibility
plotUnacceptable=0      # plot the negative (unacceptable) models too
plotExtended=1          # plot also the extended models (pivoted and constant)
plotallticks=1          # write all ticks (for data with few bins)
plotStepModel=1         # plots the step-wise Poisson mean model


5. To generate Figs. 1, 4, and 7

python3 plots.py 

Figs. 1 and 4 do not need any additional data. 
For Fig. 7, the following data are provided:

accept0=np.genfromtxt("acceptRatio-a0.txt")
accept1=np.genfromtxt("acceptRatio-a+0.01.txt")
accept2=np.genfromtxt("acceptRatio-a-0.01.txt")

The three ASCII files: 
acceptRatio-a0.txt, acceptRatio-a+0.01.txt and acceptRatio-a-0.01.txt
can be generated using the script cstatFraction.py:

python3 cstatFraction.py

Notice that the parameter "modelNumber" in the script

# ===================================
# decide which model to simulate
modelNumber=3
# 1: constant model
# 2: increasing model
# 3: decreasing model
# ==================================

must be set according to the model that needs to be simulated. Also notice
that each run of the simulation will yield slightly different results, so that
the Figure in the paper will not be reproduced exactly.
The script cstatFraction.py generates a file named "acceptRatio.txt"
that needs to be re-named after each run, before generating Fig. 7.

xbin-book.dat0000664000175000017500000000002514066252675011610 0ustar  maxmax0.5 1.5 2.5 3.5 4.5 
data-book.txt0000664000175000017500000000001214066267532011622 0ustar  maxmax0 0 1 0 1
params-book.dat0000654000175000017500000000000614066252772012127 0ustar  maxmax0 5 1