1133 trials

scintellometry/io/mark4.py

@@ -99,6 +99,8 @@ def __init__(self, raw_files, channels, fedge, fedge_at_top,
             self.npol = self.nvlbichan if channels is None else 1
         if not (1 <= self.npol <= 2):
             warnings.warn("Should use 1 or 2 channels for folding!")
+        if not isinstance(reftime, Time):
+            reftime = Time('J'+str(reftime), scale='utc')
         # PAYLOADSIZE refers to the number of bits per frame per VLBI channel.
         self.framesize = PAYLOADSIZE * self.ntrack // 8
         # Comment from C code:
@@ -207,7 +209,6 @@ def record_read(self, count, blank=True):
 
         if self.npol == 2:
             data = data.view('f4,f4')
-
         return data if self.decimation == 1 else data[::self.decimation]
 
     def ntint(self, nchan):
@@ -464,8 +465,8 @@ def init_luts():
     lut1bit = lut2level[(b >> i) & 1]
     i = np.arange(4)
     # fanout 1 @ 8/16t, fanout 4 @ 32/64t
-    s = i*2
-    m = s+1
+    s = i*2 # 0 2 4 6
+    m = s+1 # 1 3 5 7
     lut2bit1 = lut4level[(b >> s & 1) +
                          (b >> m & 1) * 2]
     # fanout 2 @ 8/16t, fanout 1 @ 32/64t
@@ -504,5 +505,29 @@ def decode_2bit_64track_fanout4(frame, channels=None):
     # Another transpose ensures samples are the first dimension.
     return lut2bit1[frame.T].reshape(frame.shape[1], -1).T
 
+def decode_2bit_32track_fanout2(frame, channels=None):
+    """Decode the frame, assuming 32 tracks using 2 bits, fan-out 2.
+
+    Optionally select some VLBI channels (by default, all 8 are returned).
+    """
+    # reshaping to have 4*8=32-bit words
+    frame = frame.view(np.uint8).reshape(-1, 4)
+    # The look-up table splits each data byte into 2 measurements for 2 channels 
+    # (i.e 2 pols), items 0 and 1 are measurement 1, 2+3 measurement 2 for same chans
+    # Using transpose ensures channels are first, then time samples, then
+    # those 4 measurements, so the first reshape splits the two measurements
+    # the transpose ensures samples are the first dimension, channels are the last.
+    # transpose to (samples, 2times,2pols,4chans)
+    #frame=lut2bit3[frame.T].reshape(4,frame.shape[0], 2, 2).transpose(1,2,3,0)
+    # next reshape puts pols and chans in order, last one orders time samples
+    #frame=frame.reshape(frame.shape[0], 2, 8).reshape(-1,8)
+    # martens
+    frame = (lut2bit3.take(frame, axis=0).reshape(-1, 4, 2, 2)
+             .transpose(3, 1, 0, 2).reshape(8, -1).T)
+    # Correct ordering, at the same time possibly selecting specific channels.
+    reorder = np.array([0, 1, 2, 3, 4, 5, 6, 7])
+    return frame[:, reorder if channels is None else reorder[channels]]
+
 # Decoders keyed by (nbit, ntrack, fanout).
-DECODERS = {(2, 64, 4): decode_2bit_64track_fanout4}
+DECODERS = {(2, 64, 4): decode_2bit_64track_fanout4,
+            (2, 32, 2): decode_2bit_32track_fanout2}

scintellometry/meta/reduction.py

@@ -17,7 +17,7 @@
 def reduce(telescope, obskey, tstart, tend, nchan, ngate, ntbin, ntw_min,
            rfi_filter_raw=None, fref=None, dedisperse=None,
            rfi_filter_power=None, do_waterfall=True, do_foldspec=True,
-           verbose=True):
+           verbose=True, conf='observations.conf'):
 
     comm = MPI.COMM_WORLD
     if dedisperse == 'None':
@@ -31,7 +31,7 @@ def reduce(telescope, obskey, tstart, tend, nchan, ngate, ntbin, ntw_min,
     if dedisperse is not None and fref is None:
         raise ValueError("Need to give reference frequency to dedisperse to")
 
-    obs = obsdata()
+    obs = obsdata(conf)
     if verbose > 3 and comm.rank == 0:
         print(obs)
     # find nearest observation to 'date',
@@ -220,6 +220,9 @@ def CL_parser():
         "They specify which observation run to process "
         "(consistent with [telescope], [[date]] entries\n"
         "in observations.conf), and the start and finish timestamps.")
+    d_parser.add_argument(
+        '--conf', type=str, default='observations.conf',
+        help='File that contains the observation setup, i.e. observations.conf.')
     d_parser.add_argument(
         '-t','--telescope', type=str, default='gmrt',
         help="The data to reduce. One of ['gmrt', 'kairo', 'lofar', 'arochime'].")

scintellometry/trials/aro_1133_14june2014/arochime.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#PBS -l nodes=32:ppn=8,walltime=1:40:00
+#PBS -N fold_2048_secs
+#PBS -m abe
+
+NP=256
+OMP=4 # number of threads per node (4 available)
+
+module purge
+module load gcc/4.8.1 binutils 
+module load intel/16.0.3 python/2.7.8
+module load  intelmpi/5.0.3.048 hdf5/187-v18-intelmpi-intel
+module load fftw/3.3.4-intel-impi
+
+# DIRECTORY TO RUN - $PBS_O_WORKDIR is directory job was submitted from
+cd $PBS_O_WORKDIR
+
+# got a funny error regarding python eggs, maybe cannot write to $HOME from compute nodes?
+export PYTHON_EGG_CACHE=/scratch2/p/pen/franzk/
+
+# PIN THE MPI DOMAINS ACCORDING TO OMP
+export I_MPI_PIN_DOMAIN=omp
+
+# EXECUTION COMMAND; -np = nodes*ppn
+echo "----------------------"
+echo "STARTING in directory $PWD"
+date
+echo "np ${NP}, rpn ${RPN}, omp ${OMP}"
+# EXECUTION COMMAND; -np = nodes*processes_per_nodes; --byhost forces a round robin of nodes.
+export OMP_NUM_THREADS=${OMP}
+export PYTHONPATH=/home/p/pen/franzk/git/scintellometry:/home/p/pen/franzk/git/pulsar:/home/p/pen/franzk/git/baseband:${PYTHONPATH}
+time mpirun -np ${NP} /scinet/gpc/tools/Python/Python278-shared-intel/bin/python /home/p/pen/franzk/git/scintellometry/scintellometry/trials/aro_1133_14june2014/reduce_data.py --telescope arochime -d 2014-06-14T17:20:41 --duration 2048 --ntbin 128 -v -v -v -v --dedisperse incoherent --conf /home/p/pen/franzk/git/scintellometry/scintellometry/trials/aro_1133_14june2014/observations.conf
+
+echo "ENDED"
+date
+

scintellometry/trials/aro_1133_14june2014/data/polycob1133+16_aro.dat

@@ -0,0 +1,90 @@
+1136+1551  13-Jun-14  140000.00   56821.58333333330             4.840738  0.947 -9.068
+    757478625.832062    0.841812442909  aro  300   12   327.000                
+  9.32381393834854638e-10 -4.92761375210590789e-03  9.06257097567023861e-08
+ -1.06502117374139576e-10 -1.41385353839905824e-13  9.57176564006278581e-17
+  9.20770033870556048e-20  4.75366938937007619e-22 -1.28310257702428499e-25
+ -1.95369472989835006e-26  2.06459127566571948e-30  2.72580397824484002e-31
+1136+1551  13-Jun-14  190000.00   56821.79166666660             4.840738  0.944 -9.234
+    757493776.982184    0.841812442909  aro  300   12   327.000                
+  4.48719015495031132e-13 -4.91210331853789671e-03 -4.59596522814305059e-08
+ -1.52096577268382891e-10  7.67834281983725832e-14  1.49547565492490699e-16
+ -6.29331101067352670e-20 -3.71811507939195795e-22  7.59060038809759783e-25
+  1.11249895346192016e-26 -1.35215689904043585e-29 -1.51974354301995792e-31
+1136+1551  13-Jun-14       0.00   56821.99999999990             4.840739  0.955 -9.181
+    757508928.125207    0.841812442909  aro  300   12   327.000                
+ -1.22530340295207541e-09 -4.96757227190318049e-03 -1.10960090431611572e-07
+  2.90215955614772204e-11  1.80729471564964388e-13 -2.62578826232235304e-17
+ -1.45393078537426483e-19 -2.57361658529266455e-22  1.52944411880291413e-24
+  1.47709830945507473e-26 -2.45158172172532731e-29 -2.58251068632635145e-31
+1136+1551  14-Jun-14   50000.00   56822.20833333330             4.840739  0.964 -9.149
+    757524079.251026    0.841812442909  aro  300   12   327.000                
+  9.13532910622331593e-11 -5.00948896074644191e-03 -7.43941777774942351e-09
+  1.67021524597481872e-10  1.55309028519022602e-14 -1.57017090039737874e-16
+ -1.19521209300961364e-20 -1.03074609904535209e-22  9.82450715718999895e-26
+  4.67254363077452100e-27 -1.57481831868356838e-30 -3.95753975071976915e-32
+1136+1551  14-Jun-14  100000.00   56822.41666666660             4.840739  0.956 -9.127
+    757539230.375736    0.841812442909  aro  300   12   327.000                
+  1.21636884526684447e-09 -4.97342606138753088e-03  1.10562308674698985e-07
+  5.64514642626464282e-11 -1.72695086828205097e-13 -5.51609635712001998e-17
+  1.28277720282890513e-19  5.10797656001222912e-23 -1.02239171700537835e-24
+  1.35803270669173754e-27  1.71920143708875023e-29 -4.52975009898161262e-32
+1136+1551  14-Jun-14  140000.00   56822.58333333330             4.840739  0.946 -9.152
+    757551351.287848    0.841812442909  aro  300   12   327.000                
+  1.63088842011895884e-10 -4.92049664730113273e-03  8.97802688571801174e-08
+ -1.08769044265394832e-10 -1.39211452144899926e-13  1.09026938771440786e-16
+  7.55865059122440825e-20 -4.39136134354881931e-22  6.57187695048233337e-25
+  1.43880470957137784e-26 -1.26585344593723109e-29 -1.97618449465995959e-31
+1136+1551  14-Jun-14  190000.00   56822.79166666660             4.840739  0.943 -9.159
+    757566502.439991    0.841812442909  aro  300   12   327.000                
+ -1.10039580122771506e-09 -4.90580734172750554e-03 -4.73196592689996084e-08
+ -1.50849695652928980e-10  7.93223054280481496e-14  1.45073936433879597e-16
+ -3.00211593329509521e-20 -9.23740102626660499e-23 -9.27380637277007217e-25
+  3.51285848277119650e-28  1.59407799549926537e-29  1.69576533862980121e-33
+1136+1551  14-Jun-14       0.00   56822.99999999990             4.840739  0.954 -9.176
+    757581653.584832    0.841812442909  aro  300   12   327.000                
+  1.08465594982516525e-09 -4.96152345152668130e-03 -1.10175458864260141e-07
+  3.18466404046548276e-11  1.80154560072935006e-13 -2.93598282730067158e-17
+ -1.36380724256327578e-19 -6.14218602292834722e-23  1.02062353822667313e-24
+  2.52348720299768179e-27 -1.64898337762208684e-29 -3.29827028093890034e-32
+1136+1551  15-Jun-14   50000.00   56823.20833333330             4.840740  0.962 -9.061
+    757596804.712602    0.841812442909  aro  300   12   327.000                
+ -1.10544336146376427e-09 -5.00236387965799414e-03 -5.04104606869456563e-09
+  1.67293507008691458e-10  1.25563231845502096e-14 -1.66047808652761560e-16
+ -2.78325713607861689e-20  5.64183722670103514e-22  8.29737419385603597e-25
+ -1.81504893899649462e-26 -1.18866053050568461e-29  2.47291362017573091e-31
+1136+1551  15-Jun-14  100000.00   56823.41666666660             4.840740  0.955 -9.229
+    757611955.839647    0.841812442909  aro  300   12   327.000                
+  3.99968396147694282e-11 -4.96512147391967723e-03  1.11639883146633128e-07
+  5.36953820871298380e-11 -1.73527051780606720e-13 -4.74452425771802706e-17
+  9.72192253321734402e-20 -2.83966739974381755e-22  7.10334058474527703e-25
+  1.12714585370097356e-26 -1.39427149818851363e-29 -1.55330319960450425e-31
+1136+1551  15-Jun-14  133000.00   56823.56250000000             4.840740  0.945 -9.022
+    757622561.638829    0.841812442909  aro  300   12   327.000                
+  9.69015103775315305e-10 -4.91777666996577687e-03  9.81285048299436659e-08
+ -9.35333237121785431e-11 -1.52176899949457787e-13  8.93798583630905578e-17
+  1.18181284067382359e-19 -6.50027967457656864e-23 -9.23529771909163626e-25
+  2.31302169138467447e-27  1.33337275612848336e-29 -4.83700728026852903e-32
+1136+1551  15-Jun-14  183000.00   56823.77083333330             4.840740  0.941 -9.275
+    757637712.792818    0.841812442909  aro  300   12   327.000                
+ -3.12297307330130703e-11 -4.89578287025965384e-03 -3.47952014387837159e-08
+ -1.58153085183806597e-10  6.01897829706703802e-14  1.53602345137489969e-16
+ -3.48768463484735030e-20 -1.88616574922068425e-22 -1.41570437453877743e-25
+  2.49142784740041344e-27  2.35903722104460328e-30 -7.86353016339277417e-33
+1136+1551  15-Jun-14  233000.00   56823.97916666660             4.840740  0.951 -9.086
+    757652863.941466    0.841812442909  aro  300   12   327.000                
+ -1.01622248313049224e-09 -4.94759864369369133e-03 -1.11496415061994412e-07
+  1.29824937992971309e-11  1.82944535412796723e-13 -2.24830543954110478e-17
+ -1.40643306518926734e-19  7.79033063942587654e-22  1.12687348174782308e-24
+ -2.62328498248291143e-26 -1.72669473722828762e-29  3.29526977864781764e-31
+1136+1551  16-Jun-14   43000.00   56824.18750000000             4.840741  0.960 -9.123
+    757668015.072847    0.841812442909  aro  300   12   327.000                
+  4.86967925625993522e-11 -4.99337302306895880e-03 -1.76090887831348139e-08
+  1.64868991589585506e-10  3.31436875550971191e-14 -1.54446909202158741e-16
+ -2.73459994616673259e-20 -1.48474703675648636e-22  3.14867462981041388e-25
+  6.43409919905214524e-27 -4.97944529672112400e-30 -6.72138609180172142e-32
+1136+1551  16-Jun-14   93000.00   56824.39583333330             4.840741  0.954 -9.096
+    757683166.201558    0.841812442909  aro  300   12   327.000                
+  1.41634752145307560e-09 -4.96217157414962154e-03  1.07175711885422482e-07
+  7.14497496322804846e-11 -1.65765875440841678e-13 -7.00933781808750837e-17
+  1.05091422661428585e-19  1.25195674426569656e-22 -1.11634612450015173e-25
+ -1.75457135075847809e-27  1.97606434638002543e-30  4.13485654516401743e-33

scintellometry/trials/aro_1133_14june2014/data/polycob1133+16_jb.dat

@@ -0,0 +1,90 @@
+1136+1551  13-Jun-14   90000.00   56821.37500000000             4.840738  0.948 -9.103
+    757463474.698330    0.841812442909   jb  300   12   327.000                
+  9.03029940663801352e-10 -4.93309608069333883e-03  7.76259142609126858e-08
+ -9.28484897710612497e-11 -1.20633895926165035e-13  9.45205389822780669e-17
+  6.54208814612551699e-20 -4.70878236879138686e-22  4.44335154381288000e-25
+  1.40855047859200954e-26 -7.06489188142828103e-30 -1.65823593125474583e-31
+1136+1551  13-Jun-14  140000.00   56821.58333333330             4.840738  0.946 -9.054
+    757478625.846135    0.841812442909   jb  300   12   327.000                
+ -7.35661515380242673e-10 -4.92014476423253272e-03 -4.02536952976439396e-08
+ -1.30309707680011358e-10  6.75294225572186976e-14  1.16721149542220828e-16
+ -3.97926564094198214e-20  5.11989809497639509e-22 -6.82564454032150895e-25
+ -1.83843024803141920e-26  1.70529366558858397e-29  2.23007827707485694e-31
+1136+1551  13-Jun-14  190000.00   56821.79166666660             4.840738  0.955 -9.265
+    757493776.987726    0.841812442909   jb  300   12   327.000                
+  3.95257673652797635e-11 -4.96803934812748219e-03 -9.51778351246062400e-08
+  2.63714326294684563e-11  1.55209188560181233e-13 -2.23552119443951113e-17
+ -1.01155232295083876e-19 -1.73063908105491680e-22  1.45407342509277660e-25
+  5.21382691976447229e-27 -2.02905104075203323e-30 -4.97544462100739669e-32
+1136+1551  14-Jun-14       0.00   56822.00000000000             4.840739  0.962 -9.062
+    757508928.114566    0.841812442909   jb  300   12   327.000                
+  1.01463278618493389e-09 -5.00360866817334705e-03 -5.21543424147252079e-09
+  1.43989205395933967e-10  1.13127348219465847e-14 -1.34384975800863614e-16
+  3.64573351764909247e-20 -3.31050523978546285e-22 -1.57601666418043834e-24
+  1.81752360936516019e-26  2.06347743387276559e-29 -2.93717011237200830e-31
+1136+1551  14-Jun-14   50000.00   56822.20833333330             4.840739  0.956 -9.252
+    757524079.240641    0.841812442909   jb  300   12   327.000                
+  2.36592698068419056e-11 -4.97195673306916833e-03  9.57840074511435962e-08
+  4.72788003154487218e-11 -1.49470619878879819e-13 -4.47498509707407025e-17
+  1.18133848080869663e-19  9.81766902418092442e-23 -6.87801014577610919e-25
+ -6.55378948886706668e-27  6.33569416162054939e-30  1.37469474281764553e-31
+1136+1551  14-Jun-14   90000.00   56822.37500000000             4.840739  0.947 -9.078
+    757536200.152211    0.841812442909   jb  300   12   327.000                
+ -3.38174057564367693e-10 -4.92633575455839884e-03  7.69433156669594674e-08
+ -9.47056942053578081e-11 -1.18805348558224664e-13  9.13392995068762186e-17
+  5.72622923041900677e-20 -1.33476537719757816e-22  7.08870364331363624e-25
+  4.54317822436331683e-27 -9.65928463948994255e-30 -7.53698387383181648e-32
+1136+1551  14-Jun-14  140000.00   56822.58333333330             4.840739  0.945 -9.138
+    757551351.301949    0.841812442909   jb  300   12   327.000                
+  6.17476337677563513e-10 -4.91405798596497578e-03 -4.13602621188710605e-08
+ -1.29288821407846996e-10  6.99154994595041594e-14  1.24511428452858816e-16
+ -4.89939025957922382e-20 -1.10412839686384457e-22  2.88615691187274727e-25
+  1.98728358970945655e-27 -5.96706802477194152e-30 -2.83416824546171881e-32
+1136+1551  14-Jun-14  190000.00   56822.79166666660             4.840739  0.954 -9.036
+    757566502.445312    0.841812442909   jb  300   12   327.000                
+ -2.49326159157436333e-09 -4.96212226155123663e-03 -9.44275901076557157e-08
+  2.88430567739815325e-11  1.54624701539159703e-13 -3.20457063261016897e-17
+ -9.53171041809835941e-20  4.13779843699570287e-22  1.20073443672564980e-26
+ -1.48317092254251993e-26 -9.63698361095494327e-31  2.00088131924918533e-31
+1136+1551  15-Jun-14       0.00   56823.00000000000             4.840739  0.961 -9.148
+    757581653.574052    0.841812442909   jb  300   12   327.000                
+ -4.24084422331009328e-10 -4.99672260792418006e-03 -3.09406896063930987e-09
+  1.44139054846387814e-10  9.64452678475531304e-15 -1.37303264938922209e-16
+ -5.71180804276234282e-20  8.35388375047655198e-23  2.33961085138762451e-24
+ -1.91849767597432162e-27 -3.87204240053799646e-29  3.38098740850772061e-32
+1136+1551  15-Jun-14   50000.00   56823.20833333330             4.840740  0.955 -8.979
+    757596804.702367    0.841812442909   jb  300   12   327.000                
+ -1.81030012050072423e-09 -4.96402552787190369e-03  9.67512749407058761e-08
+  4.49968326870196138e-11 -1.49495724823353369e-13 -5.57204455743007254e-17
+  3.95745856986493153e-20  1.17177412673574005e-21  2.61384467429229466e-24
+ -4.55464884075179204e-26 -4.41954871571463485e-29  6.59382551386364142e-31
+1136+1551  15-Jun-14   83000.00   56823.35416666660             4.840740  0.946 -9.158
+    757607410.501082    0.841812442909   jb  300   12   327.000                
+ -4.66995284673435429e-11 -4.92317622386802961e-03  8.42772255469174378e-08
+ -8.17343237618700046e-11 -1.29926410361496997e-13  8.26236102671900634e-17
+  7.11469574147113083e-20 -3.73016559492074670e-22  6.20279687345273029e-25
+  1.19986983545083723e-26 -1.24297119536000385e-29 -1.59548018564135183e-31
+1136+1551  15-Jun-14  133000.00   56823.56249999990             4.840740  0.943 -9.138
+    757622561.652666    0.841812442909   jb  300   12   327.000                
+ -1.21345078988404808e-09 -4.90455747853723104e-03 -3.05579546596335515e-08
+ -1.35700833682223947e-10  5.34643849642196607e-14  1.28816554676269549e-16
+ -5.57059420682917063e-20 -5.25285650076899255e-23  1.11421702521972718e-24
+  2.64490384838267165e-27 -1.84336767445607218e-29 -7.23707385751338980e-32
+1136+1551  15-Jun-14  183000.00   56823.77083333320             4.840740  0.952 -9.264
+    757637712.799565    0.841812442909   jb  300   12   327.000                
+ -4.95200322070080162e-12 -4.94928144093737584e-03 -9.56154988224288432e-08
+  1.25353233399220154e-11  1.57302422939331442e-13 -1.02858201630338916e-17
+ -1.10154734021724116e-19 -1.38657735208521791e-23  5.26607554555111017e-25
+ -3.37099417973672292e-27 -8.33792636191478003e-30  9.88407958716488117e-32
+1136+1551  15-Jun-14  233000.00   56823.97916666660             4.840740  0.959 -9.063
+    757652863.931666    0.841812442909   jb  300   12   327.000                
+  1.40694516412657612e-10 -4.98813975796640920e-03 -1.38696239342462554e-08
+  1.42279579536838920e-10  2.69525086569920364e-14 -1.38265183092615417e-16
+ -1.99065466590495000e-20  2.84294528780756936e-22  1.82946528226411553e-25
+ -9.09087936135061925e-27 -3.00096034200500770e-30  1.30108055546205186e-31
+1136+1551  16-Jun-14   43000.00   56824.18749999990             4.840741  0.954 -9.269
+    757668015.061676    0.841812442909   jb  300   12   327.000                
+ -5.42563962998067427e-11 -4.96057962664167660e-03  9.30689922403727917e-08
+  6.02434401664581417e-11 -1.43327293291659858e-13 -5.52035105412823755e-17
+  7.13040633074846222e-20 -5.56679769589276537e-23  9.82627450273525629e-25
+ -6.24407740372376448e-28 -1.70112570749173136e-29  5.33886275788427621e-32

scintellometry/trials/aro_1133_14june2014/get_polycos

@@ -0,0 +1,14 @@
+#!/bin/bash
+psr=${1:-b1133+16}
+#cp -p ../../ephemerides/psr$psr.par .
+grep -n TZRFRQ psr$psr.par || \
+cat <<EOD >>psr$psr.par
+TZRFREQ 1400.
+TZRMJD 56000.
+TZRSITE coe
+EOD
+for site in 'aro' 'jb'; do
+  tempo2 -tempo1 -f psr$psr.par -polyco "56822 56824 300 12 12 $site 327.0"
+  mv polyco_new.dat data/polyco${psr}_$site.dat
+  rm polyco.tim newpolyco.dat
+done