Why is clk-phase not zero is unresolved

Merge branch 'why-is-clk-phase-not-zero-centered' into main

.gitmodules

@@ -1,6 +1,6 @@
 [submodule "simulations/airshower_beacon_simulation/earsim"]
-	path = simulations/airshower_beacon_simulation/lib/earsim
+	path = airshower_beacon_simulation/lib/earsim
 	url = https://gitlab.com/harmscho/earsim.git
 [submodule "simulations/airshower_beacon_simulation/lib/atmocal"]
-	path = simulations/airshower_beacon_simulation/lib/atmocal
+	path = airshower_beacon_simulation/lib/atmocal
 	url = https://gitlab.com/harmscho/AtmosphereCal

airshower_beacon_simulation/Makefile

@@ -27,35 +27,56 @@ SEED ?= 12345
 
 all: beacon clocks phases findks vary-fixes reconstruct
 
-beacon:
+beacon: generate-beacon signal-to-noise
+phases: beacon-phase clock-phase baseline-phase antenna-phase
+
+######
+
+generate-beacon:
 	./aa_generate_beacon.py -N ${TRACE_N} -P ${TRACE_PRE} -n ${NOISE_SIGMA} -a ${BEAC_AMP} -f ${BEAC_F} -l ${PB_LOW} -u ${PB_HIGH} -d ${BEAC_DECAY} --data-dir ${DATA_DIR} --input-fname ${INPUT_DIR} | tee ${FIG_DIR}/aa.log
-	./ab_modify_clocks.py 0 --data-dir ${DATA_DIR} | tee ${FIG_DIR}/ab.log
+	./ab_modify_clocks.py 0 --no-save-clocks --data-dir ${DATA_DIR} | tee ${FIG_DIR}/ab.log
+
+signal-to-noise:
 	./ac_show_signal_to_noise.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
 	./view_beaconed_antenna.py 72 -p x -p y -p z -p n -p b --ft --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
 
-clocks:
+#
+new-clocks:
 	./ab_modify_clocks.py ${CLK_DEV} --seed ${SEED} --gaussian --data-dir ${DATA_DIR}
 
-phases:
+clocks:
+	./ab_modify_clocks.py ${CLK_DEV} --read-clocks-file --seed ${SEED} --gaussian --data-dir ${DATA_DIR}
+
+reset-clocks:
+	./ab_modify_clocks.py 0 --data-dir ${DATA_DIR}
+
+#
+beacon-phase:
 	./ba_measure_beacon_phase.py --N-mask ${N_MASK} --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
+
+clock-phase:
 	./bb_measure_clock_phase.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
+
+baseline-phase:
 	./bc_baseline_phase_deltas.py ${REF_ANTS} --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
+
+antenna-phase:
 	./bd_antenna_phase_deltas.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
 
+#
 findks:
 	./ca_period_from_shower.py --input-fname ${INPUT_DIR} --max-k ${MAX_K} --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR} -l ${PB_LOW} -u ${PB_HIGH}
 	./cb_report_measured_antenna_time_offsets.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
 
+#
 vary-fixes:
 	./dc_grid_power_time_fixes.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR} --input-fname ${INPUT_DIR}
 
+#
 reconstruct:
 	./da_reconstruction.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR} --input-fname ${INPUT_DIR}
 	./db_longitudinal_figure.py --no-show-plots --fig-dir=${FIG_DIR} --data-dir ${DATA_DIR}
 
 dist-clean:
-	rm -vf ${DATA_DIR}/antennas.hdf5
+	rm -vf ${DATA_DIR}/
-	rm -vf ${DATA_DIR}/ca_breaked_run
+	rm -vf ${FIG_DIR}/
-	rm -vf ${DATA_DIR}/res.pkl
-	rm -vf ${DATA_DIR}/clocks.csv
-	rm -vf ${DATA_DIR}/tx.json

airshower_beacon_simulation/aa_generate_beacon.py

@@ -18,6 +18,7 @@ import lib
 # {{{ vim marker
 tx_fname = 'tx.json'
 antennas_fname = 'antennas.hdf5'
+snr_fname = 'snr.json'
 c_light = lib.c_light
 
 def read_antenna_clock_repair_offsets(antennas, mode='all', freq_name=None):
@@ -49,6 +50,17 @@ def read_antenna_clock_repair_offsets(antennas, mode='all', freq_name=None):
 
     return time_offsets
 
+def write_snr_file(fname, snrs):
+    with open(fname, 'w') as fp:
+        return json.dump(
+                {'mean': np.mean(snrs), 'std': np.std(snrs), 'values': snrs},
+                fp
+                )
+
+def read_snr_file(fname):
+    with open(fname, 'r') as fp:
+        return json.load(fp)
+
 def write_tx_file(fname, tx, f_beacon, **kwargs):
     with open(fname, 'w') as fp:
         return json.dump(
@@ -309,6 +321,7 @@ if __name__ == "__main__":
 
     # Noise properties
     noise_sigma = args.noise_sigma # mu V/m set to None to ignore
+    unique_noise_realisations = True # a new noise realisation per antenna vs. single noise realisation shared across antennas
 
     # Beacon properties
     beacon_amplitudes = np.array(args.beacon_amplitudes) # mu V/m
@@ -369,6 +382,7 @@ if __name__ == "__main__":
     init_antenna_hdf5(antennas_fname, tx, f_beacon)
 
     # make beacon per antenna
+    noise_realisation = np.array([0])
     for i, antenna in enumerate(ev.antennas):
         #TODO: allow to change the samplerate (2, 4, 8 ns)
 
@@ -397,9 +411,9 @@ if __name__ == "__main__":
         beacon = 1e-6 * lib.beacon_from(tx, antenna, f_beacon, antenna.t, c_light=c_light, radiate_rsq=beacon_radiate_rsq) # mu V/m
 
         # noise realisation
-        noise_realisation = 0
+        if unique_noise_realisations or (noise_realisation == 0).all(): # either create one for every antenna, or generate a single one
-        if noise_sigma is not None:
+            print("Noise realisation!")
-            noise_realisation = 1e-6 * rng.normal(0, noise_sigma, size=len(beacon)) # mu V/m
+            noise_realisation = 1e-6 * rng.normal(0, noise_sigma or 0, size=len(antenna.t)) # mu V/m
 
         # Collect all data to be saved (with the first 3 values the E fields)
         traces = np.array([antenna.Ex, antenna.Ey, antenna.Ez, beacon, noise_realisation])

airshower_beacon_simulation/ab_modify_clocks.py

@@ -26,6 +26,8 @@ if __name__ == "__main__":
     parser.add_argument('-s', '--seed', type=int, nargs='?', default=None, help='Fix seed if supplied.')
     parser.add_argument('--uniform', action='store_const', const='uniform', dest='dist_type')
     parser.add_argument('--gaussian',   action='store_const', const='gauss', dest='dist_type')
+    parser.add_argument('-r','--read-clocks-file', action='store_true', dest='read_clocks_file')
+    parser.add_argument('--no-save-clocks', action='store_false', dest='save_clocks')
     parser.set_defaults(dist_type='gauss')
 
     parser.add_argument('--data-dir', type=str, default="./data", help='Path to Data Directory. (Default: %(default)s)')
@@ -62,7 +64,11 @@ if __name__ == "__main__":
 
         N_antennas = len(group.keys())
 
-        if True:
+        if args.read_clocks_file and path.isfile(clocks_fname): # read clock deviations from file
+            print(f"Reading clocks from {clocks_fname}.")
+            clock_offsets = np.loadtxt(clocks_fname)
+
+        elif True: # random clock deviations
             print(f"Modifying clocks upto {max_clock_offset}ns.")
             clock_offsets = np.zeros( N_antennas )
             if args.dist_type == 'uniform': # uniform
@@ -110,6 +116,7 @@ if __name__ == "__main__":
             h5ant['E_AxB'][0, :] += clk_offset
 
     # save to simple csv
+    if args.save_clocks:
         np.savetxt(clocks_fname, clock_offsets)
 
     print("Antenna clocks modified in " + str(antennas_fname))

airshower_beacon_simulation/ac_show_signal_to_noise.py

@@ -42,6 +42,7 @@ if __name__ == "__main__":
     fname_dir = args.data_dir
     antennas_fname = path.join(fname_dir, beacon.antennas_fname)
     tx_fname = path.join(fname_dir, beacon.tx_fname)
+    snr_fname = path.join(fname_dir, beacon.snr_fname)
 
     # create fig_dir
     if fig_dir:
@@ -92,6 +93,9 @@ if __name__ == "__main__":
         N_samples = len(antennas[0].beacon)
         beacon_snrs = [ lib.signal_to_noise(myfilter(beacon_amp*ant.beacon), myfilter(ant.noise), samplerate=1/dt, signal_band=beacon_pb, noise_band=noise_pb) for i, ant in enumerate(antennas) ]
 
+        # write mean and std to file
+        beacon.write_snr_file(snr_fname, beacon_snrs)
+
         fig, ax = plt.subplots(figsize=figsize)
         ax.set_title(f"Maximum Beacon/Noise SNR (N_samples:{N_samples:.1e})")
         ax.set_xlabel("Antenna no.")

airshower_beacon_simulation/ba_measure_beacon_phase.py

@@ -12,6 +12,8 @@ import numpy as np
 
 import aa_generate_beacon as beacon
 import lib
+from lib import figlib
+
 
 if __name__ == "__main__":
     from os import path
@@ -49,6 +51,7 @@ if __name__ == "__main__":
     ####
     fname_dir = args.data_dir
     antennas_fname = path.join(fname_dir, beacon.antennas_fname)
+    snr_fname = path.join(fname_dir, beacon.snr_fname)
 
     fig_dir = args.fig_dir # set None to disable saving
 
@@ -83,7 +86,8 @@ if __name__ == "__main__":
 
         N_antennas = len(group.keys())
         # just for funzies
-        found_data = np.zeros((N_antennas, 3))
+        found_data = np.zeros((N_antennas, 3)) # freq, phase, amp
+        noise_data = np.zeros((N_antennas, 2)) # phase, amp
 
         # Determine frequency and phase
         for i, name in enumerate(group.keys()):
@@ -133,6 +137,7 @@ if __name__ == "__main__":
             # TODO: refine masking
             # use beacon but remove where E_AxB-Beacon != 0
             # Uses the first traces as reference
+            t_mask = 0
             if N_mask and orients[0] != 'B':
                 N_pre, N_post = N_mask//2, N_mask//2
 
@@ -166,6 +171,18 @@ if __name__ == "__main__":
                 beacon_phases = [ 2*np.pi*t0*f_beacon ]
                 amps = [ 3e-7 ]
 
+            # Also try to find the phase from the noise trace if available
+            if len(h5ant[traces_key]) > 4:
+                noise_trace = h5ant[traces_key][5]
+                if np.any(t_mask): # Mask the same area
+                    noise_trace = noise_trace[t_mask]
+
+                real, imag = lib.direct_fourier_transform(f_beacon, t_trace, noise_trace)
+                noise_phase = np.arctan2(imag, real)
+                noise_amp = (real**2 + imag**2)**0.5
+
+                noise_data[i] = noise_phase, noise_amp
+
             # choose highest amp
             idx = np.argmax(amps)
             if False and len(beacon_phases) > 1:
@@ -246,10 +263,16 @@ if __name__ == "__main__":
             h5attrs['amplitude'] = amplitude
             h5attrs['orientation'] = orientation
 
+            if noise_phase:
+                h5attrs['noise_phase'] = noise_phase
+                h5attrs['noise_amp'] = noise_amp
+
     print("Beacon Phases, Amplitudes and Frequencies written to", antennas_fname)
 
     # show histogram of found frequencies
     if show_plots or fig_dir:
+        snrs = beacon.read_snr_file(snr_fname)
+
         if True or allow_frequency_fitting:
             fig, ax = plt.subplots(figsize=figsize)
             ax.set_xlabel("Frequency")
@@ -260,12 +283,45 @@ if __name__ == "__main__":
                 fig.savefig(path.join(fig_dir, path.basename(__file__) + f".hist_freq.pdf"))
 
         if True:
-            fig, ax = plt.subplots(figsize=figsize)
+            fig, _ = figlib.fitted_histogram_figure(found_data[:,2], fit_distr=[], mean_snr=snrs['mean'])
-            ax.set_xlabel("Amplitudes")
+            ax = fig.axes[0]
+            ax.set_xlabel("Amplitude")
             ax.set_ylabel("Counts")
             ax.hist(found_data[:,2], bins='sqrt', density=False)
+            ax.legend()
             if fig_dir:
                 fig.savefig(path.join(fig_dir, path.basename(__file__) + f".hist_amp.pdf"))
 
+        if (noise_data[0] != 0).any():
+            if True:
+                fig, ax = plt.subplots(figsize=figsize)
+                ax.set_title("Noise Phases")
+                ax.set_xlabel("Phase")
+                ax.set_ylabel("#")
+                ax.hist(noise_data[:,0], bins='sqrt', density=False)
+                ax.legend()
+                if fig_dir:
+                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f".noise.hist_phase.pdf"))
+
+            if True:
+                fig, ax = plt.subplots(figsize=figsize)
+                ax.set_title("Noise Phase vs Amplitude")
+                ax.set_xlabel("Phase")
+                ax.set_ylabel("Amplitude [a.u.]")
+                ax.plot(noise_data[:,0], noise_data[:,1], ls='none', marker='x')
+                if fig_dir:
+                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f".noise.phase_vs_amp.pdf"))
+
+            if True:
+                fig, _ = figlib.fitted_histogram_figure(noise_data[:,1], fit_distr=['rice', 'rayleigh'], mean_snr=snrs['mean'])
+                ax = fig.axes[0]
+                ax.set_title("Noise Amplitudes")
+                ax.set_xlabel("Amplitude [a.u.]")
+                ax.set_ylabel("#")
+                ax.legend()
+
+                if fig_dir:
+                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f".noise.hist_amp.pdf"))
+
     if show_plots:
         plt.show()

airshower_beacon_simulation/bb_measure_clock_phase.py

@@ -0,0 +1,197 @@
+#!/usr/bin/env python3
+# vim: fdm=indent ts=4
+
+import h5py
+from itertools import combinations, zip_longest
+import matplotlib.pyplot as plt
+import numpy as np
+
+import aa_generate_beacon as beacon
+import lib
+from lib import figlib
+
+if __name__ == "__main__":
+    from os import path
+    import sys
+
+    import os
+    import matplotlib
+    if os.name == 'posix' and "DISPLAY" not in os.environ:
+        matplotlib.use('Agg')
+
+    from scriptlib import MyArgumentParser
+    parser = MyArgumentParser()
+    args = parser.parse_args()
+
+    figsize = (12,8)
+    c_light = lib.c_light
+    show_plots = args.show_plots
+
+    remove_absolute_phase_offset_first_antenna = True # takes precedence
+    remove_absolute_phase_offset_minimum = True
+
+    ####
+    fname_dir = args.data_dir
+    antennas_fname = path.join(fname_dir, beacon.antennas_fname)
+    snr_fname = path.join(fname_dir, beacon.snr_fname)
+
+    fig_dir = args.fig_dir # set None to disable saving
+
+    if not path.isfile(antennas_fname):
+        print("Antenna file cannot be found, did you try generating a beacon?")
+        sys.exit(1)
+
+    # Read in antennas from file
+    f_beacon, tx, antennas = beacon.read_beacon_hdf5(antennas_fname)
+
+    # Make sure at least one beacon has been identified
+    if not hasattr(antennas[0], 'beacon_info') or len(antennas[0].beacon_info) == 0:
+        print(f"No analysed beacon found for {antennas[0].name}, try running the beacon phase analysis script first.")
+        sys.exit(1)
+
+    #
+    N_beacon_freqs = len(antennas[0].beacon_info)
+
+    for freq_name in antennas[0].beacon_info.keys():
+        beacon_phases = np.empty( (len(antennas)) )
+        for i, ant in enumerate(antennas):
+            beacon_phases[i] = ant.beacon_info[freq_name]['beacon_phase']
+
+        f_beacon = antennas[0].beacon_info[freq_name]['freq']
+
+        clock_phases = lib.remove_antenna_geometry_phase(tx, antennas, f_beacon, beacon_phases, c_light=c_light)
+
+        # Remove the phase from one antenna
+        # this is a free parameter
+        # (only required for absolute timing)
+        if remove_absolute_phase_offset_first_antenna or remove_absolute_phase_offset_minimum:
+            if remove_absolute_phase_offset_first_antenna: # just take the first phase
+                minimum_phase = clock_phases[0]
+            else: # take the minimum
+                minimum_phase = np.min(clock_phases, axis=-1)
+
+            clock_phases -= minimum_phase
+            clock_phases = lib.phase_mod(clock_phases)
+
+        # Save to antennas in file
+        with h5py.File(antennas_fname, 'a') as fp:
+            h5group = fp['antennas']
+
+            for i, ant in enumerate(antennas):
+                h5ant = fp['antennas'][ant.name]
+
+                h5beacon_freq = h5ant['beacon_info'][freq_name]
+                h5beacon_freq.attrs['clock_phase'] = clock_phases[i]
+
+        # Plot True Phases at their locations
+        if show_plots or fig_dir:
+            actual_clock_phases = lib.phase_mod(np.array([ -2*np.pi*a.attrs['clock_offset']*f_beacon for a in antennas ]))
+            for i in range(2):
+                plot_residuals = i == 1
+                spatial_unit='m'
+
+                antenna_locs = list(zip(*[(ant.x, ant.y) for ant in antennas]))
+
+                scatter_kwargs = {}
+                scatter_kwargs['cmap'] = 'inferno'
+
+                # Measurements
+                if not plot_residuals:
+                    title='Clock phases'
+                    color_label='$\\varphi(\\sigma_t)$ [rad]'
+
+                    fname_extra='measured.'
+
+                    #scatter_kwargs['vmin'] = -np.pi
+                    #scatter_kwargs['vmax'] = +np.pi
+
+                # Plot Clock Phases - True Clock Phases at their location
+                else:
+                    title='Clock phase Residuals'
+                    color_label='$\\Delta\\varphi(\\sigma_t) = \\varphi_{meas} - \\varphi_{true}$ [rad]'
+
+                    fname_extra='residuals.'
+
+                    # Modify actual_clock_phases, the same way as clock_phases
+                    # was modified
+                    if remove_absolute_phase_offset_first_antenna or remove_absolute_phase_offset_minimum:
+                        if remove_absolute_phase_offset_first_antenna: # just take the first phase
+                            minimum_phase = actual_clock_phases[0]
+                        else: # take the minimum
+                            minimum_phase = np.min(actual_clock_phases, axis=-1)
+
+                        actual_clock_phases -= minimum_phase
+                        actual_clock_phases = lib.phase_mod(actual_clock_phases)
+
+                    clock_phase_residuals = lib.phase_mod(clock_phases - actual_clock_phases)
+
+                if not plot_residuals:
+                    loc_c = clock_phases
+                else:
+                    loc_c = clock_phase_residuals
+
+                ##
+                ## Geometrical Plot
+                ##
+                fig, ax = plt.subplots(figsize=figsize)
+
+                ax.set_xlabel('x' if spatial_unit is None else 'x [{}]'.format(spatial_unit))
+                ax.set_ylabel('y' if spatial_unit is None else 'y [{}]'.format(spatial_unit))
+
+                fig.suptitle(title+'\nf_beacon= {:2.0f}MHz'.format(f_beacon*1e3))
+
+                sc = ax.scatter(*antenna_locs, c=loc_c, **scatter_kwargs)
+                fig.colorbar(sc, ax=ax, label=color_label)
+
+                if False:
+                    for i, ant in enumerate(antennas):
+                        ax.text(ant.x, ant.y, ant.name)
+
+                if not True:
+                    ax.plot(tx.x, tx.y, 'X', color='k', markeredgecolor='white')
+
+                if fig_dir:
+                    fig.tight_layout()
+                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f".geom.{fname_extra}F{freq_name}.pdf"))
+
+                ##
+                ## Histogram
+                ##
+                snrs = beacon.read_snr_file(snr_fname)
+
+                fig = figlib.phase_comparison_figure(
+                        loc_c,
+                        actual_clock_phases,
+                        plot_residuals=plot_residuals,
+                        f_beacon=f_beacon,
+                        figsize=figsize,
+                        fit_gaussian=plot_residuals,
+                        mean_snr=snrs['mean']
+                        )
+
+                if plot_residuals:
+                    fig.suptitle("Difference between Measured and True Clock phases")
+                else:
+                    fig.suptitle("Comparison Measured and True Clock Phases")
+
+                axs = fig.get_axes()
+                axs[-1].set_xlabel(f'Antenna {title} {color_label}')
+
+                #
+                i=0
+                secax = axs[i].child_axes[0]
+                secax.set_xlabel('Time $\\Delta\\varphi/(2\\pi f_{beac})$ [ns]')
+
+                #
+                i=1
+                axs[i].set_ylabel("Antenna no.")
+
+                # Save figure
+                if fig_dir:
+                    fig.tight_layout()
+                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f".{fname_extra}F{freq_name}.pdf"))
+
+    print(f"True phases written to", antennas_fname)
+
+    if show_plots:
+        plt.show()

airshower_beacon_simulation/bc_baseline_phase_deltas.py

@@ -8,6 +8,7 @@ import numpy as np
 
 import aa_generate_beacon as beacon
 import lib
+from lib import figlib
 
 
 if __name__ == "__main__":
@@ -108,45 +109,42 @@ if __name__ == "__main__":
 
         for i in range(2):
             plot_residuals = i == 1
-            colors = ['blue', 'orange']
 
-            fig, axs = plt.subplots(2, 1, sharex=True, figsize=figsize)
+            true_phases = my_phase_diffs
+            measured_phases = phase_diffs[:,1]
 
-            if True:
+            hist_kwargs = {}
-                phase2time = lambda x: x/(2*np.pi*f_beacon)
+            if plot_residuals:
-                time2phase = lambda x: 2*np.pi*x*f_beacon
+                measured_phases = lib.phase_mod(measured_phases - true_phases)
-                secax = axs[0].secondary_xaxis('top', functions=(phase2time, time2phase))
+                hist_kwargs['histtype'] = 'stepfilled'
-                secax.set_xlabel('Time $\\Delta\\varphi/(2\\pi f_{beac})$ [ns]')
+
+            fig = figlib.phase_comparison_figure(
+                    measured_phases,
+                    true_phases,
+                    plot_residuals=plot_residuals,
+                    f_beacon=f_beacon,
+                    figsize=figsize,
+                    hist_kwargs=hist_kwargs,
+                    fit_gaussian=plot_residuals,
+                    )
+
+            axs = fig.get_axes()
 
             if plot_residuals:
-                phase_residuals = lib.phase_mod(phase_diffs[:,1] - my_phase_diffs)
-
                 fig.suptitle("Difference between Measured and Actual phase difference\n for Baselines (i,j" + (')' if not ref_ant_id else '='+str([ int(a.name) for a in ref_ants])+')'))
                 axs[-1].set_xlabel("Baseline Phase Residual $\\Delta\\varphi_{ij_{meas}} - \\Delta\\varphi_{ij_{true}}$ [rad]")
             else:
                 fig.suptitle("Comparison Measured and Actual phase difference\n for Baselines (i,j"  + (')' if not ref_ant_id else '='+str([ int(a.name) for a in ref_ants])+')'))
                 axs[-1].set_xlabel("Baseline Phase $\\Delta\\varphi_{ij}$ [rad]")
 
-
+            #
             i=0
-            axs[i].set_ylabel("#")
+            secax = axs[i].child_axes[0]
-            if plot_residuals:
+            secax.set_xlabel('Time $\\Delta\\varphi/(2\\pi f_{beac})$ [ns]')
-                axs[i].hist(phase_residuals, bins='sqrt', density=False, alpha=0.8, color=colors[0])
-            else:
-                axs[i].hist(phase_diffs[:,1], bins='sqrt', density=False, alpha=0.8, color=colors[0], ls='solid' , histtype='step', label='Measured')
-                axs[i].hist(my_phase_diffs,   bins='sqrt', density=False, alpha=0.8, color=colors[1], ls='dashed', histtype='step', label='Actual')
-
 
+            #
             i=1
             axs[i].set_ylabel("Baseline no.")
-            if plot_residuals:
-                axs[i].plot(phase_residuals, np.arange(N_base), alpha=0.6, ls='none', marker='x', color=colors[0])
-            else:
-                axs[i].plot(phase_diffs[:,1], np.arange(N_base), alpha=0.8, color=colors[0], ls='none', marker='x', label='calculated')
-                axs[i].plot(my_phase_diffs,   np.arange(N_base), alpha=0.8, color=colors[1], ls='none', marker='+', label='actual time shifts')
-
-                axs[i].legend()
-            fig.tight_layout()
 
             if fig_dir:
                 extra_name = "measured"

airshower_beacon_simulation/bd_antenna_phase_deltas.py

@@ -7,9 +7,11 @@ import matplotlib.pyplot as plt
 from matplotlib.colors import Normalize
 import matplotlib as mpl
 import numpy as np
+import json
 
 import aa_generate_beacon as beacon
 import lib
+from lib import figlib
 
 
 if __name__ == "__main__":
@@ -118,6 +120,11 @@ if __name__ == "__main__":
     mean_clock_phase = np.nanmean(clock_phase_matrix[my_mask], axis=0)
     std_clock_phase  = np.nanstd( clock_phase_matrix[my_mask], axis=0)
 
+    # Remove the mean from the matrix
+    if False:
+        clock_phase_matrix = clock_phase_matrix - np.mean(mean_clock_phase)
+        mean_clock_phase = np.nanmean(clock_phase_matrix[my_mask], axis=0)
+        std_clock_phase  = np.nanstd( clock_phase_matrix[my_mask], axis=0)
 
     # Show resulting matrix as figure
     if True:
@@ -174,45 +181,40 @@ if __name__ == "__main__":
         global_phase_shift = actual_antenna_phase_shifts[0] - mean_clock_phase[0]
         actual_antenna_phase_shifts = lib.phase_mod(actual_antenna_phase_shifts - global_phase_shift )
 
+        fit_info = {}
         for i in range(2):
             plot_residuals = i == 1
-            colors = ['blue', 'orange']
+            true_phases = actual_antenna_phase_shifts
+            measured_phases = mean_clock_phase
 
-            fig, axs = plt.subplots(2, 1, sharex=True, figsize=figsize)
+            hist_kwargs = {}
+            if plot_residuals:
+                measured_phases = lib.phase_mod(measured_phases - actual_antenna_phase_shifts)
 
-            if True:
+            fig, _fit_info = figlib.phase_comparison_figure(
-                phase2time = lambda x: x/(2*np.pi*f_beacon)
+                    measured_phases,
-                time2phase = lambda x: 2*np.pi*x*f_beacon
+                    true_phases,
-                secax = axs[0].secondary_xaxis('top', functions=(phase2time, time2phase))
+                    plot_residuals=plot_residuals,
-                secax.set_xlabel('Time $\\Delta\\varphi/(2\\pi f_{beac})$ [ns]')
+                    f_beacon=f_beacon,
+                    figsize=figsize,
+                    hist_kwargs=hist_kwargs,
+                    fit_gaussian=plot_residuals,
+                    return_fit_info = True,
+                    )
+
+            axs = fig.get_axes()
 
             if plot_residuals:
-                phase_residuals = lib.phase_mod(mean_clock_phase - actual_antenna_phase_shifts)
                 fig.suptitle("Difference between Measured and Actual phases (minus global phase)\n for Antenna $i$")
-                axs[-1].set_xlabel("Antenna Phase Residual $\\Delta_\\varphi$")
+                axs[-1].set_xlabel("Antenna Mean Phase Residual $\\Delta_\\varphi$")
             else:
                 fig.suptitle("Comparison Measured and Actual phases (minus global phase)\n for Antenna $i$")
-                axs[-1].set_xlabel("Antenna Phase $\\varphi$")
+                axs[-1].set_xlabel("Antenna Mean Phase $\\varphi$")
-
-
-            i=0
-            axs[i].set_ylabel("#")
-            if plot_residuals:
-                axs[i].hist(phase_residuals, bins='sqrt', alpha=0.8, color=colors[0])
-            else:
-                axs[i].hist(mean_clock_phase,    bins='sqrt', density=False, alpha=0.8, color=colors[0], ls='solid' , histtype='step', label='Measured')
-                axs[i].hist(actual_antenna_phase_shifts, bins='sqrt', density=False, alpha=0.8, color=colors[1], ls='dashed', histtype='step', label='Actual')
-
 
             i=1
             axs[i].set_ylabel("Antenna no.")
-            if plot_residuals:
+            #axs[i].errorbar(mean_clock_phase, np.arange(N_ant), yerr=std_clock_phase, marker='4', alpha=0.7, ls='none', color=colors[0], label='Measured')
-                axs[i].plot(phase_residuals, np.arange(N_ant), alpha=0.6, ls='none', marker='x', color=colors[0])
-            else:
-                axs[i].errorbar(mean_clock_phase, np.arange(N_ant), yerr=std_clock_phase, marker='4', alpha=0.7, ls='none', color=colors[0], label='Measured')
-                axs[i].plot(actual_antenna_phase_shifts,  antenna_names, ls='none', marker='3', alpha=0.8, color=colors[1], label='Actual')
 
-                axs[i].legend()
             fig.tight_layout()
 
             if fig_dir:
@@ -221,6 +223,19 @@ if __name__ == "__main__":
                     extra_name = "residuals"
                 fig.savefig(path.join(fig_dir, path.basename(__file__) + f".phase.{extra_name}.pdf"))
 
+            # Save fit_info to data file
+            if fname_dir and plot_residuals:
+                with open(path.join(fname_dir, 'phase_time_residuals.json'), 'w') as fp:
+                    json.dump(
+                        {
+                            'mean': np.mean(measured_phases),
+                            'std': np.std(measured_phases),
+                            'values': measured_phases.tolist(),
+                        },
+                        fp)
+
+
+
     ##########################
     ##########################
     ##########################

airshower_beacon_simulation/cb_report_measured_antenna_time_offsets.py

@@ -10,6 +10,7 @@ import numpy as np
 from os import path
 
 import aa_generate_beacon as beacon
+from lib import figlib
 
 if __name__ == "__main__":
     import sys
@@ -76,15 +77,28 @@ if __name__ == "__main__":
 
         for i in range(2):
             plot_residuals = i == 1
-            colors = ['blue', 'orange']
 
-            fig, axs = plt.subplots(2, 1, sharex=True, figsize=figsize)
 
-            if True:
+            true_phases = actual_time_shifts
-                phase2time = lambda x: x/(2*np.pi*f_beacon)
+            measured_phases = measured_time_shifts
-                time2phase = lambda x: 2*np.pi*x*f_beacon
+
-                secax = axs[0].secondary_xaxis('top', functions=(time2phase, phase2time))
+            hist_kwargs = {}
-                secax.set_xlabel('Phase $2\\pi t f_{beac}$ [rad]')
+            if plot_residuals:
+                measured_phases = measured_phases - true_phases
+                hist_kwargs['histtype'] = 'stepfilled'
+
+            fig = figlib.phase_comparison_figure(
+                    measured_phases,
+                    true_phases,
+                    plot_residuals=plot_residuals,
+                    f_beacon=f_beacon,
+                    figsize=figsize,
+                    hist_kwargs=hist_kwargs,
+                    secondary_axis='phase',
+                    fit_gaussian=True,
+                    )
+
+            axs = fig.get_axes()
 
             if plot_residuals:
                 time_shift_residuals = measured_time_shifts - actual_time_shifts
@@ -94,26 +108,6 @@ if __name__ == "__main__":
                 fig.suptitle("Comparison Measured and Actual clock offset")
                 axs[-1].set_xlabel("Antenna Time Offset $t_c = \\left(\\frac{\\Delta\\varphi}{2\\pi} + k\\right) / f_{beac}$ [ns]")
 
-            i=0
-            axs[i].set_ylabel("#")
-            if plot_residuals:
-                axs[i].hist(time_shift_residuals, bins='sqrt', alpha=0.8, color=colors[0])
-            else:
-                axs[i].hist(measured_time_shifts, bins='sqrt', density=False, alpha=0.8, color=colors[0], ls='solid' , histtype='step', label='Measured')
-                axs[i].hist(actual_time_shifts,   bins='sqrt', density=False, alpha=0.8, color=colors[1], ls='dashed', histtype='step', label='Actual')
-
-
-            i=1
-            axs[i].set_ylabel("Antenna no.")
-            if plot_residuals:
-                axs[i].plot(time_shift_residuals, np.arange(N_ant), alpha=0.6, ls='none', marker='x', color=colors[0])
-            else:
-                axs[i].errorbar(measured_time_shifts, np.arange(N_ant), yerr=None, marker='4', alpha=0.7, ls='none', color=colors[0], label='Measured')
-                axs[i].plot(actual_time_shifts,  antenna_names, ls='none', marker='3', alpha=0.8, color=colors[1], label='Actual')
-
-                axs[i].legend()
-            fig.tight_layout()
-
             if fig_dir:
                 extra_name = "comparison"
                 if plot_residuals:

airshower_beacon_simulation/da_reconstruction.py

@@ -46,6 +46,7 @@ if __name__ == "__main__":
     fig_subdir = path.join(fig_dir, 'reconstruction')
     show_plots = args.show_plots
 
+    apply_signal_window_from_max = True
     remove_beacon_from_traces = True
 
     ####

airshower_beacon_simulation/dc_grid_power_time_fixes.py

@@ -19,6 +19,65 @@ import lib
 from lib import rit
 
 
+def save_overlapping_traces_figure(test_location, ev, N_plot = 30, wx=200, title_extra=None, fname_distinguish='', fig_dir=None, **fig_kwargs):
+    P, t_, a_, a_sum, t_sum = rit.pow_and_time(test_location, ev, dt=1)
+
+    fig, axs = plt.subplots(**fig_kwargs)
+    axs.set_title("Antenna traces" + (("\n" + title_extra) if title_extra is not None else '') )
+    axs.set_xlabel("Time [ns]")
+    axs.set_ylabel("Amplitude [$\\mu V/m$]")
+    if False:
+        text_loc = (0.02, 0.95)
+        axs.text(*text_loc, '[' + ', '.join(['{:.2e}'.format(x) for x in test_location]) + ']', ha='left', transform=axs.transAxes)
+
+    a_max = [ np.amax(ant.E_AxB) for ant in ev.antennas ]
+    power_sort_idx = np.argsort(a_max)
+
+    for i, idx in enumerate(reversed(power_sort_idx)):
+        if i >= N_plot:
+            break
+
+        alpha = max(0.4, 1/N_plot)
+
+        axs.plot(t_[idx], a_[idx], color='r', alpha=alpha, lw=2)
+
+    if fig_dir:
+        if fname_distinguish:
+            fname_distinguish = "." + fname_distinguish
+
+        fig.tight_layout()
+
+        fig.savefig(path.join(fig_dir, path.basename(__file__) + f'{fname_distinguish}.trace_overlap.{case}.pdf'))
+        fig.savefig(path.join(fig_dir, path.basename(__file__) + f'{fname_distinguish}.trace_overlap.{case}.png'), transparent=True)
+
+        # Take center between t_low and t_high
+        if True:
+            orig_xlims = axs.get_xlim()
+
+            if not True: # t_high and t_low from strongest signal
+                t_low = np.min(t_[power_sort_idx[-1]])
+                t_high = np.max(t_[power_sort_idx[-1]])
+            else: # take t_high and t_low from plotted signals
+                a = [np.min(t_[idx]) for idx in power_sort_idx[-N_plot:]]
+                t_low  = np.nanmin(a)
+                b = [np.max(t_[idx]) for idx in power_sort_idx[-N_plot:]]
+                t_high = np.nanmax(b)
+
+                if False:
+                    axs.plot(a, [0]*N_plot, 'gx', ms=10)
+                    axs.plot(b, [0]*N_plot, 'b+', ms=10)
+
+            center_x = (t_high - t_low)/2 + t_low
+
+            low_xlim  = max(orig_xlims[0], center_x - wx)
+            high_xlim = min(orig_xlims[1], center_x + wx)
+
+            axs.set_xlim(low_xlim, high_xlim)
+            fig.savefig(path.join(fig_dir, path.basename(__file__) + f'{fname_distinguish}.trace_overlap.zoomed.{case}.pdf'))
+            fig.savefig(path.join(fig_dir, path.basename(__file__) + f'{fname_distinguish}.trace_overlap.zoomed.{case}.png'), transparent=True)
+
+    return fig
+
 if __name__ == "__main__":
     valid_cases = ['no_offset', 'repair_none', 'repair_phases', 'repair_all']
 
@@ -95,6 +154,9 @@ if __name__ == "__main__":
             'scale02d': scale02d,
             }
 
+    N_plot = 30
+    trace_zoom_wx = 100
+
     plot_titling = {
             'no_offset': "no clock offset",
             'repair_none': "unrepaired clock offset",
@@ -175,6 +237,9 @@ if __name__ == "__main__":
     backup_antenna_t = [ ant.t for ant in ev.antennas ]
     backup_antenna_t_AxB = [ ant.t_AxB for ant in ev.antennas ]
 
+    fig = save_overlapping_traces_figure([0,0,0], ev, N_plot=1, wx=trace_zoom_wx, title_extra = plot_titling[case], fname_distinguish=f'single', fig_dir=fig_dir, figsize=figsize)
+    plt.close(fig)
+
     with joblib.parallel_backend("loky"):
         for case in wanted_cases:
             print(f"Starting {case} figure")
@@ -195,73 +260,29 @@ if __name__ == "__main__":
 
                 if i == 0:
                     # Specifically compare the times
-                    print(bak_ants[i].t[0], ev.antennas[i].t[0], ev.antennas[i].t[0], ev.antennas[i].attrs['clock_offset'], measured_offsets[i])
+                    print("backup time, time with measured_offset, true clock offset, measured clock offset")
+                    print(bak_ants[i].t[0], ev.antennas[i].t[0], ev.antennas[i].attrs['clock_offset'], measured_offsets[i])
 
             #
             # Plot overlapping traces at 0,0,0
             #
-            if True:
+            fig = save_overlapping_traces_figure([0,0,0], ev, N_plot=N_plot, wx=trace_zoom_wx, title_extra = plot_titling[case], fname_distinguish=f'{case}.0', fig_dir=fig_dir, figsize=figsize)
-                P, t_, a_, a_sum, t_sum = rit.pow_and_time([0,0,0], ev, dt=1)
+            plt.close(fig)
-
-                fig, axs = plt.subplots(figsize=figsize)
-                axs.set_title("Antenna traces" + "\n" + plot_titling[case])
-                axs.set_xlabel("Time [ns]")
-                axs.set_ylabel("Amplitude [$\\mu V/m$]")
-
-                a_max = [ np.amax(ant.E_AxB) for ant in ev.antennas ]
-                power_sort_idx = np.argsort(a_max)
-
-                N_plot = 30
-                for i, idx in enumerate(reversed(power_sort_idx)):
-                    if i > N_plot:
-                        break
-
-                    alpha = max(0.4, 1/len(a_))
-
-                    axs.plot(t_[idx], a_[idx], color='r', alpha=alpha, lw=2)
-
-                if fig_dir:
-                    fig.tight_layout()
-
-                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f'.trace_overlap.{case}.pdf'))
-                    fig.savefig(path.join(fig_dir, path.basename(__file__) + f'.trace_overlap.{case}.png'), transparent=True)
-
-                    # Take center between t_low and t_high
-                    if True:
-                        orig_xlims = axs.get_xlim()
-
-                        if not True: # t_high and t_low from strongest signal
-                            t_low = np.min(t_[power_sort_idx[-1]])
-                            t_high = np.max(t_[power_sort_idx[-1]])
-                        else: # take t_high and t_low from plotted signals
-                            a = [np.min(t_[idx]) for idx in power_sort_idx[-N_plot:]]
-                            axs.plot(a, [0]*N_plot, 'gx', ms=10)
-                            t_low  = np.nanmin(a)
-                            b = [np.max(t_[idx]) for idx in power_sort_idx[-N_plot:]]
-                            axs.plot(b, [0]*N_plot, 'b+', ms=10)
-                            t_high = np.nanmax(b)
-
-                        center_x = (t_high - t_low)/2 + t_low
-                        wx = 200
-
-                        low_xlim  = max(orig_xlims[0], center_x - wx)
-                        high_xlim = min(orig_xlims[1], center_x + wx)
-
-                        axs.set_xlim(low_xlim, high_xlim)
-                        fig.savefig(path.join(fig_dir, path.basename(__file__) + f'.trace_overlap.zoomed.{case}.pdf'))
-                        fig.savefig(path.join(fig_dir, path.basename(__file__) + f'.trace_overlap.zoomed.{case}.png'), transparent=True)
-
-
-                if True:
-                    continue
 
             # Measure power on grid
+            # and plot overlapping traces at position with highest power
             for scalename, scale in scales.items():
                 wx, wy = scale, scale
                 print(f"Starting grid measurement for figure {case} with {scalename}")
-                xx, yy, p, maxp = rit.shower_plane_slice(ev, X=X, Nx=Nx, Ny=Nx, wx=wx, wy=wy)
+                xx, yy, p, maxp_loc = rit.shower_plane_slice(ev, X=X, Nx=Nx, Ny=Nx, wx=wx, wy=wy, zgr=zgr)
 
-                fig, axs = rit.slice_figure(ev, X, xx, yy, p, mode='sp')
+                fig, axs = rit.slice_figure(ev, X, xx, yy, p, mode='sp', scatter_kwargs=dict(
+                    vmax=1e5,
+                    vmin=0,
+                    s=150,
+                    cmap='inferno',
+#                    edgecolor='black',
+                    ))
 
                 suptitle = fig._suptitle.get_text()
                 fig.suptitle("")
@@ -271,7 +292,24 @@ if __name__ == "__main__":
                 if fig_dir:
                     fig.tight_layout()
                     fig.savefig(path.join(fig_dir, path.basename(__file__) + f'.X{X}.{case}.{scalename}.pdf'))
+                plt.close(fig)
+
+                #
+                # Plot overlapping traces at highest power of each scale
+                #
+                fig = save_overlapping_traces_figure(maxp_loc, ev, N_plot=N_plot, wx=trace_zoom_wx, title_extra = plot_titling[case] + ', ' + scalename + ' best', fname_distinguish=scalename+'.best', fig_dir=fig_dir, figsize=figsize)
+
+                #
+                # and plot overlapping traces at two other locations
+                #
+                if True:
+                    for dist in [ 0.5, 5, 10, 50, 100]:
+                        # only add distance horizontally
+                        location = maxp_loc + np.sqrt(dist*1e3)*np.array([1,1,0])
+
+                        fig = save_overlapping_traces_figure(location, ev, N_plot=N_plot, wx=wx, title_extra = plot_titling[case] + ', ' + scalename + f', x + {dist}km', fname_distinguish=f'{scalename}.{dist}', fig_dir=fig_dir, figsize=figsize)
+                        plt.close(fig)
+
 
     if args.show_plots:
         plt.show()
-

airshower_beacon_simulation/lib/earsim

@@ -0,0 +1 @@
+Subproject commit 6ef809020477cf78415b9fa733d4fbb4a74102a1

airshower_beacon_simulation/lib/figlib.py

@@ -0,0 +1,219 @@
+import matplotlib.pyplot as plt
+import numpy as np
+
+import scipy.stats as stats
+from itertools import zip_longest
+
+def phase_comparison_figure(
+        measured_phases,
+        true_phases,
+        plot_residuals=True,
+        f_beacon=None,
+        hist_kwargs={},
+        sc_kwargs={},
+        text_kwargs={},
+        colors=['blue', 'orange'],
+        legend_on_scatter=True,
+        secondary_axis='time',
+        fit_gaussian=False,
+        mean_snr=None,
+        return_fit_info=False,
+        **fig_kwargs
+        ):
+    """
+    Create a figure comparing measured_phase against true_phase
+    by both plotting the values, and the residuals.
+    """
+    default_fig_kwargs = dict(sharex=True)
+    default_hist_kwargs = dict(bins='sqrt', density=False, alpha=0.8, histtype='step')
+    default_text_kwargs = dict(fontsize=14, verticalalignment='top')
+    default_sc_kwargs = dict(alpha=0.6, ls='none')
+
+    fig_kwargs = {**default_fig_kwargs, **fig_kwargs}
+    hist_kwargs = {**default_hist_kwargs, **hist_kwargs}
+    text_kwargs = {**default_text_kwargs, **text_kwargs}
+    sc_kwargs = {**default_sc_kwargs, **sc_kwargs}
+
+    do_hist_plot = hist_kwargs is not False
+    do_scatter_plot = sc_kwargs is not False
+
+    fig, axs = plt.subplots(0+do_hist_plot+do_scatter_plot, 1, **fig_kwargs)
+
+    if not hasattr(axs, '__len__'):
+        axs = [axs]
+
+    if f_beacon and secondary_axis in ['phase', 'time']:
+        phase2time = lambda x: x/(2*np.pi*f_beacon)
+        time2phase = lambda x: 2*np.pi*x*f_beacon
+
+        if secondary_axis == 'time':
+            functions = (phase2time, time2phase)
+            label = 'Time $\\varphi/(2\\pi f_{beac})$ [ns]'
+        else:
+            functions = (time2phase, phase2time)
+            label = 'Phase $2\\pi t f_{beac}$ [rad]'
+
+        secax = axs[0].secondary_xaxis('top', functions=functions)
+
+    # Histogram
+    fit_info = {}
+    if do_hist_plot:
+        i=0
+
+        axs[i].set_ylabel("#")
+
+        this_kwargs = dict(
+                ax = axs[i],
+                text_kwargs=text_kwargs,
+                hist_kwargs={**hist_kwargs, **dict(label='Measured', color=colors[0], ls='solid')},
+                mean_snr=mean_snr,
+                )
+
+        if fit_gaussian:
+            this_kwargs['fit_distr'] = 'gaussian'
+
+        _, fit_info = fitted_histogram_figure(
+                measured_phases,
+                **this_kwargs
+                )
+
+        if not plot_residuals: # also plot the true clock phases
+            _bins = fit_info['bins']
+            axs[i].hist(true_phases, color=colors[1], label='Actual', ls='dashed', **{**hist_kwargs, **dict(bins=_bins)})
+
+    # Scatter plot
+    if do_scatter_plot:
+        i=1
+        axs[i].set_ylabel("Antenna no.")
+        axs[i].plot(measured_phases, np.arange(len(measured_phases)), marker='x' if plot_residuals else '3', color=colors[0], label='Measured', **sc_kwargs)
+
+        if not plot_residuals: # also plot the true clock phases
+            axs[i].plot(true_phases, np.arange(len(true_phases)), marker='4', color=colors[1], label='Actual', **sc_kwargs)
+
+        if not plot_residuals and legend_on_scatter:
+            axs[i].legend()
+
+    fig.tight_layout()
+
+    if return_fit_info:
+        return fig, fit_info
+
+    return fig
+
+
+def fitted_histogram_figure(
+        amplitudes,
+        fit_distr = None,
+        calc_chisq = True,
+        text_kwargs={},
+        hist_kwargs={},
+        mean_snr = None,
+        ax = None,
+        **fig_kwargs
+    ):
+    """
+    Create a figure showing $amplitudes$ as a histogram.
+    If fit_distr is a (list of) string, also fit the respective
+    distribution function and show the parameters on the figure.
+    """
+    default_hist_kwargs = dict(bins='sqrt', density=False, alpha=0.8, histtype='step', label='hist')
+    default_text_kwargs = dict(fontsize=14, verticalalignment='top')
+
+    if isinstance(fit_distr, str):
+        fit_distr = [fit_distr]
+
+    hist_kwargs = {**default_hist_kwargs, **hist_kwargs}
+    text_kwargs = {**default_text_kwargs, **text_kwargs}
+
+    if ax is None:
+        fig, ax = plt.subplots(1,1, **fig_kwargs)
+    else:
+        fig = ax.get_figure()
+
+    text_kwargs['transform'] = ax.transAxes
+
+    counts, bins, _patches = ax.hist(amplitudes, **hist_kwargs)
+
+    fit_info = []
+    if fit_distr:
+        min_x = min(amplitudes)
+        max_x = max(amplitudes)
+
+        dx = bins[1] - bins[0]
+        scale = len(amplitudes) * dx
+
+        xs = np.linspace(min_x, max_x)
+
+        for distr in fit_distr:
+            fit_params2text_params = lambda x: x
+
+            if 'rice' == distr:
+                name = "Rice"
+                param_names = [ "$\\nu$", "$\\sigma$" ]
+                distr_func = stats.rice
+
+                fit_params2text_params = lambda x: (x[0]*x[1], x[1])
+
+            elif 'gaussian' == distr:
+                name = "Norm"
+                param_names = [ "$\\mu$", "$\\sigma$" ]
+                distr_func = stats.norm
+
+            elif 'rayleigh' == distr:
+                name = "Rayleigh"
+                param_names = [ "$\\sigma$" ]
+                distr_func = stats.rayleigh
+
+                fit_params2text_params = lambda x: (x[0]+x[1]/2,)
+
+            else:
+                raise ValueError('Unknown distribution function '+distr)
+
+            label = name +"(" + ','.join(param_names) + ')'
+
+            fit_params = distr_func.fit(amplitudes)
+            fit_ys = distr_func.pdf(xs, *fit_params)
+
+            ax.plot(xs, fit_ys*scale, label=label)
+
+            chisq_strs = []
+            if calc_chisq:
+                ct = np.diff(distr_func.cdf(bins, *fit_params))*np.sum(counts)
+                c2t = stats.chisquare(counts, ct, ddof=len(fit_params))
+                chisq_strs = [
+                        f"$\\chi^2$/dof = {c2t[0]: .2g}/{len(fit_params)}"
+                        ]
+
+            # change parameters if needed
+            text_fit_params = fit_params2text_params(fit_params)
+
+            text_str = "\n".join(
+                [label]
+                +
+                [ f"{param} = {value: .2e}" for param, value in zip_longest(param_names, text_fit_params, fillvalue='?') ]
+                +
+                chisq_strs
+                )
+
+            this_info = {
+                    'name': name,
+                    'param_names': param_names,
+                    'param_values': text_fit_params,
+                    'text_str': text_str,
+                }
+
+            if chisq_strs:
+                this_info['chisq'] = c2t[0]
+                this_info['dof'] = len(fit_params)
+
+            fit_info.append(this_info)
+
+        loc = (0.02, 0.95)
+        ax.text(*loc, "\n\n".join([info['text_str'] for info in fit_info]), **{**text_kwargs, **dict(ha='left')})
+
+    if mean_snr:
+        text_str = f"$\\langle SNR \\rangle$ = {mean_snr: .1e}"
+        loc = (0.98, 0.95)
+        ax.text(*loc, text_str, **{**text_kwargs, **dict(ha='right')})
+
+    return fig, dict(fit_info=fit_info, counts=counts, bins=bins)

airshower_beacon_simulation/lib/rit.py

@@ -58,6 +58,8 @@ def pow_and_time(test_loc,ev,dt=1.0):
         a_sum = np.add(a_sum,a_int)
     if len(a_sum) != 0:
         P = np.sum(np.square(np.absolute(np.fft.fft(a_sum))))
+        # normalise P with the length of the traces
+        P = P/( t_sum[-1] - t_sum[0])
     else:
         print("ERROR, a_sum lenght = 0",
               "tmin ",t_min,
@@ -112,11 +114,21 @@ def shower_plane_slice(e,X=750.,Nx=10,Ny=10,wx=1e3,wy=1e3,xoff=0,yoff=0,zgr=0,n_
 
     return xx,yy,p,locs[np.argmax(p)]
 
-def slice_figure(e,X,xx,yy,p,mode='horizontal'):
+def slice_figure(e,X,xx,yy,p,mode='horizontal', scatter_kwargs={}, colorbar_kwargs={'label':'Power'}):
+    scatter_kwargs = {
+            **dict(
+                cmap='Spectral_r',
+                alpha=0.9,
+                s=30
+                ),
+            **scatter_kwargs
+            }
+
+
     fig, axs = plt.subplots(1,figsize=(10,8))
     fig.suptitle(r'E = %.1f EeV, $\theta$ = %.1f$^\circ$, $\phi$ = %.1f$^\circ$ X = %.f'%(e.energy,e.zenith,e.azimuth,X))
-    sc = axs.scatter(xx/1e3,yy/1e3,c=p,cmap='Spectral_r',alpha=0.6)
+    sc = axs.scatter(xx/1e3,yy/1e3,c=p,**scatter_kwargs)
-    fig.colorbar(sc,ax=axs)
+    fig.colorbar(sc,ax=axs, **colorbar_kwargs)
     zgr = 0 + e.core[2]
     dX = atm.distance_to_slant_depth(np.deg2rad(e.zenith),X,zgr)
     xc = np.sin(np.deg2rad(e.zenith))*np.cos(np.deg2rad(e.azimuth))* dX

airshower_beacon_simulation/matrix_base/bin/collect_figures.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+
+import os
+import re
+import tempfile
+
+def parse_env_file(fname):
+    envs = {}
+    env_reg = re.compile('^(?:\s*export)?\s*(.*)\s*=\s*(.*)\s*$')
+
+    with open(fname, 'r') as f:
+        for line in f:
+            if '=' not in line:
+                continue
+
+            m = env_reg.match(line)
+            envs[m.group(1)] = m.group(2)
+
+    return envs
+
+def mutilate_figure_name(fig_fname, envs):
+    return fig_fname
+
+if __name__ == "__main__":
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser()
+    #parser.add_argument('-f', '--figures', nargs='*')
+    parser.add_argument("-d", "--directories", nargs='*')
+    parser.add_argument('out_dir', default='./figures', type=str)
+
+    args = parser.parse_args()
+
+    os.makedirs(args.out_dir, exist_ok=True)
+
+    #with open(args.out_file, 'w') as fp:
+    if True:
+        for d in args.directories:
+            d = os.path.realpath(d)
+            fig_dir = os.path.join(d, 'figures')
+            env_fname = os.path.join(d, 'env.sh')
+
+            if not os.path.exists(fig_dir):
+                print(f"Cannot find {fig_dir}")
+                continue
+
+            ## parse properties from env.sh
+            #envs = parse_env_file(env_fname)
+            #print(envs, fig_dir)
+
+            for f in os.listdir(fig_dir):
+                fname, ext = os.path.splitext(f)
+
+                dname = os.path.basename(d)
+
+                if ext not in ['.pdf', '.png']:
+                    continue
+
+                link_name = fname + "_" + dname + ext
+
+                target = os.path.realpath(os.path.join(fig_dir, f))
+                tmpLink = tempfile.mktemp(dir=args.out_dir)
+
+                os.symlink(target, tmpLink)
+                os.rename(tmpLink, os.path.join(args.out_dir, link_name))
+
+

airshower_beacon_simulation/matrix_base/bin/make_matrix.py

@@ -17,10 +17,17 @@ for options in product(baselines, clock_devs, noise_sigmas, trace_lengths):
 
     # Make directory
     if path.exists(dirname):
-        print(f"{dirname} already exists! continuing..")
+        print(f"{dirname} already exists! continuing anyway..")
 
     os.makedirs(dirname, exist_ok=True)
 
+    # Soft link clock file if available
+    if True:
+        os.makedirs(path.join(dirname, 'data'), exist_ok=True)
+
+        if not path.isfile(path.join(dirname, 'data/clocks.csv')):
+            os.symlink(f'../../c{clk_dev}_clocks.csv', path.join(dirname, 'data/clocks.csv'))
+
     # Setup config.mk
     with open(path.join(dirname, 'env.sh'), 'w') as fp:
         template = f"""

airshower_beacon_simulation/matrix_base/bin/time_res_vs_snr.py

@@ -0,0 +1,217 @@
+#!/usr/bin/env python3
+
+
+import numpy as np
+import matplotlib.pyplot as plt
+import json
+
+from scipy import special
+
+
+# Mimic import aa_generate_beacon as beacon
+beacon = lambda: None
+
+def read_snr_file(fname):
+    with open(fname, 'r') as fp:
+        return json.load(fp)
+
+def read_tx_file(fname):
+    with open(fname, 'r') as fp:
+        data = json.load(fp)
+
+    f_beacon = data['f_beacon']
+    tx = data['tx']
+
+    del data['f_beacon']
+    del data['tx']
+
+    return tx, f_beacon, data
+
+
+beacon.snr_fname = 'snr.json'
+beacon.tx_fname = 'tx.json'
+beacon.read_snr_file = read_snr_file
+beacon.read_tx_file = read_tx_file
+
+def read_snr_directories(data_directories, tx_fname=beacon.tx_fname, snr_fname=beacon.snr_fname, phase_res_fname='phase_time_residuals.json'):
+    # Read in snr
+    snrs = np.zeros(len(data_directories), dtype=float)
+    time_residuals = np.zeros( (len(snrs), 2), dtype=float)
+
+
+    tx, f_beacon, _ = beacon.read_tx_file(path.join(data_directories[0], tx_fname))
+    for i, data_dir in enumerate(data_directories):
+        # Read SNR file
+        snr_data = read_snr_file(path.join(data_dir, snr_fname))
+
+        # Open antennas file
+        with open(path.join(data_dir, phase_res_fname), 'r') as fp:
+            time_res_data = json.load(fp)
+
+        snrs[i] = snr_data['mean']
+
+        time_residuals[i] = time_res_data['mean'], time_res_data['std']
+
+    return f_beacon, snrs, time_residuals
+
+
+## Math
+def expectation(x, pdfx):
+    dx = x[1] - x[0]
+    return np.sum(x*pdfx*dx)
+
+def variance(x, pdfx):
+    mu = expectation(x, pdfx)
+    dx = x[1] - x[0]
+    return np.sum(x**2 *pdfx*dx) - mu**2
+
+def phase_distribution(theta, sigma, s):
+    k = s/sigma
+    ct = np.cos(theta)
+    st = np.sin(theta)
+    pipi = 2*np.pi
+
+    return (np.exp(-k**2/2)/pipi) \
+            + (
+                (pipi**-0.5)*k*np.exp(-(k*st)**2/2)
+                * ((1.+special.erf(k*ct*2**-0.5))*ct/2)
+            )
+
+def phase_distribution_gauss(theta, sigma, s):
+    k = s/sigma
+    return 1/np.sqrt(2*np.pi) * k *np.exp( -(k*theta)**2/2)
+
+
+
+if __name__ == "__main__":
+    from os import path
+    import sys
+
+    import os
+    import matplotlib
+    if os.name == 'posix' and "DISPLAY" not in os.environ:
+        matplotlib.use('Agg')
+
+    from argparse import ArgumentParser
+    parser = ArgumentParser()
+
+    figsize = (12,8)
+
+    # Multiple Data Directories
+    parser.add_argument("-d", dest='data_directories', default=[], nargs='*')
+
+    # Whether to show plots
+    group1 = parser.add_mutually_exclusive_group(required=False)
+    group1.add_argument('--show-plots', action='store_true', default=True, help='Default: %(default)s')
+    group1.add_argument('--no-show-plots', dest='show-plots', action='store_false')
+ 
+    # Figures directory
+    parser.add_argument('--fig-dir', type=str, default='./figures', help='Set None to disable saving figures. (Default: %(default)s)')
+
+    args = parser.parse_args()
+    
+    show_plots = args.show_plots
+
+    if not args.data_directories:
+        parser.error("At least one data directory should be specified.")
+
+    f_beacon, snrs, time_residuals = read_snr_directories(args.data_directories)
+
+    phase2time = lambda x: x/(2*np.pi*f_beacon)
+    time2phase = lambda x: 2*np.pi*x*f_beacon
+
+    fig, ax = plt.subplots(figsize=(8,6))
+    ax.set_title("Beacon ({:.2f}MHz) Simulation".format(f_beacon*1e3))
+    ax.set_xlabel('Signal to Noise ratio')
+    ax.set_ylabel('Time Accuracy $\\sigma(t)$ [ns]')
+
+    # group per directories per N
+    if True:
+        import re
+
+        dirdict = {}
+        N_re = re.compile(r'_N(\d+)_')
+
+        for d in args.data_directories:
+            m = N_re.findall(d)[0]
+
+            if m not in dirdict:
+                dirdict[m] = []
+
+            dirdict[m].append(d)
+
+    
+        dirlist = dirdict.values()
+
+    # plot data directories as one group
+    else:
+        dirlist = [args.data_directories]
+ 
+    for dirlist in dirlist:
+        f_beacon, snrs, time_residuals = read_snr_directories(dirlist)
+
+        # plot data
+        ax.plot(snrs*np.sqrt(np.pi/2), phase2time(time_residuals[:,1]), ls='none', marker='o')
+
+    # Add secondary axis
+    if True:
+        if False and secondary_axis == 'time':
+            functions = (phase2time, time2phase)
+            label = 'Time Accuracy $\\sigma_t\\varphi/(2\\pi f_{beac})$ [ns]'
+        else:
+            functions = (time2phase, phase2time)
+            label = 'Phase Accuracy $\\sigma_\\varphi$ [rad]'
+
+        secax = ax.secondary_yaxis('right', functions=functions)
+        secax.set_ylabel(label)
+ 
+    # logscales
+    if True:
+        ax.set_xscale('log')
+        ax.set_yscale('log')
+
+    # plot phasor snr
+    if True:
+        thetas = np.linspace(-np.pi, np.pi, 500)
+        sigma = 1
+        _snr_min = min(10, min(snrs))
+        _snr_max = min(100, max(snrs))
+
+        if ax.get_xscale() == 'log': #log
+            _snrs = np.logspace(np.log10(_snr_min), np.log10(_snr_max))
+        else: #linear
+            _snrs = np.linspace(_snr_min, _snr_max)
+
+        # Phasor Rice
+        phasor_pdfs = np.array([phase_distribution(thetas, sigma, s) for s in _snrs ])
+        phasor_sigma = np.sqrt(np.array([variance(thetas, pdf) for pdf in phasor_pdfs]))
+
+        ax.plot(_snrs, phase2time(phasor_sigma), label='Rice')
+
+        if True: # plot a pdf
+            fig2, ax2 = plt.subplots()
+            for idx in [0, len(_snrs)//4, len(_snrs)//2, -1]:
+                ax2.plot(thetas, phasor_pdfs[idx], label='s = {:.1f}'.format(_snrs[idx]))
+            ax2.set_xlabel('$\\theta$')
+            ax2.set_ylabel('$p(\\theta)$')
+            ax2.legend()
+
+        # Gauss Phasor
+        phasor_pdfs = np.array([phase_distribution_gauss(thetas, sigma, s) for s in _snrs ])
+        phasor_sigma = np.sqrt(np.array([variance(thetas, pdf) for pdf in phasor_pdfs]))
+
+        ax.plot(_snrs, phase2time(phasor_sigma), label='Gauss')
+        ax.legend()
+
+    # Set limit on x values
+    if not True or ax.get_xscale() != 'log':
+        ax.set_xlim(0, 100)
+
+    if args.fig_dir:
+        fig.tight_layout()
+        fig.savefig(path.join(args.fig_dir, "time_res_vs_snr.pdf"))
+
+    if args.show_plots:
+        plt.show()
+
+

simulations/airshower_beacon_simulation/bb_measure_clock_phase.py

@@ -1,148 +0,0 @@
-#!/usr/bin/env python3
-# vim: fdm=indent ts=4
-
-import h5py
-from itertools import combinations, zip_longest
-import matplotlib.pyplot as plt
-import numpy as np
-
-import aa_generate_beacon as beacon
-import lib
-
-if __name__ == "__main__":
-    from os import path
-    import sys
-
-    import os
-    import matplotlib
-    if os.name == 'posix' and "DISPLAY" not in os.environ:
-        matplotlib.use('Agg')
-
-    from scriptlib import MyArgumentParser
-    parser = MyArgumentParser()
-    args = parser.parse_args()
-
-    figsize = (12,8)
-    c_light = lib.c_light
-    show_plots = args.show_plots
-
-    remove_absolute_phase_offset_first_antenna = True # takes precedence
-    remove_absolute_phase_offset_minimum = True
-
-    ####
-    fname_dir = args.data_dir
-    antennas_fname = path.join(fname_dir, beacon.antennas_fname)
-
-    fig_dir = args.fig_dir # set None to disable saving
-
-    if not path.isfile(antennas_fname):
-        print("Antenna file cannot be found, did you try generating a beacon?")
-        sys.exit(1)
-
-    # Read in antennas from file
-    f_beacon, tx, antennas = beacon.read_beacon_hdf5(antennas_fname)
-
-    # Make sure at least one beacon has been identified
-    if not hasattr(antennas[0], 'beacon_info') or len(antennas[0].beacon_info) == 0:
-        print(f"No analysed beacon found for {antennas[0].name}, try running the beacon phase analysis script first.")
-        sys.exit(1)
-
-    #
-    N_beacon_freqs = len(antennas[0].beacon_info)
-
-    for freq_name in antennas[0].beacon_info.keys():
-        beacon_phases = np.empty( (len(antennas)) )
-        for i, ant in enumerate(antennas):
-            beacon_phases[i] = ant.beacon_info[freq_name]['beacon_phase']
-
-        f_beacon = antennas[0].beacon_info[freq_name]['freq']
-
-        clock_phases = lib.remove_antenna_geometry_phase(tx, antennas, f_beacon, beacon_phases, c_light=c_light)
-
-        # Remove the phase from one antenna
-        # this is a free parameter
-        # (only required for absolute timing)
-        if remove_absolute_phase_offset_first_antenna or remove_absolute_phase_offset_minimum:
-            if remove_absolute_phase_offset_first_antenna: # just take the first phase
-                minimum_phase = clock_phases[0]
-            else: # take the minimum
-                minimum_phase = np.min(clock_phases, axis=-1)
-
-            clock_phases -= minimum_phase
-            clock_phases = lib.phase_mod(clock_phases)
-
-        # Save to antennas in file
-        with h5py.File(antennas_fname, 'a') as fp:
-            h5group = fp['antennas']
-
-            for i, ant in enumerate(antennas):
-                h5ant = fp['antennas'][ant.name]
-
-                h5beacon_freq = h5ant['beacon_info'][freq_name]
-                h5beacon_freq.attrs['clock_phase'] = clock_phases[i]
-
-        # Plot True Phases at their locations
-        if show_plots or fig_dir:
-            fig, ax = plt.subplots(figsize=figsize)
-            spatial_unit='m'
-            fig.suptitle('Clock phases\nf_beacon= {:2.0f}MHz'.format(f_beacon*1e3))
-
-            antenna_locs = list(zip(*[(ant.x, ant.y) for ant in antennas]))
-            ax.set_xlabel('x' if spatial_unit is None else 'x [{}]'.format(spatial_unit))
-            ax.set_ylabel('y' if spatial_unit is None else 'y [{}]'.format(spatial_unit))
-            scatter_kwargs = {}
-            scatter_kwargs['cmap'] = 'inferno'
-            #scatter_kwargs['vmin'] = -np.pi
-            #scatter_kwargs['vmax'] = +np.pi
-            color_label='$\\varphi(\\sigma_t)$ [rad]'
-
-            sc = ax.scatter(*antenna_locs, c=clock_phases, **scatter_kwargs)
-            fig.colorbar(sc, ax=ax, label=color_label)
-
-            if False:
-                for i, ant in enumerate(antennas):
-                    ax.text(ant.x, ant.y, ant.name)
-
-            if not True:
-                ax.plot(tx.x, tx.y, 'X', color='k', markeredgecolor='white')
-
-            if fig_dir:
-                fig.savefig(path.join(fig_dir, path.basename(__file__) + f".F{freq_name}.pdf"))
-
-        # Plot True Phases - Actual True Phases at their location
-        if show_plots or fig_dir:
-            fig, ax = plt.subplots(figsize=figsize)
-            fig.suptitle('Clock phase Residuals\nf_beacon={:2.0f}MHz'.format(f_beacon*1e3))
-
-            actual_clock_phases = np.array([ -2*np.pi*a.attrs['clock_offset']*f_beacon for a in antennas ])
-
-            # Modify actual_clock_phases, the same way as clock_phases
-            # was modified
-            if remove_absolute_phase_offset_first_antenna or remove_absolute_phase_offset_minimum:
-                if remove_absolute_phase_offset_first_antenna: # just take the first phase
-                    minimum_phase = actual_clock_phases[0]
-                else: # take the minimum
-                    minimum_phase = np.min(actual_clock_phases, axis=-1)
-
-                actual_clock_phases -= minimum_phase
-                actual_clock_phases = lib.phase_mod(actual_clock_phases)
-
-            clock_phase_residuals = lib.phase_mod(clock_phases - actual_clock_phases)
-
-            antenna_locs = list(zip(*[(ant.x, ant.y) for ant in antennas]))
-            ax.set_xlabel('x' if spatial_unit is None else 'x [{}]'.format(spatial_unit))
-            ax.set_ylabel('y' if spatial_unit is None else 'y [{}]'.format(spatial_unit))
-            scatter_kwargs = {}
-            scatter_kwargs['cmap'] = 'inferno'
-            color_label='$\\Delta\\varphi(\\sigma_t) = \\varphi_{meas} - \\varphi_{true}$ [rad]'
-
-            sc = ax.scatter(*antenna_locs, c=clock_phase_residuals, **scatter_kwargs)
-            fig.colorbar(sc, ax=ax, label=color_label)
-
-            if fig_dir:
-                fig.savefig(path.join(fig_dir, path.basename(__file__) + f".residual.F{freq_name}.pdf"))
-
-    print(f"True phases written to", antennas_fname)
-
-    if show_plots:
-        plt.show()

simulations/airshower_beacon_simulation/lib/earsim

@@ -1 +0,0 @@
-Subproject commit a5abe360fa5210be6ab423ecf2b5f783ae45b675