diff --git a/scripts/checkPulseData.py b/scripts/checkPulseData.py index e0d9ad7..d826906 100644 --- a/scripts/checkPulseData.py +++ b/scripts/checkPulseData.py @@ -54,7 +54,7 @@ return np.array(prune(pulse)) -def monotonic(x): +def is_monotonic(x): if not len(x): return False dx = np.diff(x) @@ -93,6 +93,12 @@ class Table(dict): + '''Dictionary tweaked for conversion into pandas DataFrame + + Dictionary conversion into DataFrames is faster than + appending rows to a DataFrame, as pointed out here: + http://stackoverflow.com/a/17496530/3324012 + ''' def __init__(self, *keylist): super(dict, self).__init__() @@ -142,20 +148,7 @@ not_found = StringTemplate('{LAY}/{SEC} on fill {FILL}: calibration step {CS} not found for ns={NS}') not_found = not_found.format(LAY=layer) - # Init data frame - data_frame = pd.DataFrame(columns=['Detector', - 'Sector', - 'Fill', - 'N strips', - 'V bias', - 'MPVs']) - # dic = {'Detector': [], - # 'Sector': [], - # 'Fill': [], - # 'N strips': [], - # 'V bias': [], - # 'MPVs': []} - + # Init dictionary for data frame table = Table('Detector', 'Sector', 'Fill', 'N strips', 'V bias', 'MPVs') # Loop on data @@ -180,19 +173,9 @@ CS=cstep)) pulse.append(None) pulse = build_pulse(pulse) - if monotonic(pulse): - # Append this occurrence to the data frame - # Probably it would be faster to do as suggested here: - # http://stackoverflow.com/a/17496530/3324012 - # data_frame.loc[len(data_frame)] = [detector, - # sector, - # fill, - # ns, - # vmap[vstep], - # pulse] - - table.append(detector, sector, fill, ns, vmap[vstep], pulse) - + # Append this occurrence to the data frame dictionary + table.append(detector, sector, fill, ns, vmap[vstep], pulse) + if is_monotonic(pulse): if VERBOSE: pulse = [round(v,2) for v in pulse] print(warning.format(NS=ns, @@ -202,5 +185,10 @@ PULSE=pulse)) datafile.Close() - # data_frame[data_frame['MPVs'].apply(monotonic)] + # Create the pandas DataFrame from the dictionary data_frame = pd.DataFrame.from_dict(table) + # Select the instances where pulse data are monotonic + monotonics = data_frame[data_frame['MPVs'].apply(is_monotonic)] + + import pickle + pickle.dump(monotonics, open('monotonics.pkl', 'wb'))