From ed7678d74ac8b8e8e1eaad4ed62d5ecc110f6e2b Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:19:04 -0400 Subject: [PATCH 01/12] Add swp_spc_l02l1.py --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 971 +++++++++++++++++++++ 1 file changed, 971 insertions(+) create mode 100644 src/pyfaradaycup/pipeline/swp_spc_l02l1.py diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py new file mode 100644 index 0000000..3bacedc --- /dev/null +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -0,0 +1,971 @@ +""" +# $URL: file:///psp/psp_swp_spc_code_repository/trunk/swp_spc_l02l1.py $ +# $LastChangedRevision: 97 $ +# $LastChangedDate: 2020-08-04 09:20:42 -0400 (Tue, 04 Aug 2020) $ +# $LastChangedBy: acase $ +""" + +import numpy as np +import argparse, sys, os, datetime, math, glob + +try: + from spacepy import pycdf +except: + print(sys.exc_info()) + print("***ERRORR*** Could not import pycdf from spacepy") + print( + "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" + ) + sys.exit() +import ccsds_reader_pipeline as cc +import distutils.dir_util +import spiceypy + +# Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft +# into L0.5 or L1 CDF files + +# Requirements: Must have a reference to a skeleton file for the ApID that you wish to convert_one + +# Input: path to a binary L0 file + +# Output: saves a CDF file + +# Revision History +# 2020/02/03 - Fix timing bug in 0x351 that arose when switching to spiceypy. Add capability to produce 0x352 (time series) files. Will require update to 0x352 (time series) skeleton also. Update bug in naming of L1 files coming from gzip (spacecraft files, probably). Remove versioning from skeleton files (since SVN is taking care of that). +# 2020/01/29 - Use spiceypy to adjust time of each measurement to SCET; remove unused command-line argument options +# Change method by which newest skeleton files are found; various other cleanups +# 2019/08/27 - Calling new ccsds_reader that separates out s/c from instr. packets +# - Also, new ccsds_reader will use new packet finder (much faster), and does not scan through packets like before +# 2019/08/23 - Fixed bug that was reading in the oldest rather than newest skeleton file +# - Added capability to read 0x081 and 0x262 packets from s/c hsk files. Requires link to SC_HK.blk block definition file +# - Added revision history + + +def main( + l0file="", + l1dir="", + logdir="", + spacecraft=False, + ptp=False, + gzip=False, + apidreq=0, + overwrite=False, + verbose=False, +): + """Convert a single L0 file to L1""" + + # Try to create a filename for the new CDF that we're going to create + l0dirname = os.path.dirname(l0file) + l0basename = os.path.basename(l0file) + if l1dir == "": + l1dir = ( + l0dirname # use input L0 directory for L1 files, if nothing else specified + ) + + # Get a version of filename with no extension + l0file_noext = os.path.splitext(l0basename)[0] + if l0file_noext[-3:] == "ptp": + l0file_noext = os.path.splitext(l0file_noext)[0] + + # Open a log file to write to + nowdt = datetime.datetime.now() + if logdir == "": + logdir = l1dir # use L1 file output directory for log file, if nothing else specified + distutils.dir_util.mkpath( + logdir + ) # in case the directory doesn't exist, this will create it + logpath = os.path.join( + logdir, + "swp_spc_l02l1_{:04.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}.log".format( + nowdt.year, nowdt.month, nowdt.day, nowdt.hour, nowdt.minute, nowdt.second + ), + ) + try: + global logfile + logfile = open(logpath, "w") + except: + print("\n***ERROR*** Could not open log file!\n") + sys.exit(1) + + # Write some information to the log file + statusmsg("scriptname = swp_spc_l02l1.py", verbose=verbose) + statusmsg("timerun = " + nowdt.isoformat(), verbose=verbose) + statusmsg("l0file = " + l0file, verbose=verbose) + statusmsg("l1dir = " + l1dir, verbose=verbose) + statusmsg("spacecraft = " + repr(spacecraft), verbose=verbose) + statusmsg("ptp = " + repr(ptp), verbose=verbose) + statusmsg("gzip = " + repr(gzip), verbose=verbose) + statusmsg("apid = " + hex(apidreq), verbose=verbose) + statusmsg("overwrite = " + repr(overwrite), verbose=verbose) + + # Make sure the L0 file exists and is readable + try: + foo = open(l0file, "r") + foo.close() + statusmsg("L0 file exists and is readable") + except IOError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Input L0 file could not be read...exiting", + screen=True, + verbose=verbose, + ) + import pdb + + pdb.set_trace() + sys.exit() + + # Load in Leap Second Kernel + statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest leap second kernel...") + tls_path = get_newest_kernel(tls=True) + if not tls_path: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not find leap second kernel...exiting" + ) + sys.exit() + else: + try: + statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(tls_path)) + spiceypy.furnsh(tls_path) + except: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not furnsh leap second kernel...exiting" + ) + sys.exit() + + # Load in S/C Clock Kernel + statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest S/C clock kernel...") + sclk_path = get_newest_kernel(sclk=True) + if not sclk_path: + statusmsg("***ERROR*** [swp_spc_l02l1.py] Could not find SCLK kernel...exiting") + sys.exit() + else: + try: + statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(sclk_path)) + spiceypy.furnsh(sclk_path) + except: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not furnsh SCLK kernel...exiting" + ) + sys.exit() + + # Read in the L0 file into a python SPC data structure + if spacecraft: + statusmsg("Event = Starting reading file: spacecraft") + l0data = cc.read_file_sc(path=l0file, ptp=ptp, verbose=verbose, gzip=gzip) + else: + statusmsg("Event = Starting reading file: non-spacecraft (instrument)") + l0data = cc.read_file(path=l0file, verbose=verbose, gzip=gzip) + statusmsg("Event = Finished reading file") + + # Loop through the APIDs that we got + for apid in l0data.keys(): + statusmsg("Event = Beginning APID: {:}".format(hex(apid))) + + if apid == 0x07B: + statusmsg( + "***WARNING*** [swp_spc_l02l1] APID 0x07B CDFs not yet implemented", + screen=True, + verbose=verbose, + ) + continue + + # Make sure we need to do this apid + if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: + statusmsg("No packets found for this apid.") + continue # skip this apid if there were no packets received + if (apidreq != 0) & (apidreq != apid): + statusmsg("This apid not requested by user") + continue # skip this apid if user only wanted one apid and this isn't it + + # Filename for the L1 file we're about to write for this apid + l1path = os.path.join( + l1dir, + l0file_noext + + "_APID{:}_L1.cdf".format(str(hex(apid)[2:].zfill(3)).upper()), + ) + statusmsg("About to write: " + l1path) + + # Make sure the skeleton file exists and is readable + try: + skeleton_filename = get_newest_skeleton(apid) + foo = open(skeleton_filename, "r") + foo.close() + statusmsg("Skeleton to be used: " + skeleton_filename) + except IOError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Skeleton file could not be read...moving to next apid", + screen=True, + verbose=verbose, + ) + statusmsg("Tried to use skeleton file: " + skeleton_filename) + continue + except TypeError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={:} could not be found...moving to next apid".format( + hex(apid) + ), + screen=True, + verbose=verbose, + ) + continue + + # See if the CDF file already exists + try: + # try to open and close it + statusmsg("Using L1 path: " + l1path, screen=True, verbose=verbose) + foo = open(l1path) + foo.close() + + # if we get here, this file already exists; so delete it, if desired + statusmsg( + "***INFO*** [swp_spc_l02l1] L1 CDF file ({:}) already exists".format( + l1path + ), + screen=True, + verbose=verbose, + ) + if overwrite: + statusmsg( + "***INFO*** [swp_spc_l02l1] Overwriting existing L1 CDF", + screen=True, + verbose=verbose, + ) + os.remove(l1path) + else: + statusmsg( + "***ERROR*** [swp_spc_l02l1] L1 CDF already exists, and overwrite (-o option) was not requested...exiting.", + screen=True, + verbose=verbose, + ) + raise (SystemExit) + except SystemExit: + sys.exit() + except IOError: + pass # Apparently the file did not exist already + except: + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + statusmsg( + "\n***ERROR*** [swp_spc_l02l1] Could not check existence/delete L1 CDF file path. Exiting...\n", + screen=True, + verbose=verbose, + ) + sys.exit() + + # Create a new CDF file from the provided skeleton + try: + cdf = pycdf.CDF(l1path, skeleton_filename) + except "CDFError": + statusmsg( + "\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={:})...continuing to next APID\n).".format( + apid + ), + screen=True, + verbose=verbose, + ) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) + continue + + # Run a different procedure to put data into CDF file depending on APID + cdfproc = { + 0x081: cdf35e_35f, + 0x1DE: cdf35e_35f, + 0x254: cdf35e_35f, + 0x256: cdf35e_35f, + 0x257: cdf35e_35f, + 0x262: cdf35e_35f, + 0x351: cdf351_353_354, + 0x352: cdf352, + 0x353: cdf351_353_354, + 0x354: cdf351_353_354, + 0x35E: cdf35e_35f, + 0x35F: cdf35e_35f, + } + try: + cdfproc[apid](cdf, l0data[apid], verbose=verbose) + except: + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + statusmsg( + "***WARNING*** [swp_spc_l02l1] CDF not processed for APID={:}".format( + hex(apid) + ), + screen=True, + verbose=verbose, + ) + continue + + # Close the CDF + # import pdb; pdb.set_trace() + cdf.close() + + statusmsg( + "***INFO*** [swp_spc_l02l1] Script complete.", screen=True, verbose=verbose + ) + + # Close the log file + logfile.close() + + +##################################################### +## +##################################################### +def cdf35e_35f(cdf, dat, verbose=False): + """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" + + # Calculate MET from the variables in the L0 data + # MET of each NYS + if "CCSDS_MET" in dat.keys(): + scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPC_SUBSEC"]) + elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["FSW_HK_HK_INST_TPSH_MET_SEC"], + dat["FSW_HK_HK_INST_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["PDU_PRIO94_TPSH_MET_SEC"], + dat["PDU_PRIO94_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["HK_HIGH_TPSH_MET_SEC"], dat["HK_HIGH_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["HK_FSWL_TPSH_MET_SEC"], dat["HK_FSWL_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["HK_LOW_TPSH_MET_SEC"], dat["HK_LOW_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): + scet = secsubsec2scet( + dat["RIU_DERIVED_TPSH_MET_SEC"], + dat["RIU_DERIVED_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + + else: + statusmsg("Failed: could not create Epoch variable") + return + + # Fill in values for each variable + keys = cdf.keys() + dat["Epoch"] = scet + + for key in keys: + try: + cdf[key] = dat[key] # create variable and insert data + except KeyError: + if key not in dat.keys(): + cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] + except: + import pdb + + pdb.set_trace() + statusmsg( + "Failed : Key:{:} failed insert into CDF".format(key), + screen=True, + verbose=verbose, + ) + statusmsg(sys.exc_info()) + + +##################################################### +## +##################################################### +def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): + """Fill up a CDF with SCI, ALL, or RSS data.""" + # Take data sorted by NYS, and produce one long variable with all data + + # APID of this packet + apid = dat["CCSDS_ApID"][0] + + # Each different packet will require a different variable to calculate + # The number of measurements each NYS + if apid == 0x351: + length_var = "A1S" + elif apid == 0x353: + length_var = "ASIN" + elif apid == 0x354: + length_var = "ARSS" + + # Calculate MET from the variables in the L0 data + # MET of each NYS + scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) + + # MET of each measurement (to be filled in in the future) + scet_exp = [] + + # Same keys as original data dictionary, but will hold one variable per key + # instead of one for every NYS for every key + dat_exp = {} + for key in dat.keys(): + dat_exp[key] = [] + + # Create the 'Epoch' variable in our data array + dat_exp["Epoch"] = [] + + # Loop through each NYS + itst_warned = False + for i in range(len(scet)): + this_scet = scet[i] + + # Number of ticks each measurement takes (1024 ticks per NYS) + ticks_per_meas = dat["SW_SPC_INTTIME"][i] + dat["SW_SPC_SERVTIME"][i] + + # Make sure ST and IT are allowed values + if (math.log(ticks_per_meas, 2)) % 1 != 0: + # the SPC FPGA will default to IT=6, ST=2 (the power-on defaults) if a non-integer power of 2 IT+ST is requested + ticks_per_meas = 8 + + if not itst_warned: + statusmsg( + "***WARNING*** [swp_spc_l02l1] The reported IT+ST is not an even power of 2. Using IT=6,ST=2...", + screen=True, + verbose=verbose, + ) + itst_warned = True + + # throw out packets if the IT and ST are different than both previous and next packets + # this is almost surely an improperly identified packet that probably + # isn't even an SPC packet, but got decommutated as such + try: + if ( + (dat["SW_SPC_INTTIME"][i] != dat["SW_SPC_INTTIME"][i - 1]) + & (dat["SW_SPC_INTTIME"][i] != dat["SW_SPC_INTTIME"][i + 1]) + & (dat["SW_SPC_SERVTIME"][i] != dat["SW_SPC_SERVTIME"][i - 1]) + & (dat["SW_SPC_SERVTIME"][i] != dat["SW_SPC_SERVTIME"][i + 1]) + ): + statusmsg( + "***WARNING***IT+ST not 2^n, and not same as prev. and next values...so skipping this packet.", + screen=True, + verbose=verbose, + ) + continue + except IndexError: + statusmsg( + "***WARNING***IT+ST not 2^n, and not same as prev. and next values...so skipping this packet.", + screen=True, + verbose=verbose, + ) + continue + + # Time that each measurement took this NYS + tm_per_meas = (1.0 / 1171.875) * ticks_per_meas + + # Number of measurements this NYS + nmeas = len(dat[length_var][i]) + + # Expected number of measurements in a NYS + exp_nmeas = 1024.0 / 1171.875 / tm_per_meas + + # Calculate the time array for this NYS + add_time = np.arange(0, tm_per_meas * (nmeas - 0.1), tm_per_meas) + + # See if there were any times when we had retraces + win = np.array(dat["WINDOW"][i]) + rtpix = (np.where((win[1:] - win[:-1]) < 0)[0]) + 1 + + # Add on some time for each of the retraces + # We don't need to do this if the expected number of measurements is equal to the number of measurements we received + # This is because of the possibility that HV DAC tables were not loaded (probably only on the ground). + # In that case, the FPGA does not actually take the time to do a retrace since it does not have to slew to a new DAC value + # It doesn't matter that it is slewing to a new 'Window', since every window will have the same DAC value + # There is a slight bug here in that the 'final' NYS after a HALT is sent, will likely be a partial packet + # so we might get tricked on our logical check here for the final packet when we do not have DAC tables loaded + if nmeas != exp_nmeas: + for thisrtpix in rtpix: + add_time[thisrtpix:] += tm_per_meas + + # If we're in an AllGain packet, then the beginning of the packet might not be the beginning of the NYS (which is the time noted in the header) + if apid == 0x351: + pktnum = dat["SW_SPC_PKTNUM"][i] + # if pktnum==0: import pdb; pdb.set_trace() + if pktnum != 0: + if len(dat_exp["Epoch"]) == 0: + continue # if file started on pktnum other than zero, then we can't know precise timing for the first 1-3 packets + + add_time += (dat_exp["Epoch"][-1] / 1e9 - this_scet / 1e9) + tm_per_meas + + # in case retrace was at end of last packet + if (win[0] - dat_exp["WINDOW"][-1]) < 0: + add_time += tm_per_meas + + # Extend the new expanded dt + dscet_extend = [this_scet + 1e9 * thisaddtime for thisaddtime in add_time] + dat_exp["Epoch"].extend(dscet_extend) + + # Extend each of the data arrays + for key in dat.keys(): + try: + dat_exp[key].extend(dat[key][i]) + except TypeError: + expanded = np.ones(nmeas) * dat[key][i] + dat_exp[key].extend(expanded) + + if nocdf: + return dat_exp + else: + # Fill in the CDF + keys = cdf.keys() + + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: + statusmsg( + "Failed : Key:{:} failed insert into CDF".format(key), + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + import pdb + + pdb.set_trace() + + +##################################################### +## +##################################################### +def cdf352(cdf, dat, nocdf=False, verbose=False): + try: + # Calculate SCET from the variables in the L0 data + dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) + + # Same keys as original data dictionary, but will hold one variable per key + # instead of one for every NYS for every key + dat_exp = {} + for key in dat.keys(): + if key[-4:] == "_000": + continue + dat_exp[key] = [] + + dat_exp["VAR0"] = [] + dat_exp["VAR1"] = [] + dat_exp["VAR2"] = [] + dat_exp["VAR3"] = [] + dat_exp["VAR0_NAME"] = [] + dat_exp["VAR1_NAME"] = [] + dat_exp["VAR2_NAME"] = [] + dat_exp["VAR3_NAME"] = [] + + # Var names for each possible channel that might be contained in the packet + avars = ["A0", "A1", "A2", "A3"] + bvars = ["B0", "B1", "B2", "B3"] + cvars = ["C0", "C1", "C2", "C3"] + dvars = ["D0", "D1", "D2", "D3"] + hk1vars = ["HV_DAC_IN", "P3p3_Vmon", "P12_Vmon", "N12_Vmon"] + hk2vars = ["HV_Out", "Rail_Ctrl", "P5_Vmon", "N5_Vmon"] + + # To convert from the ID number in each packet header to the variables that + # the packet actually contains values for + coll2var = {1: avars, 2: bvars, 4: cvars, 8: dvars, 16: hk1vars, 32: hk2vars} + + # Create the 'Epoch' variable in our data array + dat_exp["Epoch"] = [] + + # Time that each measurement took this NYS + tm_per_meas = 1.0 / 32.0 / 1171.875 + + for i in range(len(dt)): + # MET for this NY second + thisdt = dt[i] + + # Collector (or HSK values) that are being used this NYS + # An integer that references which varibles are actually contained in the packet + coll_used = dat["SPC_TIMESERCOLL"][i] + + # Number of measurements this NYS + nmeas = len(dat["G0_000"][i]) # should always be 20*32=640 + + # Packet start time + pkt_start = dat["SPC_TIMESERTICK"][i] / 1171.875 + + # Calculate the time array for this NYS + add_time = pkt_start + np.arange( + 0, tm_per_meas * (nmeas - 0.1), tm_per_meas + ) + + try: + if coll_used not in coll2var.keys(): + raise ValueError( + "Value: {:} not in coll2var.keys()".format(coll_used) + ) # probably a corrupt packet + + dat_exp["VAR0_NAME"].extend( + [coll2var[coll_used][0] for foo in range(nmeas)] + ) + dat_exp["VAR1_NAME"].extend( + [coll2var[coll_used][1] for foo in range(nmeas)] + ) + dat_exp["VAR2_NAME"].extend( + [coll2var[coll_used][2] for foo in range(nmeas)] + ) + dat_exp["VAR3_NAME"].extend( + [coll2var[coll_used][3] for foo in range(nmeas)] + ) + + dat_exp["VAR0"].extend(dat["G0_000"][i]) + dat_exp["VAR1"].extend(dat["G1_000"][i]) + dat_exp["VAR2"].extend(dat["G2_000"][i]) + dat_exp["VAR3"].extend(dat["G3_000"][i]) + + except: + statusmsg( + "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + continue + + # Extend the expanded dt + dt_extend = [thisdt + sec * 1e9 for sec in add_time] + dat_exp["Epoch"].extend(dt_extend) + + # Extend each of the data arrays + for key in dat.keys(): + if key[-4:] == "_000": + continue + try: + dat_exp[key].extend(dat[key][i]) + except TypeError: + expanded = np.ones(nmeas) * dat[key][i] + dat_exp[key].extend(expanded) + + if nocdf: + return dat_exp + else: + # Fill in the CDF + keys = cdf.keys() + + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: + statusmsg( + "Failed : Key:{:} failed insert into CDF".format(key), + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + except: + print(sys.exc_info()) + import pdb + + pdb.set_trace() + + return () + + +##################################################### +## +##################################################### +def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): + """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" + sec_str = ["{:1.0f}".format(i) for i in sec] + subsec_str_base50000 = [ + "{:05.0f}".format(int(i * 50000 / 65536)) for i in subsec + ] # SWEAP has subseconds in 1/65536's of a second + if spacecraft: + subsec_str_base50000 = [ + "{:05.0f}".format(int(i * 50000 / 256)) for i in subsec + ] # S/C has subseconds in 1/256's of a second + + ephem_sec_j2000 = [ + spiceypy.scs2e(-96, sec_str[i] + ":" + subsec_str_base50000[i]) + for i in range(len(sec_str)) + ] + ephem_nanosec_j2000 = [np.round(1e9 * i) for i in ephem_sec_j2000] + + return ephem_nanosec_j2000 + + +##################################################### +## +##################################################### +def statusmsg(string, screen=False, file=True, verbose=False): + """Output status message to screen or logfile (default to file, but not screen)""" + nowdtstr = datetime.datetime.now().isoformat() + if file: + logfile.write(nowdtstr + ", " + string + "\n") + if screen: + if verbose: + print(string) + + +##################################################### +## +##################################################### +def get_newest_kernel(tls=False, sclk=False, verbose=False): + """Find the path to the newest NAIF TLS (leap second) kernel file""" + # Make sure we chose exactly one of the options + if tls + sclk != 1: + return False + + # Search in the MOC data product directory for newest file + if tls: + globdir = "/psp/data/moc_data_products/leap_second_kernel/" + globstr = globdir + "naif00[0-9][0-9].tls" + ndigits = 2 + elif sclk: + globdir = "/psp/data/moc_data_products/operations_sclk_kernel/" + globstr = globdir + "spp_sclk_[0-9][0-9][0-9][0-9].tsc" + ndigits = 4 + + files = glob.glob(globstr) + + # isolate version numbers from the file path and find newest + if tls: + versions = [int(i[-4 - ndigits : -4]) for i in files] + elif sclk: + versions = [int(i[-4 - ndigits : -4]) for i in files] + try: + maxind = np.argmax(versions) + except ValueError: + statusmsg("***ERROR*** Could not find kernel versions") + print(sys.exc_info()) + import pdb + + pdb.set_trace() + return False + + # return path to newest file + path = files[maxind] + return path + + +##################################################### +## +##################################################### +def get_newest_skeleton(apid, verbose=False): + """Find the path to the newest skeleton CDF file""" + + return "cdf_skeletons/psp_swp_spc_l1_{:}_skeleton.cdf".format( + hex(apid)[2:].zfill(3) + ) + + # The remaining code in this function is from when we used skeleton file numbers with a version # in them + # and we had to search for the most recent (highest) version + + # Search for newest file + # globstr = 'cdf_skeletons/spp_apid_{:}_sweap_00000000t000000_v[0-9][0-9].cdf'.format(hex(apid)[2:].zfill(3)) + # ndigits = 2 + + # files = glob.glob(globstr) + + # isolate version numbers from the file path and find newest + # versions = [int(i[-4-ndigits:-4]) for i in files] + + # try: + # maxind = np.argmax(versions) + # except ValueError: + # statusmsg('***ERROR*** Could not find skeleton versions') + # return(False) + + # return path to newest file + # path = files[maxind] + + # return(path) + + +##################################################### +### +##################################################### +def setup(): + """Get user command-line input and set things up""" + + # defaults + l0file_default = "" + l0dir_default = "" + l1dir_default = "" + logdir_default = "" + apid_default = "0" + + # Get User Input + parser = argparse.ArgumentParser(description="") + parser.add_argument( + "-v", + "--verbose", + default=False, + action="store_true", + help="Increase verbosity", + required=False, + ) + parser.add_argument( + "-gz", + "--gzip", + default=False, + action="store_true", + help="Read in L0 file as gzip", + required=False, + ) + parser.add_argument( + "-sc", + "--spacecraft", + default=False, + action="store_true", + help="Look for S/C packets", + required=False, + ) + parser.add_argument( + "-b", + "--batch", + default=False, + action="store_true", + help="Convert all L0 files in same directory as selected", + required=False, + ) + parser.add_argument( + "-r", + "--recursive", + default=False, + action="store_true", + help="Convert all L0 files in given directory and in all subdirectories", + required=False, + ) + parser.add_argument( + "-p", + "--ptp", + default=False, + action="store_true", + help="Indicate that input L0 file is a PTP file", + required=False, + ) + parser.add_argument( + "-o", + "--overwrite", + default=False, + action="store_true", + help="Overwrite existing L1 CDF file, if necessary", + required=False, + ) + parser.add_argument( + "-stc", + "--stcorrect", + default=False, + action="store_true", + help="If ST is wrong (FPGA bug if ST set higher than 2), then try to correct it)", + required=False, + ) + parser.add_argument( + "-a", + "--apid", + help="APID to create L1 file for [0==all] [default={:}]".format(apid_default), + required=False, + default=apid_default, + type=str, + ) + parser.add_argument( + "-l0", + "--l0file", + help="Input L0 File [default={:}]".format(l0file_default), + required=False, + default=l0file_default, + ) + parser.add_argument( + "-d", + "--l0dir", + help="Input L0 Directory (for use with -b or -r [default={:}]".format( + l0dir_default + ), + required=False, + default=l0dir_default, + ) + parser.add_argument( + "-dl1", + "--l1dir", + help="Output L1 Directory [default={:}]".format(l1dir_default), + required=False, + default=l1dir_default, + ) + parser.add_argument( + "-dlog", + "--logdir", + help="Output for Log Files [default={:}]".format(logdir_default), + required=False, + default=logdir_default, + ) + + # Read in the arguments + args = parser.parse_args() + + # Version of the data product + args.version = 2 + + # Make sure we got a good argument set + if (args.batch == 0) & (args.recursive == 0): + if args.l0file == "": + statusmsg( + "***ERROR*** You must provide --l0file, if not using -b or -r", + screen=True, + verbose=verbose, + ) + else: + if args.l0dir == "": + statusmsg( + "***ERROR*** You must provide --l0dir if using -b or -r", + screen=True, + verbose=verbose, + ) + + # Convert APID to an integer (it is read as a string from the command line) + try: + if args.apid[0:2] == "0x": + base = 16 + else: + base = 10 + args.apid = int(args.apid, base) + except TypeError: + statusmsg( + "Trouble parsing desired APID....exiting.", screen=True, verbose=verbose + ) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) + sys.exit() + + # Make sure the environmental variable reference to the data directory is set and readable + try: + datadir = os.environ["PSP_DATA_DIR"] + except: + raise KeyError( + "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" + ) + + if not os.path.exists(datadir): + raise ValueError( + "Directory specified in env. variable PSP_DATA_DIR does not exist" + ) + + # Return to main routine + return args + + +############################################ +#### +############################################ +if __name__ == "__main__": + args = setup() + main( + l0file=args.l0file, + l1dir=args.l1dir, + logdir=args.logdir, + spacecraft=args.spacecraft, + ptp=args.ptp, + gzip=args.gzip, + apidreq=args.apid, + overwrite=args.overwrite, + verbose=args.verbose, + ) From e6692f3e904e3aacc1733164c153784e13d88d83 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:20:46 -0400 Subject: [PATCH 02/12] Make changes through ruff check and ruff format --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 387 ++++++++++----------- 1 file changed, 186 insertions(+), 201 deletions(-) diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index 3bacedc..faabb58 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -3,22 +3,29 @@ # $LastChangedRevision: 97 $ # $LastChangedDate: 2020-08-04 09:20:42 -0400 (Tue, 04 Aug 2020) $ # $LastChangedBy: acase $ -""" +""" # noqa: D400 + +import argparse +import datetime +import glob +import math +import os +import sys import numpy as np -import argparse, sys, os, datetime, math, glob try: from spacepy import pycdf -except: - print(sys.exc_info()) - print("***ERRORR*** Could not import pycdf from spacepy") - print( +except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + print("***ERROR*** Could not import pycdf from spacepy") # noqa: T201 + print( # noqa: T201 "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" ) sys.exit() -import ccsds_reader_pipeline as cc import distutils.dir_util + +import ccsds_reader_pipeline as cc import spiceypy # Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft @@ -41,50 +48,47 @@ # - Added revision history -def main( - l0file="", - l1dir="", - logdir="", - spacecraft=False, - ptp=False, - gzip=False, - apidreq=0, - overwrite=False, - verbose=False, +def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 + l0file="", # noqa: ANN001 + l1dir="", # noqa: ANN001 + logdir="", # noqa: ANN001 + spacecraft=False, # noqa: ANN001, FBT002 + ptp=False, # noqa: ANN001, FBT002 + gzip=False, # noqa: ANN001, FBT002 + apidreq=0, # noqa: ANN001 + overwrite=False, # noqa: ANN001, FBT002 + verbose=False, # noqa: ANN001, FBT002 ): - """Convert a single L0 file to L1""" - + """Convert a single L0 file to L1""" # noqa: D400 # Try to create a filename for the new CDF that we're going to create - l0dirname = os.path.dirname(l0file) - l0basename = os.path.basename(l0file) + l0dirname = os.path.dirname(l0file) # noqa: PTH120 + l0basename = os.path.basename(l0file) # noqa: PTH119 if l1dir == "": l1dir = ( l0dirname # use input L0 directory for L1 files, if nothing else specified ) # Get a version of filename with no extension - l0file_noext = os.path.splitext(l0basename)[0] + l0file_noext = os.path.splitext(l0basename)[0] # noqa: PTH122 if l0file_noext[-3:] == "ptp": - l0file_noext = os.path.splitext(l0file_noext)[0] + l0file_noext = os.path.splitext(l0file_noext)[0] # noqa: PTH122 # Open a log file to write to - nowdt = datetime.datetime.now() + nowdt = datetime.datetime.now() # noqa: DTZ005 if logdir == "": logdir = l1dir # use L1 file output directory for log file, if nothing else specified distutils.dir_util.mkpath( logdir ) # in case the directory doesn't exist, this will create it - logpath = os.path.join( + logpath = os.path.join( # noqa: PTH118 logdir, - "swp_spc_l02l1_{:04.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}{:02.0f}.log".format( - nowdt.year, nowdt.month, nowdt.day, nowdt.hour, nowdt.minute, nowdt.second - ), + f"swp_spc_l02l1_{nowdt.year:04.0f}{nowdt.month:02.0f}{nowdt.day:02.0f}{nowdt.hour:02.0f}{nowdt.minute:02.0f}{nowdt.second:02.0f}.log", ) try: - global logfile - logfile = open(logpath, "w") - except: - print("\n***ERROR*** Could not open log file!\n") + global logfile # noqa: PLW0603 + logfile = open(logpath, "w") # noqa: PTH123, SIM115 + except: # noqa: E722 + print("\n***ERROR*** Could not open log file!\n") # noqa: T201 sys.exit(1) # Write some information to the log file @@ -100,18 +104,18 @@ def main( # Make sure the L0 file exists and is readable try: - foo = open(l0file, "r") + foo = open(l0file) # noqa: PTH123, SIM115 foo.close() statusmsg("L0 file exists and is readable") - except IOError: + except OSError: statusmsg( "***ERROR*** [swp_spc_l02l1.py] Input L0 file could not be read...exiting", screen=True, verbose=verbose, ) - import pdb + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 sys.exit() # Load in Leap Second Kernel @@ -124,9 +128,9 @@ def main( sys.exit() else: try: - statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(tls_path)) + statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {tls_path}") spiceypy.furnsh(tls_path) - except: + except: # noqa: E722 statusmsg( "***ERROR*** [swp_spc_l02l1.py] Could not furnsh leap second kernel...exiting" ) @@ -140,9 +144,9 @@ def main( sys.exit() else: try: - statusmsg("***INFO*** [swp_spc_l02l1.py] Using: {:}".format(sclk_path)) + statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {sclk_path}") spiceypy.furnsh(sclk_path) - except: + except: # noqa: E722 statusmsg( "***ERROR*** [swp_spc_l02l1.py] Could not furnsh SCLK kernel...exiting" ) @@ -158,10 +162,10 @@ def main( statusmsg("Event = Finished reading file") # Loop through the APIDs that we got - for apid in l0data.keys(): - statusmsg("Event = Beginning APID: {:}".format(hex(apid))) + for apid in l0data.keys(): # noqa: SIM118 + statusmsg(f"Event = Beginning APID: {hex(apid)}") - if apid == 0x07B: + if apid == 0x07B: # noqa: PLR2004 statusmsg( "***WARNING*** [swp_spc_l02l1] APID 0x07B CDFs not yet implemented", screen=True, @@ -170,7 +174,7 @@ def main( continue # Make sure we need to do this apid - if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: + if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: # noqa: RUF015 statusmsg("No packets found for this apid.") continue # skip this apid if there were no packets received if (apidreq != 0) & (apidreq != apid): @@ -178,20 +182,19 @@ def main( continue # skip this apid if user only wanted one apid and this isn't it # Filename for the L1 file we're about to write for this apid - l1path = os.path.join( + l1path = os.path.join( # noqa: PTH118 l1dir, - l0file_noext - + "_APID{:}_L1.cdf".format(str(hex(apid)[2:].zfill(3)).upper()), + l0file_noext + f"_APID{str(hex(apid)[2:].zfill(3)).upper()}_L1.cdf", # noqa: FURB116 ) statusmsg("About to write: " + l1path) # Make sure the skeleton file exists and is readable try: skeleton_filename = get_newest_skeleton(apid) - foo = open(skeleton_filename, "r") + foo = open(skeleton_filename) # noqa: PTH123, SIM115 foo.close() statusmsg("Skeleton to be used: " + skeleton_filename) - except IOError: + except OSError: statusmsg( "***ERROR*** [swp_spc_l02l1.py] Skeleton file could not be read...moving to next apid", screen=True, @@ -201,9 +204,7 @@ def main( continue except TypeError: statusmsg( - "***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={:} could not be found...moving to next apid".format( - hex(apid) - ), + f"***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={hex(apid)} could not be found...moving to next apid", screen=True, verbose=verbose, ) @@ -213,14 +214,12 @@ def main( try: # try to open and close it statusmsg("Using L1 path: " + l1path, screen=True, verbose=verbose) - foo = open(l1path) + foo = open(l1path) # noqa: PTH123, SIM115 foo.close() # if we get here, this file already exists; so delete it, if desired statusmsg( - "***INFO*** [swp_spc_l02l1] L1 CDF file ({:}) already exists".format( - l1path - ), + f"***INFO*** [swp_spc_l02l1] L1 CDF file ({l1path}) already exists", screen=True, verbose=verbose, ) @@ -230,19 +229,19 @@ def main( screen=True, verbose=verbose, ) - os.remove(l1path) + os.remove(l1path) # noqa: PTH107 else: statusmsg( "***ERROR*** [swp_spc_l02l1] L1 CDF already exists, and overwrite (-o option) was not requested...exiting.", screen=True, verbose=verbose, ) - raise (SystemExit) + raise (SystemExit) # noqa: TRY301 except SystemExit: sys.exit() - except IOError: + except OSError: pass # Apparently the file did not exist already - except: + except: # noqa: E722 statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) statusmsg( "\n***ERROR*** [swp_spc_l02l1] Could not check existence/delete L1 CDF file path. Exiting...\n", @@ -254,11 +253,9 @@ def main( # Create a new CDF file from the provided skeleton try: cdf = pycdf.CDF(l1path, skeleton_filename) - except "CDFError": + except "CDFError": # noqa: B030 statusmsg( - "\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={:})...continuing to next APID\n).".format( - apid - ), + f"\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={apid})...continuing to next APID\n).", screen=True, verbose=verbose, ) @@ -282,12 +279,10 @@ def main( } try: cdfproc[apid](cdf, l0data[apid], verbose=verbose) - except: + except: # noqa: E722 statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) statusmsg( - "***WARNING*** [swp_spc_l02l1] CDF not processed for APID={:}".format( - hex(apid) - ), + f"***WARNING*** [swp_spc_l02l1] CDF not processed for APID={hex(apid)}", screen=True, verbose=verbose, ) @@ -308,38 +303,37 @@ def main( ##################################################### ## ##################################################### -def cdf35e_35f(cdf, dat, verbose=False): - """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" - +def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, ANN201, C901, FBT002 + """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 # Calculate MET from the variables in the L0 data # MET of each NYS - if "CCSDS_MET" in dat.keys(): + if "CCSDS_MET" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPC_SUBSEC"]) - elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): + elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["FSW_HK_HK_INST_TPSH_MET_SEC"], dat["FSW_HK_HK_INST_TPSH_MET_SUBSEC"], spacecraft=True, ) - elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): + elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["PDU_PRIO94_TPSH_MET_SEC"], dat["PDU_PRIO94_TPSH_MET_SUBSEC"], spacecraft=True, ) - elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): + elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["HK_HIGH_TPSH_MET_SEC"], dat["HK_HIGH_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): + elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["HK_FSWL_TPSH_MET_SEC"], dat["HK_FSWL_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): + elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["HK_LOW_TPSH_MET_SEC"], dat["HK_LOW_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): + elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 scet = secsubsec2scet( dat["RIU_DERIVED_TPSH_MET_SEC"], dat["RIU_DERIVED_TPSH_MET_SUBSEC"], @@ -357,15 +351,15 @@ def cdf35e_35f(cdf, dat, verbose=False): for key in keys: try: cdf[key] = dat[key] # create variable and insert data - except KeyError: - if key not in dat.keys(): + except KeyError: # noqa: PERF203 + if key not in dat.keys(): # noqa: SIM118 cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] - except: - import pdb + except: # noqa: E722 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 statusmsg( - "Failed : Key:{:} failed insert into CDF".format(key), + f"Failed : Key:{key} failed insert into CDF", screen=True, verbose=verbose, ) @@ -375,7 +369,7 @@ def cdf35e_35f(cdf, dat, verbose=False): ##################################################### ## ##################################################### -def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): +def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0915, RET503 """Fill up a CDF with SCI, ALL, or RSS data.""" # Take data sorted by NYS, and produce one long variable with all data @@ -384,11 +378,11 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # Each different packet will require a different variable to calculate # The number of measurements each NYS - if apid == 0x351: + if apid == 0x351: # noqa: PLR2004 length_var = "A1S" - elif apid == 0x353: + elif apid == 0x353: # noqa: PLR2004 length_var = "ASIN" - elif apid == 0x354: + elif apid == 0x354: # noqa: PLR2004 length_var = "ARSS" # Calculate MET from the variables in the L0 data @@ -396,12 +390,12 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) # MET of each measurement (to be filled in in the future) - scet_exp = [] + scet_exp = [] # noqa: F841 # Same keys as original data dictionary, but will hold one variable per key # instead of one for every NYS for every key dat_exp = {} - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 dat_exp[key] = [] # Create the 'Epoch' variable in our data array @@ -416,7 +410,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): ticks_per_meas = dat["SW_SPC_INTTIME"][i] + dat["SW_SPC_SERVTIME"][i] # Make sure ST and IT are allowed values - if (math.log(ticks_per_meas, 2)) % 1 != 0: + if (math.log(ticks_per_meas, 2)) % 1 != 0: # noqa: FURB163 # the SPC FPGA will default to IT=6, ST=2 (the power-on defaults) if a non-integer power of 2 IT+ST is requested ticks_per_meas = 8 @@ -480,7 +474,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): add_time[thisrtpix:] += tm_per_meas # If we're in an AllGain packet, then the beginning of the packet might not be the beginning of the NYS (which is the time noted in the header) - if apid == 0x351: + if apid == 0x351: # noqa: PLR2004 pktnum = dat["SW_SPC_PKTNUM"][i] # if pktnum==0: import pdb; pdb.set_trace() if pktnum != 0: @@ -498,45 +492,44 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): dat_exp["Epoch"].extend(dscet_extend) # Extend each of the data arrays - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 try: dat_exp[key].extend(dat[key][i]) - except TypeError: + except TypeError: # noqa: PERF203 expanded = np.ones(nmeas) * dat[key][i] dat_exp[key].extend(expanded) if nocdf: return dat_exp - else: - # Fill in the CDF - keys = cdf.keys() + # Fill in the CDF + keys = cdf.keys() - # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) - epochloc = np.where(np.array(keys) == "Epoch")[0] - if len(epochloc) != 0: - keys.pop(epochloc[0]) - keys.insert(0, "Epoch") + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") - for key in keys: - try: - # insert data - cdf[key] = dat_exp[key] - except: - statusmsg( - "Failed : Key:{:} failed insert into CDF".format(key), - screen=True, - verbose=verbose, - ) - statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - import pdb + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: # noqa: E722, PERF203 + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 ##################################################### ## ##################################################### -def cdf352(cdf, dat, nocdf=False, verbose=False): +def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915 try: # Calculate SCET from the variables in the L0 data dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) @@ -544,7 +537,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # Same keys as original data dictionary, but will hold one variable per key # instead of one for every NYS for every key dat_exp = {} - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 if key[-4:] == "_000": continue dat_exp[key] = [] @@ -581,7 +574,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): thisdt = dt[i] # Collector (or HSK values) that are being used this NYS - # An integer that references which varibles are actually contained in the packet + # An integer that references which variables are actually contained in the packet coll_used = dat["SPC_TIMESERCOLL"][i] # Number of measurements this NYS @@ -596,9 +589,9 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): ) try: - if coll_used not in coll2var.keys(): - raise ValueError( - "Value: {:} not in coll2var.keys()".format(coll_used) + if coll_used not in coll2var: + raise ValueError( # noqa: TRY003 + f"Value: {coll_used} not in coll2var.keys()" # noqa: EM102 ) # probably a corrupt packet dat_exp["VAR0_NAME"].extend( @@ -619,7 +612,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): dat_exp["VAR2"].extend(dat["G2_000"][i]) dat_exp["VAR3"].extend(dat["G3_000"][i]) - except: + except: # noqa: E722 statusmsg( "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" ) @@ -631,7 +624,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): dat_exp["Epoch"].extend(dt_extend) # Extend each of the data arrays - for key in dat.keys(): + for key in dat.keys(): # noqa: SIM118 if key[-4:] == "_000": continue try: @@ -642,31 +635,30 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): if nocdf: return dat_exp - else: - # Fill in the CDF - keys = cdf.keys() - - # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) - epochloc = np.where(np.array(keys) == "Epoch")[0] - if len(epochloc) != 0: - keys.pop(epochloc[0]) - keys.insert(0, "Epoch") - for key in keys: - try: - # insert data - cdf[key] = dat_exp[key] - except: - statusmsg( - "Failed : Key:{:} failed insert into CDF".format(key), - screen=True, - verbose=verbose, - ) - statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - except: - print(sys.exc_info()) - import pdb + # Fill in the CDF + keys = cdf.keys() - pdb.set_trace() + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: # noqa: E722, PERF203 + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + except: # noqa: E722 + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 + + pdb.set_trace() # noqa: T100 return () @@ -674,15 +666,15 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): ##################################################### ## ##################################################### -def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): - """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" - sec_str = ["{:1.0f}".format(i) for i in sec] +def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 + """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # noqa: D400 + sec_str = [f"{i:1.0f}" for i in sec] subsec_str_base50000 = [ - "{:05.0f}".format(int(i * 50000 / 65536)) for i in subsec + f"{int(i * 50000 / 65536):05.0f}" for i in subsec ] # SWEAP has subseconds in 1/65536's of a second if spacecraft: subsec_str_base50000 = [ - "{:05.0f}".format(int(i * 50000 / 256)) for i in subsec + f"{int(i * 50000 / 256):05.0f}" for i in subsec ] # S/C has subseconds in 1/256's of a second ephem_sec_j2000 = [ @@ -691,27 +683,27 @@ def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): ] ephem_nanosec_j2000 = [np.round(1e9 * i) for i in ephem_sec_j2000] - return ephem_nanosec_j2000 + return ephem_nanosec_j2000 # noqa: RET504 ##################################################### ## ##################################################### -def statusmsg(string, screen=False, file=True, verbose=False): - """Output status message to screen or logfile (default to file, but not screen)""" - nowdtstr = datetime.datetime.now().isoformat() +def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, ANN201, FBT002 + """Output status message to screen or logfile (default to file, but not screen)""" # noqa: D400 + nowdtstr = datetime.datetime.now().isoformat() # noqa: DTZ005 if file: logfile.write(nowdtstr + ", " + string + "\n") - if screen: + if screen: # noqa: SIM102 if verbose: - print(string) + print(string) # noqa: T201 ##################################################### ## ##################################################### -def get_newest_kernel(tls=False, sclk=False, verbose=False): - """Find the path to the newest NAIF TLS (leap second) kernel file""" +def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 + """Find the path to the newest NAIF TLS (leap second) kernel file""" # noqa: D400 # Make sure we chose exactly one of the options if tls + sclk != 1: return False @@ -726,37 +718,32 @@ def get_newest_kernel(tls=False, sclk=False, verbose=False): globstr = globdir + "spp_sclk_[0-9][0-9][0-9][0-9].tsc" ndigits = 4 - files = glob.glob(globstr) + files = glob.glob(globstr) # noqa: PTH207 # isolate version numbers from the file path and find newest - if tls: - versions = [int(i[-4 - ndigits : -4]) for i in files] - elif sclk: + if tls or sclk: versions = [int(i[-4 - ndigits : -4]) for i in files] try: maxind = np.argmax(versions) except ValueError: statusmsg("***ERROR*** Could not find kernel versions") - print(sys.exc_info()) - import pdb + print(sys.exc_info()) # noqa: T201 + import pdb # noqa: PLC0415, T100 - pdb.set_trace() + pdb.set_trace() # noqa: T100 return False # return path to newest file path = files[maxind] - return path + return path # noqa: RET504 ##################################################### ## ##################################################### -def get_newest_skeleton(apid, verbose=False): - """Find the path to the newest skeleton CDF file""" - - return "cdf_skeletons/psp_swp_spc_l1_{:}_skeleton.cdf".format( - hex(apid)[2:].zfill(3) - ) +def get_newest_skeleton(apid, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 + """Find the path to the newest skeleton CDF file""" # noqa: D400 + return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # noqa: FURB116 # The remaining code in this function is from when we used skeleton file numbers with a version # in them # and we had to search for the most recent (highest) version @@ -785,9 +772,8 @@ def get_newest_skeleton(apid, verbose=False): ##################################################### ### ##################################################### -def setup(): - """Get user command-line input and set things up""" - +def setup(): # noqa: ANN201 + """Get user command-line input and set things up""" # noqa: D400 # defaults l0file_default = "" l0dir_default = "" @@ -864,7 +850,7 @@ def setup(): parser.add_argument( "-a", "--apid", - help="APID to create L1 file for [0==all] [default={:}]".format(apid_default), + help=f"APID to create L1 file for [0==all] [default={apid_default}]", required=False, default=apid_default, type=str, @@ -872,30 +858,28 @@ def setup(): parser.add_argument( "-l0", "--l0file", - help="Input L0 File [default={:}]".format(l0file_default), + help=f"Input L0 File [default={l0file_default}]", required=False, default=l0file_default, ) parser.add_argument( "-d", "--l0dir", - help="Input L0 Directory (for use with -b or -r [default={:}]".format( - l0dir_default - ), + help=f"Input L0 Directory (for use with -b or -r [default={l0dir_default}]", required=False, default=l0dir_default, ) parser.add_argument( "-dl1", "--l1dir", - help="Output L1 Directory [default={:}]".format(l1dir_default), + help=f"Output L1 Directory [default={l1dir_default}]", required=False, default=l1dir_default, ) parser.add_argument( "-dlog", "--logdir", - help="Output for Log Files [default={:}]".format(logdir_default), + help=f"Output for Log Files [default={logdir_default}]", required=False, default=logdir_default, ) @@ -912,41 +896,42 @@ def setup(): statusmsg( "***ERROR*** You must provide --l0file, if not using -b or -r", screen=True, - verbose=verbose, - ) - else: - if args.l0dir == "": - statusmsg( - "***ERROR*** You must provide --l0dir if using -b or -r", - screen=True, - verbose=verbose, + verbose=verbose, # noqa: F821 ) + elif args.l0dir == "": + statusmsg( + "***ERROR*** You must provide --l0dir if using -b or -r", + screen=True, + verbose=verbose, # noqa: F821 + ) # Convert APID to an integer (it is read as a string from the command line) try: - if args.apid[0:2] == "0x": + if args.apid[0:2] == "0x": # noqa: SIM108 base = 16 else: base = 10 args.apid = int(args.apid, base) except TypeError: statusmsg( - "Trouble parsing desired APID....exiting.", screen=True, verbose=verbose + "Trouble parsing desired APID....exiting.", + screen=True, + verbose=verbose, # noqa: F821 ) - statusmsg(sys.exc_info(), screen=True, verbose=verbose) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) # noqa: F821 sys.exit() # Make sure the environmental variable reference to the data directory is set and readable try: datadir = os.environ["PSP_DATA_DIR"] - except: - raise KeyError( - "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" + except: # noqa: E722 + raise KeyError( # noqa: B904, TRY003 + "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" # noqa: EM101 ) - if not os.path.exists(datadir): - raise ValueError( - "Directory specified in env. variable PSP_DATA_DIR does not exist" + if not os.path.exists(datadir): # noqa: PTH110 + raise ValueError( # noqa: TRY003 + "Directory specified in env. variable PSP_DATA_DIR does not exist" # noqa: EM101 ) # Return to main routine From dad85934ad69367dcdda3d33f0caf82e762ff04a Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:26:25 -0400 Subject: [PATCH 03/12] Minor cleanup of file --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 35 ++++++++++------------ 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index faabb58..41b0b95 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -5,6 +5,18 @@ # $LastChangedBy: acase $ """ # noqa: D400 +__all__ = [ + "cdf35e_35f", + "cdf351_353_354", + "cdf352", + "get_newest_kernel", + "get_newest_skeleton", + "main", + "secsubsec2scet", + "setup", + "statusmsg", +] + import argparse import datetime import glob @@ -300,10 +312,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 logfile.close() -##################################################### -## -##################################################### -def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, ANN201, C901, FBT002 +def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, FBT002 """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 # Calculate MET from the variables in the L0 data # MET of each NYS @@ -526,10 +535,8 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 pdb.set_trace() # noqa: T100 -##################################################### -## -##################################################### -def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915 + +def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, FBT002 try: # Calculate SCET from the variables in the L0 data dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) @@ -663,9 +670,6 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, return () -##################################################### -## -##################################################### def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # noqa: D400 sec_str = [f"{i:1.0f}" for i in sec] @@ -686,9 +690,6 @@ def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN00 return ephem_nanosec_j2000 # noqa: RET504 -##################################################### -## -##################################################### def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, ANN201, FBT002 """Output status message to screen or logfile (default to file, but not screen)""" # noqa: D400 nowdtstr = datetime.datetime.now().isoformat() # noqa: DTZ005 @@ -699,9 +700,6 @@ def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, print(string) # noqa: T201 -##################################################### -## -##################################################### def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 """Find the path to the newest NAIF TLS (leap second) kernel file""" # noqa: D400 # Make sure we chose exactly one of the options @@ -738,9 +736,6 @@ def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, AN return path # noqa: RET504 -##################################################### -## -##################################################### def get_newest_skeleton(apid, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 """Find the path to the newest skeleton CDF file""" # noqa: D400 return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # noqa: FURB116 From 3b91f1b996709f3b0dbcd4336f45753988d3ae04 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Fri, 4 Sep 2026 20:30:58 -0400 Subject: [PATCH 04/12] Update imports --- src/pyfaradaycup/pipeline/__init__.py | 2 ++ src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py index e69de29..6e86e0f 100644 --- a/src/pyfaradaycup/pipeline/__init__.py +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -0,0 +1,2 @@ +from . import swp_spc_l02l1 +from . import ccsds_reader_pipeline diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index 41b0b95..4673164 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -29,17 +29,21 @@ try: from spacepy import pycdf except: # noqa: E722 + # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) + # then we don't need this. print(sys.exc_info()) # noqa: T201 print("***ERROR*** Could not import pycdf from spacepy") # noqa: T201 print( # noqa: T201 "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" ) sys.exit() + import distutils.dir_util -import ccsds_reader_pipeline as cc import spiceypy +import pyfaradaycup.pipeline.ccsds_reader_pipeline as cc + # Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft # into L0.5 or L1 CDF files @@ -535,7 +539,6 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 pdb.set_trace() # noqa: T100 - def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, FBT002 try: # Calculate SCET from the variables in the L0 data From f4e171cd3a894774b85c4da6537a7898d1eab22b Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 16:52:38 -0400 Subject: [PATCH 05/12] Remove flake8-tidy-imports.banned-api settings These are mostly like changing `typing.Callable` to `collections.abc.Callable` in type annotations, and are unlikely to come up anymore. --- pyproject.toml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a6e867c..90346fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -291,20 +291,6 @@ lint.flake8-import-conventions.banned-from = [ "warnings", ] lint.flake8-tidy-imports.ban-relative-imports = "all" -lint.flake8-tidy-imports.banned-api."typing.Callable".msg = "Deprecated alias. Change to collections.abc.Callable" -lint.flake8-tidy-imports.banned-api."typing.Collection".msg = "Deprecated alias. Change to collections.abc.Collection" -lint.flake8-tidy-imports.banned-api."typing.DefaultDict".msg = "Deprecated alias. Change to collections.defaultdict" -lint.flake8-tidy-imports.banned-api."typing.Dict".msg = "Deprecated alias. Change to dict" -lint.flake8-tidy-imports.banned-api."typing.Generator".msg = "Deprecated alias. Change to collections.abc.Generator" -lint.flake8-tidy-imports.banned-api."typing.Iterable".msg = "Deprecated alias. Change to collections.abc.Iterable" -lint.flake8-tidy-imports.banned-api."typing.Iterator".msg = "Deprecated alias. Change to collections.abc.Iterator" -lint.flake8-tidy-imports.banned-api."typing.List".msg = "Deprecated alias. Change to list" -lint.flake8-tidy-imports.banned-api."typing.Mapping".msg = "Deprecated alias. Change to collections.abc.mapping" -lint.flake8-tidy-imports.banned-api."typing.MutableMapping".msg = "Deprecated alias. Change to collections.abc.MutableMapping" -lint.flake8-tidy-imports.banned-api."typing.Sequence".msg = "Deprecated alias. Change to collections.abc.Sequence" -lint.flake8-tidy-imports.banned-api."typing.Set".msg = "Deprecated alias. Change to set" -lint.flake8-tidy-imports.banned-api."typing.Tuple".msg = "Deprecated alias. Change to tuple" -lint.flake8-tidy-imports.banned-api."typing.Type".msg = "Deprecated alias. Change to type" lint.isort.known-first-party = [ "pyfaradaycup" ] lint.pydocstyle.convention = "numpy" lint.pylint.max-positional-args = 6 From 83b6438c9cad4e3886fd39fefa33ee178868571e Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 17:22:55 -0400 Subject: [PATCH 06/12] Update import --- src/pyfaradaycup/pipeline/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py index 6e86e0f..09e2f76 100644 --- a/src/pyfaradaycup/pipeline/__init__.py +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -1,2 +1 @@ -from . import swp_spc_l02l1 -from . import ccsds_reader_pipeline +from . import ccsds_reader_pipeline, swp_spc_l02l1 From d2424199630fe497f16059364895064fa519b070 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 17:23:50 -0400 Subject: [PATCH 07/12] Update import and type annotation --- src/pyfaradaycup/pipeline/__init__.py | 2 +- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py index 09e2f76..af0c8c1 100644 --- a/src/pyfaradaycup/pipeline/__init__.py +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -1 +1 @@ -from . import ccsds_reader_pipeline, swp_spc_l02l1 +from pyfaradaycup.pipeline import ccsds_reader_pipeline, swp_spc_l02l1 diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index 4673164..b225c46 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -316,7 +316,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 logfile.close() -def cdf35e_35f(cdf, dat, verbose=False): # noqa: ANN001, FBT002 +def cdf35e_35f(cdf, dat, verbose=False) -> None: # noqa: ANN001, FBT002 """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 # Calculate MET from the variables in the L0 data # MET of each NYS From eea85eb983ea70ad532b11c7bd24f30ce18bd55b Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 17:24:30 -0400 Subject: [PATCH 08/12] Add noqa statements --- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index b225c46..f33efec 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -29,7 +29,7 @@ try: from spacepy import pycdf except: # noqa: E722 - # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) + # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) # noqa: FIX002, TD002, TD003 # then we don't need this. print(sys.exc_info()) # noqa: T201 print("***ERROR*** Could not import pycdf from spacepy") # noqa: T201 @@ -316,7 +316,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 logfile.close() -def cdf35e_35f(cdf, dat, verbose=False) -> None: # noqa: ANN001, FBT002 +def cdf35e_35f(cdf, dat, verbose=False) -> None: # noqa: ANN001, C901, FBT002 """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 # Calculate MET from the variables in the L0 data # MET of each NYS @@ -539,7 +539,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 pdb.set_trace() # noqa: T100 -def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, FBT002 +def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915 try: # Calculate SCET from the variables in the L0 data dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) From d88d1fc61b4389dde9d062ffa3207c2718a19433 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 17:45:09 -0400 Subject: [PATCH 09/12] Add ty:ignore comments --- noxfile.py | 6 +++--- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/noxfile.py b/noxfile.py index f135b56..d1147ba 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,9 +24,9 @@ import os import pathlib -import nox -import nox.command -import nox_uv +import nox # ty:ignore[unresolved-import] +import nox.command # ty:ignore[unresolved-import] +import nox_uv # ty:ignore[unresolved-import] nox.options.default_venv_backend = "uv" diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index f33efec..ab965fa 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -40,7 +40,7 @@ import distutils.dir_util -import spiceypy +import spiceypy # ty:ignore[unresolved-import] import pyfaradaycup.pipeline.ccsds_reader_pipeline as cc @@ -101,7 +101,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 f"swp_spc_l02l1_{nowdt.year:04.0f}{nowdt.month:02.0f}{nowdt.day:02.0f}{nowdt.hour:02.0f}{nowdt.minute:02.0f}{nowdt.second:02.0f}.log", ) try: - global logfile # noqa: PLW0603 + global logfile # noqa: PLW0603 # ty:ignore[unresolved-global] logfile = open(logpath, "w") # noqa: PTH123, SIM115 except: # noqa: E722 print("\n***ERROR*** Could not open log file!\n") # noqa: T201 @@ -269,7 +269,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 # Create a new CDF file from the provided skeleton try: cdf = pycdf.CDF(l1path, skeleton_filename) - except "CDFError": # noqa: B030 + except "CDFError": # noqa: B030 # ty:ignore[invalid-exception-caught] statusmsg( f"\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={apid})...continuing to next APID\n).", screen=True, @@ -894,13 +894,13 @@ def setup(): # noqa: ANN201 statusmsg( "***ERROR*** You must provide --l0file, if not using -b or -r", screen=True, - verbose=verbose, # noqa: F821 + verbose=verbose, # noqa: F821 # ty:ignore[unresolved-reference] ) elif args.l0dir == "": statusmsg( "***ERROR*** You must provide --l0dir if using -b or -r", screen=True, - verbose=verbose, # noqa: F821 + verbose=verbose, # noqa: F821 # ty:ignore[unresolved-reference] ) # Convert APID to an integer (it is read as a string from the command line) @@ -914,9 +914,9 @@ def setup(): # noqa: ANN201 statusmsg( "Trouble parsing desired APID....exiting.", screen=True, - verbose=verbose, # noqa: F821 + verbose=verbose, # noqa: F821 # ty:ignore[unresolved-reference] ) - statusmsg(sys.exc_info(), screen=True, verbose=verbose) # noqa: F821 + statusmsg(sys.exc_info(), screen=True, verbose=verbose) # noqa: F821 # ty:ignore[unresolved-reference] sys.exit() # Make sure the environmental variable reference to the data directory is set and readable From 0d557ffc822713ff0ed287765bf3514017fe5385 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 18:22:38 -0400 Subject: [PATCH 10/12] Change noqa comments -> ruff:ignore --- .../pipeline/ccsds_reader_pipeline.py | 226 +++++++++--------- 1 file changed, 113 insertions(+), 113 deletions(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index 38a8697..196a995 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -3,7 +3,7 @@ # $LastChangedRevision: 103 $ # $LastChangedDate: 2020-08-13 08:42:52 -0400 (Thu, 13 Aug 2020) $ # $LastChangedBy: acase $ -""" # noqa: D400 +""" # ruff:ignore[D400] __all__ = [ "apid_obj", @@ -35,45 +35,45 @@ ######################################### -def read_stdin(ptp=False, verbose=False): # noqa: ANN001, ANN201, FBT002 - """Parse binary stream on stdin""" # noqa: D400 +def read_stdin(ptp=False, verbose=False): # ruff:ignore[ANN001, ANN201, FBT002] + """Parse binary stream on stdin""" # ruff:ignore[D400] ######################################### -def file2bytestr(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def file2bytestr(path="", verbose=False, gzip=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] try: if gzip: - import gzip # noqa: PLC0415 + import gzip # ruff:ignore[PLC0415] with gzip.open(path, "rb") as f: bytestr = f.read() - return bytestr # noqa: RET504 - with open(path, "rb") as f: # noqa: PTH123 + return bytestr # ruff:ignore[RET504] + with open(path, "rb") as f: # ruff:ignore[PTH123] bytestr = f.read() - return bytestr # noqa: RET504, TRY300 + return bytestr # ruff:ignore[RET504, TRY300] - except: # noqa: E722 - print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") # noqa: T201 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") # ruff:ignore[T201] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] sys.exit() ######################################### -def choose_file(path="", ptp=False, verbose=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def choose_file(path="", ptp=False, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] # make sure file exists try: - open(path).close() # noqa: PTH123 - except: # noqa: E722 - print("***ERROR*** File can not be read...will give option to choose file") # noqa: T201 + open(path).close() # ruff:ignore[PTH123] + except: # ruff:ignore[E722] + print("***ERROR*** File can not be read...will give option to choose file") # ruff:ignore[T201] path = "" # pop up a dialog to choose a file if path=='' # path = 'C:\\Users\\comra_000\\SWEAP\\SPC\\FEU\\Testing\\20150228_UCB_SPC_FEU_LVPS_PTP_data\\PTP_data.dat' if path == "": - print("***ERROR*** Must define a file path") # noqa: T201 + print("***ERROR*** Must define a file path") # ruff:ignore[T201] # root = Tkinter.Tk() # root.withdraw() # path = tkFileDialog.askopenfilename() @@ -82,7 +82,7 @@ def choose_file(path="", ptp=False, verbose=False): # noqa: ANN001, ANN201, ARG ######################################### -def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] # get a filename if not specified path = choose_file(path) @@ -92,7 +92,7 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: # define the apids that are ok wrapper_apids = range(0x348, 0x351) - if spconly: # noqa: SIM108 + if spconly: # ruff:ignore[SIM108] ok_apids = [0x351, 0x352, 0x353, 0x354, 0x35E, 0x35F] else: ok_apids = range(0x351, 0x3A0, 1) @@ -126,13 +126,13 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ) try: pkt_starts = pkt_inds[:, 0] - except: # noqa: E722 + except: # ruff:ignore[E722] return data - npackets = len(pkt_starts) # noqa: F841 + npackets = len(pkt_starts) # ruff:ignore[F841] # Loop through each packet beginning and decommutate it - for i_pointer, pointer in enumerate(pkt_starts): # noqa: B007 + for i_pointer, pointer in enumerate(pkt_starts): # ruff:ignore[B007] wrap_cchead = parse_ccsds_head(bytestr[pointer : pointer + 10]) data_cchead = parse_ccsds_head(bytestr[pointer + 12 : pointer + 22]) data["wrap_met"].append(wrap_cchead["CCSDS_MET"]) @@ -146,8 +146,8 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ######################################### -def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901, FBT002 - """Read a CCSDS File and return data structure""" # noqa: D400 +def read_file(path="", verbose=False, gzip=False): # ruff:ignore[ANN001, ANN201, C901, FBT002] + """Read a CCSDS File and return data structure""" # ruff:ignore[D400] # get a filename if not specified path = choose_file(path) @@ -198,7 +198,7 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 ) try: pkt_starts = pkt_inds[:, 0] - except: # noqa: E722 + except: # ruff:ignore[E722] return data npackets = len(pkt_starts) @@ -209,13 +209,13 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr( # noqa: F841 + foo = read_bytestr( # ruff:ignore[F841] bytestr, pointer + 12, data, apidformat, pktcnt, verbose=verbose ) # Update status nowtime = time.time() - if (nowtime - updatetime) > 0.5: # noqa: PLR2004 + if (nowtime - updatetime) > 0.5: # ruff:ignore[PLR2004] sys.stdout.write( "\b" * 40 + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." @@ -228,7 +228,7 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" ) sys.stdout.write("Packet Summary\n") - for thisapid in pktcnt[0].keys(): # noqa: SIM118 + for thisapid in pktcnt[0].keys(): # ruff:ignore[SIM118] sys.stdout.write( f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" ) @@ -238,8 +238,8 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 ######################################### -def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0915 - """Read a CCSDS File and return data structure""" # noqa: D400 +def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0915] + """Read a CCSDS File and return data structure""" # ruff:ignore[D400] # get a filename if not specified path = choose_file(path) @@ -251,7 +251,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 # Those versions (and respective dates) are listed in the L1 APID257 file # That file is created via psp_sc_hsk_257_l052l1.py # Corresponding SC_HK files that we will read in are in ./sc_hk_def/ - with open("/psp/data/sc_hsk/L1/APID257_combined.txt") as f: # noqa: PTH123 + with open("/psp/data/sc_hsk/L1/APID257_combined.txt") as f: # ruff:ignore[PTH123] lines = f.readlines() vers_dt = np.array([dateutil.parser.isoparse(line.split(",")[0]) for line in lines]) versions = np.array([line.split(",")[1].strip() for line in lines]) @@ -269,7 +269,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 # and thus which SC_HK.blk file to use # we'll assume the first bytes in the file are a header try: - if ptp: # noqa: SIM108 + if ptp: # ruff:ignore[SIM108 cchead = parse_ccsds_head(bytestr[17:]) else: cchead = parse_ccsds_head(bytestr) @@ -278,8 +278,8 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 | (cchead["CCSDS_PacketType"] != 0) | (cchead["CCSDS_SecHdrFlag"] != 1) ): - raise ValueError("CCSDS header values not as expected") # noqa: EM101, TRY003 - file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( # noqa: DTZ001 + raise ValueError("CCSDS header values not as expected") # ruff:ignore[EM101, TRY003] + file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( # ruff:ignore[DTZ001] seconds=cchead["CCSDS_MET"] ) try: @@ -287,10 +287,10 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - print("Could not find which SC_HK file to use based on packet header") # noqa: T201 - print("Attempting to find correct date based on filename/path") # noqa: T201 + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + print("Could not find which SC_HK file to use based on packet header") # ruff:ignore[T201] + print("Attempting to find correct date based on filename/path") # ruff:ignore[T201] try: match = re.search( os.path.sep @@ -301,7 +301,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 path, ).span() # ty: ignore[unresolved-attribute] file_dt = ( - datetime.datetime( # noqa: DTZ001 + datetime.datetime( # ruff:ignore[DTZ001] int(path[match[0] + 1 : match[0] + 5]), 1, 1 ) + datetime.timedelta(days=int(path[match[0] + 6 : match[0] + 9]) - 1) @@ -311,8 +311,8 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: # noqa: E722 - print( # noqa: T201 + except: # ruff:ignore[E722] + print( # ruff:ignore[T201] "***WARNING*** Could not find date based on filename...using most recent" ) sc_hk_filename = sc_hk_filenames[-1] # ty: ignore[invalid-argument-type] @@ -329,7 +329,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 apidformat[apid], lengths[apid] = get_layout_sc( apid, verbose=verbose, - filename=os.path.join("sc_hk_def", sc_hk_filename), # noqa: PTH118 + filename=os.path.join("sc_hk_def", sc_hk_filename), # ruff:ignore[PTH118] ) if apidformat[apid]: data[apid] = {} @@ -371,7 +371,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 "2B", (2048 + inst_ap & 0xFF00) >> 8, 2048 + inst_ap & 0x00FF ) pattern += b".." - if inst_ap == 0x256: # noqa: PLR2004 + if inst_ap == 0x256: # ruff:ignore[PLR2004] # because the length shown in SPP.SC.HK.XX.YY.ZZ_GWW.blk doesn't correspond to packet length # we just hard-code the length # As of 2020/06/08 there were only two different possible sizes of 0x256 packets 0x098d and 0x0a91 @@ -395,7 +395,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 ) try: pkt_starts = pkt_inds[:, 0] - except: # noqa: E722 + except: # ruff:ignore[E722] return data npackets = len(pkt_starts) @@ -405,13 +405,13 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr( # noqa: F841 + foo = read_bytestr( # ruff:ignore[F841] bytestr, pointer + offset_bytes, data, apidformat, pktcnt, verbose=verbose ) # Update status nowtime = time.time() - if (nowtime - updatetime) > 0.5: # noqa: PLR2004 + if (nowtime - updatetime) > 0.5: # ruff:ignore[PLR2004] sys.stdout.write( "\b" * 40 + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." @@ -424,7 +424,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" ) sys.stdout.write("Packet Summary\n") - for thisapid in pktcnt[0].keys(): # noqa: SIM118 + for thisapid in pktcnt[0].keys(): # ruff:ignore[SIM118] sys.stdout.write( f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" ) @@ -434,14 +434,14 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 ######################################### -def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0913, RET503 - """Take a hex string and find packets""" # noqa: D400 +def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0913, RET503] + """Take a hex string and find packets""" # ruff:ignore[D400] # Parse the CCSDS header try: ccsds_head = parse_ccsds_head(bytestr[pointer : pointer + 10]) except ValueError: if verbose: - print("Full CCSDS Header Not Present") # noqa: T201 + print("Full CCSDS Header Not Present") # ruff:ignore[T201] return () apid = ccsds_head["CCSDS_ApID"] pkt_len = ccsds_head["CCSDS_PacketLen"] @@ -449,30 +449,30 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # # Verify that the CCSDS header is valid if ccsds_head["CCSDS_Version"] != 0: if verbose: - print("CCSDS Version is invalid") # noqa: T201 + print("CCSDS Version is invalid") # ruff:ignore[T201] return () if ccsds_head["CCSDS_PacketType"] != 0: if verbose: - print("CCSDS Type is invalid") # noqa: T201 + print("CCSDS Type is invalid") # ruff:ignore[T201] return () if ccsds_head["CCSDS_SecHdrFlag"] != 1: if verbose: - print("CCSDS Secondary Header flag is invalid") # noqa: T201 + print("CCSDS Secondary Header flag is invalid") # ruff:ignore[T201] return () # Make sure the full packet is here if pointer + pkt_len + 7 > len(bytestr): if verbose: - print("Full CCSDS packet not available at end of bytestr") # noqa: T201 + print("Full CCSDS packet not available at end of bytestr") # ruff:ignore[T201] return () # This packet only (no PTP header and no wrapper header (if they existed)) thispkt = bytestr[pointer : pointer + pkt_len + 7] # make sure we know how to decom this packet - if apid in apidformat.keys(): # noqa: SIM118 + if apid in apidformat.keys(): # ruff:ignore[SIM118] # count this as a good packet pktcnt[0][apid] += 1 @@ -481,7 +481,7 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # thispkt, data, apidformat, apid, ccsds_head ) # could send this off to a parallel task? Might try that if too slow this way - elif apid in pktcnt[1].keys(): # noqa: SIM118 + elif apid in pktcnt[1].keys(): # ruff:ignore[SIM118] pktcnt[1][apid] += 1 else: pktcnt[1][apid] = 1 @@ -489,18 +489,18 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # return () # we shouldn't make it here - import pdb # noqa: PLC0415, T100 + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] ######################################### -def parse_ccsds_head(bytestr, verbose=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def parse_ccsds_head(bytestr, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] bytearr = struct.unpack("B" * len(bytestr), bytestr) exp_length = 10 if len(bytearr) < exp_length: - raise ValueError("CCSDS header is not as long as expected") # noqa: EM101, TRY003 + raise ValueError("CCSDS header is not as long as expected") # ruff:ignore[EM101, TRY003] head = {} head["CCSDS_Version"] = bytearr[0] >> 5 @@ -519,21 +519,21 @@ def parse_ccsds_head(bytestr, verbose=False): # noqa: ANN001, ANN201, ARG001, D ######################################### -def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # noqa: ANN001, ANN201, ARG001, C901, FBT002, PLR0912, PLR0913 - """Parse one CCSDS packet""" # noqa: D400 +def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, C901, FBT002, PLR0912, PLR0913] + """Parse one CCSDS packet""" # ruff:ignore[D400] # The format for this APIDs packet list form = apidformat[apid] thisdat = data[apid] # Convert to a bit string bytearr = struct.unpack("B" * len(bytestr), bytestr) - str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) # noqa: FURB116 + str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) # ruff:ignore[FURB116] # For SWEAP packets, we just have each mnemonic listed and each bit length # So we have to step through them in order # Take care of the variables in sw_data (the repeating bit of the packet) separately pointer = 0 - if hasattr(form, "sw_data_vars"): # noqa: SIM108 + if hasattr(form, "sw_data_vars"): # ruff:ignore[SIM108] sw_data_vars_len = len(form.sw_data_vars) else: sw_data_vars_len = 0 @@ -565,7 +565,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no thisbin = str_bin[startbit:endbit] try: thisval = int(thisbin, 2) - except: # noqa: E722 + except: # ruff:ignore[E722] # print(sys.exc_info()) thisval = -999 thisdat[thisname].append(thisval) @@ -573,17 +573,17 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no # If the full packet isn't here, then don't bother parsing if len(bytearr) * 8.0 < sum(form.bits): - print(f"short packet: {hex(apid)}") # noqa: T201 + print(f"short packet: {hex(apid)}") # ruff:ignore[T201] return for i_bit, bit in enumerate(form.bits[0 : len(form.bits) - sw_data_vars_len]): thisbin = str_bin[pointer : pointer + bit] try: thisval = int(thisbin, 2) - except: # noqa: E722 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] thisval = -999 thisname = form.names[i_bit] @@ -610,9 +610,9 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no try: thisval = int(thisbin, 2) except ValueError: - import pdb # noqa: PLC0415, T100 + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] thisval = -999 thisname = form.sw_data_vars[i] @@ -624,8 +624,8 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no ######################################### -class apid_obj: # noqa: D101, N801 - def __init__(self): # noqa: ANN204 +class apid_obj: # ruff:ignore[D101, N801] + def __init__(self): # ruff:ignore[ANN204] self.names = [] self.bits = [] self.bytestart = [] @@ -638,34 +638,34 @@ def __init__(self): # noqa: ANN204 ######################################### -def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 +def get_layout(apid, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, FBT002] try: - file = open("sweap_tlm.blk") # noqa: PTH123, SIM115 - except: # noqa: E722 + file = open("sweap_tlm.blk") # ruff:ignore[PTH123, SIM115] + except: # ruff:ignore[E722] if verbose: - print( # noqa: T201 + print( # ruff:ignore[T201] "***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py" ) try: thisdir = os.path.realpath(__file__) thisdir = "\\".join(thisdir.split("\\")[0:-1]) - file = open(thisdir + "\\sweap_tlm.blk") # noqa: PTH123, SIM115 - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + file = open(thisdir + "\\sweap_tlm.blk") # ruff:ignore[PTH123, SIM115] + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] lines = file.readlines() for i, line in enumerate(lines): - if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): # noqa: FURB116 + if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): # ruff:ignore[FURB116] if verbose: - print(f"APID {hex(apid)[2:]} Format Found".upper()) # noqa: FURB116, T201 + print(f"APID {hex(apid)[2:]} Format Found".upper()) # ruff:ignore[FURB116, T201] thisapid = apid_obj() thisapid.apid = apid # ty: ignore[unresolved-attribute] - line = "" # so that the while loop will start out ok # noqa: PLW2901 + line = "" # so that the while loop will start out ok # ruff:ignore[PLW2901] while line[0:4] != "APID": - i += 1 # noqa: PLW2901 - line = lines[i] # noqa: PLW2901 + i += 1 # ruff:ignore[PLW2901] + line = lines[i] # ruff:ignore[PLW2901] try: if line.strip()[0] not in ["(", "{", "}", ")"]: pieces = re.split(",|;", line.strip()) @@ -680,11 +680,11 @@ def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 thisapid.sw_data_vars = [] # ty: ignore[unresolved-attribute] except IndexError: break - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722 + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] start = np.array( [0] + [sum(thisapid.bits[0:i]) for i in range(1, len(thisapid.bits))] @@ -699,36 +699,36 @@ def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 return thisapid # if we didn't find that APID - print( # noqa: T201 - f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # noqa: FURB116 + print( # ruff:ignore[T201 + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # ruff:ignore[FURB116] ) return None ######################################### -def get_layout_sc(apid, verbose=False, filename=""): # noqa: ANN001, ANN201, C901, D103, FBT002 +def get_layout_sc(apid, verbose=False, filename=""): # ruff:ignore[ANN001, ANN201, C901, D103, FBT002] try: - file = open(filename) # noqa: PTH123, SIM115 - print(f"using sc_hk file: {filename}") # noqa: T201 - except: # noqa: E722 - print("could not open SC HK BLK file") # noqa: T201 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + file = open(filename) # ruff:ignore[PTH123, SIM115] + print(f"using sc_hk file: {filename}") # ruff:ignore[T201] + except: # ruff:ignore[E722] + print("could not open SC HK BLK file") # ruff:ignore[T201] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] lines = file.readlines() for i, line in enumerate(lines): - if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": # noqa: FURB116 + if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": # ruff:ignore[FURB116] if verbose: - print(f"APID {hex(apid)[2:]} Format Found".upper()) # noqa: FURB116, T201 + print(f"APID {hex(apid)[2:]} Format Found".upper()) # ruff:ignore[FURB116, T201] thisapid = apid_obj() thisapid.apid = apid # ty: ignore[unresolved-attribute] - line = "" # noqa: PLW2901 + line = "" # ruff:ignore[PLW2901] while line[0:4] != "SC_H": - i += 1 # noqa: PLW2901 - line = lines[i].strip() # noqa: PLW2901 + i += 1 # ruff:ignore[PLW2901] + line = lines[i].strip() # ruff:ignore[PLW2901] if line[0:8] == "( Block[": length = int(line.split("[")[1].split("]")[0]) try: @@ -745,15 +745,15 @@ def get_layout_sc(apid, verbose=False, filename=""): # noqa: ANN001, ANN201, C9 thisapid.bits.append(int(pieces[3].strip())) except IndexError: break - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] return (thisapid, length) # if we didn't find that APID - print( # noqa: T201 - f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # noqa: FURB116 + print( # ruff:ignore[T201] + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # ruff:ignore[FURB116] ) return None From c6d532823c1043e00d172c16e070596aaef3c2c2 Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 18:40:01 -0400 Subject: [PATCH 11/12] Change noqa comments -> ruff:ignore --- .../pipeline/ccsds_reader_pipeline.py | 6 +- src/pyfaradaycup/pipeline/swp_spc_l02l1.py | 226 +++++++++--------- 2 files changed, 116 insertions(+), 116 deletions(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index 196a995..7fd8709 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -269,7 +269,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # ruff:ignore[ # and thus which SC_HK.blk file to use # we'll assume the first bytes in the file are a header try: - if ptp: # ruff:ignore[SIM108 + if ptp: # ruff:ignore[SIM108] cchead = parse_ccsds_head(bytestr[17:]) else: cchead = parse_ccsds_head(bytestr) @@ -680,7 +680,7 @@ def get_layout(apid, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, thisapid.sw_data_vars = [] # ty: ignore[unresolved-attribute] except IndexError: break - except: # ruff:ignore[E722 + except: # ruff:ignore[E722] print(sys.exc_info()) # ruff:ignore[T201] import pdb # ruff:ignore[PLC0415, T100] @@ -699,7 +699,7 @@ def get_layout(apid, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, return thisapid # if we didn't find that APID - print( # ruff:ignore[T201 + print( # ruff:ignore[T201] f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # ruff:ignore[FURB116] ) return None diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py index ab965fa..f9254f5 100644 --- a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -3,7 +3,7 @@ # $LastChangedRevision: 97 $ # $LastChangedDate: 2020-08-04 09:20:42 -0400 (Tue, 04 Aug 2020) $ # $LastChangedBy: acase $ -""" # noqa: D400 +""" # ruff:ignore[D400] __all__ = [ "cdf35e_35f", @@ -28,12 +28,12 @@ try: from spacepy import pycdf -except: # noqa: E722 - # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) # noqa: FIX002, TD002, TD003 +except: # ruff:ignore[E722] + # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) # ruff:ignore[FIX002, TD002, TD003] # then we don't need this. - print(sys.exc_info()) # noqa: T201 - print("***ERROR*** Could not import pycdf from spacepy") # noqa: T201 - print( # noqa: T201 + print(sys.exc_info()) # ruff:ignore[T201] + print("***ERROR*** Could not import pycdf from spacepy") # ruff:ignore[T201] + print( # ruff:ignore[T201] "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" ) sys.exit() @@ -64,47 +64,47 @@ # - Added revision history -def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 - l0file="", # noqa: ANN001 - l1dir="", # noqa: ANN001 - logdir="", # noqa: ANN001 - spacecraft=False, # noqa: ANN001, FBT002 - ptp=False, # noqa: ANN001, FBT002 - gzip=False, # noqa: ANN001, FBT002 - apidreq=0, # noqa: ANN001 - overwrite=False, # noqa: ANN001, FBT002 - verbose=False, # noqa: ANN001, FBT002 +def main( # ruff:ignore[ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917] + l0file="", # ruff:ignore[ANN001] + l1dir="", # ruff:ignore[ANN001] + logdir="", # ruff:ignore[ANN001] + spacecraft=False, # ruff:ignore[ANN001, FBT002] + ptp=False, # ruff:ignore[ANN001, FBT002] + gzip=False, # ruff:ignore[ANN001, FBT002] + apidreq=0, # ruff:ignore[ANN001] + overwrite=False, # ruff:ignore[ANN001, FBT002] + verbose=False, # ruff:ignore[ANN001, FBT002] ): - """Convert a single L0 file to L1""" # noqa: D400 + """Convert a single L0 file to L1""" # ruff:ignore[D400] # Try to create a filename for the new CDF that we're going to create - l0dirname = os.path.dirname(l0file) # noqa: PTH120 - l0basename = os.path.basename(l0file) # noqa: PTH119 + l0dirname = os.path.dirname(l0file) # ruff:ignore[PTH120] + l0basename = os.path.basename(l0file) # ruff:ignore[PTH119] if l1dir == "": l1dir = ( l0dirname # use input L0 directory for L1 files, if nothing else specified ) # Get a version of filename with no extension - l0file_noext = os.path.splitext(l0basename)[0] # noqa: PTH122 + l0file_noext = os.path.splitext(l0basename)[0] # ruff:ignore[PTH122] if l0file_noext[-3:] == "ptp": - l0file_noext = os.path.splitext(l0file_noext)[0] # noqa: PTH122 + l0file_noext = os.path.splitext(l0file_noext)[0] # ruff:ignore[PTH122] # Open a log file to write to - nowdt = datetime.datetime.now() # noqa: DTZ005 + nowdt = datetime.datetime.now() # ruff:ignore[DTZ005] if logdir == "": logdir = l1dir # use L1 file output directory for log file, if nothing else specified distutils.dir_util.mkpath( logdir ) # in case the directory doesn't exist, this will create it - logpath = os.path.join( # noqa: PTH118 + logpath = os.path.join( # ruff:ignore[PTH118] logdir, f"swp_spc_l02l1_{nowdt.year:04.0f}{nowdt.month:02.0f}{nowdt.day:02.0f}{nowdt.hour:02.0f}{nowdt.minute:02.0f}{nowdt.second:02.0f}.log", ) try: - global logfile # noqa: PLW0603 # ty:ignore[unresolved-global] - logfile = open(logpath, "w") # noqa: PTH123, SIM115 - except: # noqa: E722 - print("\n***ERROR*** Could not open log file!\n") # noqa: T201 + global logfile # ruff:ignore[PLW0603] # ty:ignore[unresolved-global] + logfile = open(logpath, "w") # ruff:ignore[PTH123, SIM115] + except: # ruff:ignore[E722] + print("\n***ERROR*** Could not open log file!\n") # ruff:ignore[T201] sys.exit(1) # Write some information to the log file @@ -120,7 +120,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 # Make sure the L0 file exists and is readable try: - foo = open(l0file) # noqa: PTH123, SIM115 + foo = open(l0file) # ruff:ignore[PTH123, SIM115] foo.close() statusmsg("L0 file exists and is readable") except OSError: @@ -129,9 +129,9 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 screen=True, verbose=verbose, ) - import pdb # noqa: PLC0415, T100 + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] sys.exit() # Load in Leap Second Kernel @@ -146,7 +146,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 try: statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {tls_path}") spiceypy.furnsh(tls_path) - except: # noqa: E722 + except: # ruff:ignore[E722] statusmsg( "***ERROR*** [swp_spc_l02l1.py] Could not furnsh leap second kernel...exiting" ) @@ -162,7 +162,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 try: statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {sclk_path}") spiceypy.furnsh(sclk_path) - except: # noqa: E722 + except: # ruff:ignore[E722] statusmsg( "***ERROR*** [swp_spc_l02l1.py] Could not furnsh SCLK kernel...exiting" ) @@ -178,10 +178,10 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 statusmsg("Event = Finished reading file") # Loop through the APIDs that we got - for apid in l0data.keys(): # noqa: SIM118 + for apid in l0data.keys(): # ruff:ignore[SIM118] statusmsg(f"Event = Beginning APID: {hex(apid)}") - if apid == 0x07B: # noqa: PLR2004 + if apid == 0x07B: # ruff:ignore[PLR2004] statusmsg( "***WARNING*** [swp_spc_l02l1] APID 0x07B CDFs not yet implemented", screen=True, @@ -190,7 +190,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 continue # Make sure we need to do this apid - if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: # noqa: RUF015 + if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: # ruff:ignore[RUF015] statusmsg("No packets found for this apid.") continue # skip this apid if there were no packets received if (apidreq != 0) & (apidreq != apid): @@ -198,16 +198,16 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 continue # skip this apid if user only wanted one apid and this isn't it # Filename for the L1 file we're about to write for this apid - l1path = os.path.join( # noqa: PTH118 + l1path = os.path.join( # ruff:ignore[PTH118] l1dir, - l0file_noext + f"_APID{str(hex(apid)[2:].zfill(3)).upper()}_L1.cdf", # noqa: FURB116 + l0file_noext + f"_APID{str(hex(apid)[2:].zfill(3)).upper()}_L1.cdf", # ruff:ignore[FURB116] ) statusmsg("About to write: " + l1path) # Make sure the skeleton file exists and is readable try: skeleton_filename = get_newest_skeleton(apid) - foo = open(skeleton_filename) # noqa: PTH123, SIM115 + foo = open(skeleton_filename) # ruff:ignore[PTH123, SIM115] foo.close() statusmsg("Skeleton to be used: " + skeleton_filename) except OSError: @@ -230,7 +230,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 try: # try to open and close it statusmsg("Using L1 path: " + l1path, screen=True, verbose=verbose) - foo = open(l1path) # noqa: PTH123, SIM115 + foo = open(l1path) # ruff:ignore[PTH123, SIM115] foo.close() # if we get here, this file already exists; so delete it, if desired @@ -245,19 +245,19 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 screen=True, verbose=verbose, ) - os.remove(l1path) # noqa: PTH107 + os.remove(l1path) # ruff:ignore[PTH107] else: statusmsg( "***ERROR*** [swp_spc_l02l1] L1 CDF already exists, and overwrite (-o option) was not requested...exiting.", screen=True, verbose=verbose, ) - raise (SystemExit) # noqa: TRY301 + raise (SystemExit) # ruff:ignore[TRY301] except SystemExit: sys.exit() except OSError: pass # Apparently the file did not exist already - except: # noqa: E722 + except: # ruff:ignore[E722] statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) statusmsg( "\n***ERROR*** [swp_spc_l02l1] Could not check existence/delete L1 CDF file path. Exiting...\n", @@ -269,7 +269,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 # Create a new CDF file from the provided skeleton try: cdf = pycdf.CDF(l1path, skeleton_filename) - except "CDFError": # noqa: B030 # ty:ignore[invalid-exception-caught] + except "CDFError": # ruff:ignore[B030] # ty:ignore[invalid-exception-caught] statusmsg( f"\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={apid})...continuing to next APID\n).", screen=True, @@ -295,7 +295,7 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 } try: cdfproc[apid](cdf, l0data[apid], verbose=verbose) - except: # noqa: E722 + except: # ruff:ignore[E722] statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) statusmsg( f"***WARNING*** [swp_spc_l02l1] CDF not processed for APID={hex(apid)}", @@ -316,37 +316,37 @@ def main( # noqa: ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917 logfile.close() -def cdf35e_35f(cdf, dat, verbose=False) -> None: # noqa: ANN001, C901, FBT002 - """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # noqa: D400 +def cdf35e_35f(cdf, dat, verbose=False) -> None: # ruff:ignore[ANN001, C901, FBT002] + """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # ruff:ignore[D400] # Calculate MET from the variables in the L0 data # MET of each NYS - if "CCSDS_MET" in dat.keys(): # noqa: SIM118 + if "CCSDS_MET" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPC_SUBSEC"]) - elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 + elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet( dat["FSW_HK_HK_INST_TPSH_MET_SEC"], dat["FSW_HK_HK_INST_TPSH_MET_SUBSEC"], spacecraft=True, ) - elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 + elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet( dat["PDU_PRIO94_TPSH_MET_SEC"], dat["PDU_PRIO94_TPSH_MET_SUBSEC"], spacecraft=True, ) - elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 + elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet( dat["HK_HIGH_TPSH_MET_SEC"], dat["HK_HIGH_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 + elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet( dat["HK_FSWL_TPSH_MET_SEC"], dat["HK_FSWL_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 + elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet( dat["HK_LOW_TPSH_MET_SEC"], dat["HK_LOW_TPSH_MET_SUBSEC"], spacecraft=True ) - elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): # noqa: SIM118 + elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] scet = secsubsec2scet( dat["RIU_DERIVED_TPSH_MET_SEC"], dat["RIU_DERIVED_TPSH_MET_SUBSEC"], @@ -364,13 +364,13 @@ def cdf35e_35f(cdf, dat, verbose=False) -> None: # noqa: ANN001, C901, FBT002 for key in keys: try: cdf[key] = dat[key] # create variable and insert data - except KeyError: # noqa: PERF203 - if key not in dat.keys(): # noqa: SIM118 + except KeyError: # ruff:ignore[PERF203] + if key not in dat.keys(): # ruff:ignore[SIM118] cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] - except: # noqa: E722 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] statusmsg( f"Failed : Key:{key} failed insert into CDF", screen=True, @@ -382,7 +382,7 @@ def cdf35e_35f(cdf, dat, verbose=False) -> None: # noqa: ANN001, C901, FBT002 ##################################################### ## ##################################################### -def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0915, RET503 +def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0915, RET503] """Fill up a CDF with SCI, ALL, or RSS data.""" # Take data sorted by NYS, and produce one long variable with all data @@ -391,11 +391,11 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 # Each different packet will require a different variable to calculate # The number of measurements each NYS - if apid == 0x351: # noqa: PLR2004 + if apid == 0x351: # ruff:ignore[PLR2004] length_var = "A1S" - elif apid == 0x353: # noqa: PLR2004 + elif apid == 0x353: # ruff:ignore[PLR2004] length_var = "ASIN" - elif apid == 0x354: # noqa: PLR2004 + elif apid == 0x354: # ruff:ignore[PLR2004] length_var = "ARSS" # Calculate MET from the variables in the L0 data @@ -403,12 +403,12 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) # MET of each measurement (to be filled in in the future) - scet_exp = [] # noqa: F841 + scet_exp = [] # ruff:ignore[F841] # Same keys as original data dictionary, but will hold one variable per key # instead of one for every NYS for every key dat_exp = {} - for key in dat.keys(): # noqa: SIM118 + for key in dat.keys(): # ruff:ignore[SIM118] dat_exp[key] = [] # Create the 'Epoch' variable in our data array @@ -423,7 +423,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 ticks_per_meas = dat["SW_SPC_INTTIME"][i] + dat["SW_SPC_SERVTIME"][i] # Make sure ST and IT are allowed values - if (math.log(ticks_per_meas, 2)) % 1 != 0: # noqa: FURB163 + if (math.log(ticks_per_meas, 2)) % 1 != 0: # ruff:ignore[FURB163] # the SPC FPGA will default to IT=6, ST=2 (the power-on defaults) if a non-integer power of 2 IT+ST is requested ticks_per_meas = 8 @@ -487,7 +487,7 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 add_time[thisrtpix:] += tm_per_meas # If we're in an AllGain packet, then the beginning of the packet might not be the beginning of the NYS (which is the time noted in the header) - if apid == 0x351: # noqa: PLR2004 + if apid == 0x351: # ruff:ignore[PLR2004] pktnum = dat["SW_SPC_PKTNUM"][i] # if pktnum==0: import pdb; pdb.set_trace() if pktnum != 0: @@ -505,10 +505,10 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 dat_exp["Epoch"].extend(dscet_extend) # Extend each of the data arrays - for key in dat.keys(): # noqa: SIM118 + for key in dat.keys(): # ruff:ignore[SIM118] try: dat_exp[key].extend(dat[key][i]) - except TypeError: # noqa: PERF203 + except TypeError: # ruff:ignore[PERF203] expanded = np.ones(nmeas) * dat[key][i] dat_exp[key].extend(expanded) @@ -527,19 +527,19 @@ def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN20 try: # insert data cdf[key] = dat_exp[key] - except: # noqa: E722, PERF203 + except: # ruff:ignore[E722, PERF203] statusmsg( f"Failed : Key:{key} failed insert into CDF", screen=True, verbose=verbose, ) statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - import pdb # noqa: PLC0415, T100 + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] -def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915 +def cdf352(cdf, dat, nocdf=False, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915] try: # Calculate SCET from the variables in the L0 data dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) @@ -547,7 +547,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, # Same keys as original data dictionary, but will hold one variable per key # instead of one for every NYS for every key dat_exp = {} - for key in dat.keys(): # noqa: SIM118 + for key in dat.keys(): # ruff:ignore[SIM118] if key[-4:] == "_000": continue dat_exp[key] = [] @@ -600,8 +600,8 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, try: if coll_used not in coll2var: - raise ValueError( # noqa: TRY003 - f"Value: {coll_used} not in coll2var.keys()" # noqa: EM102 + raise ValueError( # ruff:ignore[TRY003] + f"Value: {coll_used} not in coll2var.keys()" # ruff:ignore[EM102] ) # probably a corrupt packet dat_exp["VAR0_NAME"].extend( @@ -622,7 +622,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, dat_exp["VAR2"].extend(dat["G2_000"][i]) dat_exp["VAR3"].extend(dat["G3_000"][i]) - except: # noqa: E722 + except: # ruff:ignore[E722] statusmsg( "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" ) @@ -634,7 +634,7 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, dat_exp["Epoch"].extend(dt_extend) # Extend each of the data arrays - for key in dat.keys(): # noqa: SIM118 + for key in dat.keys(): # ruff:ignore[SIM118] if key[-4:] == "_000": continue try: @@ -657,24 +657,24 @@ def cdf352(cdf, dat, nocdf=False, verbose=False): # noqa: ANN001, ANN201, C901, try: # insert data cdf[key] = dat_exp[key] - except: # noqa: E722, PERF203 + except: # ruff:ignore[E722, PERF203] statusmsg( f"Failed : Key:{key} failed insert into CDF", screen=True, verbose=verbose, ) statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] return () -def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 - """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # noqa: D400 +def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, FBT002] + """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # ruff:ignore[D400] sec_str = [f"{i:1.0f}" for i in sec] subsec_str_base50000 = [ f"{int(i * 50000 / 65536):05.0f}" for i in subsec @@ -690,21 +690,21 @@ def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # noqa: ANN00 ] ephem_nanosec_j2000 = [np.round(1e9 * i) for i in ephem_sec_j2000] - return ephem_nanosec_j2000 # noqa: RET504 + return ephem_nanosec_j2000 # ruff:ignore[RET504] -def statusmsg(string, screen=False, file=True, verbose=False): # noqa: ANN001, ANN201, FBT002 - """Output status message to screen or logfile (default to file, but not screen)""" # noqa: D400 - nowdtstr = datetime.datetime.now().isoformat() # noqa: DTZ005 +def statusmsg(string, screen=False, file=True, verbose=False): # ruff:ignore[ANN001, ANN201, FBT002] + """Output status message to screen or logfile (default to file, but not screen)""" # ruff:ignore[D400] + nowdtstr = datetime.datetime.now().isoformat() # ruff:ignore[DTZ005] if file: logfile.write(nowdtstr + ", " + string + "\n") - if screen: # noqa: SIM102 + if screen: # ruff:ignore[SIM102] if verbose: - print(string) # noqa: T201 + print(string) # ruff:ignore[T201] -def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 - """Find the path to the newest NAIF TLS (leap second) kernel file""" # noqa: D400 +def get_newest_kernel(tls=False, sclk=False, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, FBT002] + """Find the path to the newest NAIF TLS (leap second) kernel file""" # ruff:ignore[D400] # Make sure we chose exactly one of the options if tls + sclk != 1: return False @@ -719,7 +719,7 @@ def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, AN globstr = globdir + "spp_sclk_[0-9][0-9][0-9][0-9].tsc" ndigits = 4 - files = glob.glob(globstr) # noqa: PTH207 + files = glob.glob(globstr) # ruff:ignore[PTH207] # isolate version numbers from the file path and find newest if tls or sclk: @@ -728,20 +728,20 @@ def get_newest_kernel(tls=False, sclk=False, verbose=False): # noqa: ANN001, AN maxind = np.argmax(versions) except ValueError: statusmsg("***ERROR*** Could not find kernel versions") - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] return False # return path to newest file path = files[maxind] - return path # noqa: RET504 + return path # ruff:ignore[RET504] -def get_newest_skeleton(apid, verbose=False): # noqa: ANN001, ANN201, ARG001, FBT002 - """Find the path to the newest skeleton CDF file""" # noqa: D400 - return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # noqa: FURB116 +def get_newest_skeleton(apid, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, FBT002] + """Find the path to the newest skeleton CDF file""" # ruff:ignore[D400] + return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # ruff:ignore[FURB116] # The remaining code in this function is from when we used skeleton file numbers with a version # in them # and we had to search for the most recent (highest) version @@ -770,8 +770,8 @@ def get_newest_skeleton(apid, verbose=False): # noqa: ANN001, ANN201, ARG001, F ##################################################### ### ##################################################### -def setup(): # noqa: ANN201 - """Get user command-line input and set things up""" # noqa: D400 +def setup(): # ruff:ignore[ANN201] + """Get user command-line input and set things up""" # ruff:ignore[D400] # defaults l0file_default = "" l0dir_default = "" @@ -894,18 +894,18 @@ def setup(): # noqa: ANN201 statusmsg( "***ERROR*** You must provide --l0file, if not using -b or -r", screen=True, - verbose=verbose, # noqa: F821 # ty:ignore[unresolved-reference] + verbose=verbose, # ruff:ignore[F821] # ty:ignore[unresolved-reference] ) elif args.l0dir == "": statusmsg( "***ERROR*** You must provide --l0dir if using -b or -r", screen=True, - verbose=verbose, # noqa: F821 # ty:ignore[unresolved-reference] + verbose=verbose, # ruff:ignore[F821] # ty:ignore[unresolved-reference] ) # Convert APID to an integer (it is read as a string from the command line) try: - if args.apid[0:2] == "0x": # noqa: SIM108 + if args.apid[0:2] == "0x": # ruff:ignore[SIM108] base = 16 else: base = 10 @@ -914,22 +914,22 @@ def setup(): # noqa: ANN201 statusmsg( "Trouble parsing desired APID....exiting.", screen=True, - verbose=verbose, # noqa: F821 # ty:ignore[unresolved-reference] + verbose=verbose, # ruff:ignore[F821] # ty:ignore[unresolved-reference] ) - statusmsg(sys.exc_info(), screen=True, verbose=verbose) # noqa: F821 # ty:ignore[unresolved-reference] + statusmsg(sys.exc_info(), screen=True, verbose=verbose) # ruff:ignore[F821] # ty:ignore[unresolved-reference] sys.exit() # Make sure the environmental variable reference to the data directory is set and readable try: datadir = os.environ["PSP_DATA_DIR"] - except: # noqa: E722 - raise KeyError( # noqa: B904, TRY003 - "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" # noqa: EM101 + except: # ruff:ignore[E722] + raise KeyError( # ruff:ignore[B904, TRY003] + "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" # ruff:ignore[EM101] ) - if not os.path.exists(datadir): # noqa: PTH110 - raise ValueError( # noqa: TRY003 - "Directory specified in env. variable PSP_DATA_DIR does not exist" # noqa: EM101 + if not os.path.exists(datadir): # ruff:ignore[PTH110] + raise ValueError( # ruff:ignore[TRY003] + "Directory specified in env. variable PSP_DATA_DIR does not exist" # ruff:ignore[EM101] ) # Return to main routine From c0f80d7df1562e5fdb2ed459d53f347a8aac34fa Mon Sep 17 00:00:00 2001 From: Nick Murphy Date: Wed, 9 Sep 2026 18:45:55 -0400 Subject: [PATCH 12/12] pre-commit --- src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index 7fd8709..d0ec20e 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -662,7 +662,9 @@ def get_layout(apid, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, print(f"APID {hex(apid)[2:]} Format Found".upper()) # ruff:ignore[FURB116, T201] thisapid = apid_obj() thisapid.apid = apid # ty: ignore[unresolved-attribute] - line = "" # so that the while loop will start out ok # ruff:ignore[PLW2901] + line = ( + "" # so that the while loop will start out ok # ruff:ignore[PLW2901] + ) while line[0:4] != "APID": i += 1 # ruff:ignore[PLW2901] line = lines[i] # ruff:ignore[PLW2901]