diff --git a/noxfile.py b/noxfile.py index f135b56..d1147ba 100644 --- a/noxfile.py +++ b/noxfile.py @@ -24,9 +24,9 @@ import os import pathlib -import nox -import nox.command -import nox_uv +import nox # ty:ignore[unresolved-import] +import nox.command # ty:ignore[unresolved-import] +import nox_uv # ty:ignore[unresolved-import] nox.options.default_venv_backend = "uv" diff --git a/src/pyfaradaycup/pipeline/__init__.py b/src/pyfaradaycup/pipeline/__init__.py index e69de29..af0c8c1 100644 --- a/src/pyfaradaycup/pipeline/__init__.py +++ b/src/pyfaradaycup/pipeline/__init__.py @@ -0,0 +1 @@ +from pyfaradaycup.pipeline import ccsds_reader_pipeline, swp_spc_l02l1 diff --git a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py index 38a8697..d0ec20e 100644 --- a/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py +++ b/src/pyfaradaycup/pipeline/ccsds_reader_pipeline.py @@ -3,7 +3,7 @@ # $LastChangedRevision: 103 $ # $LastChangedDate: 2020-08-13 08:42:52 -0400 (Thu, 13 Aug 2020) $ # $LastChangedBy: acase $ -""" # noqa: D400 +""" # ruff:ignore[D400] __all__ = [ "apid_obj", @@ -35,45 +35,45 @@ ######################################### -def read_stdin(ptp=False, verbose=False): # noqa: ANN001, ANN201, FBT002 - """Parse binary stream on stdin""" # noqa: D400 +def read_stdin(ptp=False, verbose=False): # ruff:ignore[ANN001, ANN201, FBT002] + """Parse binary stream on stdin""" # ruff:ignore[D400] ######################################### -def file2bytestr(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def file2bytestr(path="", verbose=False, gzip=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] try: if gzip: - import gzip # noqa: PLC0415 + import gzip # ruff:ignore[PLC0415] with gzip.open(path, "rb") as f: bytestr = f.read() - return bytestr # noqa: RET504 - with open(path, "rb") as f: # noqa: PTH123 + return bytestr # ruff:ignore[RET504] + with open(path, "rb") as f: # ruff:ignore[PTH123] bytestr = f.read() - return bytestr # noqa: RET504, TRY300 + return bytestr # ruff:ignore[RET504, TRY300] - except: # noqa: E722 - print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") # noqa: T201 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + print("***ERROR*** [ccsds_reader_pipeline] Could not read in file...exiting") # ruff:ignore[T201] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] sys.exit() ######################################### -def choose_file(path="", ptp=False, verbose=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def choose_file(path="", ptp=False, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] # make sure file exists try: - open(path).close() # noqa: PTH123 - except: # noqa: E722 - print("***ERROR*** File can not be read...will give option to choose file") # noqa: T201 + open(path).close() # ruff:ignore[PTH123] + except: # ruff:ignore[E722] + print("***ERROR*** File can not be read...will give option to choose file") # ruff:ignore[T201] path = "" # pop up a dialog to choose a file if path=='' # path = 'C:\\Users\\comra_000\\SWEAP\\SPC\\FEU\\Testing\\20150228_UCB_SPC_FEU_LVPS_PTP_data\\PTP_data.dat' if path == "": - print("***ERROR*** Must define a file path") # noqa: T201 + print("***ERROR*** Must define a file path") # ruff:ignore[T201] # root = Tkinter.Tk() # root.withdraw() # path = tkFileDialog.askopenfilename() @@ -82,7 +82,7 @@ def choose_file(path="", ptp=False, verbose=False): # noqa: ANN001, ANN201, ARG ######################################### -def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] # get a filename if not specified path = choose_file(path) @@ -92,7 +92,7 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: # define the apids that are ok wrapper_apids = range(0x348, 0x351) - if spconly: # noqa: SIM108 + if spconly: # ruff:ignore[SIM108] ok_apids = [0x351, 0x352, 0x353, 0x354, 0x35E, 0x35F] else: ok_apids = range(0x351, 0x3A0, 1) @@ -126,13 +126,13 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ) try: pkt_starts = pkt_inds[:, 0] - except: # noqa: E722 + except: # ruff:ignore[E722] return data - npackets = len(pkt_starts) # noqa: F841 + npackets = len(pkt_starts) # ruff:ignore[F841] # Loop through each packet beginning and decommutate it - for i_pointer, pointer in enumerate(pkt_starts): # noqa: B007 + for i_pointer, pointer in enumerate(pkt_starts): # ruff:ignore[B007] wrap_cchead = parse_ccsds_head(bytestr[pointer : pointer + 10]) data_cchead = parse_ccsds_head(bytestr[pointer + 12 : pointer + 22]) data["wrap_met"].append(wrap_cchead["CCSDS_MET"]) @@ -146,8 +146,8 @@ def wrapper_status(path="", verbose=False, gzip=False, spconly=False): # noqa: ######################################### -def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901, FBT002 - """Read a CCSDS File and return data structure""" # noqa: D400 +def read_file(path="", verbose=False, gzip=False): # ruff:ignore[ANN001, ANN201, C901, FBT002] + """Read a CCSDS File and return data structure""" # ruff:ignore[D400] # get a filename if not specified path = choose_file(path) @@ -198,7 +198,7 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 ) try: pkt_starts = pkt_inds[:, 0] - except: # noqa: E722 + except: # ruff:ignore[E722] return data npackets = len(pkt_starts) @@ -209,13 +209,13 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr( # noqa: F841 + foo = read_bytestr( # ruff:ignore[F841] bytestr, pointer + 12, data, apidformat, pktcnt, verbose=verbose ) # Update status nowtime = time.time() - if (nowtime - updatetime) > 0.5: # noqa: PLR2004 + if (nowtime - updatetime) > 0.5: # ruff:ignore[PLR2004] sys.stdout.write( "\b" * 40 + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." @@ -228,7 +228,7 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" ) sys.stdout.write("Packet Summary\n") - for thisapid in pktcnt[0].keys(): # noqa: SIM118 + for thisapid in pktcnt[0].keys(): # ruff:ignore[SIM118] sys.stdout.write( f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" ) @@ -238,8 +238,8 @@ def read_file(path="", verbose=False, gzip=False): # noqa: ANN001, ANN201, C901 ######################################### -def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0915 - """Read a CCSDS File and return data structure""" # noqa: D400 +def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0915] + """Read a CCSDS File and return data structure""" # ruff:ignore[D400] # get a filename if not specified path = choose_file(path) @@ -251,7 +251,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 # Those versions (and respective dates) are listed in the L1 APID257 file # That file is created via psp_sc_hsk_257_l052l1.py # Corresponding SC_HK files that we will read in are in ./sc_hk_def/ - with open("/psp/data/sc_hsk/L1/APID257_combined.txt") as f: # noqa: PTH123 + with open("/psp/data/sc_hsk/L1/APID257_combined.txt") as f: # ruff:ignore[PTH123] lines = f.readlines() vers_dt = np.array([dateutil.parser.isoparse(line.split(",")[0]) for line in lines]) versions = np.array([line.split(",")[1].strip() for line in lines]) @@ -269,7 +269,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 # and thus which SC_HK.blk file to use # we'll assume the first bytes in the file are a header try: - if ptp: # noqa: SIM108 + if ptp: # ruff:ignore[SIM108] cchead = parse_ccsds_head(bytestr[17:]) else: cchead = parse_ccsds_head(bytestr) @@ -278,8 +278,8 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 | (cchead["CCSDS_PacketType"] != 0) | (cchead["CCSDS_SecHdrFlag"] != 1) ): - raise ValueError("CCSDS header values not as expected") # noqa: EM101, TRY003 - file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( # noqa: DTZ001 + raise ValueError("CCSDS header values not as expected") # ruff:ignore[EM101, TRY003] + file_dt = datetime.datetime(2010, 1, 1) + datetime.timedelta( # ruff:ignore[DTZ001] seconds=cchead["CCSDS_MET"] ) try: @@ -287,10 +287,10 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - print("Could not find which SC_HK file to use based on packet header") # noqa: T201 - print("Attempting to find correct date based on filename/path") # noqa: T201 + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + print("Could not find which SC_HK file to use based on packet header") # ruff:ignore[T201] + print("Attempting to find correct date based on filename/path") # ruff:ignore[T201] try: match = re.search( os.path.sep @@ -301,7 +301,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 path, ).span() # ty: ignore[unresolved-attribute] file_dt = ( - datetime.datetime( # noqa: DTZ001 + datetime.datetime( # ruff:ignore[DTZ001] int(path[match[0] + 1 : match[0] + 5]), 1, 1 ) + datetime.timedelta(days=int(path[match[0] + 6 : match[0] + 9]) - 1) @@ -311,8 +311,8 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 except IndexError: good_time = 0 sc_hk_filename = sc_hk_filenames[versions[good_time]] - except: # noqa: E722 - print( # noqa: T201 + except: # ruff:ignore[E722] + print( # ruff:ignore[T201] "***WARNING*** Could not find date based on filename...using most recent" ) sc_hk_filename = sc_hk_filenames[-1] # ty: ignore[invalid-argument-type] @@ -329,7 +329,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 apidformat[apid], lengths[apid] = get_layout_sc( apid, verbose=verbose, - filename=os.path.join("sc_hk_def", sc_hk_filename), # noqa: PTH118 + filename=os.path.join("sc_hk_def", sc_hk_filename), # ruff:ignore[PTH118] ) if apidformat[apid]: data[apid] = {} @@ -371,7 +371,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 "2B", (2048 + inst_ap & 0xFF00) >> 8, 2048 + inst_ap & 0x00FF ) pattern += b".." - if inst_ap == 0x256: # noqa: PLR2004 + if inst_ap == 0x256: # ruff:ignore[PLR2004] # because the length shown in SPP.SC.HK.XX.YY.ZZ_GWW.blk doesn't correspond to packet length # we just hard-code the length # As of 2020/06/08 there were only two different possible sizes of 0x256 packets 0x098d and 0x0a91 @@ -395,7 +395,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 ) try: pkt_starts = pkt_inds[:, 0] - except: # noqa: E722 + except: # ruff:ignore[E722] return data npackets = len(pkt_starts) @@ -405,13 +405,13 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 # Loop through each packet beginning and decommutate it for i_pointer, pointer in enumerate(pkt_starts): - foo = read_bytestr( # noqa: F841 + foo = read_bytestr( # ruff:ignore[F841] bytestr, pointer + offset_bytes, data, apidformat, pktcnt, verbose=verbose ) # Update status nowtime = time.time() - if (nowtime - updatetime) > 0.5: # noqa: PLR2004 + if (nowtime - updatetime) > 0.5: # ruff:ignore[PLR2004] sys.stdout.write( "\b" * 40 + f"{(np.double(i_pointer)) / npackets * 100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec." @@ -424,7 +424,7 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 "\b" * 40 + f"{100.0:5.1f}% Complete. ET={nowtime - starttime:6.2f} sec.\n\n" ) sys.stdout.write("Packet Summary\n") - for thisapid in pktcnt[0].keys(): # noqa: SIM118 + for thisapid in pktcnt[0].keys(): # ruff:ignore[SIM118] sys.stdout.write( f"\tAPID {hex(thisapid)}: found {pktcnt[0][thisapid]:7.0f} packets\n" ) @@ -434,14 +434,14 @@ def read_file_sc(path="", verbose=False, ptp=False, gzip=False): # noqa: ANN001 ######################################### -def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # noqa: ANN001, ANN201, C901, FBT002, PLR0912, PLR0913, RET503 - """Take a hex string and find packets""" # noqa: D400 +def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0913, RET503] + """Take a hex string and find packets""" # ruff:ignore[D400] # Parse the CCSDS header try: ccsds_head = parse_ccsds_head(bytestr[pointer : pointer + 10]) except ValueError: if verbose: - print("Full CCSDS Header Not Present") # noqa: T201 + print("Full CCSDS Header Not Present") # ruff:ignore[T201] return () apid = ccsds_head["CCSDS_ApID"] pkt_len = ccsds_head["CCSDS_PacketLen"] @@ -449,30 +449,30 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # # Verify that the CCSDS header is valid if ccsds_head["CCSDS_Version"] != 0: if verbose: - print("CCSDS Version is invalid") # noqa: T201 + print("CCSDS Version is invalid") # ruff:ignore[T201] return () if ccsds_head["CCSDS_PacketType"] != 0: if verbose: - print("CCSDS Type is invalid") # noqa: T201 + print("CCSDS Type is invalid") # ruff:ignore[T201] return () if ccsds_head["CCSDS_SecHdrFlag"] != 1: if verbose: - print("CCSDS Secondary Header flag is invalid") # noqa: T201 + print("CCSDS Secondary Header flag is invalid") # ruff:ignore[T201] return () # Make sure the full packet is here if pointer + pkt_len + 7 > len(bytestr): if verbose: - print("Full CCSDS packet not available at end of bytestr") # noqa: T201 + print("Full CCSDS packet not available at end of bytestr") # ruff:ignore[T201] return () # This packet only (no PTP header and no wrapper header (if they existed)) thispkt = bytestr[pointer : pointer + pkt_len + 7] # make sure we know how to decom this packet - if apid in apidformat.keys(): # noqa: SIM118 + if apid in apidformat.keys(): # ruff:ignore[SIM118] # count this as a good packet pktcnt[0][apid] += 1 @@ -481,7 +481,7 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # thispkt, data, apidformat, apid, ccsds_head ) # could send this off to a parallel task? Might try that if too slow this way - elif apid in pktcnt[1].keys(): # noqa: SIM118 + elif apid in pktcnt[1].keys(): # ruff:ignore[SIM118] pktcnt[1][apid] += 1 else: pktcnt[1][apid] = 1 @@ -489,18 +489,18 @@ def read_bytestr(bytestr, pointer, data, apidformat, pktcnt, verbose=False): # return () # we shouldn't make it here - import pdb # noqa: PLC0415, T100 + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] ######################################### -def parse_ccsds_head(bytestr, verbose=False): # noqa: ANN001, ANN201, ARG001, D103, FBT002 +def parse_ccsds_head(bytestr, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, D103, FBT002] bytearr = struct.unpack("B" * len(bytestr), bytestr) exp_length = 10 if len(bytearr) < exp_length: - raise ValueError("CCSDS header is not as long as expected") # noqa: EM101, TRY003 + raise ValueError("CCSDS header is not as long as expected") # ruff:ignore[EM101, TRY003] head = {} head["CCSDS_Version"] = bytearr[0] >> 5 @@ -519,21 +519,21 @@ def parse_ccsds_head(bytestr, verbose=False): # noqa: ANN001, ANN201, ARG001, D ######################################### -def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # noqa: ANN001, ANN201, ARG001, C901, FBT002, PLR0912, PLR0913 - """Parse one CCSDS packet""" # noqa: D400 +def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, C901, FBT002, PLR0912, PLR0913] + """Parse one CCSDS packet""" # ruff:ignore[D400] # The format for this APIDs packet list form = apidformat[apid] thisdat = data[apid] # Convert to a bit string bytearr = struct.unpack("B" * len(bytestr), bytestr) - str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) # noqa: FURB116 + str_bin = "".join([bin(i)[2:].zfill(8) for i in bytearr]) # ruff:ignore[FURB116] # For SWEAP packets, we just have each mnemonic listed and each bit length # So we have to step through them in order # Take care of the variables in sw_data (the repeating bit of the packet) separately pointer = 0 - if hasattr(form, "sw_data_vars"): # noqa: SIM108 + if hasattr(form, "sw_data_vars"): # ruff:ignore[SIM108] sw_data_vars_len = len(form.sw_data_vars) else: sw_data_vars_len = 0 @@ -565,7 +565,7 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no thisbin = str_bin[startbit:endbit] try: thisval = int(thisbin, 2) - except: # noqa: E722 + except: # ruff:ignore[E722] # print(sys.exc_info()) thisval = -999 thisdat[thisname].append(thisval) @@ -573,17 +573,17 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no # If the full packet isn't here, then don't bother parsing if len(bytearr) * 8.0 < sum(form.bits): - print(f"short packet: {hex(apid)}") # noqa: T201 + print(f"short packet: {hex(apid)}") # ruff:ignore[T201] return for i_bit, bit in enumerate(form.bits[0 : len(form.bits) - sw_data_vars_len]): thisbin = str_bin[pointer : pointer + bit] try: thisval = int(thisbin, 2) - except: # noqa: E722 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] thisval = -999 thisname = form.names[i_bit] @@ -610,9 +610,9 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no try: thisval = int(thisbin, 2) except ValueError: - import pdb # noqa: PLC0415, T100 + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] thisval = -999 thisname = form.sw_data_vars[i] @@ -624,8 +624,8 @@ def parse_pkt(bytestr, data, apidformat, apid, ccsds_head, verbose=False): # no ######################################### -class apid_obj: # noqa: D101, N801 - def __init__(self): # noqa: ANN204 +class apid_obj: # ruff:ignore[D101, N801] + def __init__(self): # ruff:ignore[ANN204] self.names = [] self.bits = [] self.bytestart = [] @@ -638,34 +638,36 @@ def __init__(self): # noqa: ANN204 ######################################### -def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 +def get_layout(apid, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, FBT002] try: - file = open("sweap_tlm.blk") # noqa: PTH123, SIM115 - except: # noqa: E722 + file = open("sweap_tlm.blk") # ruff:ignore[PTH123, SIM115] + except: # ruff:ignore[E722] if verbose: - print( # noqa: T201 + print( # ruff:ignore[T201] "***INFO*** No local 'sweap_tlm.blk' found...using the one near ccsds_reader_pipeline.py" ) try: thisdir = os.path.realpath(__file__) thisdir = "\\".join(thisdir.split("\\")[0:-1]) - file = open(thisdir + "\\sweap_tlm.blk") # noqa: PTH123, SIM115 - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + file = open(thisdir + "\\sweap_tlm.blk") # ruff:ignore[PTH123, SIM115] + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] lines = file.readlines() for i, line in enumerate(lines): - if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): # noqa: FURB116 + if line[0:8] == f"APID_{hex(apid)[2:].zfill(3)}".upper(): # ruff:ignore[FURB116] if verbose: - print(f"APID {hex(apid)[2:]} Format Found".upper()) # noqa: FURB116, T201 + print(f"APID {hex(apid)[2:]} Format Found".upper()) # ruff:ignore[FURB116, T201] thisapid = apid_obj() thisapid.apid = apid # ty: ignore[unresolved-attribute] - line = "" # so that the while loop will start out ok # noqa: PLW2901 + line = ( + "" # so that the while loop will start out ok # ruff:ignore[PLW2901] + ) while line[0:4] != "APID": - i += 1 # noqa: PLW2901 - line = lines[i] # noqa: PLW2901 + i += 1 # ruff:ignore[PLW2901] + line = lines[i] # ruff:ignore[PLW2901] try: if line.strip()[0] not in ["(", "{", "}", ")"]: pieces = re.split(",|;", line.strip()) @@ -680,11 +682,11 @@ def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 thisapid.sw_data_vars = [] # ty: ignore[unresolved-attribute] except IndexError: break - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] start = np.array( [0] + [sum(thisapid.bits[0:i]) for i in range(1, len(thisapid.bits))] @@ -699,36 +701,36 @@ def get_layout(apid, verbose=False): # noqa: ANN001, ANN201, C901, D103, FBT002 return thisapid # if we didn't find that APID - print( # noqa: T201 - f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # noqa: FURB116 + print( # ruff:ignore[T201] + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # ruff:ignore[FURB116] ) return None ######################################### -def get_layout_sc(apid, verbose=False, filename=""): # noqa: ANN001, ANN201, C901, D103, FBT002 +def get_layout_sc(apid, verbose=False, filename=""): # ruff:ignore[ANN001, ANN201, C901, D103, FBT002] try: - file = open(filename) # noqa: PTH123, SIM115 - print(f"using sc_hk file: {filename}") # noqa: T201 - except: # noqa: E722 - print("could not open SC HK BLK file") # noqa: T201 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + file = open(filename) # ruff:ignore[PTH123, SIM115] + print(f"using sc_hk file: {filename}") # ruff:ignore[T201] + except: # ruff:ignore[E722] + print("could not open SC HK BLK file") # ruff:ignore[T201] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] lines = file.readlines() for i, line in enumerate(lines): - if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": # noqa: FURB116 + if line[0:11] == f"SC_HK_0x{hex(apid)[2:].zfill(3).upper()}": # ruff:ignore[FURB116] if verbose: - print(f"APID {hex(apid)[2:]} Format Found".upper()) # noqa: FURB116, T201 + print(f"APID {hex(apid)[2:]} Format Found".upper()) # ruff:ignore[FURB116, T201] thisapid = apid_obj() thisapid.apid = apid # ty: ignore[unresolved-attribute] - line = "" # noqa: PLW2901 + line = "" # ruff:ignore[PLW2901] while line[0:4] != "SC_H": - i += 1 # noqa: PLW2901 - line = lines[i].strip() # noqa: PLW2901 + i += 1 # ruff:ignore[PLW2901] + line = lines[i].strip() # ruff:ignore[PLW2901] if line[0:8] == "( Block[": length = int(line.split("[")[1].split("]")[0]) try: @@ -745,15 +747,15 @@ def get_layout_sc(apid, verbose=False, filename=""): # noqa: ANN001, ANN201, C9 thisapid.bits.append(int(pieces[3].strip())) except IndexError: break - except: # noqa: E722 - print(sys.exc_info()) # noqa: T201 - import pdb # noqa: PLC0415, T100 + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] - pdb.set_trace() # noqa: T100 + pdb.set_trace() # ruff:ignore[T100] return (thisapid, length) # if we didn't find that APID - print( # noqa: T201 - f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # noqa: FURB116 + print( # ruff:ignore[T201] + f"***ERROR*** [ccsds_reader_pipeline] Did not find APID {hex(apid)[2:]}".upper() # ruff:ignore[FURB116] ) return None diff --git a/src/pyfaradaycup/pipeline/swp_spc_l02l1.py b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py new file mode 100644 index 0000000..f9254f5 --- /dev/null +++ b/src/pyfaradaycup/pipeline/swp_spc_l02l1.py @@ -0,0 +1,954 @@ +""" +# $URL: file:///psp/psp_swp_spc_code_repository/trunk/swp_spc_l02l1.py $ +# $LastChangedRevision: 97 $ +# $LastChangedDate: 2020-08-04 09:20:42 -0400 (Tue, 04 Aug 2020) $ +# $LastChangedBy: acase $ +""" # ruff:ignore[D400] + +__all__ = [ + "cdf35e_35f", + "cdf351_353_354", + "cdf352", + "get_newest_kernel", + "get_newest_skeleton", + "main", + "secsubsec2scet", + "setup", + "statusmsg", +] + +import argparse +import datetime +import glob +import math +import os +import sys + +import numpy as np + +try: + from spacepy import pycdf +except: # ruff:ignore[E722] + # TODO: If we are using newer version of SpacePy (>= 0.3, give or take) # ruff:ignore[FIX002, TD002, TD003] + # then we don't need this. + print(sys.exc_info()) # ruff:ignore[T201] + print("***ERROR*** Could not import pycdf from spacepy") # ruff:ignore[T201] + print( # ruff:ignore[T201] + "\t You must have the environmental variable CDF_LIB set, perhaps to /opt/cdf/lib?" + ) + sys.exit() + +import distutils.dir_util + +import spiceypy # ty:ignore[unresolved-import] + +import pyfaradaycup.pipeline.ccsds_reader_pipeline as cc + +# Purpose: Convert binary "level-zero" or "ssr" files that come from the SWEM or Spacecraft +# into L0.5 or L1 CDF files + +# Requirements: Must have a reference to a skeleton file for the ApID that you wish to convert_one + +# Input: path to a binary L0 file + +# Output: saves a CDF file + +# Revision History +# 2020/02/03 - Fix timing bug in 0x351 that arose when switching to spiceypy. Add capability to produce 0x352 (time series) files. Will require update to 0x352 (time series) skeleton also. Update bug in naming of L1 files coming from gzip (spacecraft files, probably). Remove versioning from skeleton files (since SVN is taking care of that). +# 2020/01/29 - Use spiceypy to adjust time of each measurement to SCET; remove unused command-line argument options +# Change method by which newest skeleton files are found; various other cleanups +# 2019/08/27 - Calling new ccsds_reader that separates out s/c from instr. packets +# - Also, new ccsds_reader will use new packet finder (much faster), and does not scan through packets like before +# 2019/08/23 - Fixed bug that was reading in the oldest rather than newest skeleton file +# - Added capability to read 0x081 and 0x262 packets from s/c hsk files. Requires link to SC_HK.blk block definition file +# - Added revision history + + +def main( # ruff:ignore[ANN201, C901, PLR0912, PLR0913, PLR0915, PLR0917] + l0file="", # ruff:ignore[ANN001] + l1dir="", # ruff:ignore[ANN001] + logdir="", # ruff:ignore[ANN001] + spacecraft=False, # ruff:ignore[ANN001, FBT002] + ptp=False, # ruff:ignore[ANN001, FBT002] + gzip=False, # ruff:ignore[ANN001, FBT002] + apidreq=0, # ruff:ignore[ANN001] + overwrite=False, # ruff:ignore[ANN001, FBT002] + verbose=False, # ruff:ignore[ANN001, FBT002] +): + """Convert a single L0 file to L1""" # ruff:ignore[D400] + # Try to create a filename for the new CDF that we're going to create + l0dirname = os.path.dirname(l0file) # ruff:ignore[PTH120] + l0basename = os.path.basename(l0file) # ruff:ignore[PTH119] + if l1dir == "": + l1dir = ( + l0dirname # use input L0 directory for L1 files, if nothing else specified + ) + + # Get a version of filename with no extension + l0file_noext = os.path.splitext(l0basename)[0] # ruff:ignore[PTH122] + if l0file_noext[-3:] == "ptp": + l0file_noext = os.path.splitext(l0file_noext)[0] # ruff:ignore[PTH122] + + # Open a log file to write to + nowdt = datetime.datetime.now() # ruff:ignore[DTZ005] + if logdir == "": + logdir = l1dir # use L1 file output directory for log file, if nothing else specified + distutils.dir_util.mkpath( + logdir + ) # in case the directory doesn't exist, this will create it + logpath = os.path.join( # ruff:ignore[PTH118] + logdir, + f"swp_spc_l02l1_{nowdt.year:04.0f}{nowdt.month:02.0f}{nowdt.day:02.0f}{nowdt.hour:02.0f}{nowdt.minute:02.0f}{nowdt.second:02.0f}.log", + ) + try: + global logfile # ruff:ignore[PLW0603] # ty:ignore[unresolved-global] + logfile = open(logpath, "w") # ruff:ignore[PTH123, SIM115] + except: # ruff:ignore[E722] + print("\n***ERROR*** Could not open log file!\n") # ruff:ignore[T201] + sys.exit(1) + + # Write some information to the log file + statusmsg("scriptname = swp_spc_l02l1.py", verbose=verbose) + statusmsg("timerun = " + nowdt.isoformat(), verbose=verbose) + statusmsg("l0file = " + l0file, verbose=verbose) + statusmsg("l1dir = " + l1dir, verbose=verbose) + statusmsg("spacecraft = " + repr(spacecraft), verbose=verbose) + statusmsg("ptp = " + repr(ptp), verbose=verbose) + statusmsg("gzip = " + repr(gzip), verbose=verbose) + statusmsg("apid = " + hex(apidreq), verbose=verbose) + statusmsg("overwrite = " + repr(overwrite), verbose=verbose) + + # Make sure the L0 file exists and is readable + try: + foo = open(l0file) # ruff:ignore[PTH123, SIM115] + foo.close() + statusmsg("L0 file exists and is readable") + except OSError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Input L0 file could not be read...exiting", + screen=True, + verbose=verbose, + ) + import pdb # ruff:ignore[PLC0415, T100] + + pdb.set_trace() # ruff:ignore[T100] + sys.exit() + + # Load in Leap Second Kernel + statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest leap second kernel...") + tls_path = get_newest_kernel(tls=True) + if not tls_path: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not find leap second kernel...exiting" + ) + sys.exit() + else: + try: + statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {tls_path}") + spiceypy.furnsh(tls_path) + except: # ruff:ignore[E722] + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not furnsh leap second kernel...exiting" + ) + sys.exit() + + # Load in S/C Clock Kernel + statusmsg("***INFO*** [swp_spc_l02l1.py] Finding newest S/C clock kernel...") + sclk_path = get_newest_kernel(sclk=True) + if not sclk_path: + statusmsg("***ERROR*** [swp_spc_l02l1.py] Could not find SCLK kernel...exiting") + sys.exit() + else: + try: + statusmsg(f"***INFO*** [swp_spc_l02l1.py] Using: {sclk_path}") + spiceypy.furnsh(sclk_path) + except: # ruff:ignore[E722] + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Could not furnsh SCLK kernel...exiting" + ) + sys.exit() + + # Read in the L0 file into a python SPC data structure + if spacecraft: + statusmsg("Event = Starting reading file: spacecraft") + l0data = cc.read_file_sc(path=l0file, ptp=ptp, verbose=verbose, gzip=gzip) + else: + statusmsg("Event = Starting reading file: non-spacecraft (instrument)") + l0data = cc.read_file(path=l0file, verbose=verbose, gzip=gzip) + statusmsg("Event = Finished reading file") + + # Loop through the APIDs that we got + for apid in l0data.keys(): # ruff:ignore[SIM118] + statusmsg(f"Event = Beginning APID: {hex(apid)}") + + if apid == 0x07B: # ruff:ignore[PLR2004] + statusmsg( + "***WARNING*** [swp_spc_l02l1] APID 0x07B CDFs not yet implemented", + screen=True, + verbose=verbose, + ) + continue + + # Make sure we need to do this apid + if len(l0data[apid][list(l0data[apid].keys())[0]]) == 0: # ruff:ignore[RUF015] + statusmsg("No packets found for this apid.") + continue # skip this apid if there were no packets received + if (apidreq != 0) & (apidreq != apid): + statusmsg("This apid not requested by user") + continue # skip this apid if user only wanted one apid and this isn't it + + # Filename for the L1 file we're about to write for this apid + l1path = os.path.join( # ruff:ignore[PTH118] + l1dir, + l0file_noext + f"_APID{str(hex(apid)[2:].zfill(3)).upper()}_L1.cdf", # ruff:ignore[FURB116] + ) + statusmsg("About to write: " + l1path) + + # Make sure the skeleton file exists and is readable + try: + skeleton_filename = get_newest_skeleton(apid) + foo = open(skeleton_filename) # ruff:ignore[PTH123, SIM115] + foo.close() + statusmsg("Skeleton to be used: " + skeleton_filename) + except OSError: + statusmsg( + "***ERROR*** [swp_spc_l02l1.py] Skeleton file could not be read...moving to next apid", + screen=True, + verbose=verbose, + ) + statusmsg("Tried to use skeleton file: " + skeleton_filename) + continue + except TypeError: + statusmsg( + f"***ERROR*** [swp_spc_l02l1.py] Skeleton file for apid={hex(apid)} could not be found...moving to next apid", + screen=True, + verbose=verbose, + ) + continue + + # See if the CDF file already exists + try: + # try to open and close it + statusmsg("Using L1 path: " + l1path, screen=True, verbose=verbose) + foo = open(l1path) # ruff:ignore[PTH123, SIM115] + foo.close() + + # if we get here, this file already exists; so delete it, if desired + statusmsg( + f"***INFO*** [swp_spc_l02l1] L1 CDF file ({l1path}) already exists", + screen=True, + verbose=verbose, + ) + if overwrite: + statusmsg( + "***INFO*** [swp_spc_l02l1] Overwriting existing L1 CDF", + screen=True, + verbose=verbose, + ) + os.remove(l1path) # ruff:ignore[PTH107] + else: + statusmsg( + "***ERROR*** [swp_spc_l02l1] L1 CDF already exists, and overwrite (-o option) was not requested...exiting.", + screen=True, + verbose=verbose, + ) + raise (SystemExit) # ruff:ignore[TRY301] + except SystemExit: + sys.exit() + except OSError: + pass # Apparently the file did not exist already + except: # ruff:ignore[E722] + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + statusmsg( + "\n***ERROR*** [swp_spc_l02l1] Could not check existence/delete L1 CDF file path. Exiting...\n", + screen=True, + verbose=verbose, + ) + sys.exit() + + # Create a new CDF file from the provided skeleton + try: + cdf = pycdf.CDF(l1path, skeleton_filename) + except "CDFError": # ruff:ignore[B030] # ty:ignore[invalid-exception-caught] + statusmsg( + f"\n***ERROR*** [swp_spc_l02l1] Could not create new CDF (APID={apid})...continuing to next APID\n).", + screen=True, + verbose=verbose, + ) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) + continue + + # Run a different procedure to put data into CDF file depending on APID + cdfproc = { + 0x081: cdf35e_35f, + 0x1DE: cdf35e_35f, + 0x254: cdf35e_35f, + 0x256: cdf35e_35f, + 0x257: cdf35e_35f, + 0x262: cdf35e_35f, + 0x351: cdf351_353_354, + 0x352: cdf352, + 0x353: cdf351_353_354, + 0x354: cdf351_353_354, + 0x35E: cdf35e_35f, + 0x35F: cdf35e_35f, + } + try: + cdfproc[apid](cdf, l0data[apid], verbose=verbose) + except: # ruff:ignore[E722] + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + statusmsg( + f"***WARNING*** [swp_spc_l02l1] CDF not processed for APID={hex(apid)}", + screen=True, + verbose=verbose, + ) + continue + + # Close the CDF + # import pdb; pdb.set_trace() + cdf.close() + + statusmsg( + "***INFO*** [swp_spc_l02l1] Script complete.", screen=True, verbose=verbose + ) + + # Close the log file + logfile.close() + + +def cdf35e_35f(cdf, dat, verbose=False) -> None: # ruff:ignore[ANN001, C901, FBT002] + """Fill up a CDF with data from an SPC HSK (0x35E or 0x35F) packet or S/C HSK packet""" # ruff:ignore[D400] + # Calculate MET from the variables in the L0 data + # MET of each NYS + if "CCSDS_MET" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPC_SUBSEC"]) + elif "FSW_HK_HK_INST_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet( + dat["FSW_HK_HK_INST_TPSH_MET_SEC"], + dat["FSW_HK_HK_INST_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + elif "PDU_PRIO94_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet( + dat["PDU_PRIO94_TPSH_MET_SEC"], + dat["PDU_PRIO94_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + elif "HK_HIGH_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet( + dat["HK_HIGH_TPSH_MET_SEC"], dat["HK_HIGH_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "HK_FSWL_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet( + dat["HK_FSWL_TPSH_MET_SEC"], dat["HK_FSWL_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "HK_LOW_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet( + dat["HK_LOW_TPSH_MET_SEC"], dat["HK_LOW_TPSH_MET_SUBSEC"], spacecraft=True + ) + elif "RIU_DERIVED_TPSH_MET_SEC" in dat.keys(): # ruff:ignore[SIM118] + scet = secsubsec2scet( + dat["RIU_DERIVED_TPSH_MET_SEC"], + dat["RIU_DERIVED_TPSH_MET_SUBSEC"], + spacecraft=True, + ) + + else: + statusmsg("Failed: could not create Epoch variable") + return + + # Fill in values for each variable + keys = cdf.keys() + dat["Epoch"] = scet + + for key in keys: + try: + cdf[key] = dat[key] # create variable and insert data + except KeyError: # ruff:ignore[PERF203] + if key not in dat.keys(): # ruff:ignore[SIM118] + cdf[key] = np.ones(len(dat["Epoch"])) * cdf[key].attrs["FILLVAL"] + except: # ruff:ignore[E722] + import pdb # ruff:ignore[PLC0415, T100] + + pdb.set_trace() # ruff:ignore[T100] + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(sys.exc_info()) + + +##################################################### +## +##################################################### +def cdf351_353_354(cdf, dat, nocdf=False, verbose=False): # ruff:ignore[ANN001, ANN201, C901, FBT002, PLR0912, PLR0915, RET503] + """Fill up a CDF with SCI, ALL, or RSS data.""" + # Take data sorted by NYS, and produce one long variable with all data + + # APID of this packet + apid = dat["CCSDS_ApID"][0] + + # Each different packet will require a different variable to calculate + # The number of measurements each NYS + if apid == 0x351: # ruff:ignore[PLR2004] + length_var = "A1S" + elif apid == 0x353: # ruff:ignore[PLR2004] + length_var = "ASIN" + elif apid == 0x354: # ruff:ignore[PLR2004] + length_var = "ARSS" + + # Calculate MET from the variables in the L0 data + # MET of each NYS + scet = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) + + # MET of each measurement (to be filled in in the future) + scet_exp = [] # ruff:ignore[F841] + + # Same keys as original data dictionary, but will hold one variable per key + # instead of one for every NYS for every key + dat_exp = {} + for key in dat.keys(): # ruff:ignore[SIM118] + dat_exp[key] = [] + + # Create the 'Epoch' variable in our data array + dat_exp["Epoch"] = [] + + # Loop through each NYS + itst_warned = False + for i in range(len(scet)): + this_scet = scet[i] + + # Number of ticks each measurement takes (1024 ticks per NYS) + ticks_per_meas = dat["SW_SPC_INTTIME"][i] + dat["SW_SPC_SERVTIME"][i] + + # Make sure ST and IT are allowed values + if (math.log(ticks_per_meas, 2)) % 1 != 0: # ruff:ignore[FURB163] + # the SPC FPGA will default to IT=6, ST=2 (the power-on defaults) if a non-integer power of 2 IT+ST is requested + ticks_per_meas = 8 + + if not itst_warned: + statusmsg( + "***WARNING*** [swp_spc_l02l1] The reported IT+ST is not an even power of 2. Using IT=6,ST=2...", + screen=True, + verbose=verbose, + ) + itst_warned = True + + # throw out packets if the IT and ST are different than both previous and next packets + # this is almost surely an improperly identified packet that probably + # isn't even an SPC packet, but got decommutated as such + try: + if ( + (dat["SW_SPC_INTTIME"][i] != dat["SW_SPC_INTTIME"][i - 1]) + & (dat["SW_SPC_INTTIME"][i] != dat["SW_SPC_INTTIME"][i + 1]) + & (dat["SW_SPC_SERVTIME"][i] != dat["SW_SPC_SERVTIME"][i - 1]) + & (dat["SW_SPC_SERVTIME"][i] != dat["SW_SPC_SERVTIME"][i + 1]) + ): + statusmsg( + "***WARNING***IT+ST not 2^n, and not same as prev. and next values...so skipping this packet.", + screen=True, + verbose=verbose, + ) + continue + except IndexError: + statusmsg( + "***WARNING***IT+ST not 2^n, and not same as prev. and next values...so skipping this packet.", + screen=True, + verbose=verbose, + ) + continue + + # Time that each measurement took this NYS + tm_per_meas = (1.0 / 1171.875) * ticks_per_meas + + # Number of measurements this NYS + nmeas = len(dat[length_var][i]) + + # Expected number of measurements in a NYS + exp_nmeas = 1024.0 / 1171.875 / tm_per_meas + + # Calculate the time array for this NYS + add_time = np.arange(0, tm_per_meas * (nmeas - 0.1), tm_per_meas) + + # See if there were any times when we had retraces + win = np.array(dat["WINDOW"][i]) + rtpix = (np.where((win[1:] - win[:-1]) < 0)[0]) + 1 + + # Add on some time for each of the retraces + # We don't need to do this if the expected number of measurements is equal to the number of measurements we received + # This is because of the possibility that HV DAC tables were not loaded (probably only on the ground). + # In that case, the FPGA does not actually take the time to do a retrace since it does not have to slew to a new DAC value + # It doesn't matter that it is slewing to a new 'Window', since every window will have the same DAC value + # There is a slight bug here in that the 'final' NYS after a HALT is sent, will likely be a partial packet + # so we might get tricked on our logical check here for the final packet when we do not have DAC tables loaded + if nmeas != exp_nmeas: + for thisrtpix in rtpix: + add_time[thisrtpix:] += tm_per_meas + + # If we're in an AllGain packet, then the beginning of the packet might not be the beginning of the NYS (which is the time noted in the header) + if apid == 0x351: # ruff:ignore[PLR2004] + pktnum = dat["SW_SPC_PKTNUM"][i] + # if pktnum==0: import pdb; pdb.set_trace() + if pktnum != 0: + if len(dat_exp["Epoch"]) == 0: + continue # if file started on pktnum other than zero, then we can't know precise timing for the first 1-3 packets + + add_time += (dat_exp["Epoch"][-1] / 1e9 - this_scet / 1e9) + tm_per_meas + + # in case retrace was at end of last packet + if (win[0] - dat_exp["WINDOW"][-1]) < 0: + add_time += tm_per_meas + + # Extend the new expanded dt + dscet_extend = [this_scet + 1e9 * thisaddtime for thisaddtime in add_time] + dat_exp["Epoch"].extend(dscet_extend) + + # Extend each of the data arrays + for key in dat.keys(): # ruff:ignore[SIM118] + try: + dat_exp[key].extend(dat[key][i]) + except TypeError: # ruff:ignore[PERF203] + expanded = np.ones(nmeas) * dat[key][i] + dat_exp[key].extend(expanded) + + if nocdf: + return dat_exp + # Fill in the CDF + keys = cdf.keys() + + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: # ruff:ignore[E722, PERF203] + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + import pdb # ruff:ignore[PLC0415, T100] + + pdb.set_trace() # ruff:ignore[T100] + + +def cdf352(cdf, dat, nocdf=False, verbose=False): # ruff:ignore[ANN001, ANN201, C901, D103, FBT002, PLR0912, PLR0915] + try: + # Calculate SCET from the variables in the L0 data + dt = secsubsec2scet(dat["CCSDS_MET"], dat["SW_SPCSUBSEC"]) + + # Same keys as original data dictionary, but will hold one variable per key + # instead of one for every NYS for every key + dat_exp = {} + for key in dat.keys(): # ruff:ignore[SIM118] + if key[-4:] == "_000": + continue + dat_exp[key] = [] + + dat_exp["VAR0"] = [] + dat_exp["VAR1"] = [] + dat_exp["VAR2"] = [] + dat_exp["VAR3"] = [] + dat_exp["VAR0_NAME"] = [] + dat_exp["VAR1_NAME"] = [] + dat_exp["VAR2_NAME"] = [] + dat_exp["VAR3_NAME"] = [] + + # Var names for each possible channel that might be contained in the packet + avars = ["A0", "A1", "A2", "A3"] + bvars = ["B0", "B1", "B2", "B3"] + cvars = ["C0", "C1", "C2", "C3"] + dvars = ["D0", "D1", "D2", "D3"] + hk1vars = ["HV_DAC_IN", "P3p3_Vmon", "P12_Vmon", "N12_Vmon"] + hk2vars = ["HV_Out", "Rail_Ctrl", "P5_Vmon", "N5_Vmon"] + + # To convert from the ID number in each packet header to the variables that + # the packet actually contains values for + coll2var = {1: avars, 2: bvars, 4: cvars, 8: dvars, 16: hk1vars, 32: hk2vars} + + # Create the 'Epoch' variable in our data array + dat_exp["Epoch"] = [] + + # Time that each measurement took this NYS + tm_per_meas = 1.0 / 32.0 / 1171.875 + + for i in range(len(dt)): + # MET for this NY second + thisdt = dt[i] + + # Collector (or HSK values) that are being used this NYS + # An integer that references which variables are actually contained in the packet + coll_used = dat["SPC_TIMESERCOLL"][i] + + # Number of measurements this NYS + nmeas = len(dat["G0_000"][i]) # should always be 20*32=640 + + # Packet start time + pkt_start = dat["SPC_TIMESERTICK"][i] / 1171.875 + + # Calculate the time array for this NYS + add_time = pkt_start + np.arange( + 0, tm_per_meas * (nmeas - 0.1), tm_per_meas + ) + + try: + if coll_used not in coll2var: + raise ValueError( # ruff:ignore[TRY003] + f"Value: {coll_used} not in coll2var.keys()" # ruff:ignore[EM102] + ) # probably a corrupt packet + + dat_exp["VAR0_NAME"].extend( + [coll2var[coll_used][0] for foo in range(nmeas)] + ) + dat_exp["VAR1_NAME"].extend( + [coll2var[coll_used][1] for foo in range(nmeas)] + ) + dat_exp["VAR2_NAME"].extend( + [coll2var[coll_used][2] for foo in range(nmeas)] + ) + dat_exp["VAR3_NAME"].extend( + [coll2var[coll_used][3] for foo in range(nmeas)] + ) + + dat_exp["VAR0"].extend(dat["G0_000"][i]) + dat_exp["VAR1"].extend(dat["G1_000"][i]) + dat_exp["VAR2"].extend(dat["G2_000"][i]) + dat_exp["VAR3"].extend(dat["G3_000"][i]) + + except: # ruff:ignore[E722] + statusmsg( + "***ERROR*** Could not process 0x352 packet (probably it was a false positive ID of a 0x352 packet?)" + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + continue + + # Extend the expanded dt + dt_extend = [thisdt + sec * 1e9 for sec in add_time] + dat_exp["Epoch"].extend(dt_extend) + + # Extend each of the data arrays + for key in dat.keys(): # ruff:ignore[SIM118] + if key[-4:] == "_000": + continue + try: + dat_exp[key].extend(dat[key][i]) + except TypeError: + expanded = np.ones(nmeas) * dat[key][i] + dat_exp[key].extend(expanded) + + if nocdf: + return dat_exp + # Fill in the CDF + keys = cdf.keys() + + # Move 'Epoch' so that it is the first variable (so that we can be ISTP-compliant) + epochloc = np.where(np.array(keys) == "Epoch")[0] + if len(epochloc) != 0: + keys.pop(epochloc[0]) + keys.insert(0, "Epoch") + for key in keys: + try: + # insert data + cdf[key] = dat_exp[key] + except: # ruff:ignore[E722, PERF203] + statusmsg( + f"Failed : Key:{key} failed insert into CDF", + screen=True, + verbose=verbose, + ) + statusmsg(repr(sys.exc_info()), screen=True, verbose=verbose) + except: # ruff:ignore[E722] + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] + + pdb.set_trace() # ruff:ignore[T100] + + return () + + +def secsubsec2scet(sec, subsec, spacecraft=False, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, FBT002] + """Parse a fairly standard CCSDS time structure into decimal MET: first 4 bytes=MET seconds, second 2 bytes = MET subseconds""" # ruff:ignore[D400] + sec_str = [f"{i:1.0f}" for i in sec] + subsec_str_base50000 = [ + f"{int(i * 50000 / 65536):05.0f}" for i in subsec + ] # SWEAP has subseconds in 1/65536's of a second + if spacecraft: + subsec_str_base50000 = [ + f"{int(i * 50000 / 256):05.0f}" for i in subsec + ] # S/C has subseconds in 1/256's of a second + + ephem_sec_j2000 = [ + spiceypy.scs2e(-96, sec_str[i] + ":" + subsec_str_base50000[i]) + for i in range(len(sec_str)) + ] + ephem_nanosec_j2000 = [np.round(1e9 * i) for i in ephem_sec_j2000] + + return ephem_nanosec_j2000 # ruff:ignore[RET504] + + +def statusmsg(string, screen=False, file=True, verbose=False): # ruff:ignore[ANN001, ANN201, FBT002] + """Output status message to screen or logfile (default to file, but not screen)""" # ruff:ignore[D400] + nowdtstr = datetime.datetime.now().isoformat() # ruff:ignore[DTZ005] + if file: + logfile.write(nowdtstr + ", " + string + "\n") + if screen: # ruff:ignore[SIM102] + if verbose: + print(string) # ruff:ignore[T201] + + +def get_newest_kernel(tls=False, sclk=False, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, FBT002] + """Find the path to the newest NAIF TLS (leap second) kernel file""" # ruff:ignore[D400] + # Make sure we chose exactly one of the options + if tls + sclk != 1: + return False + + # Search in the MOC data product directory for newest file + if tls: + globdir = "/psp/data/moc_data_products/leap_second_kernel/" + globstr = globdir + "naif00[0-9][0-9].tls" + ndigits = 2 + elif sclk: + globdir = "/psp/data/moc_data_products/operations_sclk_kernel/" + globstr = globdir + "spp_sclk_[0-9][0-9][0-9][0-9].tsc" + ndigits = 4 + + files = glob.glob(globstr) # ruff:ignore[PTH207] + + # isolate version numbers from the file path and find newest + if tls or sclk: + versions = [int(i[-4 - ndigits : -4]) for i in files] + try: + maxind = np.argmax(versions) + except ValueError: + statusmsg("***ERROR*** Could not find kernel versions") + print(sys.exc_info()) # ruff:ignore[T201] + import pdb # ruff:ignore[PLC0415, T100] + + pdb.set_trace() # ruff:ignore[T100] + return False + + # return path to newest file + path = files[maxind] + return path # ruff:ignore[RET504] + + +def get_newest_skeleton(apid, verbose=False): # ruff:ignore[ANN001, ANN201, ARG001, FBT002] + """Find the path to the newest skeleton CDF file""" # ruff:ignore[D400] + return f"cdf_skeletons/psp_swp_spc_l1_{hex(apid)[2:].zfill(3)}_skeleton.cdf" # ruff:ignore[FURB116] + + # The remaining code in this function is from when we used skeleton file numbers with a version # in them + # and we had to search for the most recent (highest) version + + # Search for newest file + # globstr = 'cdf_skeletons/spp_apid_{:}_sweap_00000000t000000_v[0-9][0-9].cdf'.format(hex(apid)[2:].zfill(3)) + # ndigits = 2 + + # files = glob.glob(globstr) + + # isolate version numbers from the file path and find newest + # versions = [int(i[-4-ndigits:-4]) for i in files] + + # try: + # maxind = np.argmax(versions) + # except ValueError: + # statusmsg('***ERROR*** Could not find skeleton versions') + # return(False) + + # return path to newest file + # path = files[maxind] + + # return(path) + + +##################################################### +### +##################################################### +def setup(): # ruff:ignore[ANN201] + """Get user command-line input and set things up""" # ruff:ignore[D400] + # defaults + l0file_default = "" + l0dir_default = "" + l1dir_default = "" + logdir_default = "" + apid_default = "0" + + # Get User Input + parser = argparse.ArgumentParser(description="") + parser.add_argument( + "-v", + "--verbose", + default=False, + action="store_true", + help="Increase verbosity", + required=False, + ) + parser.add_argument( + "-gz", + "--gzip", + default=False, + action="store_true", + help="Read in L0 file as gzip", + required=False, + ) + parser.add_argument( + "-sc", + "--spacecraft", + default=False, + action="store_true", + help="Look for S/C packets", + required=False, + ) + parser.add_argument( + "-b", + "--batch", + default=False, + action="store_true", + help="Convert all L0 files in same directory as selected", + required=False, + ) + parser.add_argument( + "-r", + "--recursive", + default=False, + action="store_true", + help="Convert all L0 files in given directory and in all subdirectories", + required=False, + ) + parser.add_argument( + "-p", + "--ptp", + default=False, + action="store_true", + help="Indicate that input L0 file is a PTP file", + required=False, + ) + parser.add_argument( + "-o", + "--overwrite", + default=False, + action="store_true", + help="Overwrite existing L1 CDF file, if necessary", + required=False, + ) + parser.add_argument( + "-stc", + "--stcorrect", + default=False, + action="store_true", + help="If ST is wrong (FPGA bug if ST set higher than 2), then try to correct it)", + required=False, + ) + parser.add_argument( + "-a", + "--apid", + help=f"APID to create L1 file for [0==all] [default={apid_default}]", + required=False, + default=apid_default, + type=str, + ) + parser.add_argument( + "-l0", + "--l0file", + help=f"Input L0 File [default={l0file_default}]", + required=False, + default=l0file_default, + ) + parser.add_argument( + "-d", + "--l0dir", + help=f"Input L0 Directory (for use with -b or -r [default={l0dir_default}]", + required=False, + default=l0dir_default, + ) + parser.add_argument( + "-dl1", + "--l1dir", + help=f"Output L1 Directory [default={l1dir_default}]", + required=False, + default=l1dir_default, + ) + parser.add_argument( + "-dlog", + "--logdir", + help=f"Output for Log Files [default={logdir_default}]", + required=False, + default=logdir_default, + ) + + # Read in the arguments + args = parser.parse_args() + + # Version of the data product + args.version = 2 + + # Make sure we got a good argument set + if (args.batch == 0) & (args.recursive == 0): + if args.l0file == "": + statusmsg( + "***ERROR*** You must provide --l0file, if not using -b or -r", + screen=True, + verbose=verbose, # ruff:ignore[F821] # ty:ignore[unresolved-reference] + ) + elif args.l0dir == "": + statusmsg( + "***ERROR*** You must provide --l0dir if using -b or -r", + screen=True, + verbose=verbose, # ruff:ignore[F821] # ty:ignore[unresolved-reference] + ) + + # Convert APID to an integer (it is read as a string from the command line) + try: + if args.apid[0:2] == "0x": # ruff:ignore[SIM108] + base = 16 + else: + base = 10 + args.apid = int(args.apid, base) + except TypeError: + statusmsg( + "Trouble parsing desired APID....exiting.", + screen=True, + verbose=verbose, # ruff:ignore[F821] # ty:ignore[unresolved-reference] + ) + statusmsg(sys.exc_info(), screen=True, verbose=verbose) # ruff:ignore[F821] # ty:ignore[unresolved-reference] + sys.exit() + + # Make sure the environmental variable reference to the data directory is set and readable + try: + datadir = os.environ["PSP_DATA_DIR"] + except: # ruff:ignore[E722] + raise KeyError( # ruff:ignore[B904, TRY003] + "Environmental variable PSP_DATA_DIR could not be found...you must specify path to data directory using that environmental variable" # ruff:ignore[EM101] + ) + + if not os.path.exists(datadir): # ruff:ignore[PTH110] + raise ValueError( # ruff:ignore[TRY003] + "Directory specified in env. variable PSP_DATA_DIR does not exist" # ruff:ignore[EM101] + ) + + # Return to main routine + return args + + +############################################ +#### +############################################ +if __name__ == "__main__": + args = setup() + main( + l0file=args.l0file, + l1dir=args.l1dir, + logdir=args.logdir, + spacecraft=args.spacecraft, + ptp=args.ptp, + gzip=args.gzip, + apidreq=args.apid, + overwrite=args.overwrite, + verbose=args.verbose, + )