diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cfdeea0018..861b63e54a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -43,4 +43,4 @@ repos: hooks: - id: mypy exclude: .*(tests|docs).* - additional_dependencies: [ numpy==1.26.4 ] + additional_dependencies: [ numpy==2.3.5 ] diff --git a/docs/source/code-documentation/cdf.rst b/docs/source/algorithm-code-documentation/cdf.rst similarity index 100% rename from docs/source/code-documentation/cdf.rst rename to docs/source/algorithm-code-documentation/cdf.rst diff --git a/docs/source/code-documentation/codice.rst b/docs/source/algorithm-code-documentation/codice.rst similarity index 100% rename from docs/source/code-documentation/codice.rst rename to docs/source/algorithm-code-documentation/codice.rst diff --git a/docs/source/code-documentation/glows.rst b/docs/source/algorithm-code-documentation/glows.rst similarity index 100% rename from docs/source/code-documentation/glows.rst rename to docs/source/algorithm-code-documentation/glows.rst diff --git a/docs/source/code-documentation/hi.rst b/docs/source/algorithm-code-documentation/hi.rst similarity index 100% rename from docs/source/code-documentation/hi.rst rename to docs/source/algorithm-code-documentation/hi.rst diff --git a/docs/source/code-documentation/hit.rst b/docs/source/algorithm-code-documentation/hit.rst similarity index 100% rename from docs/source/code-documentation/hit.rst rename to docs/source/algorithm-code-documentation/hit.rst diff --git a/docs/source/code-documentation/idex.rst b/docs/source/algorithm-code-documentation/idex.rst similarity index 100% rename from docs/source/code-documentation/idex.rst rename to docs/source/algorithm-code-documentation/idex.rst diff --git a/docs/source/code-documentation/index.rst b/docs/source/algorithm-code-documentation/index.rst similarity index 91% rename from docs/source/code-documentation/index.rst rename to docs/source/algorithm-code-documentation/index.rst index 39b49150f8..98f1f5813c 100644 --- a/docs/source/code-documentation/index.rst +++ b/docs/source/algorithm-code-documentation/index.rst @@ -1,7 +1,7 @@ -.. _code-documentation: +.. _algorithm-code-documentation: -Code Documentation -================== +Algorithm Code Documentation +============================ .. currentmodule:: imap_processing @@ -15,7 +15,6 @@ Instruments .. toctree:: :maxdepth: 1 - cli codice glows hi @@ -26,6 +25,7 @@ Instruments swapi swe ultra + quicklooks Utilities --------- @@ -62,12 +62,4 @@ variable ``IMAP_DATA_DIR``. For example to use a temporary directory imap_cli --instrument codice --level 1 --data-dir /tmp/imap-data # or equivalently with an environment variable - IMAP_DATA_DIR=/tmp/imap-data imap_cli --instrument codice --level 1 - -Tools ------ - -.. toctree:: - :maxdepth: 2 - - tools/index \ No newline at end of file + IMAP_DATA_DIR=/tmp/imap-data imap_cli --instrument codice --level 1 \ No newline at end of file diff --git a/docs/source/code-documentation/lo.rst b/docs/source/algorithm-code-documentation/lo.rst similarity index 100% rename from docs/source/code-documentation/lo.rst rename to docs/source/algorithm-code-documentation/lo.rst diff --git a/docs/source/code-documentation/mag.rst b/docs/source/algorithm-code-documentation/mag.rst similarity index 100% rename from docs/source/code-documentation/mag.rst rename to docs/source/algorithm-code-documentation/mag.rst diff --git a/docs/source/algorithm-code-documentation/quicklooks.rst b/docs/source/algorithm-code-documentation/quicklooks.rst new file mode 100644 index 0000000000..7cef5e29a9 --- /dev/null +++ b/docs/source/algorithm-code-documentation/quicklooks.rst @@ -0,0 +1,154 @@ +Quicklook Generation +=========================== + +This document provides a high-level overview of the workflow and usage +of the ``QuicklookGenerator`` system. It is intended to help developers +understand how quicklook plots are produced from IMAP instrument data files. + +Overview +-------- + +Each instrument implements its own plotting logic +while sharing a common initialization, data-loading, +and dispatch workflow. + +The system is built around four major components: + +1. **Dataset loading** – converting a CDF file into an ``xarray.Dataset``. +2. **Instrument detection** – determining the correct quicklook + generator class. +3. **Abstract quicklook interface** – common API for all instruments. +4. **Instrument-specific subclasses** – actual plotting implementations. + +Workflow +-------- + +1. **User supplies a CDF file path**:: + + quicklook = get_instrument_quicklook("path/to/file.cdf") + +2. **The filename is parsed** using + ``ScienceFilePath.extract_filename_components`` to extract the + ``instrument`` field. + +3. **The correct Quicklook subclass is selected** via the + ``QuicklookGeneratorType`` enum. For example:: + + MAG → MagQuicklookGenerator + + The pattern continues for each individual instrument. + +4. **The chosen class is instantiated**, and the constructor: + + - Loads the dataset using ``dataset_into_xarray``. + - Stores instrument metadata (such as the instrument name). + - Initializes plotting-related attributes. + +5. **The user calls** ``two_dimensional_plot(variable="...")``. + This method is implemented by each subclass and acts as a routing + mechanism that decides *which* quicklook plot to generate. + + Example for MAG:: + + mag_ql = MagQuicklookGenerator("imap_mag_20250101_v01.cdf") + mag_ql.two_dimensional_plot(variable="mag sensor co-ord") + +6. **The requested plot function runs**, accessing data from the internal + ``xarray.Dataset`` and generating figures using Matplotlib. + +7. **Plots are displayed**, and the workflow ends. No data is returned— + the quicklook system produces visualizations only. + +Core Components +--------------- + +Abstract Base Class: ``QuicklookGenerator`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``QuicklookGenerator`` abstract base class provides: + +- Unified dataset loading via ``dataset_into_xarray``. +- Common metadata attributes (instrument, title, axis labels, etc.). +- Basic validation to ensure that a dataset is present. +- An abstract method ``two_dimensional_plot``, which each instrument + subclass must implement as a dispatch mechanism. + +Time & Coordinate Handling +-------------------------- + +Instrument data often stores time in **J2000 nanoseconds**. These values +are converted to UTC timestamps using:: + + convert_j2000_to_utc(time_array) + +This conversion internally uses: + +- A fixed epoch: ``2000-01-01 11:58:55.816`` +- NumPy ``timedelta64`` arithmetic +- Output as ``datetime64[ns]`` UTC timestamps + +This routine is used across all instruments providing an ``epoch`` field. + +Instrument Dispatch Logic +------------------------- + +Users should begin quicklook generation by calling:: + + quicklook = get_instrument_quicklook(filename) + +This function: + +1. Extracts the instrument identifier from the filename. +2. Uses ``QuicklookGeneratorType`` to map the instrument to a class. +3. Returns an instantiated quicklook object. + +Example:: + + >>> ql = get_instrument_quicklook("imap_swapi_20250101_v02.cdf") + >>> ql.two_dimensional_plot("count rates") + +Adding Support for New Instruments +---------------------------------- + +To extend the quicklook system: + +1. Create a new subclass:: + + class NewInstQuicklookGenerator(QuicklookGenerator): + ... + +2. Implement ``two_dimensional_plot`` as a dispatch method. +3. Add plot functions as needed. +4. Register the class in the ``QuicklookGeneratorType`` enum. +5. Ensure that ``ScienceFilePath.extract_filename_components`` correctly + identifies the instrument. + +Instrument Team Support +----------------------- + +To implement a new instrument-specific quicklook, instrument teams must +provide a minimal set of information that defines what plots are +required and how the underlying data should be interpreted. + +Required Information +~~~~~~~~~~~~~~~~~~~~ + +1. **List of quicklook plots** + - A high-level description of each plot to be generated. + - Whether each plot is 1-D (line), 2-D (spectrogram), or multi-panel. + +2. **Variables required for each plot** + - CDF variable names. + - Which CDF files contain the required variables. + +3. **Time-axis requirements (if applicable)** + - Desired time range (full file, event-based selection, rolling window). + +4. **Plot formatting preferences** + - Preferred units or scaling (linear, log). + - Desired titles, axis labels, and annotations. + +5. **Special processing rules** + - Required calibrations or unit conversions. + - Masking rules for invalid ranges or quality flags. + - Any filtering or smoothing applied before plotting. diff --git a/docs/source/code-documentation/spice.rst b/docs/source/algorithm-code-documentation/spice.rst similarity index 100% rename from docs/source/code-documentation/spice.rst rename to docs/source/algorithm-code-documentation/spice.rst diff --git a/docs/source/code-documentation/swapi.rst b/docs/source/algorithm-code-documentation/swapi.rst similarity index 100% rename from docs/source/code-documentation/swapi.rst rename to docs/source/algorithm-code-documentation/swapi.rst diff --git a/docs/source/code-documentation/swe.rst b/docs/source/algorithm-code-documentation/swe.rst similarity index 100% rename from docs/source/code-documentation/swe.rst rename to docs/source/algorithm-code-documentation/swe.rst diff --git a/docs/source/code-documentation/ultra.rst b/docs/source/algorithm-code-documentation/ultra.rst similarity index 100% rename from docs/source/code-documentation/ultra.rst rename to docs/source/algorithm-code-documentation/ultra.rst diff --git a/docs/source/external-tools/cdf/cdf_introduction.rst b/docs/source/cdf-metadata/cdf_introduction.rst similarity index 99% rename from docs/source/external-tools/cdf/cdf_introduction.rst rename to docs/source/cdf-metadata/cdf_introduction.rst index bc8f963882..c814d37ab7 100644 --- a/docs/source/external-tools/cdf/cdf_introduction.rst +++ b/docs/source/cdf-metadata/cdf_introduction.rst @@ -32,7 +32,7 @@ The internal format of CDF files are described in the `cdf specification `_. -This section is intended to act as a high level overview for the data processing architecture of the IMAP SDC, in less technical terms. - -.. image:: ../_static/architecture_overview.png - -`Up to date overview chart in Galaxy `_ - -Each science file that arrives is treated the same, regardless of level or instrument. When a file is placed in the file storage system, it triggers a step to index the file ("indexer lambda"). -This step adds the file to the database and triggers the next step in processing ("batch starter lambda"). - -This step is what determines if a instrument and level is ready for processing, by checking dependencies. For each file that arrives, the system checks to see what the downstream dependencies are - -meaning, what future files need this file in order to complete processing. For example, if a MAG L1A file arrived, this step would determine that the MAG L1B ``mago`` and ``magi`` files are dependent on -the L1A file, and therefore MAG L1B may be ready to begin processing. - -Then, for each anticipated job, the batch starter process checks to see if all the upstream dependencies are met. Although we know we have one of the upstream dependencies for an expected job, -it's possible that there are other required dependencies that have not yet arrived. If we are missing any required dependencies, then the system does not kick off the processing job. -When the missing file arrives, it will trigger the same process of checking for all upstream dependencies. This time all required dependencies will be found and the processing job will be started. - -For example, SWAPI L3 requires both SWAPI L2 files and MAG L1D (previously called L2pre) files. The SWAPI L2 job and the MAG L1D job are run independently, so there is no guarantee that they will finish -at the same time. Let's assume that the MAG L1D job finishes first, since it is the lower level. When that file arrives, one of the downstream dependencies is going to be the SWAPI L3 processing. -However, when batch starter checks the upstream dependencies for SWAPI L3, it will find that SWAPI L2 is missing. Therefore, processing won't start. Once the SWAPI L2 processing finishes, -and the SWAPI L2 file arrives, the batch starter is triggered with that file. Once again, SWAPI L3 is a downstream dependency, but this time, both upstream dependencies for SWAPI L2 are present. -Therefore, processing for SWAPI L3 can begin. - -The status of different files is recorded in the status tracking table. This table records the status of each anticipated output file as "in progress", "complete", or "failed." Through this, -we can track processing for specific files and determine if a file exists quickly. - -Dependency Config File ----------------------- - -How does the SDC track which files are dependent on others? In order to decide what the downstream or upstream dependencies of a file are, and what the nature of those dependencies are, we -need some way to request the upstream or downstream dependencies of a given file. The current dependencies between instruments are recorded in `sds-data-manager Repo `_. - -We handle and track dependencies using a CSV config file that acts like a database. This CSV config file expects a specific format, and is used to determine the upstream and downstream dependencies of each file. - -The CSV config has the following structure: - -===================== ================= ================== ================= ==================== ===================== ========================= ================ -primary_source primary_data_type primary_descriptor dependent_source dependent_data_type dependent_descriptor relationship dependency_type -===================== ================= ================== ================= ==================== ===================== ========================= ================ -mag l1a norm-mago mag l1b norm-mago HARD DOWNSTREAM -mag l1a norm-magi mag l1b norm-magi HARD DOWNSTREAM -mag l1d norm swapi l3 sci HARD DOWNSTREAM -swapi l2 sci swapi l3 sci HARD DOWNSTREAM -idex l0 raw idex l1a all HARD DOWNSTREAM -leapseconds spice historical idex l1a all HARD_NO_TRIGGER DOWNSTREAM -spacecraft_clock spice historical idex l1a all HARD_NO_TRIGGER DOWNSTREAM -hi l1a 45sensor-de hi l1b 45sensor-de HARD DOWNSTREAM -plantary_epehemeris spice historical hi l1b 45sensor-de HARD_NO_TRIGGER DOWNSTREAM -imap_frames spice historical hi l1b 45sensor-de HARD_NO_TRIGGER DOWNSTREAM -attitude spice historical hi l1b 45sensor-de HARD DOWNSTREAM -spin spin historical hi l1b 45sensor-de HARD_NO_TRIGGER DOWNSTREAM -repoint repoint historical hi l1b 45sensor-de HARD_NO_TRIGGER DOWNSTREAM -===================== ================= ================== ================= ==================== ===================== ========================= ================ - -Valid Values for Dependency Config ------------------------------------ - -Primary Source -~~~~~~~~~~~~~~~~~~ - -Primary source can be one of the following: - -.. _imap-data-init: https://github.com/IMAP-Science-Operations-Center/imap-data-access/blob/main/imap_data_access/__init__.py -.. _imap-data-validation: https://github.com/IMAP-Science-Operations-Center/imap-data-access/blob/main/imap_data_access/file_validation.py - -- IMAP instrument name listed in the ``VALID_INSTRUMENTS`` dictionary in this file: - `imap-data-access Repo `_ - -- SPICE data type listed in the ``_SPICE_DIR_MAPPING`` dictionary in this file: - `imap-data-access validation file `_ - - -Primary Data Type -~~~~~~~~~~~~~~~~~~~~ - -Primary data type can be one of the following: - -- IMAP data level listed in the ``VALID_DATALEVELS`` dictionary in this file: - `imap-data-access Repo `_ - -- ``spice`` - -- ``spin`` - -- ``repoint`` - -- ``ancillary`` - -Primary descriptor -~~~~~~~~~~~~~~~~~~~~ - -Primary descriptor can be one of the following: - -- For science or ancillary data, the descriptors are defined by the instrument and SDC. - -- For ``spice`` data types, ``historical`` and ``best`` are the valid descriptors. - -- For ``spin`` and ``repoint`` data types, ``historical`` is the only valid descriptor. - - - -Dependent Source -~~~~~~~~~~~~~~~~~~~ - -Same as primary_source, but for the dependent file. - -Dependent Data Type -~~~~~~~~~~~~~~~~~~~~ - -Same as primary_data_type, but for the dependent file. - -Dependent Descriptor -~~~~~~~~~~~~~~~~~~~~ - -Same as primary_descriptor, but for the dependent file. - -Relationship -~~~~~~~~~~~~~~~~~~~ - -- **HARD** - Triggers processing on file ingestion or a reprocessing event. - -- **HARD_NO_TRIGGER** - Required data file. However, a new version of this file doesn't trigger - processing on file ingestion. - *Example:* leapseconds kernel or frame kernel that doesn't change often. - -- **SOFT_TRIGGER** - A "nice to have" data file that **can trigger** processing on ingestion - for downstream dependencies. - Recommended only for ancillary or SPICE data files, because this may cause - unwanted reprocessing behavior. - *Example:* a calibration file that **does** significantly affect output and - should cause reprocessing of past data falling within the updated time range. - -- **SOFT_NO_TRIGGER** - A "nice to have" file that **does not trigger** processing on ingestion. - *Example:* calibration files with minor updates that you still want included - in processing for current and future data products. - -Dependency Types -~~~~~~~~~~~~~~~~~~~ - -- **DOWNSTREAM** - This is a downstream dependency, meaning that job to kick off when this file arrives. - -- **UPSTREAM** - This is an upstream dependency. This means that upstream processing is blocked on - the existence of dependent files, meaning that a file required to kick off processing for - current file. NOTE: In the dependency config file, we only specify downstream dependencies. - Then in the dependency lambda at run time, it will determine the upstream dependencies - based on the downstream dependencies. diff --git a/docs/source/code-documentation/tools/imap-data-access.rst b/docs/source/data-access/imap-data-access.rst similarity index 100% rename from docs/source/code-documentation/tools/imap-data-access.rst rename to docs/source/data-access/imap-data-access.rst diff --git a/docs/source/data-access/index.rst b/docs/source/data-access/index.rst index 80bc4b5ec8..b36b9cf7f4 100644 --- a/docs/source/data-access/index.rst +++ b/docs/source/data-access/index.rst @@ -1,4 +1,4 @@ -.. _data-access-api: +.. _data-access: Data Access API =============== @@ -7,13 +7,19 @@ The `imap-data-access is not a valid query parameter. Valid query parameters are: ['file_path', 'instrument', 'data_level', 'descriptor', 'start_date', 'end_date', 'version', 'extension']"} - -Other pages ------------ - .. toctree:: - :maxdepth: 1 - - calibration-files - data-dependency - naming-conventions + :maxdepth: 1 + :hidden: + generated/imap_data_access.io.download + generated/imap_data_access.io.query + generated/imap_data_access.io.upload diff --git a/docs/source/data-access/openapi.yml b/docs/source/data-access/openapi.yml index a59a191402..4598550542 100644 --- a/docs/source/data-access/openapi.yml +++ b/docs/source/data-access/openapi.yml @@ -1,7 +1,7 @@ openapi: 3.0.0 servers: - - description: Development IMAP SDC Server -host: https://api.dev.imap-mission.com/ + - description: Production IMAP SDC Server +host: https://api.imap-mission.com/ info: version: "0.1.0" title: IMAP SDC API @@ -229,4 +229,181 @@ paths: type: array items: type: string - format: uri \ No newline at end of file + format: uri + + '/spice-query': + get: + tags: + - SPICE Query + summary: Query for SPICE kernel metadata + operationId: spice-query + parameters: + - in: query + name: file_name + description: | + The name of a specific SPICE kernel file (e.g. ``naif0012.tls``). + required: false + schema: + type: string + - in: query + name: start_time + description: | + Coverage start time in TDB seconds since J2000. Returns kernels whose + coverage ends on or after this time. + required: false + schema: + type: string + - in: query + name: end_time + description: | + Coverage end time in TDB seconds since J2000. Returns kernels whose + coverage begins on or before this time. + required: false + schema: + type: string + - in: query + name: type + description: | + The SPICE kernel type to filter by. Accepted values are: + ``leapseconds``, ``planetary_constants``, ``imap_frames``, + ``science_frames``, ``spacecraft_clock``, ``earth_attitude``, + ``planetary_ephemeris``, ``ephemeris_reconstructed``, + ``ephemeris_nominal``, ``ephemeris_predicted``, ``ephemeris_90days``, + ``ephemeris_long``, ``ephemeris_launch``, ``attitude_history``, + ``attitude_predict``, ``pointing_attitude``. + required: false + schema: + type: string + - in: query + name: latest + description: | + If ``True``, only return the latest version of each kernel matching + the other query parameters. + required: false + schema: + type: string + - in: query + name: start_ingest_date + description: | + Filter results to kernels ingested on or after this date, in the + format ``YYYYMMDD``. + required: false + schema: + type: string + - in: query + name: end_ingest_date + description: | + Filter results to kernels ingested on or before this date, in the + format ``YYYYMMDD``. + required: false + schema: + type: string + responses: + '200': + description: Successful query — returns a list of SPICE kernel metadata objects + content: + application/json: + schema: + type: array + items: + type: object + '400': + description: Invalid query parameter or parameter value + content: + application/json: + schema: + type: string + + '/metakernel': + get: + tags: + - SPICE Query + summary: Retrieve a metakernel or list of SPICE kernel filenames for a time range + operationId: metakernel + parameters: + - in: query + name: start_time + description: | + Coverage start time. Accepts either TDB seconds since J2000 or a date + string in the format ``YYYYMMDD``. + required: true + schema: + type: string + - in: query + name: end_time + description: | + Coverage end time. Accepts either TDB seconds since J2000 or a date + string in the format ``YYYYMMDD``. + required: true + schema: + type: string + - in: query + name: spice_path + description: | + Root path for the SPICE directory. This path is prepended to all kernel file + locations in the returned metakernel. If omitted the paths are left + relative. Only applicable if ``list_files``` is ``False``. + required: false + schema: + type: string + - in: query + name: list_files + description: | + If ``True``, return only a list of kernel filenames instead of a full + metakernel file. + required: false + schema: + type: string + - in: query + name: require_coverage + description: | + If ``True``, the request will fail with a HTTP ``422`` status if there + are any gaps in coverage for the requested time range. + required: false + schema: + type: string + - in: query + name: file_types + description: | + Comma-separated list of kernel types to include. If omitted, all + available kernel types are included. Accepted values are: + ``leapseconds``, ``planetary_constants``, ``imap_frames``, + ``science_frames``, ``spacecraft_clock``, ``earth_attitude``, + ``planetary_ephemeris``, ``ephemeris_reconstructed``, + ``ephemeris_nominal``, ``ephemeris_predicted``, ``ephemeris_90days``, + ``ephemeris_long``, ``ephemeris_launch``, ``attitude_history``, + ``attitude_predict``, ``pointing_attitude``. + required: false + schema: + type: string + responses: + '200': + description: | + Successful response. Returns a metakernel file (text/plain) when + ``list_files`` is ``False`` (default), or a JSON array of kernel + filenames when ``list_files`` is ``True``. + content: + text/plain: + schema: + type: string + application/json: + schema: + type: array + items: + type: string + '404': + description: No kernel files found for the requested time range + content: + application/json: + schema: + type: string + '422': + description: | + Coverage gaps detected when ``require_coverage=True``. The response + body contains a list of the gap intervals. + content: + application/json: + schema: + type: array + items: + type: object \ No newline at end of file diff --git a/docs/source/data-access/spice-files.rst b/docs/source/data-access/spice-files.rst new file mode 100644 index 0000000000..2a840ddb6e --- /dev/null +++ b/docs/source/data-access/spice-files.rst @@ -0,0 +1,220 @@ +SPICE Files +=========== + +The IMAP SDC provides two REST API endpoints for accessing SPICE kernel data: +``/spice-query`` for querying kernel metadata and ``/metakernel`` for retrieving +a ready-to-use metakernel (or a list of kernel filenames) that covers a +requested time range. + +Both endpoints are accessible from the base URL: ``https://api.imap-mission.com`` + +.. _spice-query-endpoint: + +SPICE Query +----------- + +The ``/spice-query`` endpoint returns metadata for SPICE kernels stored in the +SDC database. Results can be filtered by kernel type, time range, ingestion +date, or filename. + +.. openapi:: openapi.yml + :group: + :include: /spice-query + +**Example Usage:** + +.. code-block:: bash + + # Query for all attitude_history kernels covering a time range + curl -X GET -H "Accept: application/json" \ + "https://api.imap-mission.com/spice-query?start_time=315576066&end_time=4575787269&type=attitude_history" + + # Query for a specific kernel by filename + curl -X GET -H "Accept: application/json" \ + "https://api.imap-mission.com/spice-query?file_name=naif0012.tls" + + # Query for the latest version of the spacecraft_clock kernel + curl -X GET -H "Accept: application/json" \ + "https://api.imap-mission.com/spice-query?type=spacecraft_clock&latest=True" + +**Possible Responses:** + +.. code-block:: json + + [ + { + "file_name": "ck/imap_2025_274_2025_276_001.ah.bc", + "file_root": "imap_2025_274_2025_276_.ah.bc", + "kernel_type": "attitude_history", + "version": 1, + "min_date_j2000": 812631669.2765242, + "max_date_j2000": 812725268.1910387, + "file_intervals_j2000": [ + [ + 812631669.2765242, + 812725268.1910387 + ] + ], + "min_date_datetime": "2025-10-01, 23:00:00", + "max_date_datetime": "2025-10-03, 00:59:59", + "file_intervals_datetime": [ + [ + "2025-10-01T23:00:00.094178+00:00", + "2025-10-03T00:59:59.008694+00:00" + ] + ], + "min_date_sclk": "1/0497055600:00000", + "max_date_sclk": "1/0497149199:00000", + "file_intervals_sclk": [ + [ + "1/0497055600:00000", + "1/0497149199:00000" + ] + ], + "sclk_kernel": "/tmp/naif0012.tls", + "lsk_kernel": "/tmp/imap_sclk_0147.tsc", + "ingestion_date": "2026-04-06, 23:57:54", + "timestamp": 1775519874 + }, + ] + +.. code-block:: json + + {"statusCode": 400, "body": " is not a valid query parameter. Valid query parameters are: ['file_name', 'start_time', 'end_time', 'type', 'latest', 'start_ingest_date', 'end_ingest_date']"} + +.. _metakernel-endpoint: + +Metakernel +---------- + +The ``/metakernel`` endpoint builds and returns a SPICE metakernel covering a +requested time range. By default the response is a ``.tm`` metakernel file that +can be loaded directly with ``spiceypy.furnsh()``. Passing ``list_files=True`` +returns a JSON list of kernel filenames instead. + +.. openapi:: openapi.yml + :group: + :include: /metakernel + +**Example Usage:** + +.. code-block:: bash + + # Retrieve a metakernel covering a time range for selected kernel types + curl -X GET -H "Accept: application/json" \ + "https://api.imap-mission.com/metakernel?start_time=797949057&end_time=798035454&file_types=leapseconds,attitude_history" + + # Get only a list of kernel filenames (no metakernel wrapper) + curl -X GET -H "Accept: application/json" \ + "https://api.imap-mission.com/metakernel?start_time=797949057&end_time=798035454&file_types=leapseconds,spacecraft_clock&list_files=True" + + # Retrieve a metakernel with a custom spice_path prefix in the output + curl -X GET -H "Accept: application/json" \ + "https://api.imap-mission.com/metakernel?start_time=797949057&end_time=798035454&spice_path=/my/path/imap" + +**Possible Responses:** + +Metakernel file (default, ``list_files`` omitted or ``False``): + +.. code-block:: text + + \begintext + + This is the most up to date Metakernel as of + 2026-04-13 18:19:21.912444+00:00. + + This attempts to cover data from + 797949057 to 798035454 + seconds since J2000. + + \begindata + + KERNELS_TO_LOAD = ( 'lsk/naif0012.tls', + 'fk/imap_130.tf', + 'fk/imap_science_120.tf', + 'sclk/imap_sclk_0147.tsc', + 'spk/de440.bsp' + ) + + \begintext + +List of filenames (``list_files=True``): + +.. code-block:: json + + ["naif0012.tls", "imap_130.tf", "imap_science_120.tf", "imap_sclk_0147.tsc", "de440.bsp"] + +Coverage gap error (``require_coverage=True`` and gaps exist): + +.. code-block:: json + + { + "statusCode": 422, + "body": { + "leapseconds_category": [], + "planetary_constants_category": [], + "imap_frames_category": [], + "science_frames_category": [], + "spacecraft_clock_category": [], + "earth_attitude_category": [], + "planetary_ephemeris_category": [], + "spacecraft_ephemeris_category": [[797990001, 798035454]], + "spacecraft_attitude_category": [[797949057, 797990000]], + "pointing_attitude_category": [] + } + } + +Each key is a SPICE kernel category. Empty lists indicate full coverage; non-empty +lists contain one or more ``[start_time, end_time]`` intervals (in TDB seconds +since J2000) where no kernel was found. + +No files found: + +.. code-block:: json + + {"statusCode": 404, "body": "No files found."} + +Kernel Types +------------ + +The following kernel type values are accepted by both the ``type`` parameter of +``/spice-query`` and the ``file_types`` parameter of ``/metakernel``: + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Kernel Type + - Description + * - ``leapseconds`` + - Leapseconds kernel (LSK) + * - ``planetary_constants`` + - Planetary constants kernel (PCK) + * - ``imap_frames`` + - IMAP spacecraft frames kernel (FK) + * - ``science_frames`` + - Science instrument frames kernel (FK) + * - ``spacecraft_clock`` + - Spacecraft clock kernel (SCLK) + * - ``earth_attitude`` + - Earth attitude kernel + * - ``planetary_ephemeris`` + - Planetary ephemeris kernel (SPK) + * - ``ephemeris_reconstructed`` + - Reconstructed spacecraft ephemeris (SPK) + * - ``ephemeris_nominal`` + - Nominal spacecraft ephemeris (SPK) + * - ``ephemeris_predicted`` + - Predicted spacecraft ephemeris (SPK) + * - ``ephemeris_90days`` + - 90-day spacecraft ephemeris (SPK) + * - ``ephemeris_long`` + - Long-term spacecraft ephemeris (SPK) + * - ``ephemeris_launch`` + - Launch ephemeris (SPK) + * - ``attitude_history`` + - Historical spacecraft attitude kernel (CK) + * - ``attitude_predict`` + - Predicted spacecraft attitude kernel (CK) + * - ``pointing_attitude`` + - Pointing attitude kernel (CK) diff --git a/docs/source/development/cli.rst b/docs/source/development/cli.rst new file mode 100644 index 0000000000..88ffca04f9 --- /dev/null +++ b/docs/source/development/cli.rst @@ -0,0 +1,10 @@ +.. _cli: + +CLI +=== + +.. currentmodule:: imap_processing + +This is the CLI for running IMAP processing per instrument. + +TODO: more information to come in the future. diff --git a/docs/source/development/data-dependency.rst b/docs/source/development/data-dependency.rst new file mode 100644 index 0000000000..d2d70b8a74 --- /dev/null +++ b/docs/source/development/data-dependency.rst @@ -0,0 +1,311 @@ +Data Dependency Management +========================== + +The IMAP Science Data Center (SDC) utilizes an event-based processing system that allows for +processing as soon as data is available. This system is designed to be flexible to +accommodate the various requirements and inter-dependencies for all 10 instruments. + +As part of our requirements, we need some way to explicitly describe the dependencies +for each file. We also need to be able to flexibly update the dependencies on a regular +basis, to accommodate changing requirements. + +Overview +-------- + +When a file lands in the SDC, it is added to our data bucket (Also called S3 or S3 bucket.) +This bucket, as the name implies, is a simple collection which contains all the files in the +SDC, organized like a file system. + +Each data file is put into a specific subfolder depending on the file name. For example, +a file named ``imap_swe_l0_sci_20240105_20240105_v00-01.pkts`` would be placed in the +``imap/swe/l0/2024/01`` folder. More information about the naming conventions can be +found in :ref:`naming-conventions`. + +When a file of any level arrives in the bucket, it triggers the rest of processing. This is +how we manage file processing within the SDC, rather than waiting until all files have arrived +or running at particular times of day. This allows us to quickly process data as soon as all the +required pieces are available to us, and create a flexible system which can easily be updated +to add exceptions or new requirements on a per-instrument or per-level basis. + +.. note:: + This document, and our tooling, uses the terms "upstream dependencies" and + "downstream dependencies" to describe the relationships between files. A + "downstream dependency" for a given file means that the current file is required for + processing of the downstream files. For example, an L2 file is a downstream dependency + of an L1 file. An "upstream dependency" is the opposite, describing a file which is required + to begin processing the current file. For example, an L1 file is an upstream dependency of an + L2 file. + +Detailed Description of File Processing +--------------------------------------- + +For explicit descriptions of the tools and technical choices of the IMAP SDC, please refer to +`this Galaxy page `_. +This section is intended to act as a high level overview for the data processing architecture of +the IMAP SDC, in less technical terms. + +.. image:: ../_static/architecture_overview.png + +`Up to date overview chart in Galaxy `_ + +Each science file that arrives is treated the same, regardless of level or instrument. When a file +is placed in the file storage system, it triggers a step to index the file ("indexer lambda"). +This step adds the file to the database and triggers the next step in processing ("batch starter lambda"). + +After indexing, the batch starter lambda is triggered in order to determine what jobs may be ready for processing. +For each file that arrives, the system checks to see what job may need to be run by looking +at the downstream dependencies are.For example, if a MAG L1A file arrived, this step would +determine that the MAG L1B ``mago`` and ``magi`` files are dependent on +the L1A file, and therefore MAG L1B may be ready to begin processing. + +Then, for each possible job, the batch starter process checks to see if all the upstream +dependencies are met. Although we know we have one of the upstream dependencies for an +expected job, it's possible that there are other required dependencies that have not yet +arrived. If we are missing any required dependencies, then the system does not kick off the +processing job. When the missing upstream dependency arrives, it will trigger the same process of checking +for all upstream dependencies. This time all required dependencies will be found and the +processing job will be started. + +The upstream look up system will determine if it has a complete list of dependencies. +Several scenarios can cause the dependency list to be incomplete. Missing files in the database +represent the primary cause. Anomalies such as Loss of Orientation Insertion (LOI) or Trajectory +Correction Maneuver (TCM) events, as well as solar wind conditions, may also result in incomplete +dependencies, though support for these scenarios is not yet implemented. Similarly, delays in +repoint data or downlink delays can cause incompleteness, but handling for these cases is also +planned for future implementation. Additionally, if any required dependencies are missing or if a +job is still in progress, the dependency list cannot be considered complete. + +For example, SWAPI L3 requires both SWAPI L2 files and MAG L1D (previously called L2pre) +files. The SWAPI L2 job and the MAG L1D job are run independently, so there is no guarantee +that they will finish at the same time. Let's assume that the MAG L1D job finishes first, +since it is the lower level. When that file arrives, one of the downstream dependencies is +going to be the SWAPI L3 processing. However, when batch starter checks the upstream +dependencies for SWAPI L3, it will find that SWAPI L2 is missing. Therefore, processing +won't start. Once the SWAPI L2 processing finishes, and the SWAPI L2 file arrives, the batch +starter is triggered with that file. Once again, SWAPI L3 is a downstream dependency, but +this time, both upstream dependencies for SWAPI L2 are present. Therefore, processing for +SWAPI L3 can begin. + +The status of each job is recorded in the status tracking table as "in progress", "complete", +or "failed." Through this, we can track processing for specific files and determine if a +file exists quickly. + +Dependency Config File +---------------------- + +How does the SDC track which files are dependent on others? In order to decide what the +downstream or upstream dependencies of a file are, and what the nature of those dependencies +are, we need some way to request the upstream or downstream dependencies of a given file. +The current dependencies between instruments are recorded in `sds-data-manager Repo +`_. + +We handle and track dependencies using a YAML config file that acts like a database. This YAML +config file expects a specific format, and is used to determine the upstream and downstream +dependencies of each product. + +Filename convention +~~~~~~~~~~~~~~~~~~~~ +imap__dependencies.yaml + + +Dependency Types +~~~~~~~~~~~~~~~~~ + +The YAML config file stores the upstream dependencies for each data product. This information +is used across all instruments to determine both upstream and downstream relationships: + +**UPSTREAM** +An upstream dependency is a file required to begin processing the current product. +The dependency config file explicitly defines these upstream dependencies for each data product. + +**DOWNSTREAM** +A downstream dependency is a product whose processing depends on the current file. +Downstream dependencies are determined at runtime by querying which products list the current +file as an upstream dependency. + +Valid Fields for Dependency Config +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. _imap-data-init: https://github.com/IMAP-Science-Operations-Center/imap-data-access/blob/main/imap_data_access/__init__.py +.. _imap-data-validation: https://github.com/IMAP-Science-Operations-Center/imap-data-access/blob/main/imap_data_access/file_validation.py + +Upstream Source +^^^^^^^^^^^^^^^ + +Upstream source can be one of the following: + +- IMAP instrument name listed in the ``VALID_INSTRUMENTS`` dictionary in this file: + `imap-data-access Repo `_ + +- SPICE data type listed in the ``_SPICE_DIR_MAPPING`` dictionary in this file: + `imap-data-access validation file `_ + + +Upstream Data Type +^^^^^^^^^^^^^^^^^^ + +Upstream data type can be one of the following: + +- IMAP data level listed in the ``VALID_DATALEVELS`` dictionary in this file: + `imap-data-access Repo `_ + +- ``spice`` + +- ``spin`` + +- ``repoint`` + +- ``ancillary`` + + +Upstream Descriptor +^^^^^^^^^^^^^^^^^^^^^ + +Upstream descriptor can be one of the following: + +- For science or ancillary data, the descriptors are defined by the instrument and SDC. + +- For ``spice`` data types, ``historical``, and ``best`` are the valid descriptors. + +- For ``spin`` and ``repoint`` data types, ``historical`` is the only valid descriptor. + +Required (Optional) +^^^^^^^^^^^^^^^^^^^ + +**Default:** ``true`` + +Specifies whether this upstream dependency must be available before a processing +job can begin. +If set to true, the product cannot be processed until this dependency is available. +If set to false, the product can be processed even if this dependency is missing. + + +Trigger_job (Optional) +^^^^^^^^^^^^^^^^^^^^^^ + +**Default:** ``true`` + +Whether the arrival of this upstream dependency should trigger a processing job. +There are cases where we do not want to start a job when certain upstream data arrives. +For example, upstream inputs such as spacecraft clock or leapseconds data should not change +frequently, and processing jobs should not be triggered every time these files are updated. +Setting this to false allows for more controlled processing and may require additional +review before updating these types of dependencies. + +(Past_days, Future_days) (Optional) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +**Default:** + +- ENA and GLOWS: ``[0p, 0p]`` +- Rest of in-situ instruments: ``[0d, 0d]`` + +Most science files are produced daily or per pointing. Due to this cadence, the default is +daily for most in-situ instruments and per pointing for ENA and GLOWS instruments. However, +this feature provides flexibility to query for upstream data beyond the daily date range of +the current product. + +Supported values for past_days and future_days fields: + +- ``p`` - pointing +- ``h`` - hourly +- ``d`` - days +- ``l`` - last_processed +- ``nd``- nearest day +- ``np``- nearest pointing + +Days can be used to support longer durations and different cadences. For example, weekly +processing can use 7 days, and yearly processing can use 365 days. + +``last_processed`` - retrieves the last x processed science data files to use to query for files needed for the current processing job. +For example, IDEX science job requires all housekeeping data since the start date of the last processed science file. + +File content structure +~~~~~~~~~~~~~~~~~~~~~~ +The YAML config has the following structure: + +.. code-block:: yaml + + (level, product_name): + - ( + upstream_source, + upstream_data_type, + upstream_product_name, + required(bool), + kickoff_job(bool), + (past_days, future_days) + ) + - ( + upstream_source, + upstream_data_type, + upstream_product_name, + required(bool), + kickoff_job(bool), + (past_days, future_days) + ) + .... + + +File content Example +~~~~~~~~~~~~~~~~~~~~~~ + +**imap_hit_dependencies.yaml** + +.. code-block:: yaml + + spice_basics: &spice_basics + - upstream_source: leapseconds + upstream_data_type: spice + upstream_descriptor: historical + kickoff_job: false + - upstream_source: spacecraft_clock + upstream_data_type: spice + upstream_descriptor: historical + kickoff_job: false + + l0_data: &l0_data + - upstream_source: hit + upstream_data_type: l0 + upstream_descriptor: raw + + (l1a, all): + - *spice_basics + - *l0_data + + (l1b, hk): + - *spice_basics + - *l0_data + +**imap_hi_dependencies.yaml** + +.. code-block:: yaml + + spice_basic: &spice_basic + - upstream_source: leapseconds + upstream_data_type: spice + upstream_descriptor: historical + kickoff_job: false + - upstream_source: spacecraft_clock + upstream_data_type: spice + upstream_descriptor: historical + kickoff_job: false + + (l1b, 45sensor-goodtimes): + - *spice_basic + - upstream_source: repoint + upstream_data_type: repoint + upstream_descriptor: historical + kickoff_job: false + - upstream_source: hi + upstream_data_type: ancillary + upstream_descriptor: 45sensor-cal-prod + - upstream_source: hi + upstream_data_type: l1a + upstream_descriptor: 45sensor-diagfee + - upstream_source: hi + upstream_data_type: l1b + upstream_descriptor: 45sensor-de + date_range: ["6np",] + - upstream_source: hi + upstream_data_type: l1b + upstream_descriptor: 45sensor-hk diff --git a/docs/source/development/doc-overview.rst b/docs/source/development/doc-overview.rst index e4db4e19f3..1824e38ae6 100644 --- a/docs/source/development/doc-overview.rst +++ b/docs/source/development/doc-overview.rst @@ -1,3 +1,5 @@ +.. _doc-overview: + Contributing to Documentation ============================= diff --git a/docs/source/development/docker.rst b/docs/source/development/docker.rst index 48003cc832..f5dd65ca80 100644 --- a/docs/source/development/docker.rst +++ b/docs/source/development/docker.rst @@ -1,5 +1,7 @@ +.. _docker: + Docker Workflow ----------------- +=============== This page describes how to build and run a Docker Image Locally and in AWS. diff --git a/docs/source/development/git-access-roles.rst b/docs/source/development/git-access-roles.rst new file mode 100644 index 0000000000..d0cf439817 --- /dev/null +++ b/docs/source/development/git-access-roles.rst @@ -0,0 +1,56 @@ +.. _git-access-roles: + +GitHub Access & Permissions Guide +====================================== + +This document outlines the permission levels available in IMAP SDC +repositories. + +Overview +======== + +GitHub provides five repository roles with varying permission levels. +Below we outline the permissions needed to contribute to IMAP SDC repositories. + +--- + +What GitHub Users Can Access +============================= + +Any GitHub user (without invitation) can: + +- ✅ View **public repositories** +- ✅ Fork public repositories +- ✅ Create issues in public repos +- ✅ Comment on public issues +- ✅ Edit titles and descriptions of your own issues +- ✅ Create pull requests +- ❌ **Cannot** add label or assignees, etc to the issues +- ❌ **Cannot** trigger unit test workflows on PRs (requires SDC approval) +- ❌ **Cannot** request reviewers on PRs (SDC must assign reviewers) +- ❌ **Cannot** push code, merge PRs, or modify issues +- ❌ **Cannot** access private repositories + +--- + +Role Permissions & Responsibilities +----------------------------------- +GitHub Read Role +~~~~~~~~~~~~~~~~~ + +Read role should give same permissions as a user without invitation for IMAP SDC public repositories. +These permissions align with IMAP project requirements and are sufficient for most L0 to L3 code +contributors based on project guidelines. + + +Additional GitHub Roles +----------------------- + +Beyond the **GitHub Read** role (which is sufficient for most contributors), GitHub +provides additional permission levels: `Triage`, `Write`, `Maintain`, and `Admin`. +Please read the GitHub's breakdown of these roles and their permissions in the +`GitHub documentation on repository roles `_. + + +**If you need permissions** please contact the IMAP SDC team to request the +appropriate access level for your role. diff --git a/docs/source/development/style-guide/checklist-for-pull-requests.rst b/docs/source/development/git-workflow-and-style-guide/checklist-for-pull-requests.rst similarity index 100% rename from docs/source/development/style-guide/checklist-for-pull-requests.rst rename to docs/source/development/git-workflow-and-style-guide/checklist-for-pull-requests.rst diff --git a/docs/source/development/style-guide/git-and-github-workflow.rst b/docs/source/development/git-workflow-and-style-guide/git-and-github-workflow.rst similarity index 100% rename from docs/source/development/style-guide/git-and-github-workflow.rst rename to docs/source/development/git-workflow-and-style-guide/git-and-github-workflow.rst diff --git a/docs/source/development/git-workflow-and-style-guide/index.rst b/docs/source/development/git-workflow-and-style-guide/index.rst new file mode 100644 index 0000000000..1b91ba9c27 --- /dev/null +++ b/docs/source/development/git-workflow-and-style-guide/index.rst @@ -0,0 +1,20 @@ +.. _git-workflow-and-style-guide: + +GitHub Workflow and Style Guide +================================ + +This section covers best practices for contributing to the IMAP project, including workflow guidelines, code style standards, and security considerations. + +.. toctree:: + :maxdepth: 1 + + checklist-for-pull-requests + Git and GitHub Workflow + poetry-environment + python-coding + python-docstrings + review-standards + security + style-guide + tools-and-library-recommendations + versioning \ No newline at end of file diff --git a/docs/source/development/style-guide/poetry-environment.rst b/docs/source/development/git-workflow-and-style-guide/poetry-environment.rst similarity index 100% rename from docs/source/development/style-guide/poetry-environment.rst rename to docs/source/development/git-workflow-and-style-guide/poetry-environment.rst diff --git a/docs/source/development/style-guide/python-coding.rst b/docs/source/development/git-workflow-and-style-guide/python-coding.rst similarity index 100% rename from docs/source/development/style-guide/python-coding.rst rename to docs/source/development/git-workflow-and-style-guide/python-coding.rst diff --git a/docs/source/development/style-guide/python-docstrings.rst b/docs/source/development/git-workflow-and-style-guide/python-docstrings.rst similarity index 100% rename from docs/source/development/style-guide/python-docstrings.rst rename to docs/source/development/git-workflow-and-style-guide/python-docstrings.rst diff --git a/docs/source/development/style-guide/review-standards.rst b/docs/source/development/git-workflow-and-style-guide/review-standards.rst similarity index 100% rename from docs/source/development/style-guide/review-standards.rst rename to docs/source/development/git-workflow-and-style-guide/review-standards.rst diff --git a/docs/source/development/style-guide/security.rst b/docs/source/development/git-workflow-and-style-guide/security.rst similarity index 100% rename from docs/source/development/style-guide/security.rst rename to docs/source/development/git-workflow-and-style-guide/security.rst diff --git a/docs/source/development/style-guide/style-guide.rst b/docs/source/development/git-workflow-and-style-guide/style-guide.rst similarity index 87% rename from docs/source/development/style-guide/style-guide.rst rename to docs/source/development/git-workflow-and-style-guide/style-guide.rst index bf3bf70cba..38445f84c1 100644 --- a/docs/source/development/style-guide/style-guide.rst +++ b/docs/source/development/git-workflow-and-style-guide/style-guide.rst @@ -29,16 +29,3 @@ these items are provided below in the guide. Contributors can refer to the :ref:`checklist for contributors and reviewers of pull requests ` for assistance in making sure pull requests are adhering to these conventions. - -.. toctree:: - :maxdepth: 1 - - git-and-github-workflow - python-coding - python-docstrings - poetry-environment - security - tools-and-library-recommendations - versioning - checklist-for-pull-requests - review-standards \ No newline at end of file diff --git a/docs/source/development/style-guide/tools-and-library-recommendations.rst b/docs/source/development/git-workflow-and-style-guide/tools-and-library-recommendations.rst similarity index 100% rename from docs/source/development/style-guide/tools-and-library-recommendations.rst rename to docs/source/development/git-workflow-and-style-guide/tools-and-library-recommendations.rst diff --git a/docs/source/development/style-guide/versioning.rst b/docs/source/development/git-workflow-and-style-guide/versioning.rst similarity index 100% rename from docs/source/development/style-guide/versioning.rst rename to docs/source/development/git-workflow-and-style-guide/versioning.rst diff --git a/docs/source/development/index.rst b/docs/source/development/index.rst index b3ef157c4c..823133dacb 100644 --- a/docs/source/development/index.rst +++ b/docs/source/development/index.rst @@ -1,5 +1,5 @@ -Development -=========== +Onboarding and Collaboration +============================ :ref:`getting-started` @@ -14,9 +14,14 @@ be versioned appropriately to correspond with the code that produced them. .. toctree:: :maxdepth: 1 - getting-started + cli + data-dependency doc-overview docker + getting-started + git-access-roles + git-workflow-and-style-guide/index + poetry release-workflow - style-guide/style-guide - technology-stack \ No newline at end of file + technology-stack + tools/index diff --git a/docs/source/external-tools/poetry.rst b/docs/source/development/poetry.rst similarity index 100% rename from docs/source/external-tools/poetry.rst rename to docs/source/development/poetry.rst diff --git a/docs/source/development/technology-stack.rst b/docs/source/development/technology-stack.rst index 76c5a39cdf..0eaed4914e 100644 --- a/docs/source/development/technology-stack.rst +++ b/docs/source/development/technology-stack.rst @@ -1,5 +1,7 @@ +.. _technology-stack: + Technology Stack ----------------- +================ This page lists the various technologies and libraries that the IMAP SDC utilizes along with a few notes on what they are used for, why they were chosen, diff --git a/docs/source/code-documentation/tools/cdf-global-attrs.rst b/docs/source/development/tools/cdf-global-attrs.rst similarity index 96% rename from docs/source/code-documentation/tools/cdf-global-attrs.rst rename to docs/source/development/tools/cdf-global-attrs.rst index a02278b5c4..1ee1ca9642 100644 --- a/docs/source/code-documentation/tools/cdf-global-attrs.rst +++ b/docs/source/development/tools/cdf-global-attrs.rst @@ -1,3 +1,5 @@ +.. _cdf-global-attrs: + Code for creating CDF attributes ================================ diff --git a/docs/source/code-documentation/tools/index.rst b/docs/source/development/tools/index.rst similarity index 80% rename from docs/source/code-documentation/tools/index.rst rename to docs/source/development/tools/index.rst index 9f68abfd1c..35bc36097e 100644 --- a/docs/source/code-documentation/tools/index.rst +++ b/docs/source/development/tools/index.rst @@ -1,3 +1,5 @@ +.. _tools: + Tools ===== @@ -8,5 +10,4 @@ These are tools written by the IMAP team that are used across multiple instrumen xtce-generator xarray-to-cdf - cdf-global-attrs - imap-data-access \ No newline at end of file + cdf-global-attrs \ No newline at end of file diff --git a/docs/source/code-documentation/tools/xarray-to-cdf.rst b/docs/source/development/tools/xarray-to-cdf.rst similarity index 100% rename from docs/source/code-documentation/tools/xarray-to-cdf.rst rename to docs/source/development/tools/xarray-to-cdf.rst diff --git a/docs/source/code-documentation/tools/xtce-generator.rst b/docs/source/development/tools/xtce-generator.rst similarity index 100% rename from docs/source/code-documentation/tools/xtce-generator.rst rename to docs/source/development/tools/xtce-generator.rst diff --git a/docs/source/external-tools/index.rst b/docs/source/external-tools/index.rst deleted file mode 100644 index e8c95cfb3e..0000000000 --- a/docs/source/external-tools/index.rst +++ /dev/null @@ -1,10 +0,0 @@ -External Tools -============== - -This page provides documentation written by the IMAP team about external tools. - -.. toctree:: - :maxdepth: 1 - - poetry - cdf/index \ No newline at end of file diff --git a/docs/source/data-access/calibration-files.rst b/docs/source/filename-convention/calibration-files.rst similarity index 100% rename from docs/source/data-access/calibration-files.rst rename to docs/source/filename-convention/calibration-files.rst diff --git a/docs/source/filename-convention/index.rst b/docs/source/filename-convention/index.rst new file mode 100644 index 0000000000..eaa3e0c1f0 --- /dev/null +++ b/docs/source/filename-convention/index.rst @@ -0,0 +1,12 @@ +.. _filename-conventions: + +Filename Conventions +==================== + +This section describes the naming conventions used for IMAP data products and files. + +.. toctree:: + :maxdepth: 1 + + Science Filename Convention + Ancillary Filename Convention diff --git a/docs/source/data-access/naming-conventions.rst b/docs/source/filename-convention/naming-conventions.rst similarity index 100% rename from docs/source/data-access/naming-conventions.rst rename to docs/source/filename-convention/naming-conventions.rst diff --git a/docs/source/index.rst b/docs/source/index.rst index b93d17800a..dd92b214fb 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -16,16 +16,18 @@ IMAP mission and being developed at To get started with the project: :ref:`getting-started`. -The explicit code interfaces and structure are described in the :ref:`code-documentation`. +The explicit code interfaces and structure are described in the :ref:`algorithm-code-documentation`. .. toctree:: :maxdepth: 1 - code-documentation/index - development/index - project-management/index - external-tools/index - data-access/index + Onboarding & Collaboration + IMAP Data Access Tool + CDF Metadata Resources + Filename Conventions + SDC Project Management + Algorithm Code Documentation + If you make use of any ``imap_processing`` code, please consider citing it in your research. `https://zenodo.org/record/11168295 `_ diff --git a/imap_processing/ancillary/ancillary_dataset_combiner.py b/imap_processing/ancillary/ancillary_dataset_combiner.py index 02e2d06a1a..104255f876 100644 --- a/imap_processing/ancillary/ancillary_dataset_combiner.py +++ b/imap_processing/ancillary/ancillary_dataset_combiner.py @@ -123,7 +123,7 @@ def convert_to_timestamped_data(self, filename: str | Path) -> TimestampedData: ) end_dt = np.datetime64(formatted_str, "D") else: - end_dt = self.expected_end_date + end_dt = self.expected_end_date # type: ignore[assignment] return TimestampedData(start_dt, end_dt, dataset, filepath.version) @@ -338,7 +338,7 @@ def __init__( ): super().__init__(ancillary_input, expected_end_date) - def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: + def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: # noqa: PLR0911 """ Convert GLOWS ancillary .dat files to xarray datasets. @@ -364,6 +364,19 @@ def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: if "excluded-regions" in filename: # Handle excluded regions (2 columns: longitude, latitude) data = np.loadtxt(filepath, comments="#") + if data.size == 0: + return xr.Dataset( + { + "ecliptic_longitude_deg": ( + ["region"], + np.array([], dtype=float), + ), + "ecliptic_latitude_deg": ( + ["region"], + np.array([], dtype=float), + ), + } + ) return xr.Dataset( { "ecliptic_longitude_deg": (["region"], data[:, 0]), @@ -412,6 +425,19 @@ def convert_file_to_dataset(self, filepath: str | Path) -> xr.Dataset: } ) + elif "l2-calibration" in filename: + # Handle calibration file (timestamp + cps_per_R float value) + with open(filepath) as f: + lines = [line.strip() for line in f if not line.startswith("#")] + identifiers = [line.split(" ", 1)[0] for line in lines] + values = [float(line.split(" ", 1)[1]) for line in lines] + return xr.Dataset( + { + "start_time_utc": (["time_block"], identifiers), + "cps_per_r": (["time_block"], values), + } + ) + elif filename.endswith(".json"): # Handle pipeline settings JSON file using the generic read_json method return self.convert_json_to_dataset(filepath) diff --git a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml index a84984d193..66c03c7bf3 100644 --- a/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l1a_variable_attrs.yaml @@ -491,7 +491,7 @@ hi-species-attrs: SCALETYP: linear UNITS: counts VALIDMAX: *max_uint32 - VALIDMIN: 0 + VALIDMIN: 0.0 VAR_TYPE: data hi-species-unc-attrs: @@ -548,7 +548,7 @@ hi_priorities_attrs: &hi_priorities_default LABLAXIS: "events" UNITS: events VALIDMAX: *real_fillval - VALIDMIN: 0 + VALIDMIN: 0.0 VAR_TYPE: data priority0: @@ -899,32 +899,32 @@ lo-pui-species-unc-attrs: # Data quality and num of events attrs de_2d_attrs: CATDESC: Direct event data - FIELDNAM: Direct Event Data - LABLAXIS: Values DEPEND_0: epoch DEPEND_1: priority + DICT_KEY: SPASE>Support>SupportQuantity:Other + FIELDNAM: Direct Event Data FILLVAL: -9223372036854775808 FORMAT: I5 + LABLAXIS: Values + SCALETYP: linear UNITS: " " - VALIDMIN: 0 VALIDMAX: 65535 + VALIDMIN: 0 VAR_TYPE: support_data - SCALETYP: linear - DICT_KEY: SPASE>Support>SupportQuantity:Other # unpacked 64-bits attrs de_3d_attrs: CATDESC: Direct event data - FIELDNAM: Direct Event Data - LABLAXIS: Values DEPEND_0: epoch DEPEND_1: priority DEPEND_2: event_num + DICT_KEY: SPASE>Support>SupportQuantity:Other + FIELDNAM: Direct Event Data FILLVAL: -9223372036854775808 FORMAT: I{num_digits} + LABLAXIS: Values + SCALETYP: linear UNITS: " " - VALIDMIN: 0 VALIDMAX: "{valid_max}" + VALIDMIN: 0 VAR_TYPE: support_data - SCALETYP: linear - DICT_KEY: SPASE>Support>SupportQuantity:Other diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-omni_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-omni_variable_attrs.yaml index 82744c0d56..8fc45b0154 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-omni_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-omni_variable_attrs.yaml @@ -10,22 +10,23 @@ max_int: &max_int 9223372036854775807 energy_attrs: &energy_default VAR_TYPE: support_data CATDESC: Geometric mean energy per nucleon + DISPLAY_TYPE: time_series FIELDNAM: Energy Table LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Characteristic # ------------------------------- Coordinates ------------------------------- epoch_delta_minus: CATDESC: Time from acquisition start to acquisition center FIELDNAM: epoch delta minus - FILLVAL: -9223372036854775808 - FORMAT: I18 + FILLVAL: *min_int + FORMAT: I19 LABLAXIS: Epoch Delta Minus SCALETYP: linear UNITS: ns @@ -37,8 +38,8 @@ epoch_delta_minus: epoch_delta_plus: CATDESC: Time from acquisition center to acquisition end FIELDNAM: epoch delta plus - FILLVAL: -9223372036854775808 - FORMAT: I18 + FILLVAL: *min_int + FORMAT: I19 LABLAXIS: Epoch Delta Plus SCALETYP: linear UNITS: ns @@ -334,11 +335,11 @@ unc_c: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for c (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -351,11 +352,11 @@ unc_fe: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for fe (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -368,11 +369,11 @@ unc_h: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for h (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -385,11 +386,11 @@ unc_he3: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for he3 (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -402,11 +403,11 @@ unc_he4: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for he4 (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -419,11 +420,11 @@ unc_junk: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for junk (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -436,11 +437,11 @@ unc_ne_mg_si: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for ne-mg-si (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -453,11 +454,11 @@ unc_o: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for o (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -470,11 +471,11 @@ unc_uh: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for uh (Root 2 spacing) VAR_TYPE: support_data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: time_series DEPEND_0: epoch @@ -484,12 +485,11 @@ unc_uh: c: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: c + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - c FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -501,12 +501,11 @@ c: fe: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: fe + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - fe FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -518,12 +517,11 @@ fe: h: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: h + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - h FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -535,12 +533,11 @@ h: he3: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: he3 + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - he3 FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -552,12 +549,11 @@ he3: he4: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: he4 + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - he4 FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -569,12 +565,11 @@ he4: junk: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: junk + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - junk FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -586,12 +581,11 @@ junk: ne_mg_si: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: ne-mg-si + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - ne-mg-si FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -603,12 +597,11 @@ ne_mg_si: o: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: o + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - o FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 @@ -620,12 +613,11 @@ o: uh: VAR_TYPE: data DEPEND_0: epoch - DISPLAY_TYPE: spectrogram - FIELDNAM: uh + DISPLAY_TYPE: time_series + FIELDNAM: Diff. Intensity - uh FILLVAL: *real_fillval - FORMAT: '%f' - LABLAXIS: Diff. Intensity - SCALETYP: linear + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' VALIDMAX: 16777216.0 VALIDMIN: 0.0 diff --git a/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml index 20b302e421..682d9f886a 100644 --- a/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-hi-sectored_variable_attrs.yaml @@ -1,6 +1,5 @@ # ----------------------------- Useful variables ----------------------------- uint8_fillval: &uint8_fillval 255 -uint32_fillval: &uint32_fillval 4294967295 real_fillval: &real_fillval -1.0e+31 min_int: &min_int -9223372036854775808 @@ -12,20 +11,20 @@ energy_attrs: &energy_default CATDESC: Geometric mean energy per nucleon FIELDNAM: Energy Table LABLAXIS: Energy - SCALETYP: log + SCALETYP: linear UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Characteristic # ------------------------------- Coordinates ------------------------------- epoch_delta_minus: CATDESC: Time from acquisition start to acquisition center FIELDNAM: epoch delta minus - FILLVAL: -9223372036854775808 - FORMAT: I18 + FILLVAL: *min_int + FORMAT: I19 LABLAXIS: Epoch Delta Minus SCALETYP: linear UNITS: ns @@ -37,8 +36,8 @@ epoch_delta_minus: epoch_delta_plus: CATDESC: Time from acquisition center to acquisition end FIELDNAM: epoch delta plus - FILLVAL: -9223372036854775808 - FORMAT: I18 + FILLVAL: *min_int + FORMAT: I19 LABLAXIS: Epoch Delta Plus SCALETYP: linear UNITS: ns @@ -50,7 +49,8 @@ epoch_delta_plus: elevation_angle: CATDESC: Elevation Angle FIELDNAM: Elevation Angle - FORMAT: '%f' + FORMAT: F32.1 + FILLVAL: *real_fillval LABLAXIS: Elevation Angle SCALETYP: linear UNITS: degrees @@ -61,8 +61,8 @@ elevation_angle: spin_sector: CATDESC: Spin Sector Index FIELDNAM: Spin Sector Index - FILLVAL: -1 - FORMAT: I2 + FILLVAL: *uint8_fillval + FORMAT: I3 LABLAXIS: " " SCALETYP: linear UNITS: " " @@ -166,12 +166,13 @@ data_quality: species_dim_attrs: &species_dim_attrs DEPEND_0: epoch - DEPEND_1: energy_cno + DEPEND_1: energy_{species} DEPEND_2: spin_sector DEPEND_3: elevation_angle - LABL_PTR_1: energy_cno_label + LABL_PTR_1: energy_{species}_label LABL_PTR_2: spin_sector_label LABL_PTR_3: elevation_angle_label + # species: cno: <<: *species_dim_attrs @@ -179,12 +180,12 @@ cno: DISPLAY_TYPE: spectrogram CATDESC: cno (x2 spacing) FIELDNAM: cno - FILLVAL: *uint32_fillval - FORMAT: '%f' - SCALETYP: linear + FILLVAL: *real_fillval + FORMAT: E14.7 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' - VALIDMAX: 16777216 - VALIDMIN: 0 + VALIDMAX: 16777216.0 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential fe: @@ -193,12 +194,12 @@ fe: DISPLAY_TYPE: spectrogram CATDESC: fe (x2 spacing) FIELDNAM: fe - FILLVAL: *uint32_fillval - FORMAT: '%f' - SCALETYP: linear + FILLVAL: *real_fillval + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' - VALIDMAX: 16777216 - VALIDMIN: 0 + VALIDMAX: 16777216.0 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential h: @@ -207,12 +208,12 @@ h: DISPLAY_TYPE: spectrogram CATDESC: h (x2 spacing) FIELDNAM: h - FILLVAL: *uint32_fillval - FORMAT: '%f' - SCALETYP: linear + FILLVAL: *real_fillval + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' - VALIDMAX: 16777216 - VALIDMIN: 0 + VALIDMAX: 16777216.0 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential he3he4: @@ -221,12 +222,12 @@ he3he4: DISPLAY_TYPE: spectrogram CATDESC: he3he4 (x2 spacing) FIELDNAM: he3he4 - FILLVAL: *uint32_fillval - FORMAT: '%f' - SCALETYP: linear + FILLVAL: *real_fillval + FORMAT: F32.1 + SCALETYP: log UNITS: '# / cm2 s sr MeV/nuc' - VALIDMAX: 16777216 - VALIDMIN: 0 + VALIDMAX: 16777216.0 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential # uncertainties: @@ -236,13 +237,13 @@ unc_cno: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for cno (x2 spacing) VAR_TYPE: data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram COORDINATE_SYSTEM: instrument frame unc_fe: @@ -251,11 +252,11 @@ unc_fe: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for fe (x2 spacing) VAR_TYPE: data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: spectrogram COORDINATE_SYSTEM: instrument frame @@ -266,11 +267,11 @@ unc_h: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for h (x2 spacing) VAR_TYPE: data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: spectrogram DEPEND_0: epoch @@ -282,11 +283,11 @@ unc_he3he4: VALIDMIN: 0.0 VALIDMAX: 4096.0 UNITS: '# / cm2 s sr MeV/nuc' - FORMAT: '%f' + FORMAT: F32.1 CATDESC: Uncertainties for he3he4 (x2 spacing) VAR_TYPE: data SI_CONVERSION: ' > ' - SCALETYP: linear + SCALETYP: log FILLVAL: *real_fillval DISPLAY_TYPE: spectrogram COORDINATE_SYSTEM: instrument frame @@ -299,10 +300,10 @@ energy_cno_minus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_cno @@ -313,10 +314,10 @@ energy_cno_plus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_cno @@ -327,10 +328,10 @@ energy_fe_minus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_fe @@ -341,10 +342,10 @@ energy_fe_plus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_fe @@ -355,10 +356,10 @@ energy_h_minus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_h @@ -369,10 +370,10 @@ energy_h_plus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_h @@ -383,10 +384,10 @@ energy_he3he4_minus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_he3he4 @@ -397,10 +398,10 @@ energy_he3he4_plus: LABLAXIS: Energy SCALETYP: log UNITS: MeV/nuc - FORMAT: '%f' + FORMAT: F32.1 FILLVAL: *real_fillval VALIDMAX: 200.0 - VALIDMIN: 0.05000000074505806 + VALIDMIN: 0.0 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:EnergyPerCharge,Qualifier:Uncertainty DEPEND_1: energy_he3he4 @@ -409,12 +410,12 @@ spin_angle: VAR_TYPE: support_data CATDESC: Spin Angle FIELDNAM: Spin Angle - SCALETYP: linear + SCALETYP: log UNITS: degrees FILLVAL: *real_fillval VALIDMAX: 360.0 VALIDMIN: 0.0 - FORMAT: '%f' + FORMAT: F32.1 DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:ArrivalDirection,Qualifier:DirectionAngle.AzimuthAngle DEPEND_1: spin_sector DEPEND_2: elevation_angle diff --git a/imap_processing/cdf/config/imap_codice_l2-lo-species_variable_attrs.yaml b/imap_processing/cdf/config/imap_codice_l2-lo-species_variable_attrs.yaml index 90c22568fa..d6f44a8748 100644 --- a/imap_processing/cdf/config/imap_codice_l2-lo-species_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_codice_l2-lo-species_variable_attrs.yaml @@ -1,12 +1,15 @@ # ----------------------------- Useful variables ----------------------------- +uint8_fillval: &uint8_fillval 255 real_fillval: &real_fillval -1.0e+31 max_float: &max_float 3.4028235e+38 +species_valid_max: &species_valid_max 1e20 # ------------------------------- Coordinates ------------------------------- elevation_angle: CATDESC: Elevation Angle FIELDNAM: Elevation Angle - FORMAT: I8 + FORMAT: F32.1 + FILLVAL: *real_fillval LABLAXIS: Elevation Angle SCALETYP: linear UNITS: degrees @@ -17,13 +20,13 @@ elevation_angle: spin_sector: CATDESC: Spin Sector Index FIELDNAM: Spin Sector Index - FILLVAL: -1 - FORMAT: I2 + FILLVAL: *uint8_fillval + FORMAT: I3 LABLAXIS: " " SCALETYP: linear UNITS: " " VALIDMIN: 0 - VALIDMAX: 24 + VALIDMAX: 12 VAR_TYPE: support_data # ------------------------------- labels ------------------------------- @@ -40,15 +43,16 @@ lo-species-attrs: DEPEND_0: epoch DEPEND_1: esa_step DEPEND_2: spin_sector - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: "Non-sunward - {species}" FILLVAL: *real_fillval - FORMAT: F13.4 + FORMAT: F32.1 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label + SCALETYP: log UNITS: "#/(cm^2-s-sr-keV/q)" VALIDMIN: 0.0 - VALIDMAX: 16777216.0 + VALIDMAX: *species_valid_max VAR_TYPE: data DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential @@ -57,15 +61,16 @@ lo-pui-species-attrs: DEPEND_0: epoch DEPEND_1: esa_step DEPEND_2: spin_sector - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: "Sunward - {species}" FILLVAL: *real_fillval - FORMAT: F13.4 + FORMAT: F32.1 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label + SCALETYP: log UNITS: "#/(cm^2-s-sr-keV/q)" VALIDMIN: 0.0 - VALIDMAX: 16777216.0 + VALIDMAX: *species_valid_max VAR_TYPE: data DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential @@ -74,15 +79,16 @@ lo-sw-species-attrs: DEPEND_0: epoch DEPEND_1: esa_step DEPEND_2: spin_sector - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: "Sunward - {species}" FILLVAL: *real_fillval - FORMAT: F13.4 + FORMAT: F32.1 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label + SCALETYP: log UNITS: "#/(cm^2-s-sr-keV/q)" VALIDMIN: 0.0 - VALIDMAX: 16777216.0 + VALIDMAX: *species_valid_max VAR_TYPE: data DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Differential @@ -91,15 +97,16 @@ lo-species-unc-attrs: DEPEND_0: epoch DEPEND_1: esa_step DEPEND_2: spin_sector - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: "Non-sunward - {species}" FILLVAL: *real_fillval - FORMAT: F20.9 + FORMAT: F32.1 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label + SCALETYP: log UNITS: "#/(cm^2-s-sr-keV/q)" VALIDMIN: 0.0 - VALIDMAX: 16777216.0 + VALIDMAX: *species_valid_max VAR_TYPE: data DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Uncertainty @@ -108,15 +115,16 @@ lo-pui-species-unc-attrs: DEPEND_0: epoch DEPEND_1: esa_step DEPEND_2: spin_sector - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: "Sunward - {species}" FILLVAL: *real_fillval - FORMAT: F20.9 + FORMAT: F32.1 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label + SCALETYP: log UNITS: "#/(cm^2-s-sr-keV/q)" VALIDMIN: 0.0 - VALIDMAX: 16777216.0 + VALIDMAX: *species_valid_max VAR_TYPE: data DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Uncertainty @@ -125,14 +133,15 @@ lo-sw-species-unc-attrs: DEPEND_0: epoch DEPEND_1: esa_step DEPEND_2: spin_sector - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: "Sunward - {species}" FILLVAL: *real_fillval - FORMAT: F20.9 + FORMAT: F32.1 LABL_PTR_1: esa_step_label LABL_PTR_2: spin_sector_label + SCALETYP: log UNITS: "#/(cm^2-s-sr-keV/q)" VALIDMIN: 0.0 - VALIDMAX: 16777216.0 + VALIDMAX: *species_valid_max VAR_TYPE: data DICT_KEY: SPASE>Particle>ParticleType:Ion,ParticleQuantity:NumberFlux,Qualifier:Uncertainty \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml index 92b763ee46..346bfe9f82 100644 --- a/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l1a_variable_attrs.yaml @@ -36,9 +36,9 @@ bins_attrs: <<: *default_attrs CATDESC: Histogram bin number FIELDNAM: Bin number - FILLVAL: -32768 - FORMAT: I5 - LABLAXIS: Counts + FILLVAL: *max_uint16 + FORMAT: I4 + LABLAXIS: Bin no. MONOTON: INCREASE SCALETYP: linear VALIDMAX: 3599 @@ -49,35 +49,35 @@ within_the_second: # Used to be per_second_attrs <<: *default_attrs VALIDMIN: 0 VALIDMAX: 50000 - CATDESC: Direct events recorded in individual seconds # TBD any ideas how to define it - FIELDNAM: Direct events within a second - FORMAT: I10 + CATDESC: Ordinal number of direct event within a second + FIELDNAM: Ordinal number of direct event + FORMAT: I5 VAR_TYPE: support_data DISPLAY_TYPE: time_series - LABLAXIS: Direct Events + LABLAXIS: Event no. direct_event_components_attrs: <<: *default_attrs - CATDESC: Components of a direct event (seconds, subseconds, impulse_length, multi_event) - FIELDNAM: Direct event components + CATDESC: Direct-event component index (second, subsecond, pulse length, multi event) + FIELDNAM: Direct-event component index FILLVAL: 255 - FORMAT: I2 - LABLAXIS: Components + FORMAT: I1 + LABLAXIS: Index VALIDMAX: 3 VALIDMIN: 0 VAR_TYPE: support_data direct_events: <<: *default_attrs - CATDESC: Direct events grouped by epoch seconds + CATDESC: Direct events grouped by seconds DEPEND_0: epoch DEPEND_1: within_the_second DEPEND_2: direct_event_components FIELDNAM: Direct events - FILLVAL: *max_uint32_min_one + FILLVAL: *max_uint32 FORMAT: I10 - LABLAXIS: Counts - VALIDMAX: *max_uint32 + LABLAXIS: Direct events + VALIDMAX: *max_uint32_min_one VALIDMIN: 0 VAR_TYPE: data @@ -86,12 +86,12 @@ histogram: CATDESC: Histogram of photon counts in scanning-circle bins DEPEND_0: epoch DEPEND_1: bins - DISPLAY_TYPE: time_series + DISPLAY_TYPE: spectrogram FIELDNAM: Histogram of photon counts - FILL_VAL: *max_uint16 - FORMAT: I4 + FILLVAL: *max_uint16 + FORMAT: I3 LABL_PTR_1: bins_label - UNITS: counts + UNITS: '#' VALIDMAX: 255 VALIDMIN: 0 VAR_TYPE: data @@ -99,83 +99,83 @@ histogram: first_spin_id: <<: *support_data_defaults CATDESC: The ordinal number of the first spin during histogram accumulation - FIELDNAM: Number of the first spin in histogram + FIELDNAM: Number of first spin FILLVAL: *max_uint32 - FORMAT: I11 - LABLAXIS: Spin number + FORMAT: I10 + LABLAXIS: Spin no. VALIDMAX: *max_uint32_min_one last_spin_id: <<: *support_data_defaults CATDESC: The ordinal number of the last spin during histogram accumulation - FIELDNAM: Number of the last spin in histogram + FIELDNAM: Number of last spin FILLVAL: *max_uint32 - FORMAT: I11 - LABLAXIS: Spin number + FORMAT: I10 + LABLAXIS: Spin no. VALIDMAX: *max_uint32_min_one imap_start_time: <<: *support_data_defaults - CATDESC: Histogram start time, IMAP-clock seconds - FIELDNAM: Histogram start time, IMAP-clock seconds + CATDESC: Histogram start time (IMAP clock) + FIELDNAM: Start time (IMAP clock) # TODO: Presumably float64 max or min should be here? - FILLVAL: *int_fillval - FORMAT: F16.6 + FILLVAL: 1.0E+31 + FORMAT: F17.6 LABLAXIS: Start time - UNITS: seconds + UNITS: s VALIDMAX: 4294967295.0 VALIDMIN: 0.0 imap_time_offset: <<: *support_data_defaults - CATDESC: Accumulation time in seconds for GLOWS histogram - FIELDNAM: Histogram accumulation time + CATDESC: Accumulation time for histogram (IMAP clock) + FIELDNAM: Accum. time (IMAP clock) # TODO: Presumably float64 max or min should be here? - FILLVAL: *int_fillval - FORMAT: F12.6 - LABLAXIS: Duration - UNITS: seconds - VALIDMAX: 4000.0 + FILLVAL: 1.0E+31 + FORMAT: F10.6 + LABLAXIS: Accum. time + UNITS: s + VALIDMAX: 999.0 VALIDMIN: 0.0 glows_start_time: <<: *support_data_defaults - CATDESC: Histogram start time, GLOWS-clock seconds - FIELDNAM: Histogram start time, GLOWS-clock seconds - FILLVAL: *int_fillval - FORMAT: F16.6 + CATDESC: Histogram start time (GLOWS clock) + FIELDNAM: Start time (GLOWS clock) + FILLVAL: 1.0E+31 + FORMAT: F17.6 LABLAXIS: Start time - UNITS: seconds + UNITS: s VALIDMAX: 4294967295.0 VALIDMIN: 0.0 glows_time_offset: <<: *support_data_defaults - CATDESC: Accumulation time in seconds for GLOWS histogram - FIELDNAM: Histogram accumulation time - FILLVAL: *int_fillval - FORMAT: F12.6 - LABLAXIS: Duration - UNITS: seconds - VALIDMAX: 4000.0 # 15.38 s per spin x 256 spins = 3937.3 s, then rounded up + CATDESC: Accumulation time for histogram (GLOWS clock) + FIELDNAM: Accum. time (GLOWS clock) + FILLVAL: 1.0E+31 + FORMAT: F10.6 + LABLAXIS: Accum. time + UNITS: s + VALIDMAX: 999.0 # 15.38 s per spin x 64 spins = 984.32 s, then rounded up VALIDMIN: 0.0 flags_set_onboard: <<: *support_data_defaults # TODO: Verify uint32 fillval and uint16 validmax CATDESC: Binary mask with histogram flags set onboard - FIELDNAM: Mask with histogram flags set onboard + FIELDNAM: Mask with flags set onboard FILLVAL: *max_uint32 - FORMAT: I6 - LABLAXIS: Mask value + FORMAT: I5 + LABLAXIS: Onboard mask VALIDMAX: *max_uint16 is_generated_on_ground: <<: *support_data_defaults - CATDESC: Flag indicating where histogram data was generated (1 - on the ground, 0 - onboard) + CATDESC: Flag indicating where histogram was generated (1 - on the ground, 0 - onboard) FIELDNAM: Histogram-creation-site flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag VALIDMAX: 1 @@ -183,10 +183,10 @@ number_of_spins_per_block: <<: *support_data_defaults CATDESC: Number of spins per block during accumulation of histogram FIELDNAM: Number of spins per block - FILLVAL: 65535 - FORMAT: I4 - LABLAXIS: Num of spins - VALIDMAX: 256 + FILLVAL: 255 + FORMAT: I2 + LABLAXIS: No. of spins + VALIDMAX: 64 VALIDMIN: 1 number_of_bins_per_histogram: @@ -194,27 +194,27 @@ number_of_bins_per_histogram: CATDESC: Number of histogram bins FIELDNAM: Number of histogram bins FILLVAL: *max_uint16 - FORMAT: I5 - LABLAXIS: Num of bins + FORMAT: I4 + LABLAXIS: No. of bins VALIDMAX: 3600 VALIDMIN: 225 number_of_events: <<: *support_data_defaults CATDESC: Total number of events/counts in the histogram - FIELDNAM: Total number of counts in histogram - FILLVAL: *int_fillval - FORMAT: I11 - LABLAXIS: Num of counts - VALIDMAX: *max_uint32 + FIELDNAM: Histogram total counts + FILLVAL: *max_uint32 + FORMAT: I10 + LABLAXIS: Total cts + VALIDMAX: *max_uint32_min_one filter_temperature_average: <<: *support_data_defaults CATDESC: Uint-encoded spin-block-averaged filter temperature FIELDNAM: Average filter temperature FILLVAL: *max_uint16 - FORMAT: I4 - LABLAXIS: Avgd Temperature + FORMAT: I3 + LABLAXIS: Temp avg VALIDMAX: 255 filter_temperature_variance: @@ -222,36 +222,37 @@ filter_temperature_variance: CATDESC: Uint-encoded spin-block-averaged variance of filter temperature FIELDNAM: Variance of filter temperature FILLVAL: *max_uint32 - FORMAT: I6 - LABLAXIS: Variance + FORMAT: I5 + LABLAXIS: Temp var VALIDMAX: *max_uint16 hv_voltage_average: <<: *support_data_defaults - CATDESC: Uint-encoded spin-block-averaged CEM voltage - FIELDNAM: Uint-encoded averaged CEM voltage - FILLVAL: *max_uint32 - FORMAT: I6 - LABLAXIS: Avg voltage - VALIDMAX: *max_uint16 + CATDESC: Uint-encoded spin-block-averaged HV voltage on CEM + FIELDNAM: Average HV voltage + FILLVAL: *max_uint16 + FORMAT: I5 + LABLAXIS: HV avg + VALIDMAX: 65534 hv_voltage_variance: <<: *support_data_defaults - CATDESC: variance of HV voltage on the CEM, uint encoded + CATDESC: Uint-encoded spin-block-averaged variance of HV voltage on CEM FIELDNAM: Uint encoded HV voltage variance - FILLVAL: *int_fillval - LABLAXIS: Variance - VALIDMAX: *max_uint32 + FILLVAL: *max_uint32 + FORMAT: I10 + LABLAXIS: HV var + VALIDMAX: *max_uint32_min_one spin_period_average: <<: *support_data_defaults CATDESC: Uint-encoded spin-block-averaged spin period DEPEND_0: epoch DISPLAY_TYPE: time_series - FIELDNAM: Uint-encoded average spin period + FIELDNAM: Average spin period FILLVAL: *max_uint32 - FORMAT: I6 - LABLAXIS: Spin period + FORMAT: I5 + LABLAXIS: Period avg UNITS: ' ' VALIDMAX: 50000 # TBC 15.38 s where 20.9712 s = 65535, rounded up VALIDMIN: 45000 # TBC 14.63 s where 20.9712 s = 65535, rounded down @@ -260,28 +261,28 @@ spin_period_average: spin_period_variance: <<: *support_data_defaults CATDESC: Uint-encoded spin-block-averaged variance of spin period - FIELDNAM: Uint-encoded variance of spin period - FILLVAL: *int_fillval + FIELDNAM: Variance of spin period + FILLVAL: *max_uint32 FORMAT: I10 - LABLAXIS: Variance - VALIDMAX: *max_uint32 + LABLAXIS: Period var + VALIDMAX: *max_uint32_min_one pulse_length_average: <<: *support_data_defaults CATDESC: Uint-encoded spin-block-averaged pulse length FIELDNAM: Averaged pulse length FILLVAL: *max_uint16 - FORMAT: I4 - LABLAXIS: Avg pulse len + FORMAT: I3 + LABLAXIS: Pulse avg VALIDMAX: 255 pulse_length_variance: <<: *support_data_defaults - CATDESC: Uint encoded spin-block-averaged variance of pulse length + CATDESC: Uint-encoded spin-block-averaged variance of pulse length FIELDNAM: Variance of pulse length FILLVAL: *max_uint32 - FORMAT: I10 - LABLAXIS: Variance + FORMAT: I5 + LABLAXIS: Pulse var VALIDMAX: *max_uint16 # End of not-in--dicts in generate_de_dataset @@ -290,112 +291,113 @@ pulse_length_variance: seq_count_in_pkts_file: <<: *support_data_defaults # TBD: problem with several values associated with one epoch value - CATDESC: Ordinal number of a packet in a sequence of multiple CCSDS packets - FIELDNAM: Packet sequence counter - FILLVAL: *max_uint32 - FORMAT: I6 - LABLAXIS: Counter + CATDESC: Ordinal number of a packet in a sequence of CCSDS packets with the same APID + FIELDNAM: Packet number in sequence + FILLVAL: *max_uint16 + FORMAT: I5 + LABLAXIS: Pkt no. VALIDMAX: 65534 # uint16_max - 1, because it must be less than VALIDMAX for number_of_de_packets number_of_de_packets: <<: *support_data_defaults - CATDESC: Number of packets for a given portion (second) of direct-event data + CATDESC: Number of packets in a given segment (second) of direct-event data FIELDNAM: Number of DE packets FILLVAL: *max_uint32 FORMAT: I5 - LABLAXIS: Num of packets - VALIDMAX: *max_uint16 + LABLAXIS: No. of pkts + VALIDMAX: 65534 # End of support data # data_every_second in glows_l1a.py imap_sclk_last_pps: <<: *support_data_defaults - CATDESC: IMAP-clock seconds for last PPS - FIELDNAM: IMAP-clock seconds for last PPS + CATDESC: Latest PPS arrival time (IMAP clock) + FIELDNAM: Latest PPS time (IMAP clock) FILLVAL: *max_uint32 - FORMAT: I11 - LABLAXIS: IMAP seconds - UNITS: seconds + FORMAT: I10 + LABLAXIS: PPS time + UNITS: s VALIDMAX: *max_uint32_min_one glows_sclk_last_pps: <<: *support_data_defaults - CATDESC: GLOWS-clock seconds for last PPS + CATDESC: Latest PPS arrival time (GLOWS clock, seconds) DISPLAY_TYPE: no_plot - FIELDNAM: GLOWS-clock seconds for last PPS + FIELDNAM: Latest PPS time (GLOWS seconds) FILLVAL: *max_uint32 - FORMAT: I11 - LABLAXIS: GLOWS seconds - UNITS: seconds + FORMAT: I10 + LABLAXIS: PPS time + UNITS: s VALIDMAX: *max_uint32_min_one glows_ssclk_last_pps: <<: *support_data_defaults - CATDESC: GLOWS-clock subseconds for last PPS + CATDESC: Latest PPS arrival time (GLOWS clock, subseconds) DISPLAY_TYPE: no_plot - FIELDNAM: GLOWS-clock subseconds for last PPS + FIELDNAM: Latest PPS time (GLOWS subseconds) FILLVAL: *max_uint32 - LABLAXIS: GLOWS subseconds + FORMAT: I7 + LABLAXIS: PPS time VALIDMAX: 1999999 imap_sclk_next_pps: <<: *support_data_defaults - CATDESC: IMAP-clock seconds for next PPS - FIELDNAM: IMAP-clock seconds for next PPS + CATDESC: Next PPS estimated arrival time (IMAP clock) + FIELDNAM: Next PPS time (IMAP clock) FILLVAL: *max_uint32 - FORMAT: I11 - LABLAXIS: IMAP seconds - UNITS: seconds + FORMAT: I10 + LABLAXIS: PPS time + UNITS: s VALIDMAX: *max_uint32_min_one catbed_heater_active: <<: *support_data_defaults - CATDESC: Catbed-heater activity flag (1 - active, 0 - not active) - FIELDNAM: Catbed-heater activity flag - FILLVAL: -128 - FORMAT: I2 + CATDESC: Repointing maneuver flag (1 - active, 0 - not active) + FIELDNAM: Repointing maneuver flag + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag spin_period_valid: <<: *support_data_defaults - CATDESC: Spin-period-validity flag (1 - valid, 0 - invalid) + CATDESC: Spin-period validity flag (1 - valid, 0 - invalid) FIELDNAM: Spin-period-validity flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag spin_phase_at_next_pps_valid: <<: *support_data_defaults CATDESC: Spin-phase-at-next-PPS validity flag (1 - valid, 0 - invalid) FIELDNAM: Spin-phase-at-next-PPS validity flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag spin_period_source: <<: *support_data_defaults CATDESC: Spin-period-source flag (0 - from ITF, 1 - estimated by GLOWS AppSW) FIELDNAM: Spin-period-source flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag spin_period: <<: *support_data_defaults - CATDESC: Uint encoded spin period value - FIELDNAM: Uint encoded spin period value + CATDESC: Uint-encoded spin period + FIELDNAM: Spin period FILLVAL: *max_uint16 - FORMAT: I6 + FORMAT: I5 LABLAXIS: Spin period VALIDMAX: 50000 # TBC 15.38 s where 20.9712 s = 65535, rounded up VALIDMIN: 45000 # TBC 14.63 s where 20.9712 s = 65535, rounded down spin_phase_at_next_pps: <<: *support_data_defaults - CATDESC: Uint encoded next spin phase value - FIELDNAM: Uint encoded next spin phase value + CATDESC: Uint-encoded next-PPS spin phase + FIELDNAM: Next-PPS spin phase FILLVAL: *max_uint32 - FORMAT: I6 + FORMAT: I5 LABLAXIS: Spin phase VALIDMAX: *max_uint16 @@ -404,83 +406,83 @@ number_of_completed_spins: CATDESC: Number of completed spins FIELDNAM: Number of completed spins FILLVAL: *max_uint32 - FORMAT: I11 - LABLAXIS: Num of spins + FORMAT: I10 + LABLAXIS: No. of spins VALIDMAX: *max_uint32_min_one filter_temperature: <<: *support_data_defaults - CATDESC: Uint encoded filter temperature + CATDESC: Uint-encoded filter temperature DISPLAY_TYPE: time_series FIELDNAM: Filter temperature FILLVAL: *max_uint32 - FORMAT: I6 + FORMAT: I5 LABLAXIS: Temperature VALIDMAX: *max_uint16 hv_voltage: <<: *support_data_defaults - CATDESC: Uint encoded CEM voltage - FIELDNAM: Uint encoded CEM voltage + CATDESC: Uint-encoded HV voltage on CEM + FIELDNAM: HV voltage FILLVAL: *max_uint32 - FORMAT: I6 - LABLAXIS: Voltage + FORMAT: I5 + LABLAXIS: HV VALIDMAX: *max_uint16 glows_time_on_pps_valid: <<: *support_data_defaults CATDESC: GLOWS-time-on-PPS-arrival validity flag (1 - valid, 0 - not valid) FIELDNAM: GLOWS time validity flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag time_status_valid: <<: *support_data_defaults - CATDESC: Time-status-data-structure-validity flag (1 - valid, 0 - invalid) - FIELDNAM: Time-status-structure-validity flag - FILLVAL: -128 - FORMAT: I2 + CATDESC: Time-status-data (ITF) validity flag (1 - valid, 0 - invalid) + FIELDNAM: Time-status-data validity flag + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag housekeeping_valid: <<: *support_data_defaults CATDESC: GLOWS housekeeping validity flag (1 - valid, 0 - invalid) FIELDNAM: Housekeeping validity flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag is_pps_autogenerated: <<: *support_data_defaults CATDESC: Flag indicating whether PPS is autogenerated (1 - autogenerated, 0 - external) - FIELDNAM: Autogenerated-PPS flag - FILLVAL: -128 - FORMAT: I2 + FIELDNAM: PPS-autogenerated flag + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag hv_test_in_progress: <<: *support_data_defaults CATDESC: HV-test-in-progress flag (1 - test is on, 0 - test is off) FIELDNAM: HV-test-in-progress flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag pulse_test_in_progress: <<: *support_data_defaults CATDESC: Pulse-test-in-progress flag (1 - test is on, 0 - test is off) FIELDNAM: Pulse-test-in-progress flag - FILLVAL: -128 # int8_min - FORMAT: I2 + FILLVAL: 255 # uint8_max + FORMAT: I1 LABLAXIS: Flag memory_error_detected: <<: *support_data_defaults CATDESC: Memory-error flag (1 - error detected, 0 - no error) FIELDNAM: Memory-error flag - FILLVAL: -128 - FORMAT: I2 + FILLVAL: 255 + FORMAT: I1 LABLAXIS: Flag # End of data_every_second @@ -492,4 +494,4 @@ missing_packets_sequence: # Used to be missing_packets_sequence FORMAT: I10 LABLAXIS: Metadata VALIDMAX: 1000000000 - VAR_TYPE: metadata \ No newline at end of file + VAR_TYPE: metadata diff --git a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml index 8cebb5a4bf..c79c9c12e2 100644 --- a/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_glows_l2_variable_attrs.yaml @@ -38,13 +38,14 @@ support_data_defaults: &support_data_defaults FORMAT: I10 RESOLUTION: ISO8601 +# UTC Strings time_data_defaults: &time_data_defaults <<: *support_data_defaults - FILLVAL: 0 - FORMAT: I1000 + FILLVAL: ' ' + FORMAT: A30 UNITS: N/A - VALIDMIN: 0 - VALIDMAX: 0 + VALIDMIN: ' ' + VALIDMAX: ' ' DICT_KEY: SPASE>Support>SupportQuantity:Temporal lightcurve_defaults: &lightcurve_defaults @@ -104,7 +105,7 @@ number_of_good_l1b_inputs: CATDESC: Number of good L1B inputs per observational day DICT_KEY: SPASE>Support>SupportQuantity:Other FIELDNAM: Number of good L1B inputs - FILLVAL: *max_uint16 + FILLVAL: -9223372036854775808 FORMAT: I5 LABLAXIS: No. of L1B VALIDMAX: 20000 # 3 days * 86400 s / 14.6 s rounded up @@ -114,7 +115,7 @@ total_l1b_inputs: CATDESC: Total number of L1B inputs per observational day DICT_KEY: SPASE>Support>SupportQuantity:Other FIELDNAM: Total number of L1B inputs - FILLVAL: *max_uint16 + FILLVAL: -9223372036854775808 FORMAT: I5 LABLAXIS: No. of L1B VALIDMAX: 20000 @@ -124,7 +125,7 @@ identifier: CATDESC: Spin pointing number to identify observational day DICT_KEY: SPASE>Support>SupportQuantity:Temporal FIELDNAM: Spin pointing number - FILLVAL: *max_uint32 + FILLVAL: -9223372036854775808 FORMAT: I5 LABLAXIS: Pointing no. VALIDMAX: 99999 @@ -133,7 +134,7 @@ flight_software_version: <<: *support_data_defaults CATDESC: GLOWS flight software version FIELDNAM: GLOWS flight software version - FILLVAL: *max_uint32 + FILLVAL: -9223372036854775808 FORMAT: I8 LABLAXIS: FSW ver VALIDMAX: 16777215 @@ -164,7 +165,7 @@ filter_temperature_average: CATDESC: Filter temperature averaged over observational day DICT_KEY: SPASE>Support>SupportQuantity:Other FIELDNAM: Average filter temperature - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F6.2 LABLAXIS: Temp UNITS: Celsius @@ -176,7 +177,7 @@ filter_temperature_std_dev: CATDESC: Standard deviation of filter temperature DICT_KEY: SPASE>Support>SupportQuantity:Housekeeping FIELDNAM: Std dev of filter temperature - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E9.3 LABLAXIS: Temp std dev UNITS: Celsius @@ -188,7 +189,7 @@ hv_voltage_average: CATDESC: CEM HV voltage averaged over observational day DICT_KEY: SPASE>Support>SupportQuantity:Housekeeping FIELDNAM: Average HV voltage - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F7.2 LABLAXIS: HV UNITS: V @@ -200,7 +201,7 @@ hv_voltage_std_dev: CATDESC: Standard deviation of CEM HV voltage DICT_KEY: SPASE>Support>SupportQuantity:Housekeeping FIELDNAM: Std dev of HV voltage - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E9.3 LABLAXIS: HV std dev UNITS: V @@ -214,7 +215,7 @@ spin_period_average: CATDESC: Spin period averaged over observational day DICT_KEY: SPASE>Support>SupportQuantity:SpinPeriod FIELDNAM: Average spin period - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F9.6 LABLAXIS: Period UNITS: s @@ -226,7 +227,7 @@ spin_period_std_dev: CATDESC: Standard deviation of spin period DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Std dev of spin period - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E9.3 LABLAXIS: Period std dev UNITS: s @@ -241,7 +242,7 @@ spin_period_ground_average: CATDESC: Spin period (ground processing) averaged over observational day DICT_KEY: SPASE>Support>SupportQuantity:SpinPeriod FIELDNAM: Average spin period (ground) - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F9.6 LABLAXIS: Period UNITS: s @@ -253,7 +254,7 @@ spin_period_ground_std_dev: CATDESC: Standard deviation of spin period (ground processing) DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Std dev of spin period (ground) - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E9.3 LABLAXIS: Period std dev UNITS: s @@ -265,7 +266,7 @@ pulse_length_average: CATDESC: Pulse length averaged over observational day DICT_KEY: SPASE>Support>SupportQuantity:Housekeeping FIELDNAM: Average pulse length - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: F5.2 LABLAXIS: Pulse UNITS: us @@ -277,7 +278,7 @@ pulse_length_std_dev: CATDESC: Standard deviation of pulse length DICT_KEY: SPASE>Support>SupportQuantity:Housekeeping FIELDNAM: Std dev of pulse length - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: E9.3 LABLAXIS: Pulse std dev UNITS: us @@ -290,7 +291,7 @@ position_angle_offset_average: CATDESC: Position angle offset averaged over observational day DICT_KEY: SPASE>Support>SupportQuantity:Positional FIELDNAM: Average position angle offset - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: F10.6 LABLAXIS: Offset angle UNITS: degrees @@ -303,7 +304,7 @@ position_angle_offset_std_dev: CATDESC: Standard deviation of position angle offset DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Std dev of position angle offset - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: E9.3 LABLAXIS: Offset std dev UNITS: degrees @@ -316,7 +317,7 @@ spin_axis_orientation_average: CATDESC: Spin axis pointing averaged over observational day (ecliptic lon and lat) DICT_KEY: SPASE>Support>SupportQuantity:SpinPhase FIELDNAM: Average spin axis pointing - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: F7.3 LABLAXIS: Lon/lat UNITS: degrees @@ -329,7 +330,7 @@ spin_axis_orientation_std_dev: CATDESC: Standard deviation of spin axis pointing DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Std dev of spin axis pointing - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: E9.3 LABLAXIS: Lon/lat std dev UNITS: degrees @@ -344,7 +345,7 @@ spacecraft_location_average: DICT_KEY: SPASE>Support>SupportQuantity:Positional DISPLAY_TYPE: no_plot FIELDNAM: Average spacecraft location - FILLVAL: 1.0e+31 + FILLVAL: -1.0e+31 FORMAT: E13.6 LABLAXIS: Loc UNITS: km @@ -359,7 +360,7 @@ spacecraft_location_std_dev: DICT_KEY: SPASE>Support>SupportQuantity:DataQuality DISPLAY_TYPE: no_plot FIELDNAM: Std dev of spacecraft location - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E9.3 LABLAXIS: Loc std dev UNITS: km @@ -374,7 +375,7 @@ spacecraft_velocity_average: DICT_KEY: SPASE>Support>SupportQuantity:Velocity DISPLAY_TYPE: no_plot FIELDNAM: Average spacecraft velocity - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E13.6 LABLAXIS: Vsc UNITS: km/s @@ -389,7 +390,7 @@ spacecraft_velocity_std_dev: DICT_KEY: SPASE>Support>SupportQuantity:DataQuality DISPLAY_TYPE: no_plot FIELDNAM: Std dev of spacecraft velocity - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: E9.3 LABLAXIS: Vsc std dev UNITS: km/s @@ -403,7 +404,7 @@ bad_time_flag_occurrences: DICT_KEY: SPASE>Support>SupportQuantity:DataQuality DISPLAY_TYPE: no_plot FIELDNAM: Occurrences of bad-time flags - FILLVAL: *max_uint16 + FILLVAL: -1.0E+31 FORMAT: I5 LABL_PTR_1: flags_label # MS: I do not understand this LABLAXIS: No. of cases @@ -414,7 +415,7 @@ spin_angle: CATDESC: Spin angle (measured from north) for bin centers DICT_KEY: SPASE>Support>SupportQuantity:SpinPhase FIELDNAM: Spin angle for bin centers - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F7.3 LABLAXIS: Spin angle UNITS: degrees @@ -430,7 +431,7 @@ photon_flux: DICT_KEY: SPASE>Wave>WaveType:Electromagnetic,WaveQuantity:Intensity,Qualifier:Scalar DISPLAY_TYPE: spectrogram FIELDNAM: Pointing-averaged photon flux - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F8.2 LABLAXIS: Flux UNITS: Rayleigh @@ -445,7 +446,7 @@ raw_histograms: DICT_KEY: SPASE>Support>SupportQuantity:Other DISPLAY_TYPE: spectrogram FIELDNAM: Histogram of counts - FILLVAL: *max_uint32 + FILLVAL: -9223372036854775808 FORMAT: I8 LABLAXIS: Counts UNITS: '#' @@ -458,7 +459,7 @@ exposure_times: DICT_KEY: SPASE>Support>SupportQuantity:Other DISPLAY_TYPE: spectrogram FIELDNAM: Exposure time per bin - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F7.2 LABLAXIS: Bin exposure UNITS: s @@ -470,7 +471,7 @@ flux_uncertainties: CATDESC: Statistical uncertainties for photon flux DICT_KEY: SPASE>Support>SupportQuantity:DataQuality FIELDNAM: Photon flux uncertainties - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F8.2 LABLAXIS: Flux uncert UNITS: Rayleigh @@ -496,7 +497,7 @@ ecliptic_lon: CATDESC: Ecliptic longitudes of bin centers DICT_KEY: SPASE>Support>SupportQuantity:Positional FIELDNAM: Ecliptic longitudes of bins - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F7.3 LABLAXIS: Bin lon UNITS: degrees @@ -509,7 +510,7 @@ ecliptic_lat: CATDESC: Ecliptic latitudes of bin centers DICT_KEY: SPASE>Support>SupportQuantity:Positional FIELDNAM: Ecliptic latitudes of bins - FILLVAL: 1.0E+31 + FILLVAL: -1.0E+31 FORMAT: F7.3 LABLAXIS: Bin lat UNITS: degrees @@ -522,7 +523,7 @@ number_of_bins: CATDESC: Number of bins in histogram DICT_KEY: SPASE>Support>SupportQuantity:Other FIELDNAM: Number of bins in histogram - FILLVAL: *max_uint16 + FILLVAL: -9223372036854775808 FORMAT: I4 LABLAXIS: No. of bins UNITS: ' ' diff --git a/imap_processing/cdf/config/imap_hi_variable_attrs.yaml b/imap_processing/cdf/config/imap_hi_variable_attrs.yaml index 5f9bf47bce..757734bb02 100644 --- a/imap_processing/cdf/config/imap_hi_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_hi_variable_attrs.yaml @@ -674,23 +674,24 @@ hi_goodtimes_esa_step: VALIDMAX: 10 hi_goodtimes_spin_bin: - <<: *default_uint8 CATDESC: Spin angle bin index FIELDNAM: Spin Bin + FILLVAL: 255 FORMAT: I2 LABLAXIS: Spin Bin UNITS: " " - VAR_TYPE: support_data VALIDMIN: 0 VALIDMAX: 89 + VAR_TYPE: support_data VAR_NOTES: > Spin angle bins numbered 0-89, covering 0-360 degrees of spacecraft spin. Each bin is 4 degrees wide. + dtype: uint8 hi_goodtimes_spin_bin_label: CATDESC: Label for spin bin FIELDNAM: Spin Bin Label - DEPEND_1: spin_bin + DEPEND_0: spin_bin FORMAT: A3 VAR_TYPE: metadata diff --git a/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml index 3012140ba6..664e57adb2 100644 --- a/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_global_cdf_attrs.yaml @@ -15,11 +15,11 @@ imap_idex_l1a_sci: Logical_source: imap_idex_l1a_sci-1week Logical_source_description: IMAP Mission IDEX Instrument Level-1A Weekly Data. -imap_idex_l1a_evt: - <<: *instrument_base - Data_type: L1A_EVT>Level-1A Event Message Data - Logical_source: imap_idex_l1a_evt - Logical_source_description: IMAP Mission IDEX Instrument Level-1A Event Message Data. +imap_idex_l1a_msg: + <<: *instrument_base + Data_type: L1A_MSG>Level-1A Event Message Data + Logical_source: imap_idex_l1a_msg + Logical_source_description: IMAP Mission IDEX Instrument Level-1A Event Message Data. imap_idex_l1a_catlst: <<: *instrument_base @@ -33,11 +33,11 @@ imap_idex_l1b_sci: Logical_source: imap_idex_l1b_sci-1week Logical_source_description: IMAP Mission IDEX Instrument Level-1B Weekly Data. -imap_idex_l1b_evt: - <<: *instrument_base - Data_type: L1B_EVT>Level-1B Event Message Data - Logical_source: imap_idex_l1b_evt - Logical_source_description: IMAP Mission IDEX Instrument Level-1B Event Message Data. +imap_idex_l1b_msg: + <<: *instrument_base + Data_type: L1B_MSG>Level-1B Event Message Data + Logical_source: imap_idex_l1b_msg + Logical_source_description: IMAP Mission IDEX Instrument Level-1B Event Message Data. imap_idex_l1b_catlst: <<: *instrument_base diff --git a/imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml index 8765d2e6a7..be5104ddcb 100644 --- a/imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1a_variable_attrs.yaml @@ -199,6 +199,7 @@ shcoarse: FIELDNAM: Secondary header coarse time LABLAXIS: Packet Generation Time (Coarse) UNITS: seconds + FILLVAL: 4294967295 shfine: <<: *trigger_base @@ -207,6 +208,13 @@ shfine: VALIDMAX: *max_uint16 LABLAXIS: Packet Generation Time (Fine) UNITS: seconds + FILLVAL: 65535 + +messages: + <<: *string_base + CATDESC: Rendered IDEX event message text + FIELDNAM: Event message text + FORMAT: A160 checksum: <<: *trigger_base @@ -214,7 +222,7 @@ checksum: FIELDNAM: Checksum UNITS: " " -idx__sci0aid: +aid: <<: *trigger_base CATDESC: Accountability identifier for this event FIELDNAM: Accountability identifier diff --git a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml index 7397b8ebe3..e65b69da0e 100644 --- a/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_idex_l1b_variable_attrs.yaml @@ -53,15 +53,63 @@ spice_base: &spice_base VALIDMAX: *spice_data_max # <=== Instrument Setting Attributes ===> -trigger_mode: +trigger_mode_lg: <<: *string_base - FIELDNAM: Trigger Mode - CATDESC: Channel and mode that triggered the event + FIELDNAM: Low Gain Trigger Mode + CATDESC: Low Gain Trigger Mode. -trigger_level: +trigger_level_lg: <<: *trigger_base - FIELDNAM: Trigger Level - CATDESC: Threshold signal level that triggered the event + FIELDNAM: Low Gain Trigger Level + CATDESC: Low Gain Trigger Level threshold. + +trigger_mode_mg: + <<: *string_base + FIELDNAM: Mid Gain Trigger Mode + CATDESC: Mid Gain Trigger Mode. + +trigger_level_mg: + <<: *trigger_base + FIELDNAM: Mid Gain Trigger Level + CATDESC: Mid Gain Trigger level threshold. + + +trigger_mode_hg: + <<: *string_base + FIELDNAM: High Gain Trigger Mode + CATDESC: High Gain Trigger Mode. + +trigger_level_hg: + <<: *trigger_base + FIELDNAM: High Trigger Level + CATDESC: High Trigger Level threshold. + +trigger_origin: + <<: *string_base + FIELDNAM: Trigger Origin + CATDESC: Trigger Origin of the event. + +pulser_on: + <<: *trigger_base + FIELDNAM: Pulser On + CATDESC: Pulser state flag derived from message events (0=off, 1=on). + LABLAXIS: Pulser On + FORMAT: I1 + FILLVAL: 255 + VAR_TYPE: support_data + VALIDMIN: 0 + VALIDMAX: 1 + +science_on: + <<: *trigger_base + FIELDNAM: Science On + CATDESC: Science acquisition state flag derived from message events (0=off, 1=on). + LABLAXIS: Science On + FORMAT: I1 + FILLVAL: 255 + VAR_TYPE: support_data + VALIDMIN: 0 + VALIDMAX: 1 tof_high: <<: *l1b_tof_base diff --git a/imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml index 942a317ba9..f43e28ea67 100644 --- a/imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_lo_global_cdf_attrs.yaml @@ -101,6 +101,18 @@ imap_lo_l1b_instrument-status-summary: Logical_source: imap_lo_l1b_instrument-status-summary Logical_source_description: IMAP Mission IMAP-Lo Instrument Level-1B Data +imap_lo_l1b_bgrates: + <<: *instrument_base + Data_type: L1B_goodtimes>Level-1B Background Rates List + Logical_source: imap_lo_l1b_bgrates + Logical_source_description: IMAP Mission IMAP-Lo Instrument Level-1B Data + +imap_lo_l1b_goodtimes: + <<: *instrument_base + Data_type: L1B_goodtimes>Level-1B Goodtimes List + Logical_source: imap_lo_l1b_good-times + Logical_source_description: IMAP Mission IMAP-Lo Instrument Level-1B Data + imap_lo_l1c_goodtimes: <<: *instrument_base Data_type: L1C_goodtimes>Level-1C Goodtimes List diff --git a/imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml b/imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml index ac926f3c03..e36fff54ea 100644 --- a/imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml +++ b/imap_processing/cdf/config/imap_ultra_global_cdf_attrs.yaml @@ -212,6 +212,54 @@ imap_ultra_l1b_90sensor-de: Logical_source: imap_ultra_l1b_90sensor-de Logical_source_description: IMAP-Ultra Instrument Level-1B Direct Event Data. +imap_ultra_l1b_45sensor-priority-1-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_1_DE>Level-1B Priority 1 Direct Event + Logical_source: imap_ultra_l1b_45sensor-priority-1-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 1 Direct Event Data. + +imap_ultra_l1b_90sensor-priority-1-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_1_DE>Level-1B Priority 1 Direct Event + Logical_source: imap_ultra_l1b_90sensor-priority-1-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 1 Direct Event Data. + +imap_ultra_l1b_45sensor-priority-2-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_2_DE>Level-1B Priority 2 Direct Event + Logical_source: imap_ultra_l1b_45sensor-priority-2-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 2 Direct Event Data. + +imap_ultra_l1b_90sensor-priority-2-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_2_DE>Level-1B Priority 2 Direct Event + Logical_source: imap_ultra_l1b_90sensor-priority-2-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 2 Direct Event Data. + +imap_ultra_l1b_45sensor-priority-3-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_3_DE>Level-1B Priority 3 Direct Event + Logical_source: imap_ultra_l1b_45sensor-priority-3-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 3 Direct Event Data. + +imap_ultra_l1b_90sensor-priority-3-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_3_DE>Level-1B Priority 3 Direct Event + Logical_source: imap_ultra_l1b_90sensor-priority-3-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 3 Direct Event Data. + +imap_ultra_l1b_45sensor-priority-4-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_4_DE>Level-1B Priority 4 Direct Event + Logical_source: imap_ultra_l1b_45sensor-priority-4-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 4 Direct Event Data. + +imap_ultra_l1b_90sensor-priority-4-de: + <<: *instrument_base + Data_type: L1B_PRIORITY_4_DE>Level-1B Priority 4 Direct Event + Logical_source: imap_ultra_l1b_90sensor-priority-4-de + Logical_source_description: IMAP-Ultra Instrument Level-1B Priority 4 Direct Event Data. + imap_ultra_l1b_45sensor-extendedspin: <<: *instrument_base Data_type: L1B_45Sensor-ExtendedSpin>Level-1B Extended Spin for Ultra45 diff --git a/imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml b/imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml index f6d3303261..438d913ae5 100644 --- a/imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ultra_l1b_variable_attrs.yaml @@ -526,6 +526,7 @@ quality_scattering: LABLAXIS: quality_scattering # TODO: come back to format UNITS: " " + DEPEND_0: spin_number quality_low_voltage: <<: *default_uint16 @@ -534,6 +535,7 @@ quality_low_voltage: LABLAXIS: quality_low_voltage # TODO: come back to format UNITS: " " + DEPEND_0: spin_number quality_high_energy: <<: *default_uint16 @@ -542,6 +544,7 @@ quality_high_energy: LABLAXIS: quality_high_energy # TODO: come back to format UNITS: " " + DEPEND_0: spin_number quality_statistics: <<: *default_uint16 @@ -550,22 +553,30 @@ quality_statistics: LABLAXIS: quality_statistics # TODO: come back to format UNITS: " " - + DEPEND_0: spin_number energy_range_edges: - <<: *default_float64 - CATDESC: Energy range edges for culling data at l1b. + CATDESC: Energy range edges used for culling data. DISPLAY_TYPE: no_plot FIELDNAM: Energy Range Edges + FILLVAL: -1.0e+31 + FORMAT: F12.4 LABLAXIS: Energy Edges UNITS: keV VALIDMIN: 0.0 - VALIDMAX: 9223372036854775807 + VALIDMAX: 5000 + VAR_TYPE: support_data + DEPEND_0: energy_range_edges_dim energy_range_flags: - <<: *default_float64 - CATDESC: Bit flags for each culling energy range + CATDESC: Bit flags describing culling energy ranges. DISPLAY_TYPE: no_plot FIELDNAM: Energy Range Flags + FILLVAL: 0 + FORMAT: I5 LABLAXIS: Range Flags - UNITS: " " \ No newline at end of file + UNITS: " " + VALIDMIN: 1 + VALIDMAX: 65534 + VAR_TYPE: support_data + DEPEND_0: energy_range_flags_dim \ No newline at end of file diff --git a/imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml b/imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml index acfbe6fc18..2db8569544 100644 --- a/imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml +++ b/imap_processing/cdf/config/imap_ultra_l1c_variable_attrs.yaml @@ -136,8 +136,11 @@ scatter_threshold: counts: <<: *default CATDESC: Counts for a spin. + VAR_NOTES: Counts healpix maps are sampled at finer resolution to help maintain the pointing accuracy for each event. + Since count maps are a binned integral quantity, they necessarily require a non-spun approach per Pointing, unlike + exposure time and sensitivities. DEPEND_1: energy_bin_geometric_mean - DEPEND_2: pixel_index + DEPEND_2: counts_pixel_index FIELDNAM: counts LABLAXIS: counts # TODO: come back to format @@ -244,6 +247,18 @@ pixel_index: VALIDMAX: *max_int VAR_TYPE: support_data +counts_pixel_index: + FILLVAL: *min_int + CATDESC: Counts variable HEALPix pixel index. + DISPLAY_TYPE: no_plot + FIELDNAM: counts_pixel_index + FORMAT: I12 + LABLAXIS: Counts Pixel Index + UNITS: " " + VALIDMIN: *min_int + VALIDMAX: *max_int + VAR_TYPE: support_data + spin_phase_step: FILLVAL: *min_int CATDESC: Spin phase step index (1ms resolution). diff --git a/imap_processing/cli.py b/imap_processing/cli.py index a9ad55c131..f629b97ed6 100644 --- a/imap_processing/cli.py +++ b/imap_processing/cli.py @@ -676,7 +676,7 @@ def do_processing( # Load conversion table (needed for both hist and DE) conversion_table_file = dependencies.get_processing_inputs( - descriptor="conversion-table-for-anc-data" + descriptor="l1b-conversion-table-for-anc-data" )[0] with open(conversion_table_file.imap_file_paths[0].construct_path()) as f: @@ -691,16 +691,16 @@ def do_processing( if "hist" in self.descriptor: # Create file lists for each ancillary type excluded_regions_files = dependencies.get_processing_inputs( - descriptor="map-of-excluded-regions" + descriptor="l1b-map-of-excluded-regions" )[0] uv_sources_files = dependencies.get_processing_inputs( - descriptor="map-of-uv-sources" + descriptor="l1b-map-of-uv-sources" )[0] suspected_transients_files = dependencies.get_processing_inputs( - descriptor="suspected-transients" + descriptor="l1b-suspected-transients" )[0] exclusions_by_instr_team_files = dependencies.get_processing_inputs( - descriptor="exclusions-by-instr-team" + descriptor="l1b-exclusions-by-instr-team" )[0] pipeline_settings = dependencies.get_processing_inputs( descriptor="pipeline-settings" @@ -758,10 +758,15 @@ def do_processing( pipeline_settings_combiner = GlowsAncillaryCombiner( pipeline_settings_input, day_buffer ) + calibration_input = dependencies.get_processing_inputs( + descriptor="l2-calibration" + )[0] + calibration_combiner = GlowsAncillaryCombiner(calibration_input, day_buffer) datasets = glows_l2( input_dataset, pipeline_settings_combiner.combined_dataset, + calibration_combiner.combined_dataset, ) return datasets @@ -772,7 +777,7 @@ class Hi(ProcessInstrument): def do_processing( # noqa: PLR0912 self, dependencies: ProcessingInputCollection - ) -> list[xr.Dataset | Path]: + ) -> list[xr.Dataset]: """ Perform IMAP-Hi specific processing. @@ -789,10 +794,6 @@ def do_processing( # noqa: PLR0912 print(f"Processing IMAP-Hi {self.data_level}") datasets: list[xr.Dataset] = [] - # Check self.repointing is not None (for mypy type checking) - if self.repointing is None: - raise ValueError("Repointing must be provided for Hi processing.") - if self.data_level == "l1a": science_files = dependencies.get_file_paths(source="hi") if len(science_files) != 1: @@ -806,6 +807,12 @@ def do_processing( # noqa: PLR0912 if l0_files: datasets = hi_l1b.housekeeping(l0_files[0]) elif "goodtimes" in self.descriptor: + # Check self.repointing is not None (for mypy type checking) + if self.repointing is None: + raise ValueError( + "Repointing must be provided for Hi Goodtimes processing." + ) + # Goodtimes processing l1b_de_paths = dependencies.get_file_paths( source="hi", data_type="l1b", descriptor="de" @@ -830,14 +837,24 @@ def do_processing( # noqa: PLR0912 f"got {len(cal_prod_paths)}" ) + l1a_diagfee_paths = dependencies.get_file_paths( + source="hi", data_type="l1a", descriptor="diagfee" + ) + if len(l1a_diagfee_paths) != 1: + raise ValueError( + f"Expected one L1A DIAG_FEE file, got {len(l1a_diagfee_paths)}" + ) + # Load CDFs before passing to hi_goodtimes l1b_de_datasets = [load_cdf(path) for path in l1b_de_paths] l1b_hk = load_cdf(l1b_hk_paths[0]) + l1a_diagfee = load_cdf(l1a_diagfee_paths[0]) datasets = hi_goodtimes.hi_goodtimes( - l1b_de_datasets, self.repointing, + l1b_de_datasets, l1b_hk, + l1a_diagfee, cal_prod_paths[0], ) else: @@ -854,19 +871,60 @@ def do_processing( # noqa: PLR0912 elif self.data_level == "l1c": if "pset" in self.descriptor: # L1C PSET processing - science_paths = dependencies.get_file_paths( - source="hi", data_type="l1b" + l1b_de_paths = dependencies.get_file_paths( + source="hi", data_type="l1b", descriptor="de" + ) + if len(l1b_de_paths) != 1: + raise ValueError( + f"Expected exactly one DE science dependency. " + f"Got {l1b_de_paths}" + ) + + # Get ancillary dependencies + anc_dependencies = dependencies.get_processing_inputs( + data_type="ancillary" ) - if len(science_paths) != 1: + if len(anc_dependencies) != 2: raise ValueError( - f"Expected only one science dependency. Got {science_paths}" + f"Expected two ancillary dependencies (cal-prod and " + f"backgrounds). Got " + f"{[anc_dep.descriptor for anc_dep in anc_dependencies]}" ) - anc_paths = dependencies.get_file_paths(data_type="ancillary") - if len(anc_paths) != 1: + + # Create mapping from descriptor to path + anc_path_dict = { + dep.descriptor.split("-", 1)[1]: dep.imap_file_paths[ + 0 + ].construct_path() + for dep in anc_dependencies + } + + # Verify we have both required ancillary files + if ( + "cal-prod" not in anc_path_dict + or "backgrounds" not in anc_path_dict + ): raise ValueError( - f"Expected only one ancillary dependency. Got {anc_paths}" + f"Missing required ancillary files. Expected 'cal-prod' and " + f"'backgrounds', got {list(anc_path_dict.keys())}" ) - datasets = hi_l1c.hi_l1c(load_cdf(science_paths[0]), anc_paths[0]) + + # Load goodtimes dependency + goodtimes_paths = dependencies.get_file_paths( + source="hi", data_type="l1b", descriptor="goodtimes" + ) + if len(goodtimes_paths) != 1: + raise ValueError( + f"Expected exactly one goodtimes dependency. " + f"Got {goodtimes_paths}" + ) + + datasets = hi_l1c.hi_l1c( + load_cdf(l1b_de_paths[0]), + anc_path_dict["cal-prod"], + load_cdf(goodtimes_paths[0]), + anc_path_dict["backgrounds"], + ) elif self.data_level == "l2": science_paths = dependencies.get_file_paths(source="hi", data_type="l1c") anc_dependencies = dependencies.get_processing_inputs(data_type="ancillary") @@ -1014,16 +1072,23 @@ def do_processing( science_files = dependencies.get_file_paths(source="idex") datasets = PacketParser(science_files[0]).data elif self.data_level == "l1b": - if len(dependency_list) != 3: + n_expected_deps = 3 if self.descriptor == "sci-1week" else 1 + if len(dependency_list) != n_expected_deps: raise ValueError( - f"Unexpected dependencies found for IDEX L1B:" - f"{dependency_list}. Expected only three dependencies." + f"Unexpected dependencies found for IDEX L1B {self.descriptor}:" + f"{dependency_list}. Expected only {n_expected_deps} dependencies." ) # get CDF file science_files = dependencies.get_file_paths(source="idex") + # Load all the science files. There should only be one, but in the case of + # multiple files, we want to make sure to load them all and select the one + # with the latest time. + science_datasets = [load_cdf(f) for f in science_files] + if not science_datasets: + raise ValueError("No science files found for IDEX L1B processing.") + latest_file = max(science_datasets, key=lambda ds: ds["epoch"].data[0]) # process data - dependency = load_cdf(science_files[0]) - datasets = [idex_l1b(dependency)] + datasets = [idex_l1b(latest_file, self.descriptor)] elif self.data_level == "l2a": if len(dependency_list) != 3: raise ValueError( @@ -1049,7 +1114,7 @@ def do_processing( sci_dependencies = [load_cdf(f) for f in sci_files] # sort science files by the first epoch value sci_dependencies.sort(key=lambda ds: ds["epoch"].values[0]) - hk_files = dependencies.get_file_paths(source="idex", descriptor="evt") + hk_files = dependencies.get_file_paths(source="idex", descriptor="msg") # Remove duplicate housekeeping files hk_dependencies = [load_cdf(dep) for dep in list(set(hk_files))] # sort housekeeping files by the first epoch value diff --git a/imap_processing/codice/codice_l1a_de.py b/imap_processing/codice/codice_l1a_de.py index abc8ad0303..25e437594e 100644 --- a/imap_processing/codice/codice_l1a_de.py +++ b/imap_processing/codice/codice_l1a_de.py @@ -49,25 +49,25 @@ def extract_initial_items_from_combined_packets( n_packets = len(packets.epoch) # Preallocate arrays - packet_version = np.zeros(n_packets, dtype=np.uint16) - spin_period = np.zeros(n_packets, dtype=np.uint16) - acq_start_seconds = np.zeros(n_packets, dtype=np.uint32) - acq_start_subseconds = np.zeros(n_packets, dtype=np.uint32) - spare_1 = np.zeros(n_packets, dtype=np.uint8) - st_bias_gain_mode = np.zeros(n_packets, dtype=np.uint8) - sw_bias_gain_mode = np.zeros(n_packets, dtype=np.uint8) - suspect = np.zeros(n_packets, dtype=np.uint8) - priority = np.zeros(n_packets, dtype=np.uint8) - compressed = np.zeros(n_packets, dtype=np.uint8) - rgfo_half_spin = np.zeros(n_packets, dtype=np.uint8) - rgfo_esa_step = np.zeros(n_packets, dtype=np.uint8) - rgfo_spin_sector = np.zeros(n_packets, dtype=np.uint8) - nso_half_spin = np.zeros(n_packets, dtype=np.uint8) - nso_spin_sector = np.zeros(n_packets, dtype=np.uint8) - nso_esa_step = np.zeros(n_packets, dtype=np.uint8) - spare_2 = np.zeros(n_packets, dtype=np.uint16) - num_events = np.zeros(n_packets, dtype=np.uint32) - byte_count = np.zeros(n_packets, dtype=np.uint32) + packet_version: np.ndarray = np.zeros(n_packets, dtype=np.uint16) + spin_period: np.ndarray = np.zeros(n_packets, dtype=np.uint16) + acq_start_seconds: np.ndarray = np.zeros(n_packets, dtype=np.uint32) + acq_start_subseconds: np.ndarray = np.zeros(n_packets, dtype=np.uint32) + spare_1: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + st_bias_gain_mode: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + sw_bias_gain_mode: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + suspect: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + priority: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + compressed: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + rgfo_half_spin: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + rgfo_esa_step: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + rgfo_spin_sector: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + nso_half_spin: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + nso_spin_sector: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + nso_esa_step: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + spare_2: np.ndarray = np.zeros(n_packets, dtype=np.uint16) + num_events: np.ndarray = np.zeros(n_packets, dtype=np.uint32) + byte_count: np.ndarray = np.zeros(n_packets, dtype=np.uint32) # Extract fields from each packet for pkt_idx in range(n_packets): @@ -342,10 +342,10 @@ def _unpack_and_store_events( num_packets = len(num_events_arr) # Preallocate arrays for concatenated events and their destination indices - all_event_bytes = np.zeros((total_events, 8), dtype=np.uint8) - event_epoch_idx = np.zeros(total_events, dtype=np.int32) - event_priority_idx = np.zeros(total_events, dtype=np.int32) - event_position_idx = np.zeros(total_events, dtype=np.int32) + all_event_bytes: np.ndarray = np.zeros((total_events, 8), dtype=np.uint8) + event_epoch_idx: np.ndarray = np.zeros(total_events, dtype=np.int32) + event_priority_idx: np.ndarray = np.zeros(total_events, dtype=np.int32) + event_position_idx: np.ndarray = np.zeros(total_events, dtype=np.int32) # Build concatenated event array and index mappings offset = 0 diff --git a/imap_processing/codice/codice_l1a_hi_sectored.py b/imap_processing/codice/codice_l1a_hi_sectored.py index 440d780958..1c84fded6f 100644 --- a/imap_processing/codice/codice_l1a_hi_sectored.py +++ b/imap_processing/codice/codice_l1a_hi_sectored.py @@ -234,7 +234,13 @@ def l1a_hi_sectored(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset: # Extract species data from decompressed data: # - (num_packets, energy_bins, spin_sector, inst_az) + # Unpacked data is in this order: + # (epoch, number_species, energy_{species}, inst_az, spin_sector) species_data = decompressed_data[:, species_index, :, :, :] + # Now transpose to put data in the correct order for writing to CDF: + # (epoch, energy_{species}, spin_sector, inst_az) + species_data = np.transpose(species_data, [0, 1, 3, 2]) + species_attrs = cdf_attrs.get_variable_attributes("hi-species-attrs") species_attrs = apply_replacements_to_attrs( species_attrs, {"species": species_name} diff --git a/imap_processing/codice/codice_l1a_ialirt_hi.py b/imap_processing/codice/codice_l1a_ialirt_hi.py index 09c8c07db1..a5dfcc1f32 100644 --- a/imap_processing/codice/codice_l1a_ialirt_hi.py +++ b/imap_processing/codice/codice_l1a_ialirt_hi.py @@ -152,7 +152,7 @@ def l1a_ialirt_hi(unpacked_dataset: xr.Dataset, lut_file: Path) -> xr.Dataset: # -> (epoch, n_spins, energy, spin_sector, inst_az) -> # finally (epoch * n_spins, energy, # spin_sector, inst_az) - decompressed_data = decompressed_data.transpose(0, 2, 1, 3, 4).reshape( + decompressed_data = decompressed_data.transpose(0, 2, 1, 4, 3).reshape( -1, chunk_size, *collapse_shape ) diff --git a/imap_processing/codice/codice_l2.py b/imap_processing/codice/codice_l2.py index 8876d6bf2f..ef2a664172 100644 --- a/imap_processing/codice/codice_l2.py +++ b/imap_processing/codice/codice_l2.py @@ -815,7 +815,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: # Add these new coordinates new_coords = { - "energy_h": l1b_dataset["energy_h"], + "energy_h": xr.DataArray( + l1b_dataset["energy_h"].values, + dims=("energy_h",), + attrs=cdf_attrs.get_variable_attributes("energy_h", check_schema=False), + ), "energy_h_label": xr.DataArray( l1b_dataset["energy_h"].values.astype(str), dims=("energy_h",), @@ -823,7 +827,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_h_label", check_schema=False ), ), - "energy_he3": l1b_dataset["energy_he3"], + "energy_he3": xr.DataArray( + l1b_dataset["energy_he3"].values, + dims=("energy_he3",), + attrs=cdf_attrs.get_variable_attributes("energy_he3", check_schema=False), + ), "energy_he3_label": xr.DataArray( l1b_dataset["energy_he3"].values.astype(str), dims=("energy_he3",), @@ -831,7 +839,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_he3_label", check_schema=False ), ), - "energy_he4": l1b_dataset["energy_he4"], + "energy_he4": xr.DataArray( + l1b_dataset["energy_he4"].values, + dims=("energy_he4",), + attrs=cdf_attrs.get_variable_attributes("energy_he4", check_schema=False), + ), "energy_he4_label": xr.DataArray( l1b_dataset["energy_he4"].values.astype(str), dims=("energy_he4",), @@ -839,7 +851,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_he4_label", check_schema=False ), ), - "energy_c": l1b_dataset["energy_c"], + "energy_c": xr.DataArray( + l1b_dataset["energy_c"].values, + dims=("energy_c",), + attrs=cdf_attrs.get_variable_attributes("energy_c", check_schema=False), + ), "energy_c_label": xr.DataArray( l1b_dataset["energy_c"].values.astype(str), dims=("energy_c",), @@ -847,7 +863,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_c_label", check_schema=False ), ), - "energy_o": l1b_dataset["energy_o"], + "energy_o": xr.DataArray( + l1b_dataset["energy_o"].values, + dims=("energy_o",), + attrs=cdf_attrs.get_variable_attributes("energy_o", check_schema=False), + ), "energy_o_label": xr.DataArray( l1b_dataset["energy_o"].values.astype(str), dims=("energy_o",), @@ -855,7 +875,13 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_o_label", check_schema=False ), ), - "energy_ne_mg_si": l1b_dataset["energy_ne_mg_si"], + "energy_ne_mg_si": xr.DataArray( + l1b_dataset["energy_ne_mg_si"].values, + dims=("energy_ne_mg_si",), + attrs=cdf_attrs.get_variable_attributes( + "energy_ne_mg_si", check_schema=False + ), + ), "energy_ne_mg_si_label": xr.DataArray( l1b_dataset["energy_ne_mg_si"].values.astype(str), dims=("energy_ne_mg_si",), @@ -863,7 +889,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_ne_mg_si_label", check_schema=False ), ), - "energy_fe": l1b_dataset["energy_fe"], + "energy_fe": xr.DataArray( + l1b_dataset["energy_fe"].values, + dims=("energy_fe",), + attrs=cdf_attrs.get_variable_attributes("energy_fe", check_schema=False), + ), "energy_fe_label": xr.DataArray( l1b_dataset["energy_fe"].values.astype(str), dims=("energy_fe",), @@ -871,7 +901,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_fe_label", check_schema=False ), ), - "energy_uh": l1b_dataset["energy_uh"], + "energy_uh": xr.DataArray( + l1b_dataset["energy_uh"].values, + dims=("energy_uh",), + attrs=cdf_attrs.get_variable_attributes("energy_uh", check_schema=False), + ), "energy_uh_label": xr.DataArray( l1b_dataset["energy_uh"].values.astype(str), dims=("energy_uh",), @@ -879,7 +913,11 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_uh_label", check_schema=False ), ), - "energy_junk": l1b_dataset["energy_junk"], + "energy_junk": xr.DataArray( + l1b_dataset["energy_junk"].values, + dims=("energy_junk",), + attrs=cdf_attrs.get_variable_attributes("energy_junk", check_schema=False), + ), "energy_junk_label": xr.DataArray( l1b_dataset["energy_junk"].values.astype(str), dims=("energy_junk",), @@ -892,7 +930,13 @@ def process_hi_omni(dependencies: ProcessingInputCollection) -> xr.Dataset: dims=("epoch",), attrs=cdf_attrs.get_variable_attributes("epoch", check_schema=False), ), + "epoch_delta_plus": l1b_dataset["epoch_delta_plus"], + "epoch_delta_minus": l1b_dataset["epoch_delta_minus"], } + + l1b_dataset["epoch"].attrs["DELTA_MINUS_VAR"] = "epoch_delta_minus" + l1b_dataset["epoch"].attrs["DELTA_PLUS_VAR"] = "epoch_delta_plus" + l1b_dataset = l1b_dataset.assign_coords(new_coords) return l1b_dataset @@ -942,7 +986,11 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: "spin_sector_label", check_schema=False ), ), - "energy_h": l1b_dataset["energy_h"], + "energy_h": xr.DataArray( + l1b_dataset["energy_h"].values, + dims=("energy_h",), + attrs=cdf_attrs.get_variable_attributes("energy_h", check_schema=False), + ), "energy_h_label": xr.DataArray( l1b_dataset["energy_h"].values.astype(str), dims=("energy_h",), @@ -950,7 +998,13 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_h_label", check_schema=False ), ), - "energy_he3he4": l1b_dataset["energy_he3he4"], + "energy_he3he4": xr.DataArray( + l1b_dataset["energy_he3he4"].values, + dims=("energy_he3he4",), + attrs=cdf_attrs.get_variable_attributes( + "energy_he3he4", check_schema=False + ), + ), "energy_he3he4_label": xr.DataArray( l1b_dataset["energy_he3he4"].values.astype(str), dims=("energy_he3he4",), @@ -958,7 +1012,13 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_he3he4_label", check_schema=False ), ), - "energy_cno": l1b_dataset["energy_cno"], + "energy_cno": xr.DataArray( + l1b_dataset["energy_cno"].values, + dims=("energy_cno",), + attrs=cdf_attrs.get_variable_attributes( + "energy_cno", check_schema=False + ), + ), "energy_cno_label": xr.DataArray( l1b_dataset["energy_cno"].values.astype(str), dims=("energy_cno",), @@ -966,7 +1026,13 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: "energy_cno_label", check_schema=False ), ), - "energy_fe": l1b_dataset["energy_fe"], + "energy_fe": xr.DataArray( + l1b_dataset["energy_fe"].values, + dims=("energy_fe",), + attrs=cdf_attrs.get_variable_attributes( + "energy_fe", check_schema=False + ), + ), "energy_fe_label": xr.DataArray( l1b_dataset["energy_fe"].values.astype(str), dims=("energy_fe",), @@ -975,6 +1041,8 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: ), ), "epoch": l1b_dataset["epoch"], + "epoch_delta_plus": l1b_dataset["epoch_delta_plus"], + "epoch_delta_minus": l1b_dataset["epoch_delta_minus"], "elevation_angle": xr.DataArray( HI_L2_ELEVATION_ANGLE, dims=("elevation_angle",), @@ -993,6 +1061,9 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: attrs=cdf_attrs.get_global_attributes("imap_codice_l2_hi-sectored"), ) + l1b_dataset["epoch"].attrs["DELTA_MINUS_VAR"] = "epoch_delta_minus" + l1b_dataset["epoch"].attrs["DELTA_PLUS_VAR"] = "epoch_delta_plus" + efficiencies_file = dependencies.get_file_paths( descriptor="l2-hi-sectored-efficiency" )[0] @@ -1046,10 +1117,14 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: ) # Replace existing species data with omni-directional intensities + species_attrs = cdf_attrs.get_variable_attributes(species, check_schema=False) + # Replace {species} in attributes with actual species name + species_attrs = apply_replacements_to_attrs(species_attrs, {"species": species}) + l2_dataset[species] = xr.DataArray( sectored_intensities.data, dims=("epoch", f"energy_{species}", "spin_sector", "elevation_angle"), - attrs=cdf_attrs.get_variable_attributes(species, check_schema=False), + attrs=species_attrs, ) # Calculate uncertainty if available species_uncertainty = f"unc_{species}" @@ -1057,12 +1132,18 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: sectored_uncertainties = l1b_dataset[species_uncertainty] / ( geometric_factor_da * species_efficiencies * energy_passbands ) + unc_species_attrs = cdf_attrs.get_variable_attributes( + species_uncertainty, check_schema=False + ) + # Replace {species} in attributes with actual species name + unc_species_attrs = apply_replacements_to_attrs( + unc_species_attrs, {"species": species} + ) + l2_dataset[species_uncertainty] = xr.DataArray( sectored_uncertainties.data, dims=("epoch", f"energy_{species}", "spin_sector", "elevation_angle"), - attrs=cdf_attrs.get_variable_attributes( - species_uncertainty, check_schema=False - ), + attrs=unc_species_attrs, ) # Calculate spin angle @@ -1085,6 +1166,16 @@ def process_hi_sectored(dependencies: ProcessingInputCollection) -> xr.Dataset: base_angles = np.asarray(L2_HI_SECTORED_ANGLE, dtype=float).reshape(n_spin, 1) spin_angle = (base_angles + elevation_offsets) % 360.0 + # We need to transpose spin_angle to put spin_angle data into correct + # dimensions. Eg. + # spin_angle[0,0] - 285 + # spin_angle[1,0] - 315 + # .... + # This is expected behavior per CoDICE because the spin angle should increments + # at 30 degree spin angle per elevation angle. Due to that, in the example, the + # column remained same but spin angle incremented by 30 degrees for each + # elevation angle. + spin_angle = spin_angle.T # Add spin angle variable using the new elevation_angle dimension l2_dataset["spin_angle"] = (("spin_sector", "elevation_angle"), spin_angle) l2_dataset["spin_angle"].attrs = cdf_attrs.get_variable_attributes( diff --git a/imap_processing/codice/utils.py b/imap_processing/codice/utils.py index fe1900a0a5..200d86b2e3 100644 --- a/imap_processing/codice/utils.py +++ b/imap_processing/codice/utils.py @@ -422,7 +422,7 @@ def calculate_acq_time_per_step( np.maximum(non_adjusted_hv_settle_per_step, min_hv_settle_ms), max_hv_settle_ms ) # initialize array of nans for acquisition time per step - acq_time_per_step = np.full(esa_step_dim, np.nan, dtype=np.float64) + acq_time_per_step: np.ndarray = np.full(esa_step_dim, np.nan, dtype=np.float64) # acquisition time per step in milliseconds # sector_time - sector_margin_ms / num_steps - hv_settle_per_step acq_time_per_step[: len(num_steps_data)] = ( diff --git a/imap_processing/ena_maps/ena_maps.py b/imap_processing/ena_maps/ena_maps.py index ed7b675235..6072b34505 100644 --- a/imap_processing/ena_maps/ena_maps.py +++ b/imap_processing/ena_maps/ena_maps.py @@ -22,7 +22,7 @@ # so we define an enum to handle the coordinate names. from imap_processing.ena_maps.utils.coordinates import CoordNames from imap_processing.spice import geometry -from imap_processing.spice.time import ttj2000ns_to_et +from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et logger = logging.getLogger(__name__) @@ -136,14 +136,15 @@ def match_coords_to_indices( if isinstance(input_object, PointingSet) and isinstance(output_object, PointingSet): raise ValueError("Cannot match indices between two PointingSet objects.") - # If event_et is not specified, use epoch of the PointingSet, if present. + # If event_et is not specified, use the midpoint of the PointingSet, if + # present. # The epoch will be in units of terrestrial time (TT) J2000 nanoseconds, # which must be converted to ephemeris time (ET) for SPICE. if event_et is None: if isinstance(input_object, PointingSet): - event_et = ttj2000ns_to_et(input_object.epoch) + event_et = input_object.midpoint_j2000_et elif isinstance(output_object, PointingSet): - event_et = ttj2000ns_to_et(output_object.epoch) + event_et = output_object.midpoint_j2000_et else: raise ValueError( "Event time must be specified if both objects are SkyMaps." @@ -301,6 +302,19 @@ def epoch(self) -> int: """ return self.data["epoch"].values[0] + @property + def midpoint_j2000_et(self) -> float: + """ + The midpoint of the pointing in ET. + + Returns + ------- + midpoint: int + The midpoint value [J2000 ET] of the pointing set. + """ + epoch_delta = self.data["epoch_delta"].values[0] + return float(ttj2000ns_to_et(self.epoch + epoch_delta / 2)) + @property def unwrapped_dims_dict(self) -> dict[str, tuple[str, ...]]: """ @@ -558,6 +572,8 @@ def __init__( np.stack((azimuth_pixel_center, elevation_pixel_center), axis=-1), dims=[CoordNames.GENERIC_PIXEL.value, CoordNames.AZ_EL_VECTOR.value], ) + # downsample counts variable to match the nside of the pointing set + self.downsample_counts() @property def num_points(self) -> int: @@ -600,6 +616,87 @@ def __repr__(self) -> str: f"num_points={self.num_points})" ) + def downsample_counts(self) -> None: + """ + Downsample the counts variable to match the pset nside. + + Counts at l1c are sampled at a finer resolution to help maintain the + pointing accuracy for each event. Since count maps are a binned integral + quantity, they necessarily require a non-spun approach per Pointing, unlike + exposure time and sensitivities. We need to downsample the counts from the + nside of the input pset counts variable (e.g. 128) to the nside of the pset. + """ + pset_data = self.data + # TODO remove this check once we reprocess all psets + # going forward, all psets should have counts_pixel_index as a coordinate. + if "counts_pixel_index" not in pset_data.dims: + logger.info( + "No counts_pixel_index found in the dataset. Skipping counts " + "downsampling." + ) + return + counts_n_pix = pset_data.sizes["counts_pixel_index"] + pset_n_pix = hp.nside2npix(self.nside) + if counts_n_pix != pset_n_pix: + # Raise an error if the nside the counts were sampled at is lower than the + # nside of the output map. We never want counts to be upsampled. + if counts_n_pix < pset_n_pix: + raise ValueError( + f"Counts in the input PSET are sampled at nside " + f"{hp.npix2nside(counts_n_pix)}, and the pset is {self.nside}. " + f"This would require upsampling the counts, which we do not want." + ) + counts_nside = hp.npix2nside(counts_n_pix) + n_energy_bins = pset_data.sizes["energy_bin_geometric_mean"] + order_diff = int(np.log2(counts_nside // self.nside)) + counts = pset_data["counts"].values[ + 0 + ] # shape: (n_energy_bins, counts_n_pix) + # Get counts in nested ordering. In nested ordering, the + # pixels that need to be binned together to go from the counts nside to + # the pset nside are contiguous in the array. + # Use nest2ring to get the indices to convert from ring to nest ordering if + # necessary. + if not self.nested: + counts_n = counts[ + :, hp.nest2ring(counts_nside, np.arange(counts_n_pix)) + ] + else: + counts_n = counts + + # reshape the counts by the amount pixels to bin together which is + # 4**order_diff because each step in order multiplies the pixel count + # by 4 + # Shape: (n_energy_bins, pset_n_pix, 4**order_diff) -> + # (n_energy_bins, pset_n_pix) + binned_counts_n = counts_n.reshape( + (n_energy_bins, pset_n_pix, 4**order_diff) + ).sum(axis=-1) + + if not self.nested: + # convert back to ring ordering if necessary and store in the + # downsampled counts array + binned_counts_n = binned_counts_n[ + :, hp.ring2nest(self.nside, np.arange(pset_n_pix)) + ] + + self.data["counts"] = xr.DataArray( + binned_counts_n[np.newaxis, :, :], + dims=( + *self.data["counts"].dims[:-1], + CoordNames.HEALPIX_INDEX.value, + ), + ) + logger.info( + f"Counts variable with nside = {counts_nside} downsampled to " + f"nside {self.nside}." + ) + else: + # Update the counts variable with the correct dims + self.data["counts"] = self.data["counts"].rename( + {CoordNames.COUNTS_HEALPIX_INDEX.value: CoordNames.HEALPIX_INDEX.value} + ) + class LoHiBasePointingSet(PointingSet): """ @@ -696,6 +793,21 @@ def __init__(self, dataset: xr.Dataset): # Update az_el_points using the base class method self.update_az_el_points() + @property + def midpoint_j2000_et(self) -> float: + """ + The midpoint of the pointing in ET. + + Returns + ------- + midpoint: int + The midpoint value [J2000 ET] of the pointing set. + """ + epoch_delta = met_to_ttj2000ns( + self.data["pointing_end_met"].data + ) - met_to_ttj2000ns(self.data["pointing_start_met"].data) + return float(ttj2000ns_to_et(self.epoch + epoch_delta / 2)) + # Define the Map classes class AbstractSkyMap(ABC): diff --git a/imap_processing/ena_maps/utils/coordinates.py b/imap_processing/ena_maps/utils/coordinates.py index 079e27a4c7..4ad217f2fb 100644 --- a/imap_processing/ena_maps/utils/coordinates.py +++ b/imap_processing/ena_maps/utils/coordinates.py @@ -12,6 +12,7 @@ class CoordNames(Enum): ENERGY_ULTRA_L1C = "energy_bin_geometric_mean" ENERGY_L2 = "energy" HEALPIX_INDEX = "pixel_index" + COUNTS_HEALPIX_INDEX = "counts_pixel_index" # The names of the az/el angular coordinates may differ between L1C and L2 data AZIMUTH_L1C = "longitude" diff --git a/imap_processing/ena_maps/utils/corrections.py b/imap_processing/ena_maps/utils/corrections.py index 71afb5536b..33d449218a 100644 --- a/imap_processing/ena_maps/utils/corrections.py +++ b/imap_processing/ena_maps/utils/corrections.py @@ -273,7 +273,7 @@ def predictor_corrector_iteration( of input. """ n_levels = observed_fluxes.shape[0] - energy_levels = np.arange(n_levels) + 1 + energy_levels: np.ndarray = np.arange(n_levels) + 1 # Initial power-law estimate from observed fluxes gamma_initial, _ = self.estimate_power_law_slope(observed_fluxes, energies) diff --git a/imap_processing/ena_maps/utils/map_utils.py b/imap_processing/ena_maps/utils/map_utils.py index ed8d8c4fa2..dd7b84208a 100644 --- a/imap_processing/ena_maps/utils/map_utils.py +++ b/imap_processing/ena_maps/utils/map_utils.py @@ -77,7 +77,7 @@ def vectorized_bincount( # dimension by an integer multiple of the number of bins. Doing so gives # each element in the additional dimensions its own set of 1D bins: index 0 # uses bins [0, minlength), index 1 uses bins [minlength, 2*minlength), etc. - offsets = np.arange(n_binsets).reshape(*non_spatial_shape, 1) * minlength + offsets: NDArray = np.arange(n_binsets).reshape(*non_spatial_shape, 1) * minlength indices_flat = (indices_bc + offsets).ravel() # Single bincount call with flattened data diff --git a/imap_processing/glows/l1a/glows_l1a.py b/imap_processing/glows/l1a/glows_l1a.py index 3a13c7af55..5a35abdeac 100644 --- a/imap_processing/glows/l1a/glows_l1a.py +++ b/imap_processing/glows/l1a/glows_l1a.py @@ -163,7 +163,7 @@ def generate_de_dataset( # TODO: Block header per second, or global attribute? # Store timestamps for each DirectEventL1a object. - time_data = np.zeros(len(de_l1a_list), dtype=np.int64) + time_data: np.ndarray = np.zeros(len(de_l1a_list), dtype=np.int64) # Each DirectEventL1A class covers 1 second of direct events data direct_events = np.zeros((len(de_l1a_list), len(de_l1a_list[0].direct_events), 4)) @@ -330,10 +330,10 @@ def generate_histogram_dataset( ] # Store timestamps for each HistogramL1A object. - time_data = np.zeros(len(hist_l1a_list), dtype=np.int64) + time_data: np.ndarray = np.zeros(len(hist_l1a_list), dtype=np.int64) # Data in lists, for each of the 25 time varying datapoints in HistogramL1A - hist_data = np.full( + hist_data: np.ndarray = np.full( (len(hist_l1a_list), GlowsConstants.STANDARD_BIN_COUNT), GlowsConstants.HISTOGRAM_FILLVAL, dtype=np.uint16, diff --git a/imap_processing/glows/l1b/glows_l1b_data.py b/imap_processing/glows/l1b/glows_l1b_data.py index a949f49506..ceb4779144 100644 --- a/imap_processing/glows/l1b/glows_l1b_data.py +++ b/imap_processing/glows/l1b/glows_l1b_data.py @@ -176,6 +176,28 @@ def __post_init__(self, pipeline_dataset: xr.Dataset) -> None: if "threshold" in var_name.lower() or "limit" in var_name.lower(): self.processing_thresholds[var_name] = pipeline_dataset[var_name].item() + def get_threshold(self, suffix: str) -> float | None: + """ + Return the threshold value whose key ends with the given suffix. + + Parameters + ---------- + suffix : str + The suffix to match against threshold keys. + + Returns + ------- + return_value : float or None + The matching threshold value, or None if no match is found. + """ + return_value = None + for descriptor, value in self.processing_thresholds.items(): + if descriptor.endswith(suffix): + return_value = float(value) + break + + return return_value + @dataclass class AncillaryExclusions: @@ -870,7 +892,7 @@ def __post_init__( # is_inside_excluded_region, is_excluded_by_instr_team, # is_suspected_transient] x 3600 bins self.histogram_flag_array = self._compute_histogram_flag_array(day_exclusions) - self.flags = np.ones((FLAG_LENGTH,), dtype=np.uint8) + self.flags = self.compute_flags(pipeline_settings) def update_spice_parameters(self) -> None: """Update SPICE parameters based on the current state.""" @@ -917,15 +939,17 @@ def update_spice_parameters(self) -> None: np.array([0, 0, 1]), SpiceFrame.IMAP_SPACECRAFT, SpiceFrame.ECLIPJ2000, - ) + ), + degrees=False, ) # Calculate circular statistics for longitude (wraps around) lon_mean = circmean(spin_axis_all_times[..., 1], low=-np.pi, high=np.pi) lon_std = circstd(spin_axis_all_times[..., 1], low=-np.pi, high=np.pi) lat_mean = circmean(spin_axis_all_times[..., 2], low=-np.pi, high=np.pi) lat_std = circstd(spin_axis_all_times[..., 2], low=-np.pi, high=np.pi) - self.spin_axis_orientation_average = np.array([lon_mean, lat_mean]) - self.spin_axis_orientation_std_dev = np.array([lon_std, lat_std]) + # Convert circular statistics to degrees and store + self.spin_axis_orientation_average = np.degrees(np.array([lon_mean, lat_mean])) + self.spin_axis_orientation_std_dev = np.degrees(np.array([lon_std, lat_std])) # Calculate spacecraft location and velocity # ------------------------------------------ @@ -979,6 +1003,70 @@ def deserialize_flags(raw: int) -> np.ndarray[int]: return flags + def compute_flags(self, pipeline_settings: PipelineSettings) -> np.ndarray: + """ + Compute the 17 bad-time flags for this histogram. + + Parameters + ---------- + pipeline_settings : PipelineSettings + Pipeline settings containing processing thresholds. + + Returns + ------- + flags : numpy.ndarray + Array of shape (FLAG_LENGTH,) with dtype uint8. 1 = good, 0 = bad. + """ + # Section 12.3.1 of the Algorithm Document: onboard generated bad-time flags. + # Flags are "stored in a 16-bit integer field. + onboard_flags: np.ndarray = ( + 1 - self.deserialize_flags(int(self.flags_set_onboard)) + ).astype(np.uint8) + + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 1. + # Informs if the histogram was generated on-board or on the ground. + # Flag 1 = onboard. + is_generated_on_ground = np.uint8(1 - int(self.is_generated_on_ground)) + + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 2. + # Checks if total count in a given histogram is far from the daily average. + # Placeholder until daily histogram is available in glows_l1b.py. + # TODO: this equation needs to be clarified. + is_beyond_daily_statistical_error = np.uint8(1) + + # Section 12.3.2 of the Algorithm Document: ground processing flags: flag 3-7. + # (1=good, 0=bad). + temp_threshold = pipeline_settings.get_threshold( + "std_dev_threshold__celsius_deg" + ) + hv_threshold = pipeline_settings.get_threshold("std_dev_threshold__volt") + spin_std_threshold = pipeline_settings.get_threshold("std_dev_threshold__sec") + pulse_threshold = pipeline_settings.get_threshold("std_dev_threshold__usec") + + is_temp_ok = np.uint8(self.filter_temperature_std_dev <= temp_threshold) + is_hv_ok = np.uint8(self.hv_voltage_std_dev <= hv_threshold) + is_spin_std_ok = np.uint8(self.spin_period_std_dev <= spin_std_threshold) + is_pulse_ok = np.uint8(self.pulse_length_std_dev <= pulse_threshold) + + # TODO: listed as TBC in Algorithm Document. + # Placeholder for now. + is_beyond_background_error = np.uint8(1) + + ground_flags = np.array( + [ + is_generated_on_ground, + is_beyond_daily_statistical_error, + is_temp_ok, + is_hv_ok, + is_spin_std_ok, + is_pulse_ok, + is_beyond_background_error, + ], + dtype=np.uint8, + ) + + return np.concatenate([onboard_flags, ground_flags]) + def flag_uv_and_excluded(self, exclusions: AncillaryExclusions) -> tuple: """ Create boolean mask where True means bin is within radius of UV source. @@ -1131,7 +1219,7 @@ def _compute_histogram_flag_array( np.ndarray Array of shape (4, 3600) with bad-angle flags for each bin. """ - histogram_flags = np.full( + histogram_flags: np.ndarray = np.full( (4, len(self.histogram)), GLOWSL1bFlags.NONE.value, dtype=np.uint8, diff --git a/imap_processing/glows/l2/glows_l2.py b/imap_processing/glows/l2/glows_l2.py index 041bf15825..8e457e815b 100644 --- a/imap_processing/glows/l2/glows_l2.py +++ b/imap_processing/glows/l2/glows_l2.py @@ -12,7 +12,13 @@ PipelineSettings, ) from imap_processing.glows.l2.glows_l2_data import HistogramL2 -from imap_processing.spice.time import et_to_datetime64, ttj2000ns_to_et +from imap_processing.glows.utils.constants import GlowsConstants +from imap_processing.spice.time import ( + et_to_datetime64, + met_to_utc, + ttj2000ns_to_et, + ttj2000ns_to_met, +) logger = logging.getLogger(__name__) @@ -20,9 +26,10 @@ def glows_l2( input_dataset: xr.Dataset, pipeline_settings_dataset: xr.Dataset, + calibration_dataset: xr.Dataset, ) -> list[xr.Dataset]: """ - Will process GLoWS L2 data from L1 data. + Will process GLOWS L2 data from L1 data. Parameters ---------- @@ -31,6 +38,9 @@ def glows_l2( pipeline_settings_dataset : xarray.Dataset Dataset containing pipeline settings from GlowsAncillaryCombiner. + calibration_dataset : xarray.Dataset + Dataset containing calibration data from + GlowsAncillaryCombiner. Returns ------- @@ -49,10 +59,17 @@ def glows_l2( pipeline_settings_dataset.sel(epoch=day, method="nearest") ) - l2 = HistogramL2(input_dataset, pipeline_settings) + l2 = HistogramL2(input_dataset, pipeline_settings, calibration_dataset) if l2.number_of_good_l1b_inputs == 0: logger.warning("No good data found in L1B dataset. Returning empty list.") return [] + elif ( + np.all(l2.daily_lightcurve.photon_flux == 0) + and np.all(l2.daily_lightcurve.flux_uncertainties == 0) + and np.all(l2.daily_lightcurve.exposure_times == 0) + ): + logger.warning("All flux and exposure times are zero. Returning empty list.") + return [] else: return [create_l2_dataset(l2, cdf_attrs)] @@ -91,7 +108,7 @@ def create_l2_dataset( ) bins = xr.DataArray( - np.arange(histogram_l2.daily_lightcurve.number_of_bins, dtype=np.uint32), + np.arange(GlowsConstants.STANDARD_BIN_COUNT, dtype=np.uint32), name="bins", dims=["bins"], attrs=attrs.get_variable_attributes("bins_dim", check_schema=False), @@ -145,6 +162,8 @@ def create_l2_dataset( "spin_axis_orientation_std_dev", ] + utc_time_variables = ["start_time", "end_time"] + for key, value in dataclasses.asdict(histogram_l2).items(): if key in ecliptic_variables: output[key] = xr.DataArray( @@ -164,7 +183,12 @@ def create_l2_dataset( dims=["epoch", "flags"], attrs=attrs.get_variable_attributes(key), ) - + elif key in utc_time_variables: + # Convert time to UTC + utc_string = [met_to_utc(ttj2000ns_to_met(value))] + output[key] = xr.DataArray( + utc_string, dims=["epoch"], attrs=attrs.get_variable_attributes(key) + ) elif key != "daily_lightcurve": val = value if type(value) is not np.ndarray: @@ -175,19 +199,27 @@ def create_l2_dataset( attrs=attrs.get_variable_attributes(key), ) + n_bins = histogram_l2.daily_lightcurve.number_of_bins for key, value in dataclasses.asdict(histogram_l2.daily_lightcurve).items(): if key == "number_of_bins": - # number_of_bins does not have n_bins dimensions. + # number_of_bins does not have a bins dimension. output[key] = xr.DataArray( np.array([value]), dims=["epoch"], attrs=attrs.get_variable_attributes(key), ) else: + # Bin arrays are chopped to number_of_bins in DailyLightcurve to + # avoid operating on FILLVAL data. Re-expand to STANDARD_BIN_COUNT + # here, filling unused bins with the variable's CDF FILLVAL. + var_attrs = attrs.get_variable_attributes(key) + fillval = var_attrs["FILLVAL"] + padded = np.full(GlowsConstants.STANDARD_BIN_COUNT, fillval) + padded[:n_bins] = value output[key] = xr.DataArray( - np.array([value]), + np.array([padded]), dims=["epoch", "bins"], - attrs=attrs.get_variable_attributes(key), + attrs=var_attrs, ) return output diff --git a/imap_processing/glows/l2/glows_l2_data.py b/imap_processing/glows/l2/glows_l2_data.py index 97ce1355ca..e90e220483 100644 --- a/imap_processing/glows/l2/glows_l2_data.py +++ b/imap_processing/glows/l2/glows_l2_data.py @@ -8,6 +8,18 @@ from imap_processing.glows import FLAG_LENGTH from imap_processing.glows.l1b.glows_l1b_data import PipelineSettings +from imap_processing.glows.utils.constants import GlowsConstants +from imap_processing.spice.geometry import ( + SpiceFrame, + frame_transform_az_el, + get_instrument_mounting_az_el, +) +from imap_processing.spice.time import ( + et_to_datetime64, + met_to_sclkticks, + sct_to_et, + ttj2000ns_to_et, +) @dataclass @@ -39,6 +51,8 @@ class DailyLightcurve: number of bins in lightcurve l1b_data : xarray.Dataset L1B data filtered by good times, good angles, and good bins. + calibration_factor : float | None + Rayleigh calibration factor used for flux calculations. """ # All variables should have n_bin elements @@ -47,15 +61,20 @@ class DailyLightcurve: raw_histograms: np.ndarray = field(init=False) exposure_times: np.ndarray = field(init=False) flux_uncertainties: np.ndarray = field(init=False) - # TODO: flag array histogram_flag_array: np.ndarray = field(init=False) - # TODO: ecliptic coordinates ecliptic_lon: np.ndarray = field(init=False) ecliptic_lat: np.ndarray = field(init=False) number_of_bins: int = field(init=False) l1b_data: InitVar[xr.Dataset] - - def __post_init__(self, l1b_data: xr.Dataset) -> None: + position_angle: InitVar[float] + calibration_factor: InitVar[float | None] + + def __post_init__( + self, + l1b_data: xr.Dataset, + position_angle: float, + calibration_factor: float | None, + ) -> None: """ Compute all the daily lightcurve variables from L1B data. @@ -64,10 +83,31 @@ def __post_init__(self, l1b_data: xr.Dataset) -> None: l1b_data : xarray.Dataset L1B data filtered by good times, good angles, and good bins for one observation day. + position_angle : float + The offset angle of the GLOWS instrument from the north spin point - this + is used in spin angle calculations. + calibration_factor : float + Calibration factor used for flux calculations, in units of counts per second + per Rayleigh. This is used to convert from raw histograms and exposure times + to physical photon flux units. """ - self.raw_histograms = self.calculate_histogram_sums(l1b_data["histogram"].data) + # number_of_bins_per_histogram is the count of valid (non-FILLVAL) bins. + # Histogram arrays from L1B are always GlowsConstants.STANDARD_BIN_COUNT + # (3600) long, with unused bins filled with GlowsConstants.HISTOGRAM_FILLVAL. + # All bin-dimensioned arrays here are chopped to number_of_bins so that + # computations only operate on valid data. glows_l2.py is responsible for + # re-expanding these arrays back to STANDARD_BIN_COUNT, filling unused bins + # with the appropriate CDF FILLVAL before writing to output. + + self.number_of_bins = ( + l1b_data["number_of_bins_per_histogram"].data[0] + if len(l1b_data["number_of_bins_per_histogram"].data) != 0 + else 0 + ) - self.number_of_bins = l1b_data["histogram"].shape[1] + self.raw_histograms = self.calculate_histogram_sums(l1b_data["histogram"].data)[ + : self.number_of_bins + ] exposure_per_epoch = ( l1b_data["spin_period_average"].data @@ -79,16 +119,22 @@ def __post_init__(self, l1b_data: xr.Dataset) -> None: self.exposure_times = np.full(self.number_of_bins, np.sum(exposure_per_epoch)) raw_uncertainties = np.sqrt(self.raw_histograms) - self.photon_flux = np.zeros(len(self.raw_histograms)) - self.flux_uncertainties = np.zeros(len(self.raw_histograms)) - - # TODO: Only where exposure counts != 0 - if len(self.exposure_times) != 0: - self.photon_flux = self.raw_histograms / self.exposure_times - self.flux_uncertainties = raw_uncertainties / self.exposure_times - - # TODO: Average this, or should they all be the same? - self.spin_angle = np.average(l1b_data["imap_spin_angle_bin_cntr"].data, axis=0) + self.photon_flux = np.zeros(self.number_of_bins) + self.flux_uncertainties = np.zeros(self.number_of_bins) + + if ( + self.number_of_bins > 0 + and self.exposure_times[0] > 0 + and calibration_factor + ): + self.photon_flux = ( + self.raw_histograms / self.exposure_times + ) / calibration_factor + self.flux_uncertainties = ( + raw_uncertainties / self.exposure_times + ) / calibration_factor + + self.spin_angle = np.zeros(0) # Apply 'OR' operation to histogram_flag_array across all # good-time L1B blocks per bin. @@ -104,8 +150,38 @@ def __post_init__(self, l1b_data: xr.Dataset) -> None: ) else: self.histogram_flag_array = np.zeros(self.number_of_bins, dtype=np.uint8) - self.ecliptic_lon = np.zeros(self.number_of_bins) - self.ecliptic_lat = np.zeros(self.number_of_bins) + + if self.number_of_bins: + # imap_spin_angle_bin_cntr is the raw IMAP spin angle ψ (0 - 360°, + # bin midpoints). + spin_angle_bin_cntr = l1b_data["imap_spin_angle_bin_cntr"].data[0][ + : self.number_of_bins + ] + # Convert ψ → ψPA (Eq. 29): position angle measured from the + # northernmost point of the scanning circle. + self.spin_angle = (spin_angle_bin_cntr - position_angle + 360.0) % 360.0 + + # Roll all bin arrays so bin 0 corresponds to the northernmost + # point (minimum ψPA). + roll = -np.argmin(self.spin_angle) + self.spin_angle = np.roll(self.spin_angle, roll) + self.raw_histograms = np.roll(self.raw_histograms, roll) + self.photon_flux = np.roll(self.photon_flux, roll) + self.exposure_times = np.roll(self.exposure_times, roll) + self.flux_uncertainties = np.roll(self.flux_uncertainties, roll) + self.histogram_flag_array = np.roll(self.histogram_flag_array, roll) + + # Get the midpoint start time covered by repointing kernels + # needed to compute ecliptic coordinates + mid_idx = len(l1b_data["imap_start_time"]) // 2 + pointing_midpoint_time_et = sct_to_et( + met_to_sclkticks(l1b_data["imap_start_time"][mid_idx].data) + ) + self.ecliptic_lon, self.ecliptic_lat = ( + self.compute_ecliptic_coords_of_bin_centers( + pointing_midpoint_time_et, self.spin_angle + ) + ) @staticmethod def calculate_histogram_sums(histograms: NDArray) -> NDArray: @@ -123,9 +199,57 @@ def calculate_histogram_sums(histograms: NDArray) -> NDArray: Sum of valid histograms across all timestamps. """ histograms = histograms.copy() - histograms[histograms == -1] = 0 + # Zero out areas where HISTOGRAM_FILLVAL (i.e. unused bins) + histograms[histograms == GlowsConstants.HISTOGRAM_FILLVAL] = 0 return np.sum(histograms, axis=0, dtype=np.int64) + @staticmethod + def compute_ecliptic_coords_of_bin_centers( + data_time_et: float, spin_angle_bin_centers: NDArray + ) -> tuple[np.ndarray, np.ndarray]: + """ + Compute the ecliptic coordinates of the histogram bin centers. + + This method transforms the instrument pointing direction for each bin + center from the IMAP Pointing frame (IMAP_DPS) to the ECLIPJ2000 frame. + + Parameters + ---------- + data_time_et : float + Ephemeris time corresponding to the midpoint of the histogram accumulation. + + spin_angle_bin_centers : numpy.ndarray + Spin angle bin centers for the histogram bins, measured in the IMAP frame, + with shape (n_bins,), and already corrected for the northernmost point of + the scanning circle. + + Returns + ------- + tuple[numpy.ndarray, numpy.ndarray] + Longitude and latitudes in the ECLIPJ2000 frame representing the pointing + direction of each histogram bin center, with shape (n_bins,). + """ + # In the IMAP frame, the azimuth corresponds to the spin angle bin centers + azimuth = spin_angle_bin_centers + + # Get elevation from instrument pointing direction in the DPS frame. + az_el_instrument_mounting = get_instrument_mounting_az_el(SpiceFrame.IMAP_GLOWS) + elevation = az_el_instrument_mounting[1] + + # Create array of azimuth, elevation coordinates in the DPS frame (n_bins, 2) + az_el = np.stack((azimuth, np.full_like(azimuth, elevation)), axis=-1) + + # Transform coordinates to ECLIPJ2000 frame using SPICE transformations. + ecliptic_coords = frame_transform_az_el( + data_time_et, + az_el, + SpiceFrame.IMAP_DPS, + SpiceFrame.ECLIPJ2000, + ) + + # Return ecliptic longitudes and latitudes as separate arrays. + return ecliptic_coords[:, 0], ecliptic_coords[:, 1] + @dataclass class HistogramL2: @@ -140,6 +264,8 @@ class HistogramL2: GLOWS histogram L1B dataset, as produced by glows_l1b.py. pipeline_settings : PipelineSettings Pipeline settings object read from ancillary file. + calibration_dataset : xr.Dataset + The cps-to-Rayleigh calibration dataset needed for flux calculations. Attributes ---------- @@ -213,8 +339,8 @@ class HistogramL2: pulse_length_std_dev: np.ndarray[np.double] spin_period_ground_average: np.ndarray[np.double] spin_period_ground_std_dev: np.ndarray[np.double] - position_angle_offset_average: np.ndarray[np.double] - position_angle_offset_std_dev: np.ndarray[np.double] + position_angle_offset_average: np.double + position_angle_offset_std_dev: np.double spin_axis_orientation_std_dev: np.ndarray[np.double] spacecraft_location_average: np.ndarray[np.double] spacecraft_location_std_dev: np.ndarray[np.double] @@ -223,7 +349,12 @@ class HistogramL2: spin_axis_orientation_average: np.ndarray[np.double] bad_time_flag_occurrences: np.ndarray - def __init__(self, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings): + def __init__( + self, + l1b_dataset: xr.Dataset, + pipeline_settings: PipelineSettings, + calibration_dataset: xr.Dataset, + ) -> None: """ Given an L1B dataset, process data into an output HistogramL2 object. @@ -233,20 +364,31 @@ def __init__(self, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings) GLOWS histogram L1B dataset, as produced by glows_l1b.py. pipeline_settings : PipelineSettings Pipeline settings object read from ancillary file. + calibration_dataset : xr.Dataset + cps-to-Rayleigh calibration dataset used for flux calculations. + coords: start_time_utc, data_vars: cps_per_r """ active_flags = np.array(pipeline_settings.active_bad_time_flags, dtype=float) + # Apply sunrise/sunset offsets to extend the night region around + # is_night transitions before selecting good blocks. + flags = self.apply_is_night_offsets( + l1b_dataset["flags"].data, + is_night_idx=GlowsConstants.IS_NIGHT_FLAG_IDX, + sunrise_offset=int(pipeline_settings.sunrise_offset), + sunset_offset=int(pipeline_settings.sunset_offset), + ) + flags_da = xr.DataArray(flags, dims=l1b_dataset["flags"].dims) + # Select the good blocks (i.e. epoch values) according to the flags. Drop any # bad blocks before processing. good_data = l1b_dataset.isel( - epoch=self.return_good_times(l1b_dataset["flags"], active_flags) + epoch=self.return_good_times(flags_da, active_flags) ) # TODO: bad angle filter # TODO: filter bad bins out. Needs to happen here while everything is still # per-timestamp. - self.daily_lightcurve = DailyLightcurve(good_data) - self.total_l1b_inputs = len(l1b_dataset["epoch"]) self.number_of_good_l1b_inputs = len(good_data["epoch"]) self.identifier = -1 # TODO: retrieve from spin table @@ -299,16 +441,12 @@ def __init__(self, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings) self.spin_period_ground_std_dev = ( good_data["spin_period_ground_average"].std(dim="epoch", keepdims=True).data ) - self.position_angle_offset_average = ( - good_data["position_angle_offset_average"] - .mean(dim="epoch", keepdims=True) - .data - ) - self.position_angle_offset_std_dev = ( - good_data["position_angle_offset_average"] - .std(dim="epoch", keepdims=True) - .data - ) + + position_angle = self.compute_position_angle() + self.position_angle_offset_average: np.double = np.double(position_angle) + + # Always zero - per algorithm doc 10.6 + self.position_angle_offset_std_dev = np.double(0.0) self.spacecraft_location_average = ( good_data["spacecraft_location_average"] .mean(dim="epoch") @@ -340,6 +478,18 @@ def __init__(self, l1b_dataset: xr.Dataset, pipeline_settings: PipelineSettings) .data[np.newaxis, :] ) + # Select calibration factor corresponding to the mid-epoch in the L1B data. + if len(good_data["epoch"].data) != 0: + calibration_factor = self.get_calibration_factor( + good_data["epoch"].data, calibration_dataset + ) + else: + calibration_factor = None # No good data available. Still proceed + + self.daily_lightcurve = DailyLightcurve( + good_data, position_angle, calibration_factor + ) + def filter_bad_bins(self, histograms: NDArray, bin_exclusions: NDArray) -> NDArray: """ Filter out bad bins from the histogram. @@ -392,3 +542,173 @@ def return_good_times(flags: xr.DataArray, active_flags: NDArray) -> NDArray: # where all the active indices == 1. good_times = np.where(np.all(flags[:, active_flags == 1] == 1, axis=1))[0] return good_times + + @staticmethod + def apply_is_night_offsets( + flags: np.ndarray, + is_night_idx: int, + sunrise_offset: int, + sunset_offset: int, + ) -> np.ndarray: + """ + Apply sunrise/sunset offsets to is_night transitions. + + Per algorithm doc v4.4.7, Sec. 3.9.1, item 2 (raw is_night: 1=night, 0=day): + + sunset_offset applies at both transitions: + >0: night shortens by N at each end (first N night epochs at sunset become + day; last N night epochs before sunrise become day) + <0: night extends by |N| at each end + + sunrise_offset is an additional adjustment at sunrise (is_night 1->0) only: + >0: night extends N histograms past the raw sunrise transition + <0: night shortens by |N| before the raw sunrise transition + + In the processed flags array: 0 = bad (night), 1 = good (day). + + Parameters + ---------- + flags : numpy.ndarray + Flags array with shape (n_epochs, FLAG_LENGTH), 0=bad, 1=good. + is_night_idx : int + Column index of the is_night flag in the flags array. + sunrise_offset : int + Additional histogram shift at the sunrise (is_night 1->0) transition. + sunset_offset : int + Histogram shift applied at both the sunset and sunrise transitions. + + Returns + ------- + numpy.ndarray + Returns the original flags array if no offsets are applied, + otherwise returns a modified copy. + + Notes + ----- + Algorithm doc v4.4.7, Sec. 3.9.1, item 2 + is_night: 1 = daytime (good), 0 = night (bad) + """ + # If sunrise_offset=0 and sunset_offset=0 then no corrections are needed + # relative to is_night transition set onboard. + if sunrise_offset == 0 and sunset_offset == 0: + return flags + + flags_with_offsets = flags.copy() + + is_night_col = flags[:, is_night_idx] + n = flags.shape[0] + diff = np.diff(is_night_col.astype(int)) + sunset_index = np.where(diff == -1)[0] + sunrise_index = np.where(diff == 1)[0] + + if sunrise_offset > 0: + # Night (flag = 0) extends by sunrise_offset relative + # to is_night 0 -> 1 transition. + for i in sunrise_index: + flags_with_offsets[ + i + 1 : min(n, i + 1 + sunrise_offset), is_night_idx + ] = 0 + + elif sunrise_offset < 0: + # Night (flag = 0) shortens by sunrise_offset relative + # to is_night 0 -> 1 transition. + for i in sunrise_index: + flags_with_offsets[ + max(0, i + 1 + sunrise_offset) : i + 1, is_night_idx + ] = 1 + + if sunset_offset > 0: + # Night (flag = 0) shortens by sunset_offset relative + # to is_night 1 -> 0 transition. + for i in sunset_index: + flags_with_offsets[ + i + 1 : min(n, i + 1 + sunset_offset), is_night_idx + ] = 1 + + elif sunset_offset < 0: + # Night (flag = 0) extends by sunset_offset relative + # to is_night 1 -> 0 transition. + for i in sunset_index: + flags_with_offsets[ + max(0, i + 1 + sunset_offset) : i + 1, is_night_idx + ] = 0 + + return flags_with_offsets + + def compute_position_angle(self) -> float: + """ + Compute the position angle based on the instrument mounting. + + This number is not expected to change significantly. It is the same for all L1B + blocks (epoch values). + + Returns + ------- + float + The GLOWS mounting position angle. + """ + # Calculation described in algorithm doc 10.6 (Eq. 30): + # psi_G_eff = 360 - psi_GLOWS + # where psi_GLOWS is the azimuth of the GLOWS boresight in the + # IMAP spacecraft frame, measured from the spacecraft x-axis. + # This angle does not change with time, as it is in the spinning IMAP frame. + # It basically defines the angle between x=0 in the IMAP frame and x=0 in the + # GLOWS instrument frame, and is defined by the physical mounting location of + # the instrument. + # delta_psi_G_eff is assumed to be 0 per instrument team decision (aka this + # doesn't move from the SPICE determined mounting angle. + glows_mounting_azimuth, _ = get_instrument_mounting_az_el(SpiceFrame.IMAP_GLOWS) + return (360.0 - glows_mounting_azimuth) % 360.0 + + @staticmethod + def get_calibration_factor( + epoch_values: np.ndarray, calibration_dataset: xr.Dataset + ) -> float: + """ + Select calibration factor for an observational day. + + The calibration factor is needed to compute flux in Rayleigh units. + There is a strong assumption that the calibration is constant for + a given observational day. + + Parameters + ---------- + epoch_values : np.ndarray + Array of epoch values from the L1B dataset, in TT J2000 nanoseconds. + calibration_dataset : xr.Dataset + Dataset containing calibration data with the following structure: + Coords: epoch (datetime64[s]) + Dims: epoch, cps_per_r_dim_0, start_time_utc_dim_0 + Data vars: "cps_per_r" and "start_time_utc" are 2D (epoch, *_dim_0) + + Note: epoch and start_time_utc do not necessarily match in size or + values + - epoch contains timestamps in the calibration data up to a defined + day buffer and start_time_utc are the timestamps for all the + calibration data entries. + - epoch is used for selecting the time block, and start_time_utc is + used for selecting the calibration value within that block. + + Returns + ------- + float + The calibration factor needed to compute flux in Rayleigh units. + """ + # Use the midpoint epoch for the observation day + mid_idx = len(epoch_values) // 2 + mid_epoch_utc = et_to_datetime64(ttj2000ns_to_et(epoch_values[mid_idx].item())) + + # Select calibration data before or equal to mid_epoch_utc using "pad" to find + # the nearest preceding entry in the calibration dataset's epoch + # coordinate which is in UTC datetime64 format. + cal_at_epoch = calibration_dataset.sel(epoch=mid_epoch_utc, method="pad") + + # start_time_utc is a data variable with its own index dimension. + # Use searchsorted to find the last entry whose start_time_utc <= mid_epoch_utc. + start_times = np.array( + cal_at_epoch["start_time_utc"].values, dtype="datetime64[ns]" + ) + nearest_idx = np.searchsorted(start_times, mid_epoch_utc, side="right") - 1 + + # Select the calibration value at the nearest index. + return float(cal_at_epoch["cps_per_r"].isel(cps_per_r_dim_0=nearest_idx)) diff --git a/imap_processing/glows/utils/constants.py b/imap_processing/glows/utils/constants.py index 13821f70ae..08714e3637 100644 --- a/imap_processing/glows/utils/constants.py +++ b/imap_processing/glows/utils/constants.py @@ -65,12 +65,15 @@ class GlowsConstants: Fill value for histogram bins (65535 for uint16) STANDARD_BIN_COUNT: int Standard number of bins per histogram (3600) + IS_NIGHT_FLAG_IDX: int + Index of the is_night flag in the bad-time flags array (0-indexed) """ SUBSECOND_LIMIT: int = 2_000_000 SCAN_CIRCLE_ANGULAR_RADIUS: float = 75.0 HISTOGRAM_FILLVAL: int = 65535 STANDARD_BIN_COUNT: int = 3600 + IS_NIGHT_FLAG_IDX: int = 6 @dataclass diff --git a/imap_processing/hi/hi_goodtimes.py b/imap_processing/hi/hi_goodtimes.py index 476b3ba96a..342e7ee9cb 100644 --- a/imap_processing/hi/hi_goodtimes.py +++ b/imap_processing/hi/hi_goodtimes.py @@ -1,7 +1,6 @@ """IMAP-HI Goodtimes processing module.""" import logging -import re from enum import IntEnum from pathlib import Path @@ -15,6 +14,7 @@ CalibrationProductConfig, CoincidenceBitmap, HiConstants, + compute_qualified_event_mask, parse_sensor_number, ) from imap_processing.quality_flags import ImapHiL1bDeFlags @@ -24,29 +24,37 @@ logger = logging.getLogger(__name__) # Structured dtype for good time intervals -INTERVAL_DTYPE = np.dtype( +INTERVAL_DTYPE: np.dtype = np.dtype( [ ("met_start", np.float64), ("met_end", np.float64), ("spin_bin_low", np.uint8), ("spin_bin_high", np.uint8), - ("n_good_bins", np.uint8), - ("esa_step", np.uint8), + ("n_bins", np.uint8), + ("esa_step_mask", np.uint16), # Bitmask for ESA steps 1-10 (bit i = step i+1) + ("cull_value", np.uint8), ] ) class CullCode(IntEnum): - """Cull reason codes for good/bad time classification.""" + """Cull reason codes for good/bad time classification (bit flags).""" GOOD = 0 - LOOSE = 1 + INCOMPLETE_SPIN = 1 << 0 # 1 + DRF = 1 << 1 # 2 + BAD_TDC_CAL = 1 << 2 # 4 + OVERFLOW = 1 << 3 # 8 + STAT_FILTER_0 = 1 << 4 # 16 + STAT_FILTER_1 = 1 << 5 # 32 + STAT_FILTER_2 = 1 << 6 # 64 def hi_goodtimes( - l1b_de_datasets: list[xr.Dataset], current_repointing: str, + l1b_de_datasets: list[xr.Dataset], l1b_hk: xr.Dataset, + l1a_diagfee: xr.Dataset, cal_product_config_path: Path, ) -> list[xr.Dataset]: """ @@ -57,23 +65,26 @@ def hi_goodtimes( 1. mark_incomplete_spin_sets - Remove incomplete 8-spin histogram periods 2. mark_drf_times - Remove times during spacecraft drift restabilization - 3. mark_overflow_packets - Remove times when DE packets overflow - 4. mark_statistical_filter_0 - Detect drastic penetrating background changes - 5. mark_statistical_filter_1 - Detect isotropic count rate increases - 6. mark_statistical_filter_2 - Detect short-lived event pulses + 3. mark_bad_tdc_cal - Remove times with failed TDC calibration + 4. mark_overflow_packets - Remove times when DE packets overflow + 5. mark_statistical_filter_0 - Detect drastic penetrating background changes + 6. mark_statistical_filter_1 - Detect isotropic count rate increases + 7. mark_statistical_filter_2 - Detect short-lived event pulses Parameters ---------- + current_repointing : str + Repointing identifier for the current pointing (e.g., "repoint00001"). + Used to identify which dataset in l1b_de_datasets is the current one. l1b_de_datasets : list[xr.Dataset] L1B DE datasets for surrounding pointings. Typically includes current plus 3 preceding and 3 following pointings (7 total). Statistical filters 0 and 1 use all datasets; other filters use only the current pointing. - current_repointing : str - Repointing identifier for the current pointing (e.g., "repoint00001"). - Used to identify which dataset in l1b_de_datasets is the current one. l1b_hk : xr.Dataset L1B housekeeping dataset containing DRF status. + l1a_diagfee : xr.Dataset + L1A DIAG_FEE dataset containing TDC calibration status. cal_product_config_path : Path Path to calibration product configuration CSV file. @@ -130,6 +141,7 @@ def hi_goodtimes( l1b_de_datasets, current_index, l1b_hk, + l1a_diagfee, cal_product_config_path, ) else: @@ -139,7 +151,7 @@ def hi_goodtimes( f"expected 7 files, got {len(l1b_de_datasets)}. " "Marking all times as bad." ) - goodtimes_ds["cull_flags"][:, :] = CullCode.LOOSE + goodtimes_ds["cull_flags"][:, :] = CullCode.INCOMPLETE_SPIN # Log final statistics stats = goodtimes_ds.goodtimes.get_cull_statistics() @@ -199,12 +211,13 @@ def _apply_goodtimes_filters( l1b_de_datasets: list[xr.Dataset], current_index: int, l1b_hk: xr.Dataset, + l1a_diagfee: xr.Dataset, cal_product_config_path: Path, ) -> None: """ Apply all goodtimes culling filters to the dataset. - Modifies goodtimes_ds in place by applying filters 1-6. + Modifies goodtimes_ds in place by applying filters 1-7. Parameters ---------- @@ -216,6 +229,8 @@ def _apply_goodtimes_filters( Index of the current pointing in l1b_de_datasets. l1b_hk : xr.Dataset L1B housekeeping dataset. + l1a_diagfee : xr.Dataset + L1A DIAG_FEE dataset containing TDC calibration status. cal_product_config_path : Path Path to calibration product configuration CSV file. """ @@ -229,11 +244,28 @@ def _apply_goodtimes_filters( stats = goodtimes_ds.goodtimes.get_cull_statistics() logger.info(f"Initial good bins: {stats['good_bins']}/{stats['total_bins']}") - # Build set of qualified coincidence types from calibration product config - qualified_coincidence_types: set[int] = set() - for coin_types in cal_product_config["coincidence_type_values"]: - qualified_coincidence_types.update(coin_types) - logger.info(f"Qualified coincidence types: {qualified_coincidence_types}") + # Pre-compute qualified event masks for each dataset + # These masks check BOTH coincidence_type AND TOF windows + for l1b_de in l1b_de_datasets: + ccsds_index = l1b_de["ccsds_index"].values + + # Handle invalid events (FILLVAL trigger_id) to avoid IndexError + # For pointings with no valid events, trigger_id will be at FILLVAL + trigger_id_fillval = l1b_de["trigger_id"].attrs.get("FILLVAL", 65535) + valid_events = l1b_de["trigger_id"].values != trigger_id_fillval + + # Initialize with -1 (won't match any config row since ESA energy steps > 0) + esa_energy_steps: np.ndarray = np.full(len(ccsds_index), -1, dtype=np.int32) + if np.any(valid_events): + esa_energy_steps[valid_events] = l1b_de["esa_energy_step"].values[ + ccsds_index[valid_events] + ] + + l1b_de["qualified_mask"] = xr.DataArray( + compute_qualified_event_mask(l1b_de, cal_product_config, esa_energy_steps), + dims=["event_met"], + ) + logger.info("Pre-computed qualified event masks for all datasets") # === Apply culling filters === @@ -245,29 +277,31 @@ def _apply_goodtimes_filters( logger.info("Applying filter: mark_drf_times") mark_drf_times(goodtimes_ds, l1b_hk) - # 3. Mark overflow packets + # 3. Mark bad TDC calibration times + logger.info("Applying filter: mark_bad_tdc_cal") + mark_bad_tdc_cal(goodtimes_ds, l1a_diagfee) + + # 4. Mark overflow packets logger.info("Applying filter: mark_overflow_packets") mark_overflow_packets(goodtimes_ds, current_l1b_de, cal_product_config) - # 4. Statistical Filter 0 - drastic background changes + # 5. Statistical Filter 0 - drastic background changes logger.info("Applying filter: mark_statistical_filter_0") mark_statistical_filter_0(goodtimes_ds, l1b_de_datasets, current_index) - # 5. Statistical Filter 1 - isotropic count rate increases + # 6. Statistical Filter 1 - isotropic count rate increases logger.info("Applying filter: mark_statistical_filter_1") mark_statistical_filter_1( goodtimes_ds, l1b_de_datasets, current_index, - qualified_coincidence_types, ) - # 6. Statistical Filter 2 - short-lived event pulses + # 7. Statistical Filter 2 - short-lived event pulses logger.info("Applying filter: mark_statistical_filter_2") mark_statistical_filter_2( goodtimes_ds, current_l1b_de, - qualified_coincidence_types, ) @@ -329,15 +363,10 @@ def create_goodtimes_dataset(l1b_de: xr.Dataset) -> xr.Dataset: # Create attributes sensor_number = parse_sensor_number(l1b_de.attrs["Logical_source"]) - match = re.match(r"repoint(?P\d{5})", l1b_de.attrs["Repointing"]) - if not match: - raise ValueError( - f"Unable to parse pointing number from l1b_de Repointing " - f"attribute: {l1b_de.attrs['Repointing']}" - ) + repointing = l1b_de.attrs.get("Repointing", "repoint-9999") attrs = { - "sensor": f"{sensor_number}sensor", - "pointing": int(match["pointing_num"]), + "Sensor": f"{sensor_number}sensor", + "Repointing": repointing, } return xr.Dataset(data_vars, coords, attrs) @@ -510,35 +539,38 @@ def mark_bad_times( # Subtract 1 to get the largest value <= met_val met_indices = np.searchsorted(met_values, met_array, side="right") - 1 - # Set cull_flags for all indices + # Set cull_flags for all indices using bitwise OR to combine flags n_times = len(met_indices) n_bins = len(bins_array) logger.debug( f"Flagging {n_times} MET time(s) x {n_bins} spin bin(s) with " f"cull code {cull}" ) - self._obj["cull_flags"].values[np.ix_(met_indices, bins_array)] = cull + self._obj["cull_flags"].values[np.ix_(met_indices, bins_array)] |= np.uint8( + cull + ) def get_good_intervals(self) -> np.ndarray: """ - Extract good time intervals for each MET timestamp. - - Creates an interval for each MET time that has good bins. Since ESA step - changes at each MET, each MET gets its own interval(s). + Extract good time intervals grouped by ESA sweep cull patterns. - If good bins wrap around the 89->0 boundary (e.g., bins 88,89,0,1), multiple - intervals are created for the same MET time, one for each contiguous set. + Groups consecutive ESA sweeps with identical cull patterns. For each group: + 1. Writes one interval for fully-good ESA steps (all 90 bins good) spanning + bins 0-89, with cull_value indicating the cull code from any bad ESAs. + 2. Writes additional intervals for each good bin region of partially-good + ESA steps, with cull_value indicating the cull code that removed bad bins. Returns ------- numpy.ndarray Structured array with dtype INTERVAL_DTYPE containing: - - met_start: MET timestamp of interval - - met_end: MET timestamp of interval (same as met_start) - - spin_bin_low: Lowest good spin bin in interval - - spin_bin_high: Highest good spin bin in interval - - n_good_bins: Number of good bins - - esa_step: ESA step for this MET + - met_start: First MET timestamp of interval + - met_end: Start of next interval (or last MET for final interval) + - spin_bin_low: Lowest spin bin in this contiguous region + - spin_bin_high: Highest spin bin in this contiguous region + - n_bins: Number of bins in this region + - esa_step_mask: Bitmask of good ESA steps (1-10) for this interval + - cull_value: Cull code for ESA steps/bins not included (0 if all good) Notes ----- @@ -546,100 +578,189 @@ def get_good_intervals(self) -> np.ndarray: document Section 2.3.2.5. """ logger.debug("Extracting good time intervals") - intervals: list[np.void] = [] - met_values = self._obj.coords["met"].values - cull_flags = self._obj["cull_flags"].values - esa_steps = self._obj["esa_step"].values + # Determine which dimension is present (epoch for CDF, met for in-memory) + time_dim = "epoch" if "epoch" in self._obj.dims else "met" + + # Get met values + met_values = self._obj["met"].values if len(met_values) == 0: logger.warning("No MET values found, returning empty intervals array") return np.array([], dtype=INTERVAL_DTYPE) - # Process each MET time - for met_idx in range(len(met_values)): - self._add_intervals_for_pattern( - intervals, - met_values[met_idx], - met_values[met_idx], # met_start == met_end - cull_flags[met_idx, :], - esa_steps[met_idx], + # Add sweep indices as a coordinate + ds = _add_sweep_indices(self._obj) + + # Compare consecutive sweeps using xarray groupby + grouped = list(ds["cull_flags"].groupby("esa_sweep")) + + # Determine pattern changes by comparing each sweep to the next + # Start with False for first sweep (no previous sweep) + pattern_changes = [False] + for i in range(len(grouped) - 1): + # The grouped list contains tuples (sweep_idx, cull_flags_ds). + # Grab just the cull_flags_ds values for comparison. + cull_curr = grouped[i][1] + cull_next = grouped[i + 1][1] + + # Compare shapes first (different lengths = different pattern) + # In a nominal Pointing, the final ESA sweep will get cut short by + # the repoint maneuver. This causes a difference in shape. + if cull_curr.shape != cull_next.shape: + pattern_changes.append(True) + else: + # Compare cull_flag values only (not coordinates) + pattern_changes.append( + not np.array_equal(cull_curr.values, cull_next.values) + ) + + # Convert to numpy array and create group IDs + pattern_changes = np.array(pattern_changes, dtype=bool) + + # Use cumsum to create group IDs + group_ids = pattern_changes.cumsum().astype(int) + + # Map group IDs to all time points using the correct dimension + group_coord = np.array([group_ids[int(s)] for s in ds["esa_sweep"].values]) + ds = ds.assign_coords(pattern_group=(time_dim, group_coord)) + + # Group by pattern_group (consecutive identical sweeps only) + intervals: list[tuple] = [] + pattern_groups = list(ds.groupby("pattern_group")) + + for i, (_, pattern_ds) in enumerate(pattern_groups): + # Get met values from the pattern dataset + pattern_met = pattern_ds["met"].values + met_start = float(pattern_met.min()) + + # met_end is start of next group, or max MET of this group if last + if i + 1 < len(pattern_groups): + next_met = pattern_groups[i + 1][1]["met"].values + met_end = float(next_met.min()) + else: + met_end = float(pattern_met.max()) + + # Get first sweep as representative (all sweeps in pattern are identical) + first_sweep_idx = pattern_ds["esa_sweep"].values[0] + first_sweep = pattern_ds.sel( + {time_dim: (pattern_ds["esa_sweep"] == first_sweep_idx)} + ) + + # Generate interval elements for this pattern + intervals.extend( + self._generate_intervals_for_pattern(first_sweep, met_start, met_end) ) logger.info(f"Extracted {len(intervals)} good time intervals") return np.array(intervals, dtype=INTERVAL_DTYPE) - def _add_intervals_for_pattern( - self, - intervals: list, - met_start: float, - met_end: float, - pattern: np.ndarray, - esa_step: int, - ) -> None: + def _generate_intervals_for_pattern( + self, sweep_ds: xr.Dataset, met_start: float, met_end: float + ) -> list[tuple]: """ - Add interval(s) for a cull_flags pattern, splitting if bins wrap around. + Generate interval elements for a sweep pattern. Parameters ---------- - intervals : list - List to append interval tuples to. + sweep_ds : xarray.Dataset + Representative sweep. met_start : float - Start MET timestamp. + Start MET for this interval group. met_end : float - End MET timestamp. - pattern : numpy.ndarray - Cull flags pattern for spin bins. - esa_step : int - ESA step for this MET. + End MET for this interval group. + + Returns + ------- + list[tuple] + List of interval tuples matching INTERVAL_DTYPE. """ - good_bins = np.nonzero(pattern == 0)[0] - - if len(good_bins) == 0: - return - - # Check for gaps in good_bins (indicating separate contiguous regions) - # Bins are contiguous if difference between consecutive bins is 1 - gaps = np.nonzero(np.diff(good_bins) > 1)[0] - - if len(gaps) == 0: - # No gaps - single contiguous region - interval = ( - met_start, - met_end, - good_bins[0], - good_bins[-1], - len(good_bins), - esa_step, + all_good_mask = 0 + partial_regions = [] + bad_cull_value = 0 + + # Process each unique ESA step + for esa_step in np.unique(sweep_ds["esa_step"].values): + esa_mask = sweep_ds["esa_step"] == esa_step + cull_pattern = sweep_ds["cull_flags"].values[esa_mask.values][0] + esa_bit = 1 << (int(esa_step) - 1) + + if np.all(cull_pattern == 0): + all_good_mask |= esa_bit + else: + bad_vals = cull_pattern[cull_pattern > 0] + if len(bad_vals) > 0: + # Aggregate all non-zero cull codes for this ESA step so that + # the region cull value reflects every flag that contributed. + region_cull = int(np.bitwise_or.reduce(bad_vals)) + bad_cull_value |= region_cull + else: + region_cull = 0 + + for bin_low, bin_high in self._find_good_bin_regions(cull_pattern): + partial_regions.append( + { + "esa_bit": esa_bit, + "bin_low": bin_low, + "bin_high": bin_high, + "cull_value": region_cull, + } + ) + + # Generate interval elements + elements = [] + + if all_good_mask > 0: + elements.append( + (met_start, met_end, 0, 89, 90, all_good_mask, bad_cull_value) ) - intervals.append(interval) - else: - # Multiple contiguous regions - split at gaps - start_idx = 0 - for gap_idx in gaps: - # Create interval for bins before the gap - bins_segment = good_bins[start_idx : gap_idx + 1] - interval = ( + + for region in partial_regions: + n_bins = region["bin_high"] - region["bin_low"] + 1 + elements.append( + ( met_start, met_end, - bins_segment[0], - bins_segment[-1], - len(bins_segment), - esa_step, + region["bin_low"], + region["bin_high"], + n_bins, + region["esa_bit"], + region["cull_value"], ) - intervals.append(interval) - start_idx = gap_idx + 1 - - # Handle final segment after last gap - bins_segment = good_bins[start_idx:] - interval = ( - met_start, - met_end, - bins_segment[0], - bins_segment[-1], - len(bins_segment), - esa_step, ) - intervals.append(interval) + + return elements + + @staticmethod + def _find_good_bin_regions(cull_pattern: np.ndarray) -> list[tuple[int, int]]: + """ + Find contiguous regions where cull_pattern == 0. + + Parameters + ---------- + cull_pattern : np.ndarray + Array of cull values for 90 spin bins. + + Returns + ------- + list[tuple[int, int]] + List of (start_bin, end_bin) tuples for good regions. + """ + regions: list[tuple[int, int]] = [] + in_good_region = False + start_bin = 0 + + for i, val in enumerate(cull_pattern): + if val == 0 and not in_good_region: + start_bin = i + in_good_region = True + elif val != 0 and in_good_region: + regions.append((start_bin, i - 1)) + in_good_region = False + + if in_good_region: + regions.append((start_bin, 89)) + + return regions def get_cull_statistics(self) -> dict: """ @@ -678,11 +799,14 @@ def get_cull_statistics(self) -> dict: def write_txt(self, output_path: Path) -> Path: """ - Write good times to text file in the format specified by algorithm document. + Write time intervals to text file in the format specified by algorithm document. Format per Section 2.3.2.5: - pointing MET_start MET_end spin_bin_low spin_bin_high sensor esa_step - [rate/sigma values...] + pointing MET_start MET_end spin_bin_low spin_bin_high sensor + esa_steps[10] cull_value + + The esa_steps field consists of 10 binary values (0 or 1) indicating whether + each ESA step (1-10) is included in this interval. Parameters ---------- @@ -694,24 +818,43 @@ def write_txt(self, output_path: Path) -> Path: pathlib.Path Path to the created file. """ - logger.info(f"Writing good times to file: {output_path}") + logger.info(f"Writing intervals to file: {output_path}") + pointing = int(self._obj.attrs["Repointing"].replace("repoint", "")) + sensor = ( + parse_sensor_number(self._obj.attrs["Logical_source"]) + if "Logical_source" in self._obj.attrs + else self._obj.attrs["Sensor"].replace("sensor", "") + ) + intervals = self.get_good_intervals() with open(output_path, "w") as f: + # Write header info + file_id = self._obj.attrs.get("Logical_file_id") + if file_id is not None: + f.write( + f"# Goodtimes txt file generated for input CDF: {file_id}" + "\n" + ) for interval in intervals: - pointing = self._obj.attrs.get("pointing", 0) - sensor = self._obj.attrs["sensor"] + # Convert esa_step_mask bitmask to 10 binary values + # Bit i represents ESA step i+1, so check bits 0-9 + esa_step_mask = int(interval["esa_step_mask"]) + esa_step_flags = " ".join( + "1" if (esa_step_mask >> i) & 1 else "0" for i in range(10) + ) # Format: - # pointing met_start met_end spin_bin_low spin_bin_high sensor esa_step + # pointing met_start met_end spin_bin_low spin_bin_high sensor + # esa_steps[10] cull_value line = ( f"{pointing:05d} " - f"{int(interval['met_start'])} " - f"{int(interval['met_end'])} " - f"{interval['spin_bin_low']} " - f"{interval['spin_bin_high']} " + f"{interval['met_start']:0.1f} " + f"{interval['met_end']:0.1f} " + f"{interval['spin_bin_low']:2d} " + f"{interval['spin_bin_high']:2d} " f"{sensor} " - f"{interval['esa_step']}" + f"{esa_step_flags} " + f"{interval['cull_value']:3d}" ) # TODO: Add rate/sigma values for each ESA step @@ -774,7 +917,6 @@ def finalize_dataset(self) -> xr.Dataset: ds[coord_name].attrs = attr_mgr.get_variable_attributes( attr_mgr_key, check_schema=False ) - ds["spin_bin"].attrs = attr_mgr.get_variable_attributes("hi_goodtimes_spin_bin") # Add variable attributes for var_name in ds.data_vars: @@ -783,7 +925,7 @@ def finalize_dataset(self) -> xr.Dataset: ) # Update global attributes - sensor_str = ds.attrs.pop("sensor") + sensor_str = ds.attrs.pop("Sensor") ds.attrs = attr_mgr.get_global_attributes("imap_hi_l1b_goodtimes_attrs") # Update Logical_source with sensor string @@ -803,7 +945,7 @@ def finalize_dataset(self) -> xr.Dataset: def mark_incomplete_spin_sets( goodtimes_ds: xr.Dataset, l1b_de: xr.Dataset, - cull_code: int = CullCode.LOOSE, + cull_code: int = CullCode.INCOMPLETE_SPIN, ) -> None: """ Filter out incomplete 8-spin histogram periods. @@ -918,7 +1060,7 @@ def mark_incomplete_spin_sets( def mark_drf_times( goodtimes_ds: xr.Dataset, hk: xr.Dataset, - cull_code: int = CullCode.LOOSE, + cull_code: int = CullCode.DRF, ) -> None: """ Remove times during spacecraft drift restabilization. @@ -991,7 +1133,7 @@ def mark_overflow_packets( goodtimes_ds: xr.Dataset, l1b_de: xr.Dataset, config_df: pd.DataFrame, - cull_code: int = CullCode.LOOSE, + cull_code: int = CullCode.OVERFLOW, ) -> None: """ Remove times when DE packets overflow with qualified events. @@ -1081,7 +1223,7 @@ def mark_overflow_packets( # - After processing all events, last_event_per_packet[P] contains the # index of the last event belonging to packet P max_packet_idx = int(np.max(ccsds_indices)) - last_event_per_packet = np.full(max_packet_idx + 1, -1, dtype=np.intp) + last_event_per_packet: np.ndarray = np.full(max_packet_idx + 1, -1, dtype=np.intp) event_indices = np.arange(len(ccsds_indices)) np.maximum.at(last_event_per_packet, ccsds_indices, event_indices) @@ -1114,6 +1256,97 @@ def mark_overflow_packets( ) +def mark_bad_tdc_cal( + goodtimes_ds: xr.Dataset, + diagfee: xr.Dataset, + cull_code: int = CullCode.BAD_TDC_CAL, + check_tdc_3: bool = False, +) -> None: + """ + Remove times with failed TDC calibration (DIAG_FEE method). + + Based on C reference: drop_bad_tdc_diagfee in culling_v2.c provided by + IMAP-Hi team. + + This function scans DIAG_FEE packets chronologically and checks the TDC + calibration status for each packet. If any TDC has failed calibration, + all times from that DIAG_FEE packet until the next DIAG_FEE packet are + marked as bad. + + Parameters + ---------- + goodtimes_ds : xr.Dataset + Goodtimes dataset to update with cull flags. + diagfee : xr.Dataset + DIAG_FEE dataset containing TDC calibration status fields: + - shcoarse: Mission Elapsed Time (MET) + - tdc1_cal_ctrl_stat: TDC1 calibration status (bit 1 = success) + - tdc2_cal_ctrl_stat: TDC2 calibration status (bit 1 = success) + - tdc3_cal_ctrl_stat: TDC3 calibration status (bit 1 = success) + cull_code : int, optional + Cull code to use for marking bad times. Default is CullCode.LOOSE. + check_tdc_3 : bool, optional + Whether to check TDC3 calibration status in addition to TDC1 and TDC2. + Default is False to match original C code behavior. + + Notes + ----- + This function modifies goodtimes_ds in place. + + Quirk: Two DIAG_FEE packets are generated when entering HVSCI mode. + The first packet is skipped if two packets appear within 10 seconds. + """ + logger.info("Running mark_bad_tdc_cal culling") + + # Based on sample code in culling_v2.c, skip this check if we have fewer + # than two diag_fee packets. + if len(diagfee.epoch) < 2: + logger.warning( + f"Insufficient DIAG_FEE packets to select good times " + f"(found {len(diagfee.epoch)}, need at least 2)" + ) + return + + diagfee_met = diagfee["shcoarse"].values + goodtimes_met = goodtimes_ds.coords["met"].values + + # Identify duplicate packets: skip if followed by another within 10 seconds + time_gaps = np.diff(diagfee_met) + is_duplicate = np.concatenate([time_gaps < 10, [False]]) + + # Identify any packets where any of the three TDC calibrations failed. + # TDC failure check (bit 1: 1=good, 0=bad) + tdc_failed = ((diagfee["tdc1_cal_ctrl_stat"].values & 2) == 0) | ( + (diagfee["tdc2_cal_ctrl_stat"].values & 2) == 0 + ) + if check_tdc_3: + tdc_failed |= (diagfee["tdc3_cal_ctrl_stat"].values & 2) == 0 + + # Only loop over non-duplicate packets with TDC failures + tdc_failed_indices = np.nonzero(~is_duplicate & tdc_failed)[0] + + n_times_removed = 0 + for i in tdc_failed_indices: + # Remove times from this DIAG_FEE packet until next. We are skipping the + # first packet of a duplicate pair, so determining the window based on the + # current packet met and next packet met covers the time window between + # non-duplicate DIAG_FEE packets. We can ignore the ~10 seconds of slop + # around duplicate packets because these packets should only be produced + # when IMAP-Hi is transitioning to HVSCI mode which means that there will + # be no DE packets being produced. + df_time = diagfee_met[i] + next_df_time = diagfee_met[i + 1] if i < len(diagfee_met) - 1 else np.inf + + in_window = (goodtimes_met >= df_time) & (goodtimes_met < next_df_time) + mets_to_cull = goodtimes_met[in_window] + + if len(mets_to_cull) > 0: + goodtimes_ds.goodtimes.mark_bad_times(met=mets_to_cull, cull=cull_code) + n_times_removed += len(mets_to_cull) + + logger.info(f"Dropped {n_times_removed} time(s) due to bad TDC calibration") + + def _get_sweep_indices(esa_step: np.ndarray) -> np.ndarray: """ Assign sweep indices to each MET based on ESA step transitions. @@ -1155,15 +1388,18 @@ def _add_sweep_indices(l1b_de: xr.Dataset) -> xr.Dataset: Parameters ---------- l1b_de : xarray.Dataset - L1B Direct Event dataset. + L1B Direct Event dataset or goodtimes dataset. Returns ------- xarray.Dataset - Dataset with esa_sweep coordinate added on epoch dimension. + Dataset with esa_sweep coordinate added on the time dimension + (either 'epoch' or 'met'). """ sweep_indices = _get_sweep_indices(l1b_de["esa_step"].values) - return l1b_de.assign_coords(esa_sweep=("epoch", sweep_indices)) + # Determine which dimension to use (epoch for CDF data, met for in-memory) + time_dim = "epoch" if "epoch" in l1b_de.dims else "met" + return l1b_de.assign_coords(esa_sweep=(time_dim, sweep_indices)) def _compute_normalized_counts_per_sweep( @@ -1212,7 +1448,7 @@ def _compute_normalized_counts_per_sweep( # Count valid AB events per sweep n_sweeps = int(l1b_de["esa_sweep"].max().values) + 1 - counts_per_sweep = np.zeros(n_sweeps, dtype=np.int64) + counts_per_sweep: np.ndarray = np.zeros(n_sweeps, dtype=np.int64) np.add.at(counts_per_sweep, event_sweep_idx[is_valid_ab.values], 1) # Normalize by number of unique ESA energy steps @@ -1251,7 +1487,7 @@ def mark_statistical_filter_0( current_index: int, threshold_factor: float = HiConstants.STAT_FILTER_0_THRESHOLD_FACTOR, tof_ab_limit_ns: int = HiConstants.STAT_FILTER_0_TOF_AB_LIMIT_NS, - cull_code: int = CullCode.LOOSE, + cull_code: int = CullCode.STAT_FILTER_0, min_pointings: int = HiConstants.STAT_FILTER_MIN_POINTINGS, ) -> None: """ @@ -1390,7 +1626,7 @@ def mark_statistical_filter_0( def _compute_qualified_counts_per_sweep( l1b_de: xr.Dataset, - qualified_coincidence_types: set[int], + qualified_mask: np.ndarray, ) -> xr.Dataset: """ Compute qualified calibration product counts per 8-spin interval and reshape. @@ -1402,8 +1638,9 @@ def _compute_qualified_counts_per_sweep( ---------- l1b_de : xarray.Dataset L1B Direct Event dataset with esa_sweep coordinate on epoch dimension. - qualified_coincidence_types : set[int] - Set of coincidence type integers that qualify for calibration products. + qualified_mask : np.ndarray + Boolean mask indicating which events qualify for calibration products. + This mask should check BOTH coincidence_type AND TOF windows. Returns ------- @@ -1416,13 +1653,12 @@ def _compute_qualified_counts_per_sweep( raise ValueError("Dataset must have esa_sweep coordinate") # Get values needed for counting - coincidence_type = l1b_de["coincidence_type"].values ccsds_index = l1b_de["ccsds_index"].values esa_sweep = l1b_de.coords["esa_sweep"].values esa_energy_step = l1b_de["esa_energy_step"].values - # Identify qualified events - is_qualified = np.isin(coincidence_type, list(qualified_coincidence_types)) + # Use pre-computed qualified mask + is_qualified = qualified_mask # Map qualified events to their packet's (esa_sweep, esa_energy_step) qualified_packet_idx = ccsds_index[is_qualified] @@ -1432,7 +1668,7 @@ def _compute_qualified_counts_per_sweep( # Count qualified events per (esa_sweep, esa_energy_step) using 2D array n_sweeps = int(esa_sweep.max()) + 1 n_esa_energy_steps = int(esa_energy_step.max()) + 1 - counts_2d = np.zeros((n_sweeps, n_esa_energy_steps), dtype=np.float64) + counts_2d: np.ndarray = np.zeros((n_sweeps, n_esa_energy_steps), dtype=np.float64) np.add.at(counts_2d, (qualified_sweep, qualified_energy_step), 1) # Remove event_met dimension and reshape using multi-index @@ -1460,7 +1696,6 @@ def _compute_qualified_counts_per_sweep( def _build_per_sweep_datasets( l1b_de_datasets: list[xr.Dataset], - qualified_coincidence_types: set[int], ) -> dict[int, xr.Dataset]: """ Build per-sweep datasets with qualified counts for each Pointing. @@ -1468,9 +1703,9 @@ def _build_per_sweep_datasets( Parameters ---------- l1b_de_datasets : list[xarray.Dataset] - List of L1B DE datasets for multiple Pointings. - qualified_coincidence_types : set[int] - Set of coincidence type integers that qualify for calibration products. + List of L1B DE datasets for multiple Pointings. Each dataset must + contain a "qualified_mask" DataArray indicating which events qualify + for calibration products. Returns ------- @@ -1484,7 +1719,7 @@ def _build_per_sweep_datasets( # Add esa_sweep coordinate and compute counts per 8-spin interval l1b_de_with_sweep = _add_sweep_indices(l1b_de) per_sweep = _compute_qualified_counts_per_sweep( - l1b_de_with_sweep, qualified_coincidence_types + l1b_de_with_sweep, l1b_de["qualified_mask"].values ) per_sweep_datasets[i] = per_sweep @@ -1684,11 +1919,10 @@ def mark_statistical_filter_1( goodtimes_ds: xr.Dataset, l1b_de_datasets: list[xr.Dataset], current_index: int, - qualified_coincidence_types: set[int], consecutive_threshold_sigma: float = HiConstants.STAT_FILTER_1_CONSECUTIVE_SIGMA, extreme_threshold_sigma: float = HiConstants.STAT_FILTER_1_EXTREME_SIGMA, min_consecutive_intervals: int = HiConstants.STAT_FILTER_1_MIN_CONSECUTIVE, - cull_code: int = CullCode.LOOSE, + cull_code: int = CullCode.STAT_FILTER_1, min_pointings: int = HiConstants.STAT_FILTER_MIN_POINTINGS, ) -> None: """ @@ -1711,11 +1945,11 @@ def mark_statistical_filter_1( Goodtimes dataset for the current Pointing to update. l1b_de_datasets : list[xarray.Dataset] List of L1B DE datasets for surrounding Pointings. Typically includes - current plus 3 preceding and 3 following Pointings. + current plus 3 preceding and 3 following Pointings. Each dataset must + contain a "qualified_mask" DataArray indicating which events qualify + for calibration products (checking both coincidence_type AND TOF). current_index : int Index of the current Pointing in l1b_de_datasets. - qualified_coincidence_types : set[int] - Set of coincidence type integers that qualify for calibration products. consecutive_threshold_sigma : float, optional Sigma multiplier for consecutive interval check. Default is HiConstants.STAT_FILTER_1_CONSECUTIVE_SIGMA. @@ -1758,9 +1992,7 @@ def mark_statistical_filter_1( ) # Step 1: Build per-sweep datasets with qualified counts for each Pointing - per_sweep_datasets = _build_per_sweep_datasets( - l1b_de_datasets, qualified_coincidence_types - ) + per_sweep_datasets = _build_per_sweep_datasets(l1b_de_datasets) # Step 2: Compute median and sigma per ESA energy step using xarray median_per_esa, sigma_per_esa = _compute_median_and_sigma_per_esa( @@ -1853,6 +2085,11 @@ def _find_event_clusters( # Find transitions: +1 = start of group, -1 = end of group diff = np.diff(padded.astype(int)) starts = np.flatnonzero(diff == 1) + # We need to adjust ends for the shortening from diffs performed. + # The window_spans array has length = n_events - min_events + 1 + # The contiguous diff adds two padding elements and np.diff shortens by 1. + # The result is that we need to add min_events and subtract 2 to get the + # correct end index. ends = np.flatnonzero(diff == -1) + min_events - 2 # Adjust for window size return list(zip(starts.tolist(), ends.tolist(), strict=False)) @@ -1888,7 +2125,9 @@ def _compute_bins_for_cluster( For example, if cluster spans bins 88-91 with n_bins=90, returns [87, 88, 89, 0, 1, 2] (with padding=1). """ - cluster_bins = nominal_bins[cluster_start : cluster_end + 1].astype(np.int32) + cluster_bins: np.ndarray = nominal_bins[cluster_start : cluster_end + 1].astype( + np.int32 + ) # Unwrap to handle clusters spanning the 0/n_bins boundary unwrapped = np.unwrap(cluster_bins, period=n_bins) @@ -1900,7 +2139,9 @@ def _compute_bins_for_cluster( bin_high = bin_max + bin_padding # Generate bin indices with wrapping using modulo - bins_to_mark = np.arange(bin_low, bin_high + 1) % n_bins + bins_to_mark: np.ndarray = np.arange(bin_low, bin_high + 1) % n_bins + + logger.debug(f"Cluster {cluster_start} to {cluster_end} bins: {bins_to_mark}") return bins_to_mark @@ -1908,11 +2149,10 @@ def _compute_bins_for_cluster( def mark_statistical_filter_2( goodtimes_ds: xr.Dataset, l1b_de: xr.Dataset, - qualified_coincidence_types: set[int], min_events: int = HiConstants.STAT_FILTER_2_MIN_EVENTS, max_time_delta: float = HiConstants.STAT_FILTER_2_MAX_TIME_DELTA, bin_padding: int = HiConstants.STAT_FILTER_2_BIN_PADDING, - cull_code: int = CullCode.LOOSE, + cull_code: int = CullCode.STAT_FILTER_2, ) -> None: """ Apply Statistical Filter 2 to detect short-lived event pulses. @@ -1942,9 +2182,8 @@ def mark_statistical_filter_2( - coincidence_type: detector coincidence bitmap - nominal_bin: spacecraft spin bin (0-89) - esa_step: ESA energy step for each packet - qualified_coincidence_types : set[int] - Set of coincidence type integers qualifying as calibration - products 1 or 2. + - qualified_mask: boolean mask indicating which events qualify for + calibration products (checking both coincidence_type AND TOF windows) min_events : int, optional Minimum events to form a pulse cluster. Default is HiConstants.STAT_FILTER_2_MIN_EVENTS. @@ -1981,19 +2220,18 @@ def mark_statistical_filter_2( # Add event-level coordinates for grouping l1b_de_with_sweep = l1b_de_with_sweep.assign_coords( - event_sweep=("event", esa_sweep[ccsds_index]), - event_step=("event", esa_step[ccsds_index]), + event_sweep=("event_met", esa_sweep[ccsds_index]), + event_step=("event_met", esa_step[ccsds_index]), ) - # Filter to qualified events - coincidence_type = l1b_de_with_sweep["coincidence_type"].values - is_qualified = np.isin(coincidence_type, list(qualified_coincidence_types)) + # Get qualified mask from the dataset + qualified_mask = l1b_de["qualified_mask"].values - if not np.any(is_qualified): + if not np.any(qualified_mask): logger.info("Statistical Filter 2: No qualified events found") return - qualified_events = l1b_de_with_sweep.isel(event=is_qualified) + qualified_events = l1b_de_with_sweep.isel(event_met=qualified_mask) n_clusters_found = 0 n_bins_marked = 0 diff --git a/imap_processing/hi/hi_l1a.py b/imap_processing/hi/hi_l1a.py index 72a7e819cc..0d32cf775d 100644 --- a/imap_processing/hi/hi_l1a.py +++ b/imap_processing/hi/hi_l1a.py @@ -365,7 +365,7 @@ def parse_direct_events(de_data: bytes) -> dict[str, npt.ArrayLike]: # word_0: full 16-bits is the de_tag # word_1: 2-bits of Trigger ID, 10-bits tof_1, upper 4-bits of tof_2 # word_2: lower 6-bits of tof_2, 10-bits of tof_3 - data_uint16 = np.reshape( + data_uint16: np.ndarray = np.reshape( np.frombuffer(de_data, dtype=">u2"), (3, -1), order="F" ).astype(np.uint16) @@ -404,11 +404,11 @@ def finish_hist_dataset(input_ds: xr.Dataset) -> xr.Dataset: dataset = input_ds.rename_vars({"shcoarse": "ccsds_met"}) dataset.epoch.attrs.update( - attr_mgr.get_variable_attributes("epoch"), + attr_mgr.get_variable_attributes("epoch", check_schema=False), ) # Add the hist_angle coordinate # Histogram data is binned in 90, 4-degree bins - attrs = attr_mgr.get_variable_attributes("hi_hist_angle") + attrs = attr_mgr.get_variable_attributes("hi_hist_angle", check_schema=False) dataset.coords.update( { "angle": xr.DataArray( @@ -535,7 +535,7 @@ def finish_memdmp_dataset(input_ds: xr.Dataset) -> xr.Dataset: dataset = dataset.rename_vars({"shcoarse": "ccsds_met"}) dataset.epoch.attrs.update( - attr_mgr.get_variable_attributes("epoch"), + attr_mgr.get_variable_attributes("epoch", check_schema=False), ) # Update existing variable attributes diff --git a/imap_processing/hi/hi_l1c.py b/imap_processing/hi/hi_l1c.py index c4802c2352..520e624d13 100644 --- a/imap_processing/hi/hi_l1c.py +++ b/imap_processing/hi/hi_l1c.py @@ -4,31 +4,33 @@ import logging from pathlib import Path -from typing import NamedTuple import numpy as np import pandas as pd import xarray as xr from numpy import typing as npt -from numpy._typing import NDArray from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import parse_filename_like from imap_processing.hi.utils import ( + BackgroundConfig, CalibrationProductConfig, HiConstants, create_dataset_variables, full_dataarray, + iter_background_events_by_config, + iter_qualified_events_by_config, parse_sensor_number, ) from imap_processing.spice.geometry import ( SpiceFrame, frame_transform, frame_transform_az_el, + get_spacecraft_to_instrument_spin_phase_offset, ) from imap_processing.spice.repoint import get_pointing_times from imap_processing.spice.spin import ( - get_instrument_spin_phase, + get_spacecraft_spin_phase, get_spin_data, ) from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et @@ -41,22 +43,24 @@ def hi_l1c( - de_dataset: xr.Dataset, calibration_prod_config_path: Path + de_dataset: xr.Dataset, + calibration_prod_config_path: Path, + goodtimes_ds: xr.Dataset, + background_config_path: Path, ) -> list[xr.Dataset]: """ High level IMAP-Hi l1c processing function. - This function will be expanded once the l1c processing is better defined. It - will need to add inputs such as Ephemerides, Goodtimes inputs, and - instrument status summary and will output a Pointing Set CDF as well as a - Goodtimes list (CDF?). - Parameters ---------- de_dataset : xarray.Dataset IMAP-Hi l1b de product. calibration_prod_config_path : pathlib.Path Calibration product configuration file. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. + background_config_path : pathlib.Path + Background configuration file. Returns ------- @@ -65,15 +69,18 @@ def hi_l1c( """ logger.info("Running Hi l1c processing") - # TODO: I am not sure what the input for Goodtimes will be so for now, - # If the input is an xarray Dataset, do pset processing - l1c_dataset = generate_pset_dataset(de_dataset, calibration_prod_config_path) + l1c_dataset = generate_pset_dataset( + de_dataset, calibration_prod_config_path, goodtimes_ds, background_config_path + ) return [l1c_dataset] def generate_pset_dataset( - de_dataset: xr.Dataset, calibration_prod_config_path: Path + de_dataset: xr.Dataset, + calibration_prod_config_path: Path, + goodtimes_ds: xr.Dataset, + background_config_path: Path, ) -> xr.Dataset: """ Generate IMAP-Hi l1c pset xarray dataset from l1b product. @@ -84,6 +91,10 @@ def generate_pset_dataset( IMAP-Hi l1b de product. calibration_prod_config_path : pathlib.Path Calibration product configuration file. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. + background_config_path : pathlib.Path + Background configuration file. Returns ------- @@ -97,6 +108,8 @@ def generate_pset_dataset( logical_source_parts = parse_filename_like(de_dataset.attrs["Logical_source"]) # read calibration product configuration file config_df = CalibrationProductConfig.from_csv(calibration_prod_config_path) + # read background configuration file + background_df = BackgroundConfig.from_csv(background_config_path) pset_dataset = empty_pset_dataset( de_dataset.ccsds_met.data.mean(), @@ -111,11 +124,22 @@ def generate_pset_dataset( ) pset_dataset.update(pset_geometry(pset_midpoint_et, logical_source_parts["sensor"])) # Bin the counts into the spin-bins - pset_dataset.update(pset_counts(pset_dataset.coords, config_df, de_dataset)) + pset_dataset.update( + pset_counts(pset_dataset.coords, config_df, de_dataset, goodtimes_ds) + ) # Calculate and add the exposure time to the pset_dataset - pset_dataset.update(pset_exposure(pset_dataset.coords, de_dataset)) - # Get the backgrounds - pset_dataset.update(pset_backgrounds(pset_dataset.coords)) + pset_dataset.update(pset_exposure(pset_dataset.coords, de_dataset, goodtimes_ds)) + + # Compute backgrounds (background counts computed internally) + pset_dataset.update( + pset_backgrounds( + pset_dataset.coords, + background_df, + de_dataset, + goodtimes_ds, + pset_dataset["exposure_times"], + ) + ) return pset_dataset @@ -323,6 +347,7 @@ def pset_counts( pset_coords: dict[str, xr.DataArray], config_df: pd.DataFrame, l1b_de_dataset: xr.Dataset, + goodtimes_ds: xr.Dataset, ) -> dict[str, xr.DataArray]: """ Bin direct events into PSET spin-bins. @@ -335,14 +360,15 @@ def pset_counts( The calibration product configuration dataframe. l1b_de_dataset : xarray.Dataset The L1B dataset for the pointing being processed. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. Returns ------- dict[str, xarray.DataArray] - Dictionary containing new exposure_times DataArray to be added to the PSET - dataset. + Dictionary containing counts DataArray. """ - # Generate exposure time variable filled with zeros + # Generate counts variable filled with zeros counts_var = create_dataset_variables( ["counts"], coords=pset_coords, @@ -363,149 +389,275 @@ def pset_counts( # Remove DEs with invalid trigger_id. This should only occur for a # pointing with no events that gets a single fill event good_mask = de_ds["trigger_id"].data != de_ds["trigger_id"].attrs["FILLVAL"] + if not np.any(good_mask): + return counts_var + # Remove DEs not in Goodtimes/angles - good_mask &= good_time_and_phase_mask( - l1b_de_dataset.event_met.values, l1b_de_dataset.spin_phase.values + # For direct events, use nominal_bin (spacecraft spin bin 0-89) to look up goodtimes + goodtimes_mask = good_time_and_phase_mask( + de_ds.event_met.values, + de_ds.nominal_bin.values, + goodtimes_ds, ) - de_ds = de_ds.isel(event_met=good_mask) + de_ds = de_ds.isel(event_met=goodtimes_mask) + + # Get esa_energy_step for each event (recorded per packet, use ccsds_index) + esa_energy_steps = l1b_de_dataset["esa_energy_step"].data[de_ds["ccsds_index"].data] # The calibration product configuration potentially has different coincidence # types for each ESA and different TOF windows for each calibration product, - # esa energy step combination. Because of this we need to filter DEs that - # belong to each combo individually. - # Loop over the esa_energy_step values first - for esa_energy, esa_df in config_df.groupby(level="esa_energy_step"): - # Create a mask for all DEs at the current esa_energy_step. - # esa_energy_step is recorded for each packet rather than for each DE, - # so we use ccsds_index to get the esa_energy_step for each DE - esa_mask = ( - l1b_de_dataset["esa_energy_step"].data[de_ds["ccsds_index"].data] - == esa_energy + # esa energy step combination. Use the shared generator to iterate over all + # config combinations and get qualified event masks. + for esa_energy, config_row, qualified_mask in iter_qualified_events_by_config( + de_ds, config_df, esa_energy_steps + ): + # Filter events using the qualified mask + filtered_de_ds = de_ds.isel(event_met=qualified_mask) + + # Bin remaining DEs into spin-bins + i_esa = np.flatnonzero(pset_coords["esa_energy_step"].data == esa_energy)[0] + # spin_phase is in the range [0, 1). Multiplying by N_SPIN_BINS and + # truncating to an integer gives the correct bin index + spin_bin_indices = (filtered_de_ds["spin_phase"].data * N_SPIN_BINS).astype(int) + # When iterating over rows of a dataframe, the names of the multi-index + # are not preserved. Below, `config_row.Index[0]` gets the + # calibration_prod value from the namedtuple representing the + # dataframe row. We map this to the array index using cal_prod_to_index. + i_cal_prod = cal_prod_to_index[config_row.Index[0]] + np.add.at( + counts_var["counts"].data[0, i_esa, i_cal_prod], + spin_bin_indices, + 1, ) - # Now loop over the calibration products for the current ESA energy - for config_row in esa_df.itertuples(): - # Remove DEs that are not at the current ESA energy and in the list - # of coincidence types for the current calibration product - type_mask = de_ds["coincidence_type"].isin( - config_row.coincidence_type_values - ) - filtered_de_ds = de_ds.isel(event_met=(esa_mask & type_mask)) - - # Use the TOF window mask to remove DEs with TOFs outside the allowed range - tof_fill_vals = { - f"tof_{detector_pair}": l1b_de_dataset[f"tof_{detector_pair}"].attrs[ - "FILLVAL" - ] - for detector_pair in CalibrationProductConfig.tof_detector_pairs - } - tof_in_window_mask = get_tof_window_mask( - filtered_de_ds, config_row, tof_fill_vals - ) - filtered_de_ds = filtered_de_ds.isel(event_met=tof_in_window_mask) - - # Bin remaining DEs into spin-bins - i_esa = np.flatnonzero(pset_coords["esa_energy_step"].data == esa_energy)[0] - # spin_phase is in the range [0, 1). Multiplying by N_SPIN_BINS and - # truncating to an integer gives the correct bin index - spin_bin_indices = (filtered_de_ds["spin_phase"].data * N_SPIN_BINS).astype( - int - ) - # When iterating over rows of a dataframe, the names of the multi-index - # are not preserved. Below, `config_row.Index[0]` gets the - # calibration_prod value from the namedtuple representing the - # dataframe row. We map this to the array index using cal_prod_to_index. - i_cal_prod = cal_prod_to_index[config_row.Index[0]] - np.add.at( - counts_var["counts"].data[0, i_esa, i_cal_prod], - spin_bin_indices, - 1, - ) + return counts_var -def get_tof_window_mask( - de_ds: xr.Dataset, prod_config_row: NamedTuple, fill_vals: dict -) -> NDArray[bool]: +def _compute_background_counts( + pset_coords: dict[str, xr.DataArray], + background_config_df: pd.DataFrame, + l1b_de_dataset: xr.Dataset, + goodtimes_ds: xr.Dataset, +) -> xr.DataArray: """ - Generate a mask indicating which DEs to keep based on TOF windows. + Compute background counts by filtering and binning direct events. + + Background counts are computed across all esa_energy_steps and spin_angle_bins + since backgrounds are isotropic and do not depend on ESA energy step or spin angle. Parameters ---------- - de_ds : xarray.Dataset - The Direct Event Dataset for the DEs to filter based on the TOF - windows. - prod_config_row : NamedTuple - A single row of the prod config dataframe represented as a named tuple. - fill_vals : dict - A dictionary containing the fill values used in the input DE TOF - dataframe values. This value should be derived from the L1B DE CDF - TOF variable attributes. + pset_coords : dict[str, xarray.DataArray] + The PSET coordinates from the xarray.Dataset. + background_config_df : pandas.DataFrame + Background configuration DataFrame with MultiIndex + (calibration_prod, background_index). + l1b_de_dataset : xarray.Dataset + The L1B dataset for the pointing being processed. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. Returns ------- - window_mask : np.ndarray - A mask with one entry per DE in the input `de_df` indicating which DEs - contain TOF values within the windows specified by `prod_config_row`. - The mask is intended to directly filter the DE dataframe. + xarray.DataArray + Background counts with dims (epoch, calibration_prod, background_index). """ - detector_pairs = CalibrationProductConfig.tof_detector_pairs - tof_in_window_mask = np.empty( - (len(detector_pairs), len(de_ds["event_met"])), dtype=bool - ) - for i_pair, detector_pair in enumerate(detector_pairs): - low_limit = getattr(prod_config_row, f"tof_{detector_pair}_low") - high_limit = getattr(prod_config_row, f"tof_{detector_pair}_high") - tof_array = de_ds[f"tof_{detector_pair}"].data - # The TOF in window mask contains True wherever the TOF is within - # the configuration low/high bounds OR the FILLVAL is present. The - # FILLVAL indicates that the detector pair was not hit. DEs with - # the incorrect coincidence_type are already filtered out and this - # implementation simplifies combining the tof_in_window_masks in - # the next step. - tof_in_window_mask[i_pair] = np.logical_or( - np.logical_and(low_limit <= tof_array, tof_array <= high_limit), - tof_array == fill_vals[f"tof_{detector_pair}"], - ) - return np.all(tof_in_window_mask, axis=0) + # Create background_counts as xarray DataArray with proper coordinates + # Note: esa_energy_step and spin_angle_bin are NOT included since backgrounds + # are isotropic and computed across all ESA steps and spin angles + background_indices = ( + background_config_df.index.get_level_values("background_index") + .unique() + .sort_values() + .values + ) + + bg_coords = { + "epoch": pset_coords["epoch"], + "calibration_prod": pset_coords["calibration_prod"], + "background_index": background_indices, + } + + background_counts = xr.DataArray( + np.zeros( + ( + len(bg_coords["epoch"]), + len(bg_coords["calibration_prod"]), + len(bg_coords["background_index"]), + ), + ), + dims=[ + "epoch", + "calibration_prod", + "background_index", + ], + coords=bg_coords, + ) + + # Process direct events + de_ds = l1b_de_dataset.drop_dims("epoch") + + good_mask = de_ds["trigger_id"].data != de_ds["trigger_id"].attrs["FILLVAL"] + if not np.any(good_mask): + return background_counts + + # Remove DEs not in Goodtimes/angles + goodtimes_mask = good_time_and_phase_mask( + de_ds.event_met.values, + de_ds.nominal_bin.values, + goodtimes_ds, + ) + de_ds = de_ds.isel(event_met=goodtimes_mask) + + n_events = len(de_ds["event_met"]) + if n_events == 0: + return background_counts + + for cal_prod in pset_coords["calibration_prod"].values: + # Check that cal_prod exists in background_config_df + if cal_prod not in background_config_df.index.get_level_values( + "calibration_prod" + ): + raise ValueError( + f"Calibration product {cal_prod} not found in background " + f"configuration. Available calibration products: " + f"{sorted(background_config_df.index.get_level_values('calibration_prod').unique().tolist())}" + ) + + # Take a cross-section of the background configuration DataFrame + # to get rows relevant to the current calibration product + cal_prod_rows = background_config_df.xs(cal_prod, level="calibration_prod") + + # Use iter_background_events_by_config to get filtered events + for config_row, filtered_de_ds in iter_background_events_by_config( + de_ds, cal_prod_rows + ): + background_idx = config_row.Index + + if len(filtered_de_ds["event_met"]) == 0: + continue + + # Count all filtered events + # (no binning by spin angle since backgrounds are isotropic) + count = len(filtered_de_ds["event_met"]) + background_counts.loc[ + dict( + epoch=pset_coords["epoch"].values[0], + calibration_prod=cal_prod, + background_index=background_idx, + ) + ] += count -def pset_backgrounds(pset_coords: dict[str, xr.DataArray]) -> dict[str, xr.DataArray]: + return background_counts + + +def pset_backgrounds( + pset_coords: dict[str, xr.DataArray], + background_config_df: pd.DataFrame, + l1b_de_dataset: xr.Dataset, + goodtimes_ds: xr.Dataset, + exposure_times: xr.DataArray, +) -> dict[str, xr.DataArray]: """ - Calculate pointing set backgrounds and background uncertainties. + Calculate pointing set backgrounds from direct events. + + Computes background counts internally by filtering and binning events + according to the background configuration, then calculates background + rates and uncertainties. Parameters ---------- pset_coords : dict[str, xarray.DataArray] The PSET coordinates from the xarray.Dataset. + background_config_df : pandas.DataFrame + Background configuration DataFrame with MultiIndex + (calibration_prod, background_index). + l1b_de_dataset : xarray.Dataset + The L1B dataset for the pointing being processed. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. + exposure_times : xarray.DataArray + Exposure times with dims (epoch, esa_energy_step, spin_angle_bin). Returns ------- dict[str, xarray.DataArray] - Dictionary containing background_rates and background_rates_unc DataArrays - to be added to the PSET dataset. + Dictionary containing background_rates and background_rates_uncertainty + DataArrays to be added to the PSET dataset. """ - # TODO: This is just a placeholder setting backgrounds to zero. The background - # algorithm will be determined in flight. attr_mgr = ImapCdfAttributes() attr_mgr.add_instrument_global_attrs("hi") attr_mgr.add_instrument_variable_attrs(instrument="hi", level=None) - return { + # Create output arrays + output_vars = { var_name: full_dataarray( var_name, attr_mgr.get_variable_attributes(f"hi_pset_{var_name}", check_schema=False), pset_coords, - fill_value=fill_val, ) - for var_name, fill_val in [ - ("background_rates", 0), - ("background_rates_uncertainty", 1), - ] + for var_name in ["background_rates", "background_rates_uncertainty"] } + # Get total exposure time + total_exposure_time = float(exposure_times.sum()) + + if total_exposure_time <= 0: + output_vars["background_rates"].values[:] = 0 + output_vars["background_rates_uncertainty"].values[:] = 0 + return output_vars + + # Compute background counts: shape (epoch, calibration_prod, background_index) + background_counts = _compute_background_counts( + pset_coords, background_config_df, l1b_de_dataset, goodtimes_ds + ) + + # Compute count rates: shape (epoch, calibration_prod, background_index) + count_rates = background_counts / total_exposure_time + + # Convert background config DataFrame to xarray Dataset + config_ds = background_config_df.to_xarray() + if not config_ds["calibration_prod"].equals(pset_coords["calibration_prod"]): + raise ValueError( + f"Calibration products in pset_coords and background_config_df " + f"do not match. pset_coords: {pset_coords['calibration_prod'].values}, " + f"background_config_df: {config_ds['calibration_prod'].values}" + ) + scaling_factors_da = config_ds["scaling_factor"] + uncertainties_da = config_ds["uncertainty"] + + # Compute scaled rates + scaled_rates = count_rates * scaling_factors_da + + # Compute uncertainties (Poisson + scaling factor, combined in quadrature) + poisson_unc = ( + np.sqrt(background_counts) / total_exposure_time + ) * scaling_factors_da + scaling_unc = count_rates * uncertainties_da + combined_unc = np.sqrt(poisson_unc**2 + scaling_unc**2) + + # Sum over background_index dimension to get final rates + total_rates = scaled_rates.sum(dim="background_index", skipna=True) + total_unc = np.sqrt((combined_unc**2).sum(dim="background_index", skipna=True)) + + # Broadcast to (epoch, esa_energy_step, calibration_prod, spin_angle_bin) + # Backgrounds are isotropic and independent of ESA step, so we + # broadcast across esa_energy_step and spin_angle_bin dimensions. + output_vars["background_rates"].values[:] = total_rates.values[ + :, np.newaxis, :, np.newaxis + ] + output_vars["background_rates_uncertainty"].values[:] = total_unc.values[ + :, np.newaxis, :, np.newaxis + ] + + return output_vars + def pset_exposure( - pset_coords: dict[str, xr.DataArray], l1b_de_dataset: xr.Dataset + pset_coords: dict[str, xr.DataArray], + l1b_de_dataset: xr.Dataset, + goodtimes_ds: xr.Dataset, ) -> dict[str, xr.DataArray]: """ Calculate PSET exposure time. @@ -516,6 +668,8 @@ def pset_exposure( The PSET coordinates from the xarray.Dataset. l1b_de_dataset : xarray.Dataset The L1B dataset for the pointing being processed. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags. Returns ------- @@ -552,15 +706,26 @@ def pset_exposure( # Clock tick MET times are accumulation "edges". To get the mean spin-phase # for a given clock tick, add 1/2 clock tick and compute spin-phase. - spin_phases = np.atleast_1d( - get_instrument_spin_phase( - clock_tick_mets + HiConstants.HALF_CLOCK_TICK_S, - SpiceFrame[f"IMAP_HI_{sensor_number}"], - ) - ) + mid_tick_mets = clock_tick_mets + HiConstants.HALF_CLOCK_TICK_S + + # Compute spacecraft spin phase first (used for goodtimes filtering) + spacecraft_spin_phase = np.atleast_1d(get_spacecraft_spin_phase(mid_tick_mets)) + + # Convert spacecraft spin phase to nominal_bins (0-89) for goodtimes lookup + nominal_bins = (spacecraft_spin_phase * 90).astype(np.int32) + nominal_bins = np.clip(nominal_bins, 0, 89) + + # Compute instrument spin phase from spacecraft spin phase + # This implementation is identical to spin.get_instrument_spin_phase and + # is replicated here to avoid querying the spin dataframe again. + instrument_frame = SpiceFrame[f"IMAP_HI_{sensor_number}"] + phase_offset = get_spacecraft_to_instrument_spin_phase_offset(instrument_frame) + spin_phases = (spacecraft_spin_phase + phase_offset) % 1.0 # Remove ticks not in good times/angles - good_mask = good_time_and_phase_mask(clock_tick_mets, spin_phases) + good_mask = good_time_and_phase_mask( + clock_tick_mets, nominal_bins, goodtimes_ds + ) spin_phases = spin_phases[good_mask] clock_tick_weights = clock_tick_weights[good_mask] @@ -687,7 +852,7 @@ def get_de_clock_ticks_for_esa_step( f"The CCSDS MET time {ccsds_met} " "is less than 8 spins from the loaded spin table data." ) - clock_tick_mets = np.arange( + clock_tick_mets: np.ndarray = np.arange( spin_start_mets[end_time_ind - 8], spin_start_mets[end_time_ind], HiConstants.DE_CLOCK_TICK_S, @@ -707,22 +872,40 @@ def get_de_clock_ticks_for_esa_step( def good_time_and_phase_mask( - tick_mets: np.ndarray, spin_phases: np.ndarray -) -> npt.NDArray: + mets: np.ndarray, + nominal_bins: np.ndarray, + goodtimes_ds: xr.Dataset, +) -> npt.NDArray[np.bool_]: """ - Filter out the clock tick times that are not in good times and angles. + Filter out times that are not in good times based on the goodtimes dataset. Parameters ---------- - tick_mets : np.ndarray - Clock-tick MET times. - spin_phases : np.ndarray - Spin phases for each clock tick. + mets : np.ndarray + MET times for each event or clock tick. + nominal_bins : np.ndarray + Spacecraft spin bins (0-89) for each event or clock tick. + goodtimes_ds : xarray.Dataset + Goodtimes dataset with cull_flags variable dimensioned (met, spin_bin). Returns ------- keep_mask : np.ndarray - Boolean mask indicating which clock ticks are in good times/phases. + Boolean mask indicating which events/ticks are in good times. """ - # TODO: Implement this once we have Goodtimes data product defined. - return np.full_like(tick_mets, True, dtype=bool) + gt_mets = goodtimes_ds["met"].values + cull_flags = goodtimes_ds["cull_flags"].values + + # Map each event/tick to the nearest goodtimes MET interval + # searchsorted with side='right' - 1 gives the largest MET <= query MET + met_indices = np.searchsorted(gt_mets, mets, side="right") - 1 + met_indices = np.clip(met_indices, 0, len(gt_mets) - 1) + + # Convert nominal_bins to int32 for indexing + spin_bins: npt.NDArray[np.int32] = nominal_bins.astype(np.int32) + + # Look up cull_flags for each event/tick + # Events are good if cull_flags == 0 + event_cull_flags = cull_flags[met_indices, spin_bins] + + return event_cull_flags == 0 diff --git a/imap_processing/hi/hi_l2.py b/imap_processing/hi/hi_l2.py index 03a5e39032..ce880445b2 100644 --- a/imap_processing/hi/hi_l2.py +++ b/imap_processing/hi/hi_l2.py @@ -593,25 +593,35 @@ def combine_maps(sky_maps: dict[str, RectangularSkyMap]) -> RectangularSkyMap: combined["counts"] = ram_ds["counts"] + anti_ds["counts"] combined["exposure_factor"] = ram_ds["exposure_factor"] + anti_ds["exposure_factor"] - # Inverse-variance weighted average for ena_intensity + # Compute weights for ram and anti-ram based on the inverse of the variance + # (uncertainty squared). Zero weights where the variance is zero or where + # ena_intensity is not finite. The latter prevents NaNs, which get replaced + # with zeros for the sum below, from contributing to the weighted average. + ram_valid_mask = np.logical_and( + ram_ds["ena_intensity_stat_uncert"] > 0, np.isfinite(ram_ds["ena_intensity"]) + ) weight_ram = xr.where( - ram_ds["ena_intensity_stat_uncert"] > 0, + ram_valid_mask, 1 / ram_ds["ena_intensity_stat_uncert"] ** 2, 0, ) + anti_valid_mask = np.logical_and( + anti_ds["ena_intensity_stat_uncert"] > 0, np.isfinite(anti_ds["ena_intensity"]) + ) weight_anti = xr.where( - anti_ds["ena_intensity_stat_uncert"] > 0, + anti_valid_mask, 1 / anti_ds["ena_intensity_stat_uncert"] ** 2, 0, ) total_weight = weight_ram + weight_anti with np.errstate(divide="ignore", invalid="ignore"): + # Inverse-variance weighted average for ena_intensity combined["ena_intensity"] = ( - ram_ds["ena_intensity"] * weight_ram - + anti_ds["ena_intensity"] * weight_anti + ram_ds["ena_intensity"].fillna(0) * weight_ram + + anti_ds["ena_intensity"].fillna(0) * weight_anti ) / total_weight - + # ena_intensity_stat_uncertainty is combined using inverse quadrature sum combined["ena_intensity_stat_uncert"] = np.sqrt(1 / total_weight) # Exposure-weighted average for systematic error diff --git a/imap_processing/hi/utils.py b/imap_processing/hi/utils.py index a8152ae228..e693ff98af 100644 --- a/imap_processing/hi/utils.py +++ b/imap_processing/hi/utils.py @@ -3,15 +3,17 @@ from __future__ import annotations import re -from collections.abc import Iterable, Sequence +from collections.abc import Generator, Iterable, Sequence from dataclasses import dataclass from enum import IntEnum from pathlib import Path +from typing import IO, Any import numpy as np import pandas as pd import xarray as xr from numpy import typing as npt +from numpy.typing import NDArray from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes @@ -311,7 +313,12 @@ class EsaEnergyStepLookupTable: def __init__(self) -> None: self.df = pd.DataFrame( - columns=["start_met", "end_met", "esa_step", "esa_energy_step"] + { + "start_met": pd.Series(dtype="float64"), + "end_met": pd.Series(dtype="float64"), + "esa_step": pd.Series(dtype="int64"), + "esa_energy_step": pd.Series(dtype="int64"), + } ) self._indexed = False @@ -449,10 +456,12 @@ def query( return results.astype(self._esa_energy_step_dtype) -@pd.api.extensions.register_dataframe_accessor("cal_prod_config") -class CalibrationProductConfig: +class _BaseConfigAccessor: """ - Register custom accessor for calibration product configuration DataFrames. + Base class for configuration DataFrame accessors. + + Provides common functionality for validating and processing configuration + DataFrames with coincidence types and TOF windows. Parameters ---------- @@ -460,19 +469,10 @@ class CalibrationProductConfig: Object to run validation and use accessor functions on. """ - index_columns = ( - "calibration_prod", - "esa_energy_step", - ) + # Subclasses must define these + index_columns: tuple[str, ...] + required_columns: tuple[str, ...] tof_detector_pairs = ("ab", "ac1", "bc1", "c1c2") - required_columns = ( - "coincidence_type_list", - *[ - f"tof_{det_pair}_{limit}" - for det_pair in tof_detector_pairs - for limit in ["low", "high"] - ], - ) def __init__(self, pandas_obj: pd.DataFrame) -> None: self._validate(pandas_obj) @@ -493,7 +493,7 @@ def _validate(self, df: pd.DataFrame) -> None: AttributeError : If the dataframe does not pass validation. """ for index_name in self.index_columns: - if index_name in df.index: + if index_name not in df.index.names: raise AttributeError( f"Required index {index_name} not present in dataframe." ) @@ -501,8 +501,6 @@ def _validate(self, df: pd.DataFrame) -> None: for col in self.required_columns: if col not in df.columns: raise AttributeError(f"Required column {col} not present in dataframe.") - # TODO: Verify that the same ESA energy steps exist in all unique calibration - # product numbers def _add_coincidence_values_column(self) -> None: """Generate and add the coincidence_type_values column to the dataframe.""" @@ -516,20 +514,57 @@ def _add_coincidence_values_column(self) -> None: axis=1, ) + @property + def calibration_product_numbers(self) -> npt.NDArray[np.int_]: + """ + Get the calibration product numbers from the current configuration. + + Returns + ------- + cal_prod_numbers : numpy.ndarray + Array of calibration product numbers from the configuration. + These are sorted in ascending order and can be arbitrary integers. + """ + return ( + self._obj.index.get_level_values("calibration_prod") + .unique() + .sort_values() + .values + ) + + +@pd.api.extensions.register_dataframe_accessor("cal_prod_config") +class CalibrationProductConfig(_BaseConfigAccessor): + """Register custom accessor for calibration product configuration DataFrames.""" + + index_columns = ( + "calibration_prod", + "esa_energy_step", + ) + required_columns = ( + "coincidence_type_list", + *[ + f"tof_{det_pair}_{limit}" + for det_pair in _BaseConfigAccessor.tof_detector_pairs + for limit in ["low", "high"] + ], + ) + @classmethod - def from_csv(cls, path: str | Path) -> pd.DataFrame: + def from_csv(cls, path: str | Path | IO[str]) -> pd.DataFrame: """ - Read configuration CSV file into a pandas.DataFrame. + Read calibration product configuration CSV file into a pandas.DataFrame. Parameters ---------- - path : str or pathlib.Path - Location of the Calibration Product configuration CSV file. + path : str or pathlib.Path or file-like object + Location of the calibration product configuration CSV file. Returns ------- dataframe : pandas.DataFrame - Validated calibration product configuration data frame. + Validated calibration product configuration DataFrame with + coincidence_type_values column added. """ df = pd.read_csv( path, @@ -537,7 +572,7 @@ def from_csv(cls, path: str | Path) -> pd.DataFrame: converters={"coincidence_type_list": lambda s: tuple(s.split("|"))}, comment="#", ) - # Force the _init_ method to run by using the namespace + # Trigger the accessor to run validation and add coincidence_type_values _ = df.cal_prod_config.number_of_products return df @@ -554,20 +589,368 @@ def number_of_products(self) -> int: """ return len(self._obj.index.unique(level="calibration_prod")) - @property - def calibration_product_numbers(self) -> npt.NDArray[np.int_]: + +@pd.api.extensions.register_dataframe_accessor("background_config") +class BackgroundConfig(_BaseConfigAccessor): + """Register custom accessor for background configuration DataFrames.""" + + index_columns = ( + "calibration_prod", + "background_index", + ) + required_columns = ( + "coincidence_type_list", + *[ + f"tof_{det_pair}_{limit}" + for det_pair in _BaseConfigAccessor.tof_detector_pairs + for limit in ["low", "high"] + ], + "scaling_factor", + "uncertainty", + ) + + @classmethod + def from_csv(cls, path: str | Path | IO[str]) -> pd.DataFrame: """ - Get the calibration product numbers from the current configuration. + Read background configuration CSV file into a pandas.DataFrame. + + Parameters + ---------- + path : str or pathlib.Path or file-like object + Location of the background configuration CSV file. Returns ------- - cal_prod_numbers : numpy.ndarray - Array of calibration product numbers from the configuration. - These are sorted in ascending order and can be arbitrary integers. + dataframe : pandas.DataFrame + Validated background configuration DataFrame with + coincidence_type_values column added. """ - return ( - self._obj.index.get_level_values("calibration_prod") - .unique() - .sort_values() - .values + df = pd.read_csv( + path, + index_col=cls.index_columns, + converters={"coincidence_type_list": lambda s: tuple(s.split("|"))}, + comment="#", ) + # Trigger the accessor to run validation and add coincidence_type_values + _ = df.background_config.calibration_product_numbers + return df + + +def get_tof_window_mask( + de_ds: xr.Dataset, + tof_windows: dict[str, tuple[float, float]], + tof_fill_vals: dict[str, float], +) -> NDArray[np.bool_]: + """ + Generate mask indicating which DEs pass TOF window checks. + + An event passes the TOF window check for a given detector pair if its TOF value + is within the (low, high) bounds OR equals the fill value (indicating the detector + pair was not hit). + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event Dataset with TOF variables (tof_ab, tof_ac1, tof_bc1, tof_c1c2). + tof_windows : dict[str, tuple[float, float]] + Dictionary mapping TOF field names to (low, high) tuples defining the + acceptable window for each TOF measurement. + tof_fill_vals : dict[str, float] + Fill values for each TOF field - events with fill values pass the check. + If not provided, fill value handling is disabled. + + Returns + ------- + mask : numpy.ndarray + Boolean mask where True = event passes all specified TOF window checks. + """ + # Start with all True mask + n_events = len(de_ds["event_met"]) if "event_met" in de_ds.dims else 0 + if n_events == 0: + return np.array([], dtype=bool) + + combined_mask: np.ndarray = np.ones(n_events, dtype=bool) + + for tof_field, (low, high) in tof_windows.items(): + tof_array = de_ds[tof_field].values + # TOF is in window if between low/high bounds OR equals fill value + in_window = (low <= tof_array) & (tof_array <= high) + in_window |= tof_array == tof_fill_vals[tof_field] + + combined_mask &= in_window + + return combined_mask + + +def filter_events_by_coincidence( + de_ds: xr.Dataset, + coincidence_types: Sequence[int], +) -> NDArray[np.bool_]: + """ + Filter events by coincidence type. + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event Dataset with coincidence_type variable. + coincidence_types : Sequence[int] + Sequence of coincidence type integers to match. + + Returns + ------- + mask : np.ndarray + Boolean mask where True = event's coincidence_type is in the provided list. + """ + if "coincidence_type" not in de_ds: + raise ValueError("Dataset must have 'coincidence_type' variable") + + coincidence_array = de_ds["coincidence_type"].values + return np.isin(coincidence_array, list(coincidence_types)) + + +def get_bin_range_with_wrap( + first_bin: int, last_bin: int, n_bins: int, extend_by: int +) -> np.ndarray: + """ + Get bin range with wraparound and optional extension. + + Computes a range of bin indices from first_bin to last_bin, optionally + extending by a padding amount on each side, with proper wraparound + handling for circular bin structures (e.g., spin bins). + + Parameters + ---------- + first_bin : int + First bin index in the range. + last_bin : int + Last bin index in the range (may be less than first_bin if wrapping). + n_bins : int + Total number of bins (bins are 0 to n_bins-1). + extend_by : int + Number of bins to add on each side of the range. + + Returns + ------- + bins : np.ndarray + Array of bin indices in the range, with wrapping handled. + """ + # Apply extension + bot = (first_bin - extend_by) % n_bins + top = (last_bin + extend_by) % n_bins + + # Check if we need to wrap + if top >= bot: + # No wrap needed + return np.arange(bot, top + 1) + else: + # Wrap around: bins from bot to n_bins-1, then 0 to top + return np.concatenate([np.arange(bot, n_bins), np.arange(0, top + 1)]) + + +def _build_tof_fill_vals(de_ds: xr.Dataset) -> dict[str, float]: + """ + Build TOF fill values dictionary from dataset attributes. + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event dataset with TOF variables containing FILLVAL attributes. + + Returns + ------- + dict[str, float] + Dictionary mapping TOF variable names to their fill values. + """ + tof_fill_vals = {} + for pair in CalibrationProductConfig.tof_detector_pairs: + tof_var = f"tof_{pair}" + tof_fill_vals[tof_var] = de_ds[tof_var].attrs.get("FILLVAL", np.nan) + return tof_fill_vals + + +def iter_qualified_events_by_config( + de_ds: xr.Dataset, + cal_product_config: pd.DataFrame, + esa_energy_steps: NDArray[np.int_], +) -> Generator[tuple[Any, Any, NDArray[np.bool_]], None, None]: + """ + Iterate over calibration config, yielding masks for qualified events. + + For each (esa_energy_step, calibration_prod) combination in the config, + yields a mask indicating which events qualify based on BOTH coincidence_type + AND TOF window checks. + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event dataset with coincidence_type and TOF variables. + TOF variables must have FILLVAL attribute for fill value handling. + cal_product_config : pandas.DataFrame + Config DataFrame with multi-index (calibration_prod, esa_energy_step). + Must have coincidence_type_values column and TOF window columns. + esa_energy_steps : np.ndarray + ESA energy step for each event in de_ds. + + Yields + ------ + esa_energy : Any + The ESA energy step value. + config_row : namedtuple + The config row from itertuples() containing calibration product settings. + qualified_mask : np.ndarray + Boolean mask where True = event qualifies for this (esa, cal_prod). + """ + n_events = len(de_ds["event_met"]) if "event_met" in de_ds.dims else 0 + + # Build TOF fill values from dataset attributes + tof_fill_vals = _build_tof_fill_vals(de_ds) + + for esa_energy, esa_df in cal_product_config.groupby(level="esa_energy_step"): + # Mask for events at this ESA energy step + esa_mask = esa_energy_steps == esa_energy if n_events > 0 else np.array([]) + + for config_row in esa_df.itertuples(): + if n_events == 0 or not np.any(esa_mask): + yield esa_energy, config_row, np.zeros(n_events, dtype=bool) + continue + + # Apply common filtering logic + filter_mask = _filter_events_by_config_row(de_ds, config_row, tof_fill_vals) + + yield esa_energy, config_row, esa_mask & filter_mask + + +def _filter_events_by_config_row( + de_ds: xr.Dataset, + config_row: Any, + tof_fill_vals: dict[str, float], +) -> NDArray[np.bool_]: + """ + Filter events by coincidence type and TOF windows for a single config row. + + Helper function to apply common filtering logic used by both + iter_qualified_events_by_config and iter_background_events_by_config. + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event dataset with coincidence_type and TOF variables. + config_row : namedtuple + Config row from DataFrame.itertuples() containing: + - coincidence_type_values: tuple of int coincidence types + - tof__low, tof__high: TOF window bounds + tof_fill_vals : dict[str, float] + Dictionary mapping TOF variable names to their fill values. + + Returns + ------- + filter_mask : numpy.ndarray + Boolean mask where True = event matches the filter criteria. + """ + # Check coincidence type + coin_mask = filter_events_by_coincidence(de_ds, config_row.coincidence_type_values) + + # Build TOF windows dict from config row + tof_windows = { + f"tof_{pair}": ( + getattr(config_row, f"tof_{pair}_low"), + getattr(config_row, f"tof_{pair}_high"), + ) + for pair in CalibrationProductConfig.tof_detector_pairs + } + + # Check TOF windows + tof_mask = get_tof_window_mask(de_ds, tof_windows, tof_fill_vals) + + return coin_mask & tof_mask + + +def iter_background_events_by_config( + de_ds: xr.Dataset, + background_config: pd.DataFrame, +) -> Generator[tuple[Any, xr.Dataset], None, None]: + """ + Iterate over background config, yielding filtered event datasets. + + For each (calibration_prod, background_index) combination in the config, + yields the filtered dataset containing only events that match BOTH + coincidence_type AND TOF window checks. + + Unlike iter_qualified_events_by_config, this does NOT filter by ESA energy + step, as background counts are accumulated across all ESA steps. + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event dataset with coincidence_type and TOF variables. + TOF variables must have FILLVAL attribute for fill value handling. + background_config : pandas.DataFrame + Config DataFrame with multi-index (calibration_prod, background_index). + Must have coincidence_type_values column and TOF window columns. + + Yields + ------ + config_row : namedtuple + The config row from itertuples() containing background settings. + filtered_ds : xarray.Dataset + Filtered dataset containing only events matching the criteria. + """ + n_events = len(de_ds["event_met"]) + + # Build TOF fill values from dataset attributes + tof_fill_vals = _build_tof_fill_vals(de_ds) + + for config_row in background_config.itertuples(): + if n_events == 0: + # Return empty dataset + yield config_row, de_ds.isel(event_met=slice(0, 0)) + continue + + # Apply common filtering logic + filter_mask = _filter_events_by_config_row(de_ds, config_row, tof_fill_vals) + + # Return filtered dataset (no ESA energy filtering) + filtered_ds = de_ds.isel(event_met=filter_mask) + yield config_row, filtered_ds + + +def compute_qualified_event_mask( + de_ds: xr.Dataset, + cal_product_config: pd.DataFrame, + esa_energy_steps: NDArray[np.int_], +) -> NDArray[np.bool_]: + """ + Compute mask of events qualifying for ANY calibration product. + + An event qualifies if it passes BOTH coincidence_type AND TOF window + checks for ANY (calibration_prod, esa_energy_step) combination in the + configuration. + + Parameters + ---------- + de_ds : xarray.Dataset + Direct Event dataset with coincidence_type and TOF variables. + TOF variables must have FILLVAL attribute for fill value handling. + cal_product_config : pandas.DataFrame + Config DataFrame with multi-index (calibration_prod, esa_energy_step). + Must have coincidence_type_values column and TOF window columns. + esa_energy_steps : np.ndarray + ESA energy step for each event in de_ds. + + Returns + ------- + qualified_mask : np.ndarray + Boolean mask - True if event qualifies for at least one cal product. + """ + n_events = len(de_ds["event_met"]) + if n_events == 0: + return np.array([], dtype=bool) + + qualified_mask: np.ndarray = np.zeros(n_events, dtype=bool) + + for _, _, mask in iter_qualified_events_by_config( + de_ds, cal_product_config, esa_energy_steps + ): + qualified_mask |= mask + + return qualified_mask diff --git a/imap_processing/hit/hit_utils.py b/imap_processing/hit/hit_utils.py index af131f53cc..55f8b92a7b 100644 --- a/imap_processing/hit/hit_utils.py +++ b/imap_processing/hit/hit_utils.py @@ -368,7 +368,7 @@ def add_energy_variables( """ updated_ds = dataset.copy() - energy_mean = np.round( + energy_mean: np.ndarray = np.round( np.mean(np.array([energy_min_values, energy_max_values]), axis=0), 3 ).astype(np.float32) @@ -424,8 +424,8 @@ def add_summed_particle_data_to_dataset( ) # Initialize arrays for energy values - energy_min = np.zeros(len(energy_ranges), dtype=np.float32) - energy_max = np.zeros(len(energy_ranges), dtype=np.float32) + energy_min: np.ndarray = np.zeros(len(energy_ranges), dtype=np.float32) + energy_max: np.ndarray = np.zeros(len(energy_ranges), dtype=np.float32) # Compute summed data and update the dataset for i, energy_range_dict in enumerate(energy_ranges): diff --git a/imap_processing/hit/l0/decom_hit.py b/imap_processing/hit/l0/decom_hit.py index 88fee0319d..357765c08a 100644 --- a/imap_processing/hit/l0/decom_hit.py +++ b/imap_processing/hit/l0/decom_hit.py @@ -286,9 +286,9 @@ def assemble_science_frames(sci_dataset: xr.Dataset) -> xr.Dataset: f"{starting_indices[0]} packets at start of file belong to science frame " f"from previous day's ccsds file" ) - last_index_of_last_frame = starting_indices[-1] + FRAME_SIZE + last_index_of_last_frame = int(starting_indices[-1]) + FRAME_SIZE if last_index_of_last_frame: - remaining_packets = total_packets - last_index_of_last_frame + remaining_packets = int(total_packets) - last_index_of_last_frame if 0 < remaining_packets < FRAME_SIZE: print( f"{remaining_packets} packets at end of file belong to science frame " diff --git a/imap_processing/hit/l1a/hit_l1a.py b/imap_processing/hit/l1a/hit_l1a.py index 3ac84b4f52..6d6a88f781 100644 --- a/imap_processing/hit/l1a/hit_l1a.py +++ b/imap_processing/hit/l1a/hit_l1a.py @@ -150,7 +150,7 @@ def subcom_sectorates(sci_dataset: xr.Dataset) -> xr.Dataset: # Update counts for science frames where data is available for i, mod_10 in enumerate(hdr_min_count_mod_10): - data_by_species_and_energy_range[mod_10]["counts"][i] = updated_dataset[ + data_by_species_and_energy_range[mod_10]["counts"][i] = updated_dataset[ # type: ignore[index] "sectorates" ].values[i] @@ -427,7 +427,7 @@ def subset_sectored_counts( ) complete_sectored_counts_dataset = sectored_counts_dataset.isel(epoch=data_indices) - epoch_per_complete_set = np.repeat( + epoch_per_complete_set: np.ndarray = np.repeat( [ complete_sectored_counts_dataset.epoch[idx : idx + bin_size].mean().item() for idx in range(0, len(complete_sectored_counts_dataset.epoch), 10) diff --git a/imap_processing/hit/l1b/hit_l1b.py b/imap_processing/hit/l1b/hit_l1b.py index 5a2e697918..cd6344ce2a 100644 --- a/imap_processing/hit/l1b/hit_l1b.py +++ b/imap_processing/hit/l1b/hit_l1b.py @@ -300,7 +300,7 @@ def sum_livetime_10min(livetime: xr.DataArray) -> xr.DataArray: livetime_10min_sum = [ livetime[i : i + 10].sum().item() for i in range(0, len(livetime) - 9, 10) ] - livetime_expanded = np.repeat(livetime_10min_sum, 10) + livetime_expanded: np.ndarray = np.repeat(livetime_10min_sum, 10) return xr.DataArray(livetime_expanded, dims=livetime.dims, coords=livetime.coords) diff --git a/imap_processing/ialirt/calculate_ingest.py b/imap_processing/ialirt/calculate_ingest.py index 94671dfab5..9eb1f7d8b0 100644 --- a/imap_processing/ialirt/calculate_ingest.py +++ b/imap_processing/ialirt/calculate_ingest.py @@ -6,7 +6,7 @@ logger = logging.getLogger(__name__) -STATIONS = ["Kiel"] +STATIONS = ["Kiel", "UKSA"] def packets_created(start_file_creation: datetime, lines: list) -> dict: @@ -123,6 +123,7 @@ def format_ingest_data(last_filename: str, log_lines: list) -> dict: start_of_time.isoformat(), end_of_time.isoformat(), ], # Overall time range of the data + "stations": list(STATIONS), **station_dict, } diff --git a/imap_processing/ialirt/generate_coverage.py b/imap_processing/ialirt/generate_coverage.py index 4233e4c22d..99bbc699d9 100644 --- a/imap_processing/ialirt/generate_coverage.py +++ b/imap_processing/ialirt/generate_coverage.py @@ -1,10 +1,8 @@ """Coverage time for each station.""" import logging -from pathlib import Path import numpy as np -import pandas as pd from imap_processing.ialirt.constants import STATIONS, StationProperties from imap_processing.ialirt.process_ephemeris import calculate_azimuth_and_elevation @@ -27,75 +25,10 @@ "DSS-56", "DSS-74", "DSS-75", + "DSS-43", ] -def parse_uksa_schedule_xlsx(xlsx_path: Path) -> list[tuple[str, str]]: - """ - Parse the UKSA (GHY-6) availability sheet and return a list of contacts. - - Parameters - ---------- - xlsx_path : Path - Path to the UKSA (GHY-6) availability sheet. - - Returns - ------- - contacts : list[tuple[str, str]] - Available contacts for UKSA (GHY-6) availability sheet. - """ - data = pd.read_excel(xlsx_path) - - # Import start and stop times. - start_dt = ( - data["Date"] - + pd.to_timedelta( - data["GHY-6 Start Availability Times (5degrees) (UTC)"].astype(str) - ) - ).to_numpy("datetime64[s]") - - stop_dt = ( - data["Date"] - + pd.to_timedelta( - data["GHY-6 Stop Availability Times (5degrees) (UTC)"].astype(str) - ) - ).to_numpy("datetime64[s]") - - # Indicates whether or not setup or teardown should be taken from contact window. - notes = data["Short due to existing booking "].fillna("") - - truncate_setup = ( - notes.eq("Yes- setup needs to be included with the window") - | notes.eq("Yes- setup and teardown needs to be included with the window") - ).to_numpy() - - truncate_teardown = ( - notes.eq("Yes- tear down needs to be included within the window") - | notes.eq("Yes- setup and teardown needs to be included with the window") - ).to_numpy() - - setup_time = data["Setup time"].iloc[0] - teardown_time = data["Tear down time"].iloc[0] - - setup_seconds = setup_time.hour * 3600 + setup_time.minute * 60 + setup_time.second - teardown_seconds = ( - teardown_time.hour * 3600 + teardown_time.minute * 60 + teardown_time.second - ) - - setup_delta = np.timedelta64(setup_seconds, "s") - teardown_delta = np.timedelta64(teardown_seconds, "s") - - # Apply adjustments - start_dt[truncate_setup] += setup_delta - stop_dt[truncate_teardown] -= teardown_delta - - # Format to strings with ms, append Z - start_str = np.datetime_as_string(start_dt, unit="ms") - stop_str = np.datetime_as_string(stop_dt, unit="ms") - - return list(zip(start_str, stop_str, strict=False)) - - def create_schedule_mask( station: StationProperties, time_range: np.ndarray ) -> np.ndarray: @@ -185,10 +118,10 @@ def generate_coverage( # noqa: PLR0912 stop_et_input = start_et_input + duration_seconds time_range = np.arange(start_et_input, stop_et_input, time_step) - total_visible_mask = np.zeros(time_range.shape, dtype=bool) + total_visible_mask: np.ndarray = np.zeros(time_range.shape, dtype=bool) # Precompute DSN outage mask for non-DSN stations - dsn_outage_mask = np.zeros(time_range.shape, dtype=bool) + dsn_outage_mask: np.ndarray = np.zeros(time_range.shape, dtype=bool) if dsn: for dsn_contacts in dsn.values(): for start, end in dsn_contacts: @@ -209,7 +142,7 @@ def generate_coverage( # noqa: PLR0912 schedule_mask = create_schedule_mask(station, time_range) visible &= schedule_mask - outage_mask = np.zeros(time_range.shape, dtype=bool) + outage_mask: np.ndarray = np.zeros(time_range.shape, dtype=bool) if outages and station_name in outages: for start, end in outages[station_name]: start_et = str_to_et(start) @@ -229,7 +162,7 @@ def generate_coverage( # noqa: PLR0912 # --- DSN Stations --- if dsn: for dsn_station, contacts in dsn.items(): - dsn_visible_mask = np.zeros(time_range.shape, dtype=bool) + dsn_visible_mask: np.ndarray = np.zeros(time_range.shape, dtype=bool) for start, end in contacts: start_et = str_to_et(start) end_et = str_to_et(end) @@ -253,7 +186,7 @@ def generate_coverage( # noqa: PLR0912 time_range[outage_mask], format_str="ISOC" ) if uksa: - uksa_visible_mask = np.zeros(time_range.shape, dtype=bool) + uksa_visible_mask: np.ndarray = np.zeros(time_range.shape, dtype=bool) for start, end in uksa: start_et = str_to_et(start) end_et = str_to_et(end) diff --git a/imap_processing/ialirt/l0/process_hit.py b/imap_processing/ialirt/l0/process_hit.py index 52f219c69f..c76e43374e 100644 --- a/imap_processing/ialirt/l0/process_hit.py +++ b/imap_processing/ialirt/l0/process_hit.py @@ -11,7 +11,7 @@ find_groups, ) from imap_processing.ialirt.utils.time import calculate_time -from imap_processing.spice.time import met_to_ttj2000ns, met_to_utc +from imap_processing.spice.time import met_to_ttj2000ns logger = logging.getLogger(__name__) @@ -132,6 +132,7 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]: """ hit_data = [] incomplete_groups = [] + status_groups = [] # Subsecond time conversion specified in 7516-9054 GSW-FSW ICD. # Value of SCLK subseconds, unsigned, (LSB = 1/256 sec) @@ -151,12 +152,7 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]: ] if np.any(status_values == 0): - logger.info( - f"Off-nominal value detected at " - f"missing or duplicate pkt_counter values: " - f"{group}" - ) - continue + status_groups.append(group) # Subcom values for the group should be 0-59 with no duplicates. subcom_values = grouped_data["hit_subcom"][ @@ -172,14 +168,6 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]: grouped_data["hit_met"][(grouped_data["group"] == group).values].values[0] ) - status_values = grouped_data["hit_status"][ - (grouped_data["group"] == group).values - ] - - if np.any(status_values == 0): - logger.info(f"Off-nominal value detected at {met_to_utc(hit_met)}") - continue - fast_rate_1 = grouped_data["hit_fast_rate_1"][ (grouped_data["group"] == group).values ] @@ -227,5 +215,9 @@ def process_hit(xarray_data: xr.Dataset) -> list[dict]: f"missing or duplicate pkt_counter values: " f"{incomplete_groups}" ) + if status_groups: + logger.warning( + f"The following hit groups have zero status values: {status_groups}" + ) return hit_data diff --git a/imap_processing/ialirt/l0/process_status.py b/imap_processing/ialirt/l0/process_status.py new file mode 100644 index 0000000000..f32007c16e --- /dev/null +++ b/imap_processing/ialirt/l0/process_status.py @@ -0,0 +1,77 @@ +"""Functions to support status processing.""" + +import logging + +import xarray as xr + +from imap_processing.ialirt.utils.grouping import ( + _populate_instrument_header_items, +) +from imap_processing.ialirt.utils.time import calculate_time +from imap_processing.spice.time import met_to_ttj2000ns + +logger = logging.getLogger(__name__) + + +def process_status(xarray_data: xr.Dataset) -> list[dict]: + """ + Create L1 data dictionary. + + Parameters + ---------- + xarray_data : xr.Dataset + Parsed data. + + Returns + ------- + status_data : list[dict] + Dictionary final data product. + """ + status_data = [] + + # Subsecond time conversion specified in 7516-9054 GSW-FSW ICD. + # Value of SCLK subseconds, unsigned, (LSB = 1/256 sec) + met = calculate_time( + xarray_data["sc_sclk_sec"], xarray_data["sc_sclk_sub_sec"], 256 + ) + + # Add required parameters. + xarray_data["met"] = met + + sc_swapi_status = xarray_data["sc_swapi_status"] + sc_mag_status = xarray_data["sc_mag_status"] + sc_hit_status = xarray_data["sc_hit_status"] + sc_codice_status = xarray_data["sc_codice_status"] + sc_lo_status = xarray_data["sc_lo_status"] + sc_hi_45_status = xarray_data["sc_hi_45_status"] + sc_hi_90_status = xarray_data["sc_hi_90_status"] + sc_ultra_45_status = xarray_data["sc_ultra_45_status"] + sc_ultra_90_status = xarray_data["sc_ultra_90_status"] + sc_swe_status = xarray_data["sc_swe_status"] + sc_idex_status = xarray_data["sc_idex_status"] + sc_glows_status = xarray_data["sc_glows_status"] + sc_autonomy_status = xarray_data["sc_autonomy"] + + for i in range(len(xarray_data["met"])): + status_data.append( + _populate_instrument_header_items(met) + | { + "instrument": "spacecraft_status", + "status_epoch": int(met_to_ttj2000ns(met[i])), + "sc_swapi_status": int(sc_swapi_status[i]), + "sc_mag_status": int(sc_mag_status[i]), + "sc_hit_status": int(sc_hit_status[i]), + "sc_codice_status": int(sc_codice_status[i]), + "sc_lo_status": int(sc_lo_status[i]), + "sc_hi_45_status": int(sc_hi_45_status[i]), + "sc_hi_90_status": int(sc_hi_90_status[i]), + "sc_ultra_45_status": int(sc_ultra_45_status[i]), + "sc_ultra_90_status": int(sc_ultra_90_status[i]), + "sc_swe_status": int(sc_swe_status[i]), + "sc_idex_status": int(sc_idex_status[i]), + "sc_glows_status": int(sc_glows_status[i]), + "sc_autonomy_status": int(sc_autonomy_status[i]), + } + ) + + return status_data diff --git a/imap_processing/ialirt/l0/process_swapi.py b/imap_processing/ialirt/l0/process_swapi.py index 79812ed1d4..1be0277d38 100644 --- a/imap_processing/ialirt/l0/process_swapi.py +++ b/imap_processing/ialirt/l0/process_swapi.py @@ -112,9 +112,9 @@ def optimize_pseudo_parameters( try: five_point_range = range(max_index - 2, max_index + 2 + 1) - xdata = energy_passbands.take(five_point_range, mode="clip") - ydata = count_rates.take(five_point_range, mode="clip") - sigma = count_rate_error.take(five_point_range, mode="clip") + xdata: np.ndarray = energy_passbands.take(five_point_range, mode="clip") + ydata: np.ndarray = count_rates.take(five_point_range, mode="clip") + sigma: np.ndarray = count_rate_error.take(five_point_range, mode="clip") curve_fit_output = curve_fit( f=count_rate, xdata=xdata, @@ -128,7 +128,7 @@ def optimize_pseudo_parameters( covariance_matrix_is_finite = np.all(np.isfinite(curve_fit_output[1])) # fit has failed if R^2 < 0.7 - yfit = count_rate(xdata, *curve_fit_output[0]) + yfit = count_rate(xdata, *curve_fit_output[0]) # type: ignore[arg-type] r2 = 1 - np.sum((ydata - yfit) ** 2) / np.sum((ydata - ydata.mean()) ** 2) r2_is_acceptable = r2 >= 0.7 diff --git a/imap_processing/ialirt/l0/process_swe.py b/imap_processing/ialirt/l0/process_swe.py index 8d2adc680b..3ae639ae5e 100644 --- a/imap_processing/ialirt/l0/process_swe.py +++ b/imap_processing/ialirt/l0/process_swe.py @@ -104,7 +104,7 @@ def prepare_raw_counts(grouped: xr.Dataset, cem_number: int = N_CEMS) -> NDArray - 7 corresponds to the 7 CEM detectors. - 30 corresponds to the 30 phi bins. """ - raw_counts = np.zeros((8, cem_number, 30), dtype=np.uint8) + raw_counts: np.ndarray = np.zeros((8, cem_number, 30), dtype=np.uint8) # Compute phi values and their corresponding bins # Example: energy steps 0-1 have the same phi; diff --git a/imap_processing/ialirt/utils/create_xarray.py b/imap_processing/ialirt/utils/create_xarray.py index 2abc5f4e78..f5a2fa1e56 100644 --- a/imap_processing/ialirt/utils/create_xarray.py +++ b/imap_processing/ialirt/utils/create_xarray.py @@ -284,7 +284,7 @@ def create_xarray_from_records(records: list[dict]) -> xr.Dataset: # noqa: PLR0 shape = [dataset.dims[d] for d in dims] - data = np.full(shape, fill, dtype=dtype) + data: np.ndarray = np.full(shape, fill, dtype=dtype) dataset[key] = xr.DataArray(data, dims=dims, attrs=attrs) for i, record in enumerate(by_inst.get("mag", [])): diff --git a/imap_processing/idex/evt_msg_decode_utils.py b/imap_processing/idex/evt_msg_decode_utils.py new file mode 100644 index 0000000000..ec3dff8828 --- /dev/null +++ b/imap_processing/idex/evt_msg_decode_utils.py @@ -0,0 +1,82 @@ +"""Helper functions for decoding event messages.""" + +import re + +# Regex to match spaces where we need to embed params in the event message templates, +# e.g. {p0}, {p1+2}, {p3:dict}, {p4+1|dict} +EMBEDDED_PARAM_RE = re.compile(r"\{p(\d+)(?:\+(\d+))?(?::[\w]+)?(?:\|(\w+))?\}") + + +def render_event_template( + event_description_template: str, params_bytes: list, msg_json_data: dict +) -> str: + """ + Produce an event message string by replacing placeholders with parameter values. + + Example template: + "Event {p0} occurred with value {p1+2|dictName}" + + This would replace {p0} with the hex value of params[0], and replace {p1+2:dictName} + with the combined hex value of params[1] and params[2] (treated as big-endian bytes) + looked up in the dictionary named "dictName" for a human-readable string, or + rendered as hex if not found in the dictionary. + + Parameters + ---------- + event_description_template : str + The event message template containing placeholders like {p0}, {p1+2}, + {p3:dict}, {p4+1|dict}. + params_bytes : list + The list of parameter values to substitute into the template. + msg_json_data : dict + Mapping of parameter values to human-readable strings for decoding event + messages. + + Returns + ------- + str + The rendered event message with placeholders replaced by parameter values. + """ + + def replace(m: re.Match[str]) -> str: + """ + Replace a single placeholder match with the corresponding parameter value. + + Parameters + ---------- + m : re.Match[str] + A regex match object for a placeholder in the template. + + Returns + ------- + str + The string to replace the placeholder with. + """ + # We are parsing the placeholder value here to determine which parameter + # to substitute and how to format them. + # group one is the parameter index e.g. p0-3 + idx = int(m.group(1)) + # group two is the optional byte length e.g. +2 (so we know how many params + # to combine) + n_bytes = int(m.group(2)) if m.group(2) else 1 + # group three is an optional dictionary name to use for decoding this parameter + dict_name = m.group(3) + value = 0 + for i in range(n_bytes): + # combine the next n_bytes params into a single integer value, treating + # them as big-endian bytes + value = (value << 8) | ( + params_bytes[idx + i] if idx + i < len(params_bytes) else 0 + ) + # If a dictionary name is provided use it to decode the value, otherwise just + # render as hex. + if dict_name: + resolved = msg_json_data.get(dict_name, {}).get( + value, f"0x{value:0{2 * n_bytes}X}" + ) + return f"{dict_name}({resolved})" # wrap with dict name + + return f"{value:02x}" + + # Replace all placeholders in the template using the replace function defined above. + return EMBEDDED_PARAM_RE.sub(replace, event_description_template) diff --git a/imap_processing/idex/idex_constants.py b/imap_processing/idex/idex_constants.py index 0f6d0e8f1f..176971aea4 100644 --- a/imap_processing/idex/idex_constants.py +++ b/imap_processing/idex/idex_constants.py @@ -88,11 +88,3 @@ class ConversionFactors(float, Enum): # Define the pointing reference frame for IDEX IDEX_EVENT_REFERENCE_FRAME = SpiceFrame.ECLIPJ2000 - - -class IDEXEvtAcquireCodes(IntEnum): - """Create ENUM for event message ints that signify science acquire events.""" - - ACQSETUP = 2 - ACQ = 3 - CHILL = 5 diff --git a/imap_processing/idex/idex_evt_msg_parsing_dictionaries.json b/imap_processing/idex/idex_evt_msg_parsing_dictionaries.json new file mode 100644 index 0000000000..be721d716e --- /dev/null +++ b/imap_processing/idex/idex_evt_msg_parsing_dictionaries.json @@ -0,0 +1,1377 @@ +{ + "logEntryIdDictionary": { + "0": "EMPTY", + "8": "PMLOG_INIT", + "10": "TASK_START", + "11": "SLICE_ERR", + "12": "PERF_PARAM_ERR", + "13": "PERF_DURR_ERR", + "14": "TASK_OVERRUN", + "15": "TASK_DONE", + "17": "EXEC_CMD", + "18": "GOT_RX_DATA", + "19": "GOOD_CCSDS", + "20": "EXEC_TIME", + "21": "DUAL_EXPIRED", + "30": "START_TX_PKT", + "32": "RX_CKSUM_ERR", + "36": "CCSDS_RX_ERR", + "37": "CMD_LEN_INVLD", + "38": "CMD_UNKNOWN", + "39": "CMD_RJCT_UNARM", + "40": "CMD_RJCT_ENGINE", + "41": "CMD_RJCT_ARGNUM", + "42": "CMD_RJCT_NULL", + "43": "CMD_RJCT_RANGE", + "44": "CMD_RJCT_MODE", + "45": "CMD_RJCT_BUSY", + "46": "CMD_NO_CMD_YET", + "48": "QBOOT_ST_CHANGE", + "49": "QBOOT_HALTED", + "50": "QBOOT_ALRDY_IDLE", + "51": "QBOOT_MEM_BUSY", + "58": "REGION_OFF_DIE", + "59": "INIT_FLASH_FOUND", + "60": "LOAD_OFF_DIE", + "61": "MEMINIT_OBJ_MISS", + "63": "INIT_CKSUM_MISMT", + "64": "UERR_SCI_DATA", + "65": "LISTED_CATALOGS", + "66": "MAKE_CATALOG_ERR", + "67": "SCI_DATA_DROPPED", + "68": "FEWER_SCI_BLOCKS", + "69": "COPYTO_NO_HALT", + "70": "AUTOSAVE_ERR", + "71": "RECOVR_MODE_RST", + "72": "RECOVR_OPER_RST", + "73": "ANALYZE_ERROR", + "74": "BOTH_MIRROR_BAD", + "75": "FOUND_SCI_DATA", + "76": "MEM_QUEUE_RJCT", + "77": "MEM_COLLAB_START", + "78": "MEM_COLLAB_SUCC", + "79": "MEM_COLLAB_FAIL", + "80": "REBUILD_SUMMARY", + "81": "CKSUM_MISMATCH", + "82": "MISMATCH_REBUILT", + "83": "FLASH_DIE_UNUSED", + "84": "NO_CHANGE_NVFSW3", + "85": "FLASH_DRIVER_ERR", + "86": "OPERATION_FAILED", + "87": "FOUND_BAD_BLOCK", + "88": "FLASH_UERR", + "89": "FLASH_CERR", + "90": "BAD_MEM_CLEANUP", + "91": "FLASH_DRIVER_ERR", + "92": "MEM_OBJ_MISSING", + "93": "BAD_MODE_EXIT", + "94": "BAD_OPER_EXIT", + "95": "SHUFFLE_ERROR", + "96": "MEM_OP_QUEUED", + "97": "MEM_FLASH_FOUND", + "98": "MEM_OP_HALTED", + "99": "MEM_OP_START", + "100": "MEM_OP_DONE", + "101": "MEM_OP_WORKING", + "102": "MEM_OP_SETUP", + "103": "MEM_OP_RUNNING", + "104": "DIR_RD_DONE", + "105": "DIR_WR_DONE", + "106": "MEM_STATE_CHG", + "107": "MEM_ALRDY_IDLE", + "108": "LOAD_SPILLOVER", + "109": "MEM_GOT_PARAM", + "110": "CKSUM_MATCH", + "111": "SHUFFLE_STATE", + "112": "SHUFFLE_DONE", + "113": "FAIL_ADD_FETCH", + "114": "POP_FETCH_TBL", + "115": "RECONCILED_FT", + "116": "DATASET_FOUND", + "117": "DATASET_NOTFOUND", + "128": "IBRAM_WDOG", + "129": "DBRAM_WDOG", + "130": "SRAM_WDOG", + "131": "EXCEPTION_WDOG", + "132": "PROC_RST_WDOG", + "133": "DECON_AUTO_DIS", + "135": "HV_CURRENT_FAULT", + "136": "HV_STATE_CHANGE", + "137": "HV_STATE_TOF_ON", + "138": "HV_VOTING_FAILED", + "139": "HV_MMR_MISMATCH", + "140": "HV_BAD_STPT_MAX", + "141": "ANA_HK_FAULTLO", + "142": "ANA_HK_FAULTHI", + "143": "HV_OSCIL_FAULT", + "144": "BOOT2OPER", + "145": "POR_RST", + "146": "CMD_RST", + "147": "BOOT_HELLO", + "148": "OPER_HELLO", + "152": "MODE_CHANGE", + "153": "TOOK_DWELL_DATA", + "154": "DWELL_COMPLETE", + "155": "STIM_COMPLETE", + "159": "HAPPY_GOODBYE", + "160": "DUBIOUS_GOODBYE", + "161": "UPK_PROC_RST", + "162": "UPK_WDOG_RST", + "163": "UPK_UNKWN_RST", + "164": "UPK_BOOT_ERR", + "192": "SCI_STATE", + "193": "MEM_COLLAB_DONE", + "194": "CATALOG_MATCH", + "196": "SCI_DONE", + "197": "SCI_PEAK_CNT", + "198": "SCI_EVENT_ID", + "199": "SCI_CAT_FULL", + "200": "SCI_PROCESS_EVT", + "203": "SCI_TRANSMIT", + "204": "SCI_DATA", + "206": "SCI_SNDEVT_AID", + "207": "SCI_SPI_READ", + "208": "SCI_CLK_FAIL", + "209": "SCI_TASK_STATE", + "211": "SCI_PARSE_SIZE", + "212": "SCI_PARSE_HDR", + "213": "SCI_PARSE_END", + "214": "SCI_PARSE_EVT", + "215": "SCI_SND_ST_ERR", + "216": "SCI_SND_CH_ERR", + "217": "SCI_PWR_ERR", + "219": "ALLOC_EXHAUSTED", + "220": "SCI_AID_PREV_USE", + "221": "PROCESS_FIRST", + "224": "SEQ_STATE_CHANGE", + "225": "SEQ_RESUME", + "226": "SEQ_TERM_AT_IDLE", + "227": "SEQ_PAUSE_IDLE", + "228": "SEQ_PAUSE_STALE", + "229": "SEQ_BUF_VERIFIED", + "230": "SEQ_ABRT_UNCLEAN", + "232": "SEQ_ERROR_STOP", + "233": "SUBSEQ_CALL_SUB", + "234": "SEQ_BAD_STATE", + "235": "SEQ_ERR_NO_SEQ", + "236": "SEQ_RECURSE_CALL", + "237": "SEQ_BUF_NOT_LOAD", + "238": "SEQ_BAD_CKSUM", + "239": "SEQ_CMD_SUCC", + "240": "FAULT_EXCESS", + "241": "FAULT_RESP_STRT", + "242": "FAULT_INTERRUPT", + "243": "FAULT_FAIL", + "244": "FAULT_IGNORED", + "245": "MODE_RESP_STRT", + "246": "MODE_RESP_FAIL", + "247": "MODE_RESP_BUSY", + "248": "SUSPEND_DONE", + "249": "ALREADY_ERASED", + "255": "ENTRY_ID_INVAL" + }, + "eventMsgDictionary": { + "8": "DES postmortem log initialized", + "10": "DES task execution started", + "11": "DES INVALID DES SLICE, INVALID ID=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "12": "DES PERFMON ASKED TO ADD INVALID ENTRY, TASK ID=0x{p0:02x}{p1:02x}, SLICE=0x{p2:02x}{p3:02x}", + "13": "DES PERFMON SUSPICIOUS DURATION, TASK=0x{p0:02x}{p1:02x}, SLICE=0x{p2:02x}{p3:02x}", + "14": "DES TASK OVERRUN, START:END SLICE=0x{p0:02x}:{p1:02x}, TASK ID={p2:02x}", + "15": "DES task execution completed", + "17": "CMD success (apid=0x{p0:02x}{p1:02x}, {p2+2|opCodeLCDictionary})", + "18": "CMD received itf, byte length=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "19": "CMD rx itf and ccsds pkt good, vc=0x{p0:02x}{p1:02x}, byte length=0x{p2:02x}{p3:02x}", + "20": "CMD processed spacecraft time message", + "21": "CMD engine arm state expired", + "30": "TLM started tlm pkt tx", + "32": "CMD CRC MISMATCH (DISCARDING), P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "36": "CMD ERROR WITH CCSDS HDR IN RX PKT (DISCARDING), CCSDS1=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "37": "CMD OUT OF BOUNDS LENGTH, hdr field=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "38": "CMD UNKNOWN, ENG={p0+2|cmdEngineDictionary}, OPCODE=0x{p2:02x}{p3:02x}", + "39": "CMD RJCT UNARM, ENG={p0+1|cmdEngineDictionary}, {p1+1|idexModeDictionary}, {p2+2|opCodeDictionary}", + "40": "CMD RJCT ENGINE, ENG={p0+1|cmdEngineDictionary}, {p1+1|idexModeDictionary}, {p2+2|opCodeDictionary}", + "41": "CMD RJCT ARG#, ENG={p0+1|cmdEngineDictionary}, {p1+1|idexModeDictionary}, {p2+2|opCodeDictionary}", + "42": "CMD RJCT NULL, ENG={p0+1|cmdEngineDictionary}, {p1+1|idexModeDictionary}, {p2+2|opCodeDictionary}", + "43": "CMD RJCT RANGE, ENG={p0+1|cmdEngineDictionary}, {p1+1|idexModeDictionary}, {p2+2|opCodeDictionary}", + "44": "CMD RJCT MODE, ENG={p0+1|cmdEngineDictionary}, {p1+1|idexModeDictionary}, {p2+2|opCodeDictionary}", + "45": "CMD RJCT BUSY, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "46": "CMD REDUCED VALIDATION CRITERIA, CCSDS=0x{p0:02x}{p1:02x}, CRC=0x{p2:02x}{p3:02x}", + "48": "QBT state change: {p0+2|qbootStateDictionary}==>{p2+2|qbootStateDictionary}", + "49": "QBT quickboot halted", + "50": "QBT quickboot halted, but already idle", + "51": "QBT unexpected memory busy state while quickbooting", + "58": "QBT FT REPORTS OBJECT IN OFF DIE, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "59": "QBT loaded {p0+2|regionLCDictionary} from block 0x{p2:02x}{p3:02x}", + "60": "QBT LOADED {p0+2|regionDictionary} FROM OFF DIE, BLOCK=0x{p2:02x}{p3:02x}", + "61": "QBT OBJECT MISSING INIT! PARAM={p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "63": "MEM CKSUM MISMATCH FOUND DURING INIT, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "64": "MEM UNCORRECTABLE EDAC ERR IN SCI DATA, BLOCK=0x{p0:02x}{p1:02x} PAGE=0x{p2:02x}{p3:02x}", + "65": "MEM listed catalogs done, catalog count=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "66": "MEM ERROR CREATING OR SAVING CATALOG, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "67": "MEM SCI DATA FILLED RESERVED MEMORY, REST DROPPED, ErrStatReg=0x{p0:02x}{p1:02x}{p2:02x}", + "68": "MEM only 0x{p0:02x}{p1:02x} blocks reserved out of 0x{p2:02x}{p3:02x} requested", + "69": "MEM allowing copyto glash operation to complete before halting, param=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "70": "MEM AUTO-SAVE ERROR {p0+1|memOpDictionary} {p1+1|regionDictionary} PARAM=0x{p2:02x} CODE=0x{p3:02x}", + "71": "MEM FOUND RESET DURING CORRUPTION SENSITIVE MODE, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "72": "MEM FOUND RESET DURING CORRUPTION SENSITIVE MEM OPER, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "73": "MEM FLASH BLOCK ANALYSIS FAILURE, Block=0x{p0:02x}{p1:02x} ERROR=0x{p2:02x}{p3:02x}", + "74": "MEM BOTH MIRRORED BLOCKS FOUND BAD AT SAME TIME, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "75": "MEM cleanup found sci data, blocks empty=0x{p0:02x}{p1:02x}, blocks w/data=0x{p2:02x}{p3:02x}", + "76": "MEM QUEUED MEM CMD REJECTED, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "77": "MEM collaboration started, {p0+1|memCollabDictionary}, param=0x{p1:02x}{p2:02x}{p3:02x}", + "78": "MEM collaboration succeeded, {p0+1|memCollabDictionary}, param=0x{p1:02x}{p2:02x}{p3:02x}", + "79": "MEM MEMORY COLLABORATION FAILED, {p0+1|memCollabCAPSDictionary}, PARAM=0x{p1:02x}{p2:02x}{p3:02x}", + "80": "MEM REBUILD ERROR SUMMARY, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "81": "MEM CKSUM MISMATCH FOR OBJECT, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "82": "MEM REBUILT MISMATCHED FT ENTRY, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "83": "MEM FLASH DIE UNPOWERED, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "84": "MEM REJECT MODIFY NVFSW3, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "85": "MEM UNEXPECTED FLASH ERROR, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "86": "MEM OPERATION FAILED, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "87": "MEM FOUND BAD BLOCK, BLOCK=0x{p0:02x}{p1:02x} ERRTYPE={p2:02x} OLDSTATE={p3:02x}", + "88": "MEM FLASH UErr FOUND, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "89": "MEM FLASH CErr FOUND, PARAM=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "90": "MEM BAD CLEANUP STATE P={p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "91": "MEM FLASH DRIVER ERROR P={p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "92": "MEM OBJECT MISSING! REGION={p0+2|regionDictionary}, BLOCK={p2:02x}{p3:02x}", + "93": "MEM FOUND CORRUPTION-SENSITIVE MODE EXITED ABNORMALLY", + "94": "MEM FOUND CORRUPTION-SENSITIVE MEM OPER EXITED ABNORMALLY", + "95": "MEM SHUFFLE OPER ERROR, STATE=0x{p0:02x}{p1:02x} BLOCK=0x{p2:02x}{p3:02x}", + "96": "MEM operation queued, {p0+2|memOpLCDictionary} {p2+2|regionLCDictionary}", + "97": "MEM found {p0+2|regionLCDictionary} in block 0x{p2:02x}{p3:02x}", + "98": "MEM operation halted", + "99": "MEM memory operation started", + "100": "MEM {p0+1|memOpLCDictionary} {p1+1|regionLCDictionary} complete, duration=0x{p2:02x}{p3:02x}", + "101": "MEM operation working P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "102": "MEM operation setup P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "103": "MEM operation running P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "104": "MEM exec raw read, value=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "105": "MEM exec raw write, addr=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "106": "MEM state change: Transition {p0+2|memStateLCDictionary} <== {p2+2|memStateLCDictionary}", + "107": "MEM got halt but already idle", + "108": "MEM load command spillover, Offset8=0x{p0:02x} Len8={p1:02x}", + "109": "MEM got param cmd received and executed, param=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "110": "MEM checksum matched expected", + "111": "MEM shuffle state change: Transition 0x{p0:02x}{p1:02x} <== 0x{p2:02x}{p3:02x}", + "112": "MEM shuffle complete, param=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "113": "MEM FETCH TABLE FAILED TO ADD, param=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "114": "MEM popped fetch table entry, param=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "115": "MEM reconciled Flash Table, number entries fixed=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "116": "MEM Science dataset with AID 0x{p0:02x}{p1:02x}{p2:02x}{p3:02x} found", + "117": "MEM SCIENCE DATASET WITH AID 0x{p0:02x}{p1:02x}{p2:02x}{p3:02x} NOT FOUND", + "128": "AUT IBRAM EDAC MERR DETECTED (WDOG RESET LIKELY), P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "129": "AUT DBRAM EDAC MERR DETECTED (WDOG RESET LIKELY), P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "130": "AUT SRAM EDAC MERR DETECTED (WDOG RESET LIKELY), P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "131": "AUT CAUSING WDOG RESET DUE TO PROCESSOR EXCEPTION, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "132": "AUT CAUSING WDOG RESET DUE TO PROCESSOR-ONLY RESET, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "133": "AUT DECON ENABLED OUTSIDE MODE, DISABLING, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "135": "AUT HVPS DET CURRENT TOO HIGH FAULT I=0x{p0:02x}{p1:02x} Limit=0x{p2:02x}{p3:02x}", + "136": "AUT hvps state changed to {p0+4|hvStateDictionary}", + "137": "AUT TOF ADCS ON BUT HVPS CHANGED {p0+1|hvStateDictionary} TO {p1+1|hvStateDictionary}, ADC0=0x{p2:02x}, ADC1=0x{p3:02x}", + "138": "AUT HVPS TRIPLE-VOTING FAILED, {p0+4|hvMmrMismatchDictionary}", + "139": "AUT HVPS FPGA MMR FAILS LAST WRITTEN VALUE CHECK, P=0x{p0:02x}{p1:02x} {p2+2|hvMmrMismatchDictionary}", + "140": "AUT HVPS SETPOINT (0x{p0:02x}{p1:02x}) GREATER THAN MAX (0x{p2:02x}{p3:02x})", + "141": "AUT ANALOG HK(0x{p0:02x}{p1:02x}) 0x{p2:02x}{p3:02x} IS FAULT LO", + "142": "AUT ANALOG HK(0x{p0:02x}{p1:02x}) 0x{p2:02x}{p3:02x} IS FAULT HI", + "143": "AUT HVPS SENSOR OSCILLATOR FAULT, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "144": "UPK transition from {p0:02x} {p1+1|idexModeLCDictionary} to {p2:02x} {p3+1|idexModeLCDictionary}", + "145": "UPK power-on reset, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "146": "UPK commanded reset, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "147": "UPK boot fsw says hello, version={p0:02x}.{p1:02x}.{p2:02x}{p3:02x}", + "148": "UPK oper fsw says hello, version={p0:02x}.{p1:02x}.{p2:02x}{p3:02x}", + "152": "UPK mode changed from {p0+2|idexModeLCDictionary} to {p2+2|idexModeLCDictionary}", + "153": "UPK analog dwell measurement recorded", + "154": "UPK analog dwell completed", + "155": "UPK stim pulser operation completed, , PulserSel=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "159": "UPK csci change imminent to {p0+4|idexModeLCDictionary} - goodbye", + "160": "UPK RESET IMMINENT TO {p0+4|idexModeDictionary} - DUBIOUS GOODBYE!", + "161": "UPK PROCESSOR ONLY RESET, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "162": "UPK WATCHDOG RESET, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "163": "UPK UNKNOWN RESET, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "164": "UPK BOOT STATUS ERROR, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "192": "SCI state change: {p0+2|sciState16Dictionary} ==> {p2+2|sciState16Dictionary}", + "193": "SCI collab {p0+2|memCollabDictionary} is {p2+2|memCollabStatLCDictionary}", + "194": "SCI event ID 0x{p0:02x}{p1:02x} found matching category 0x{p2:02x}{p3:02x}", + "196": "SCI science activity completed: {p0+4|sciState16Dictionary}", + "197": "SCI process channel: {p0+2|sciChannel16Dictionary}, peak cnt (0x{p2:02x}{p3:02x})", + "198": "SCI event trigger (0x{p0:02x}{p1:02x}) id (0x{p2:02x}{p3:02x})", + "199": "SCI process cat full AID (0x{p0:02x}{p1:02x}) count (0x{p2:02x}{p3:02x})", + "200": "SCI process event id (0x{p0:02x}{p1:02x}) Category (0x{p2:02x}{p3:02x})", + "203": "SCI transmit: id (0x{p0:02x}{p1:02x}) cat (0x{p2:02x}) content (0x{p3:02x})", + "204": "SCI data: (0x{p0:02x}{p1:02x}{p2:02x}{p3:02x})", + "206": "SCI sending event with aid (0x{p0:02x}{p1:02x}) and evt num (0x{p2:02x}{p3:02x})", + "207": "SCI spi read: adc (0x{p0:02x}) addr (0x{p1:02x}) value (0x{p2:02x}{p3:02x})", + "208": "SCI CLOCK TRAINING FAILED: Size (0x{p0:02x}{p1:02x}) retry (0x{p2:02x}{p3:02x})", + "209": "SCI TASK STATE ERROR: {p0+2|sciState16Dictionary} ==> {p2+2|sciState16Dictionary}", + "211": "SCI PARSE SIZE ERROR: channel (0x{p0:02x}{p1:02x}) size (0x{p2:02x}{p3:02x})", + "212": "SCI PARSE HEADER SIZE ERROR: packetlen (0x{p0:02x}{p1:02x}) block size (0x{p2:02x}{p3:02x})", + "213": "SCI PARSE END ERROR: STATE {p0+2|sciState16Dictionary} root (0x{p2:02x}{p3:02x})", + "214": "SCI PARSE EVENT ERROR: page (0x{p0:02x}{p1:02x}) id (0x{p2:02x}{p3:02x})", + "215": "SCI SEND STATE ERROR: STATE (0x{p0:02x}{p1:02x}{p2:02x}{p3:02x})", + "216": "SCI SEND CHANNEL ERROR: CHANNEL (0x{p0:02x}{p1:02x}{p2:02x}{p3:02x})", + "217": "SCI ADC POWER ERROR: exp {p0+1|enaDis32Dictionary} act {p1+1|enaDis32Dictionary} chan {p2+2|sciChannel16Dictionary}", + "219": "SCI BDS ALLOCATION EXHAUSTED: REMAINING={p0:02x}", + "220": "SCI AID ALREADY PRESENT: AID=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "221": "SCI DATASET NEEDS PROCESSING BEFORE TRANSMIT", + "224": "SEQ engine has changed state, eng=seqEngineDictionary(0x{p0:02x}) was={p1+1|seqEngineStateDictionary} is={p2+1|seqEngineStateDictionary} 0x{p3:02x}", + "225": "SEQ got resume command when not paused, eng={p0+2|seqEngineDictionary} state={p2+2|seqEngineStateDictionary}", + "226": "SEQ got terminate command when already idle, eng={p0+2|seqEngineDictionary} state={p2+2|seqEngineStateDictionary}", + "227": "SEQ got pause command when already idle, eng={p0+2|seqEngineDictionary} state={p2+2|seqEngineStateDictionary}", + "228": "SEQ GOT PAUSE COMMAND WHEN STALE, eng={p0+2|seqEngineDictionary} state={p2+2|seqEngineStateDictionary}", + "229": "SEQ buffer successfully verified, buf={p0+4|seqBufferDictionary}", + "230": "SEQ ABORTED BUT CANNOT CLEANUP, ENG={p0+2|seqEngineDictionary} BUF={p2+2|seqBufferDictionary}", + "232": "SEQ ENGINE STOPPED DUE TO ERROR, ENG={p0+1|seqEngineDictionary} STATE={p1+1|seqEngineStateDictionary} STOPCODE={p2+1|seqEngStopCodeDictionary} 0x{p3:02x}", + "233": "SEQ ATTEMPTED TO CALL SUBSEQ WHILE ALREADY IN SUBSEQ, eng={p0+2|seqEngineDictionary} state={p2+2|seqEngineStateDictionary}", + "234": "SEQ STATE IS UNDEFINED, STOPPING AND GOING TO IDLE, ENG={p0+2|seqEngineDictionary} STATE={p2+2|seqEngineStateDictionary}", + "235": "SEQ ATTEMPTING WORK ON SEQUENCE THAT IS NOT LOADED, ENG={p0+2|seqEngineDictionary} STATE={p2+2|seqEngineStateDictionary}", + "236": "SEQ ATTEMPTED SUBSEQ CALL FROM ONE BUF TO SAME BUF, ENG={p0+2|seqEngineDictionary} STATE={p2+2|seqEngineStateDictionary}", + "237": "SEQ ATTEMPTED OPERATION ON UNLOADED BUF, BUF={p0+4|seqBufferDictionary}", + "238": "SEQ BUF'S CALCULATED CKSUM DOES NOT MATCH HDR, BUF={p0+4|seqBufferDictionary}", + "239": "SEQ success (len=0x{p0:02x}{p1:02x}, {p2+2|opCodeLCDictionary})", + "240": "SEQ EXCESS FAULT RESPONSE COUNT: FAULT={p0+1|faultSeqDictionary} STATE={p1+1|seqState3Dictionary} ENG={p2+1|seqEngineStateDictionary} BUFID=0x{p3:02x}", + "241": "SEQ Start Fault Resp: FAULT={p0+1|faultSeqDictionary} STATE={p1+1|seqState3Dictionary} ENG={p2+1|seqEngineStateDictionary} BUFID=0x{p3:02x}", + "242": "SEQ Higher Priority Fault: FAULT={p0+1|faultSeqDictionary} STATE={p1+1|seqState3Dictionary} ENG={p2+1|seqEngineStateDictionary} BUFID=0x{p3:02x}", + "243": "SEQ Fault Verify Fail: FAULT={p0+1|faultSeqDictionary} STATE={p1+1|seqState3Dictionary} ENG={p2+1|seqEngineStateDictionary} BUFID=0x{p3:02x}", + "244": "SEQ Fault Lower Priority Ignored: FAULT={p0+1|faultSeqDictionary} STATE={p1+1|seqState3Dictionary} ENG={p2+1|seqEngineStateDictionary} BUFID=0x{p3:02x}", + "245": "SEQ mode trans started: modetrans={p0+1|modeTransDictionary} state={p1+1|seqEngineStateDictionary} eng={p2+1|seqEngineDictionary} BUFID=0x{p3:02x}", + "246": "SEQ TRANS VERIFY FAIL: MODE={p0+1|modeTransDictionary} STATE={p1+1|seqEngineStateDictionary} ENG={p2+1|seqEngineDictionary} BUFID=0x{p3:02x}", + "247": "SEQ TRANS IGNORED NOT IDLE: MODE={p0+1|modeTransDictionary} STATE={p1+1|seqEngineStateDictionary} ENG={p2+1|seqEngineDictionary} BUFID=0x{p3:02x}", + "248": "SEQ suspend complete, eng={p0+2|seqEngineDictionary}, reason={p2+2|seqSuspendEndDictionary}", + "249": "SEQ attempted to erase seq but already empty, BUF={p0+4|seqBufferDictionary}", + "254": "??? INVALID FE, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}", + "255": "??? INVALID FF, P=0x{p0:02x}{p1:02x}{p2:02x}{p3:02x}" + }, + "busyStateTypeDictionary": { + "1": "BUSY", + "0": "OK" + }, + "idexModeDictionary": { + "0": "NONE", + "1": "BOOT", + "2": "SAFE", + "3": "IDLE", + "4": "DECON", + "5": "SCIENCE", + "6": "TRANSMIT" + }, + "idexModeLCDictionary": { + "0": "none", + "1": "boot", + "2": "safe", + "3": "idle", + "4": "decon", + "5": "science", + "6": "transmit" + }, + "chgModeInputType32Dictionary": { + "2": "SAFE", + "3": "IDLE", + "4": "DECON", + "5": "SCIENCE", + "6": "TRANSMIT" + }, + "ovrflwStateTypeDictionary": { + "1": "ERR", + "0": "OK" + }, + "regionDictionary": { + "0": "SCISTART", + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3", + "4": "NVPT", + "5": "SCICONT", + "6": "NVPERST", + "7": "NVFT", + "8": "NVPM", + "9": "NVSCIT", + "10": "SCIRSVP", + "11": "NVFETCHT", + "12": "NVSCS", + "13": "NVCATALOG", + "14": "REG14", + "15": "USER", + "16": "SCRATCH", + "17": "ACTFSW", + "18": "PT", + "19": "FT", + "20": "PERST", + "21": "SCIT", + "22": "SCS", + "24": "CATALOG", + "25": "FETCHT", + "51": "BLOCK", + "63": "NONE" + }, + "regionLCDictionary": { + "0": "scistart", + "1": "nvfsw1", + "2": "nvfsw2", + "3": "nvfsw3", + "4": "nvpt", + "5": "scicont", + "6": "nvperst", + "7": "nvft", + "8": "nvpm", + "9": "nvscit", + "10": "scirsvp", + "11": "nvfetcht", + "12": "nvscs", + "13": "nvcatalog", + "14": "reg14", + "15": "user", + "16": "scratch", + "17": "actfsw", + "18": "pt", + "19": "ft", + "20": "perst", + "21": "scit", + "22": "scs", + "24": "catalog", + "25": "fetcht", + "51": "block", + "63": "none" + }, + "qbootRegion2Dictionary": { + "0": "NONE", + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3" + }, + "qbootRegion8Dictionary": { + "0": "NONE", + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3", + "63": "NONE" + }, + "memRegionDumpDictionary": { + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3", + "4": "NVPT", + "6": "NVPERST", + "7": "NVFT", + "8": "NVPM", + "9": "NVSCIT", + "11": "NVFETCHT", + "12": "NVSCS", + "13": "NVCATALOG", + "16": "SCRATCH", + "17": "ACTFSW", + "18": "PT", + "19": "FT", + "20": "PERST", + "21": "SCIT", + "22": "SCS", + "24": "CATALOG", + "25": "FETCHT", + "51": "BLOCK" + }, + "memRegionCksumDictionary": { + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3", + "4": "NVPT", + "6": "NVPERST", + "7": "NVFT", + "9": "NVSCIT", + "11": "NVFETCHT", + "12": "NVSCS", + "13": "NVCATALOG", + "16": "SCRATCH", + "17": "ACTFSW", + "18": "PT", + "19": "FT", + "20": "PERST", + "21": "SCIT", + "22": "SCS", + "24": "CATALOG", + "25": "FETCHT" + }, + "memRegionCopyToDictionary": { + "1": "NVFSW1", + "2": "NVFSW2", + "4": "NVPT", + "6": "NVPERST", + "7": "NVFT", + "9": "NVSCIT", + "11": "NVFETCHT", + "12": "NVSCS", + "13": "NVCATALOG", + "17": "ACTFSW", + "18": "PT", + "19": "FT", + "20": "PERST", + "21": "SCIT", + "22": "SCS", + "24": "CATALOG", + "25": "FETCHT", + "51": "BLOCK" + }, + "memRegionCopyFromDictionary": { + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3", + "4": "NVPT", + "6": "NVPERST", + "7": "NVFT", + "8": "NVPM", + "9": "NVSCIT", + "11": "NVFETCHT", + "12": "NVSCS", + "13": "NVCATALOG", + "17": "ACTFSW", + "18": "PT", + "19": "FT", + "20": "PERST", + "21": "SCIT", + "22": "SCS", + "24": "CATALOG", + "25": "FETCHT", + "51": "BLOCK" + }, + "memRegionEraseDictionary": { + "8": "NVPM", + "16": "SCRATCH", + "17": "ACTFSW", + "24": "CATALOG", + "25": "FETCHT", + "51": "BLOCK" + }, + "memRegionRebuildDictionary": { + "51": "BLOCK" + }, + "memRegionShuffleDictionary": { + "51": "BLOCK" + }, + "memRegionSetHdrDictionary": { + "1": "NVFSW1", + "2": "NVFSW2", + "3": "NVFSW3", + "51": "BLOCK" + }, + "memOpDictionary": { + "0": "NONE", + "1": "DUMP", + "2": "CKSUM", + "4": "ERASE", + "8": "CPYTO", + "16": "CPYFROM", + "32": "REBUILD", + "64": "SHUFFLE" + }, + "memOpLCDictionary": { + "0": "none", + "1": "dump", + "2": "cksum", + "4": "erase", + "8": "cpyto", + "16": "cpyfrom", + "32": "rebuild", + "64": "shuffle" + }, + "memStateDictionary": { + "1": "NOTINIT", + "2": "IDLE", + "4": "DUMPSRAM", + "5": "CKSUMSRAM", + "6": "ERASESRAM", + "7": "CPY2SRAM", + "8": "CPYSRAM", + "9": "SETUPSCI", + "10": "CLEANSCI", + "11": "COPYFLASH", + "12": "COPY2FLASH", + "13": "ERASEFLASH", + "14": "DUMPFLASH", + "15": "CKSUMFLASH", + "16": "REBUILD", + "17": "SHUFFLE", + "18": "CLEANUP", + "19": "FETCHSCI", + "20": "ERASESCI" + }, + "memStateLCDictionary": { + "1": "notinit", + "2": "idle", + "4": "dumpsram", + "5": "cksumsram", + "6": "erasesram", + "7": "copy2sram", + "8": "cpysram", + "9": "setupsci", + "10": "cleansci", + "11": "copyflash", + "12": "copy2flash", + "13": "eraseflash", + "14": "dumpflash", + "15": "cksumflash", + "16": "rebuild", + "17": "shuffle", + "18": "cleanup", + "19": "fetchsci", + "20": "erasesci" + }, + "memCollabCAPSDictionary": { + "0": "SAVEPMLOG", + "1": "SCISETUP", + "2": "SCICLEANUP", + "3": "SCIFIND", + "4": "SCICOPY", + "5": "SCIFETCH", + "6": "GETCATALOG", + "7": "SAVECATALOG", + "8": "ERASESCI" + }, + "memCollabDictionary": { + "0": "savepmlog", + "1": "scisetup", + "2": "scicleanup", + "3": "scifind", + "4": "scicopy", + "5": "scifetch", + "6": "getcatalog", + "7": "savecatalog", + "8": "erasesci" + }, + "memCollabStatLCDictionary": { + "0": "busy", + "1": "success", + "2": "FAIL", + "3": "NEVER" + }, + "adpMetaControlDictionary": { + "1": "START", + "2": "END" + }, + "setAllocType32Dictionary": { + "0": "SET", + "1": "ADD" + }, + "deconSelDictionary": { + "0": "THERM0", + "1": "THERM1" + }, + "logSel8Dictionary": { + "0": "EVT", + "1": "PM", + "2": "CMD" + }, + "catalogSelDictionary": { + "0": "ALL", + "1": "HDR", + "2": "PAGE2", + "3": "PAGE3", + "4": "PAGE4", + "5": "PAGE5", + "6": "PAGE6", + "7": "PAGE7", + "8": "PAGE8", + "9": "PAGE9", + "10": "PAGE10", + "11": "PAGE11", + "12": "PAGE12", + "13": "PAGE13", + "14": "PAGE14", + "15": "PAGE15", + "16": "PAGE16", + "17": "PAGE17", + "18": "PAGE18", + "19": "PAGE19", + "20": "PAGE20", + "21": "PAGE21", + "22": "PAGE22", + "23": "PAGE23", + "24": "PAGE24", + "25": "PAGE25", + "26": "PAGE26", + "27": "PAGE27", + "28": "PAGE28", + "29": "PAGE29", + "30": "PAGE30", + "31": "PAGE31", + "32": "PAGE32", + "33": "PAGE33", + "34": "PAGE34", + "35": "PAGE35", + "36": "PAGE36", + "37": "PAGE37", + "38": "PAGE38", + "39": "PAGE39", + "40": "PAGE40", + "41": "PAGE41", + "42": "PAGE42", + "43": "PAGE43", + "44": "PAGE44", + "45": "PAGE45", + "46": "PAGE46", + "47": "PAGE47", + "48": "PAGE48", + "49": "PAGE49", + "50": "PAGE50", + "51": "PAGE51", + "52": "PAGE52", + "53": "PAGE53", + "54": "PAGE54", + "55": "PAGE55", + "56": "PAGE56", + "57": "PAGE57", + "58": "PAGE58", + "59": "PAGE59", + "60": "PAGE60", + "61": "PAGE61", + "62": "PAGE62", + "63": "PAGE63" + }, + "flagSel32Dictionary": { + "1": "PWRCYC", + "2": "PANIC", + "3": "BUSY", + "4": "PWRDOWN", + "5": "SAFE", + "6": "WDOG", + "7": "SPARE0", + "8": "SPARE1", + "9": "SPARE2", + "10": "MODEINTR", + "11": "OPERINTR", + "12": "BDSALLOC" + }, + "panicStateTypeDictionary": { + "1": "PANIC", + "0": "OK" + }, + "resetTypeDictionary": { + "0": "UNKWN", + "1": "POR", + "2": "WDOG", + "3": "CMD", + "4": "PROC", + "5": "BOOT" + }, + "irqOrdinalDictionary": { + "0": "WDOG", + "1": "BRAMINST", + "2": "BRAMDATA", + "3": "SRAM", + "4": "TIMER", + "5": "WISHBONE", + "6": "TIMEMSG", + "7": "PLL", + "8": "FLASHDONE", + "15": "NONE" + }, + "injectingErrorDictionary": { + "0": "OKAY", + "1": "BADBLOCK", + "2": "CERR", + "3": "BBCERR", + "4": "MERR", + "5": "BBMERR", + "6": "INVLD", + "7": "INVLD" + }, + "triggerModeDictionary": { + "0": "None", + "1": "THOLD", + "2": "PULSE1", + "3": "PULSE2" + }, + "triggerPolarityDictionary": { + "0": "ABOVE", + "1": "BELOW" + }, + "externTriggerDictionary": { + "0": "FALL", + "1": "RISE" + }, + "sciState16Dictionary": { + "0": "IDLE", + "1": "ACQCLEANUP", + "2": "ACQSETUP", + "3": "ACQ", + "4": "CAL", + "5": "CHILL", + "6": "CLKPATTERN", + "7": "CLK", + "8": "DUMPADCSPI", + "9": "MEMCOPY", + "10": "FETCHEVT", + "11": "MEMFIND", + "12": "MEMGETCAT", + "13": "MEMSAVCAT", + "14": "PARSE", + "15": "PROCESS", + "16": "SEND", + "17": "READSPI", + "18": "TRANSMIT", + "19": "ADCINIT" + }, + "sciChannel16Dictionary": { + "0": "HS_HIGH_GAIN", + "1": "HS_LOW_GAIN", + "2": "HS_MID_GAIN", + "3": "LS_TLR", + "4": "LS_THR", + "5": "LS_ION", + "6": "MAX_CHANNEL" + }, + "sciSndEvtSt16Dictionary": { + "0": "HEADER_RDY", + "1": "HEADER_SNT", + "2": "CHANNEL_COMP_ZERO", + "3": "CHANNEL_COMP_RDY", + "4": "CHANNEL_COMP_SNT", + "5": "CHANNEL_PACK_RDY", + "6": "CHANNEL_PACK_SNT", + "7": "CHANNEL_RDY", + "8": "CHANNEL_SNT", + "9": "EVT_SEND_DONE", + "10": "EVT_MAX_STATE" + }, + "sciChanOneHotDictionary": { + "0": "NONE", + "1": "EVTHDR", + "2": "TOFHG", + "4": "TOFLG", + "8": "TOFMG", + "16": "TLR", + "32": "THR", + "64": "ION", + "127": "ALL" + }, + "depthSelTypeDictionary": { + "0": "B10", + "1": "B12" + }, + "cmdEngineDictionary": { + "0": "AUTO", + "1": "OPER", + "2": "RT" + }, + "setSeqStateDictionary": { + "1": "PAUSE", + "2": "RESUME", + "3": "TERM" + }, + "seqEngineDictionary": { + "0": "AUTO", + "1": "OPER" + }, + "seqBufferDictionary": { + "0": "BUF0", + "1": "BUF1", + "2": "BUF2", + "3": "BUF3", + "4": "BUF4", + "5": "BUF5", + "6": "BUF6", + "7": "BUF7", + "8": "BUF8", + "9": "BUF9", + "10": "BUF10", + "11": "BUF11", + "12": "BUF12", + "13": "BUF13", + "14": "BUF14", + "15": "BUF15", + "16": "BUF16", + "17": "BUF17", + "18": "BUF18", + "19": "BUF19", + "20": "BUF20", + "21": "BUF21", + "22": "BUF22", + "23": "BUF23", + "24": "BUF24", + "25": "BUF25", + "26": "BUF26", + "27": "BUF27", + "28": "BUF28", + "29": "BUF29", + "30": "BUF30", + "31": "BUF31", + "32": "BUF32", + "33": "NONE" + }, + "seqEngineStateDictionary": { + "0": "IDLE", + "1": "ACTIV", + "2": "SUSPN", + "3": "PAUSE", + "4": "STALE" + }, + "seqEngStopCodeDictionary": { + "0": "NOM", + "1": "CMD", + "2": "VRFY", + "3": "RJCT", + "4": "STALE", + "5": "ZERO" + }, + "seqEngWaitTypeDictionary": { + "0": "None", + "1": "Abs", + "2": "Rel", + "3": "Cond" + }, + "seqSuspendEndDictionary": { + "0": "None", + "1": "Abs", + "2": "Rel", + "3": "Cond", + "4": "SciIdle", + "5": "Timeout" + }, + "seqState3Dictionary": { + "0": "Clear", + "1": "Start", + "2": "Done", + "3": "FAIL", + "4": "Intr" + }, + "cmdExecStatusDictionary": { + "0": "NONE", + "1": "OKAY", + "2": "BUSY", + "3": "LENGTH", + "4": "ID", + "5": "PROT", + "6": "RANGE", + "7": "MODE", + "8": "SRC", + "9": "ARGNUM", + "10": "NULL", + "11": "CRC", + "12": "OKAY2", + "13": "TIMEMSG", + "14": "CCSDS" + }, + "opCodeDictionary": { + "0": "NONE", + "4353": "NOOP", + "34561": "RST", + "34563": "SHUTDWN", + "36352": "SETAIDBIN", + "52416": "SETFLAG", + "52417": "CLRFLAG", + "52418": "LISTCATALOGS", + "52419": "CHGMODE", + "52420": "DUMPLOG", + "52421": "DWELL", + "52422": "CFGHTR", + "52423": "DSHTR", + "52424": "HALTQB", + "52425": "DUALCMD", + "52426": "CLRCMDST", + "52427": "LOADMEM", + "52428": "RAWWRT", + "52429": "RAWREAD", + "52430": "RAWCPY", + "52431": "SETPRM", + "52432": "GETPRM", + "52433": "WRFTBL", + "52434": "DUMPFTBL", + "52436": "SETMEMHDR", + "52437": "ERASEMEM", + "52438": "CKSUMMEM", + "52439": "DUMPMEM", + "52440": "COPYTOMEM", + "52441": "COPYFROMMEM", + "52445": "ACQUIRE", + "52446": "PROCESS_XMIT", + "52447": "WRSPI", + "52448": "ERASESCI", + "52449": "SETSCIPRM", + "52450": "INITSEQ", + "52451": "STRTSEQ", + "52452": "SETSEQST", + "52453": "SUSPABS", + "52454": "SUSPREL", + "52455": "CLSUB", + "52456": "VERSEQ", + "52457": "CLRFAULTCNT", + "52458": "SHUTDWNHV", + "52459": "CLRTLMST", + "52460": "CLRMEM", + "52461": "CLRSCI", + "52462": "REBUILD", + "52463": "SHUFFLE", + "52464": "DUMPADC", + "52465": "HALTSCI", + "52466": "HALTMEM", + "52467": "CLRUK", + "52468": "PTSEQ", + "52469": "STUFFSEQ", + "52470": "ERASESEQ", + "52480": "PWRADC", + "52481": "CFGADCTOF", + "52482": "CFGADCTAR", + "52483": "CFGSCIACQ", + "52484": "CFGTRG", + "52485": "INITADC", + "52486": "CALADC", + "52487": "TRAINADCCLK", + "52488": "SETHVPWR", + "52490": "DSHVOSC", + "52491": "SETHVSETPT", + "52492": "SETHVMAX", + "52493": "CLRATN", + "52494": "CLRSEQ", + "52495": "READSPI", + "52496": "FETCHONE", + "52506": "FETCH", + "52507": "TRANSMIT", + "52508": "SENDCATALOG", + "52509": "SUSPIDLESCI", + "52510": "SETALLOC", + "52511": "ADDTOFETCH", + "52512": "CFGSTIM", + "52513": "ENSTIM", + "52514": "HALTSTIM", + "48059": "DEPLOYDOOR", + "61166": "ENAHVOSC" + }, + "opCodeLCDictionary": { + "0": "none", + "4353": "noop", + "34561": "rst", + "34563": "shutdwn", + "36352": "setaidbin", + "52416": "setflag", + "52417": "clrflag", + "52418": "listcatalogs", + "52419": "chgmode", + "52420": "dumplog", + "52421": "dwell", + "52422": "cfghtr", + "52423": "dshtr", + "52424": "haltqb", + "52425": "dualcmd", + "52426": "clrcmdst", + "52427": "loadmem", + "52428": "rawwrt", + "52429": "rawread", + "52430": "rawcpy", + "52431": "setprm", + "52432": "getprm", + "52433": "wrftbl", + "52434": "dumpftbl", + "52436": "setmemhdr", + "52437": "erasemem", + "52438": "cksummem", + "52439": "dumpmem", + "52440": "copytomem", + "52441": "copyfrommem", + "52445": "acquire", + "52446": "process_xmit", + "52447": "wrspi", + "52448": "erasesci", + "52449": "setsciprm", + "52450": "initseq", + "52451": "strtseq", + "52452": "setseqst", + "52453": "suspabs", + "52454": "susprel", + "52455": "clsub", + "52456": "verseq", + "52457": "clrfaultcnt", + "52458": "shutdwnhv", + "52459": "clrtlmst", + "52460": "clrmem", + "52461": "clrsci", + "52462": "rebuild", + "52463": "shuffle", + "52464": "dumpadc", + "52465": "haltsci", + "52466": "haltmem", + "52467": "clruk", + "52468": "ptseq", + "52469": "stuffseq", + "52470": "eraseseq", + "52480": "pwradc", + "52481": "cfgadctof", + "52482": "cfgadctar", + "52483": "cfgsciacq", + "52484": "cfgtrg", + "52485": "initadc", + "52486": "caladc", + "52487": "trainadcclk", + "52488": "sethvpwr", + "52490": "dshvosc", + "52491": "sethvsetpt", + "52492": "sethvmax", + "52493": "clratn", + "52494": "clrseq", + "52495": "readspi", + "52496": "fetchone", + "52506": "fetch", + "52507": "transmit", + "52508": "sendcatalog", + "52509": "suspidlesci", + "52510": "setalloc", + "52511": "addtofetch", + "52512": "cfgstim", + "52513": "enstim", + "52514": "haltstim", + "48059": "deploydoor", + "61166": "enahvosc" + }, + "tlmApIdLsbDictionary": { + "128": "EVTMSG", + "129": "ALIVE", + "130": "EVTLOG", + "131": "PMLOG", + "132": "CMDLOG", + "133": "HWPKT", + "134": "SWPKT", + "135": "DUMP", + "136": "DWELL" + }, + "qbootStateDictionary": { + "0": "NOATTEMPT", + "1": "UNINIT", + "2": "STARTED", + "6": "STOPPED", + "7": "WAITING", + "8": "PENDING" + }, + "qbootStateLCDictionary": { + "0": "noattempt", + "1": "uninit", + "2": "started", + "6": "stopped", + "7": "waiting", + "8": "pending" + }, + "qbootReason8Dictionary": { + "0": "NO_STOP", + "1": "FT_FAIL", + "2": "PERST_FAIL", + "3": "PT_FAIL", + "4": "NVFSW_FAIL", + "5": "SAFE_ENTRY", + "6": "PANICKING", + "7": "REBOOT_RST", + "8": "MEM_FAIL", + "9": "BYCMD", + "11": "NO_TIME", + "12": "WDOGCNT", + "13": "NONE" + }, + "blockState8Dictionary": { + "0": "UNKWN", + "16": "BAD", + "32": "EMPTY", + "48": "SCISTRT", + "49": "NVFSW1", + "50": "NVFSW2", + "51": "NVFSW3", + "52": "NVPT", + "53": "SCICONT", + "54": "NVPERST", + "55": "NVFT", + "56": "NVPMLOG", + "57": "NVSCIT", + "58": "SCIRSVD", + "59": "NVFETCHT", + "60": "NVSCS", + "61": "NVCATALOG", + "62": "SPAREE", + "63": "USER" + }, + "blockState32Dictionary": { + "0": "UNKWN", + "16": "BAD", + "32": "EMPTY", + "48": "SCISTRT", + "49": "NVFSW1", + "50": "NVFSW2", + "51": "NVFSW3", + "52": "NVPT", + "53": "SCICONT", + "54": "NVPERST", + "55": "NVFT", + "56": "NVPMLOG", + "57": "NVSCIT", + "58": "SCIRSVD", + "59": "NVFETCHT", + "60": "NVSCS", + "61": "NVCATALOG", + "62": "REG14", + "63": "USER" + }, + "hvOscillatorDictionary": { + "1": "DETECT", + "0": "SENSOR" + }, + "hvPolarityDictionary": { + "1": "NEG", + "0": "POS" + }, + "hvOutputDictionary": { + "0": "SENSOR", + "1": "RJCTN", + "2": "TARGET", + "3": "DETECT" + }, + "hvOutputOnlyDSDictionary": { + "0": "SENSOR", + "3": "DETECT" + }, + "hvOutputOnlyTRDictionary": { + "1": "RJCTN", + "2": "TARGET" + }, + "hvStateDictionary": { + "0": "OFF", + "1": "STANDBY", + "2": "RAMPUP", + "3": "ACTIVE", + "4": "RAMPDOWN" + }, + "hvMmrMismatchDictionary": { + "0": "POW_ENA", + "1": "SEN_ENA", + "2": "DET_ENA", + "3": "POL_ENA", + "4": "DET_SPT", + "5": "SEN_SPT", + "6": "TAR_SPT", + "7": "REF_SPT", + "8": "DET_MAX", + "9": "SEN_MAX", + "10": "TAR_MAX", + "11": "REF_MAX" + }, + "faultDictionary": { + "1": "ANAHKFLAG", + "4": "HVPSOSCERR", + "8": "HVPSCURR", + "16": "ANAHKLIMIT", + "64": "SPARE", + "128": "DECON0", + "256": "DECON1", + "512": "REPOINT", + "1023": "ALLFAULTS" + }, + "faultSeqDictionary": { + "0": "ABORTED_SEQ_FAULT", + "1": "HV_SEN_OSC_FAULT", + "2": "HV_DET_CUR_FAULT", + "3": "ANA_HK_FAULT", + "4": "COMM_LOSS_FAULT", + "5": "SPARE_FAULT", + "6": "DECON0_FAULT", + "7": "DECON1_FAULT", + "8": "REPOINT_FAULT", + "9": "FPGA_CLK_FAULT", + "10": "SHUTDOWN_RESET" + }, + "disEnaDictionary": { + "1": "DIS", + "0": "ENA" + }, + "enaDisDictionary": { + "1": "ENA", + "0": "DIS" + }, + "enaDis3Dictionary": { + "7": "ENA", + "0": "DIS" + }, + "enaDis8Dictionary": { + "1": "ENA", + "0": "DIS" + }, + "enaDis32Dictionary": { + "1": "ENA", + "0": "DIS" + }, + "busyIdleDictionary": { + "1": "BUSY", + "0": "IDLE" + }, + "onOffDictionary": { + "1": "ON", + "0": "OFF" + }, + "onOff32Dictionary": { + "1": "ON", + "0": "OFF" + }, + "errOkayDictionary": { + "1": "ERR", + "0": "OK" + }, + "okayErrDictionary": { + "1": "OK", + "0": "ERR" + }, + "yesNoDictionary": { + "1": "YES", + "0": "NO" + }, + "noYesDictionary": { + "1": "NO", + "0": "YES" + }, + "inOutDictionary": { + "1": "IN", + "0": "OUT" + }, + "openClosedDictionary": { + "1": "OPEN", + "0": "CLOSED" + }, + "closedOpenDictionary": { + "1": "CLOSED", + "0": "OPEN" + }, + "highLowDictionary": { + "1": "HI", + "0": "LO" + }, + "redPriDictionary": { + "1": "RED", + "0": "PRI" + }, + "deconHtrStateDictionary": { + "0": "DIS", + "1": "THERM0", + "2": "THERM1", + "3": "BOTH" + }, + "boardIdDictionary": { + "1": "EM", + "4": "EMULATOR", + "8": "FM", + "9": "SP" + }, + "nandOwnDictionary": { + "0": "FSW", + "1": "FPGA" + }, + "hvpsBoardPwrDictionary": { + "0": "DIS", + "1": "ERR", + "2": "ERR", + "3": "ERR", + "4": "ERR", + "5": "ERR", + "6": "ERR", + "7": "ENA" + }, + "modeTransDictionary": { + "0": "BOOT_TO_IDLE", + "1": "IDLE_TO_DECON", + "2": "IDLE_TO_SCIENCE", + "3": "IDLE_TO_TRANSMIT", + "4": "DECON_TO_IDLE", + "5": "SCIENCE_TO_IDLE", + "6": "TRANSMIT_TO_IDLE", + "7": "ANY_TO_SAFE", + "8": "SAFE_TO_IDLE" + } +} diff --git a/imap_processing/idex/idex_l1a.py b/imap_processing/idex/idex_l1a.py index 24f0caee70..3cc053a674 100644 --- a/imap_processing/idex/idex_l1a.py +++ b/imap_processing/idex/idex_l1a.py @@ -14,6 +14,7 @@ l1a_data.write_l1a_cdf() """ +import json import logging from enum import IntEnum from pathlib import Path @@ -24,7 +25,9 @@ import xarray as xr from xarray import Dataset +from imap_processing import imap_module_directory from imap_processing.idex.decode import rice_decode +from imap_processing.idex.evt_msg_decode_utils import render_event_template from imap_processing.idex.idex_constants import IDEXAPID from imap_processing.idex.idex_l0 import decom_packets from imap_processing.idex.idex_utils import get_idex_attrs @@ -86,23 +89,19 @@ def __init__(self, packet_file: str | Path) -> None: if science_packets: logger.info("Processing IDEX L1A Science data.") self.data.append(self._create_science_dataset(science_packets)) - datasets_by_level = {"l1a": raw_datset_by_apid, "l1b": derived_datasets_by_apid} for level, dataset in datasets_by_level.items(): - if IDEXAPID.IDEX_EVT in dataset: - logger.info(f"Processing IDEX {level} Event Message data") + # Only produce l1a products for event messages. L1b will be processed in a + # another job. + if IDEXAPID.IDEX_EVT in dataset and level == "l1a": + logger.info("Processing IDEX L1A Event Message data") data = dataset[IDEXAPID.IDEX_EVT] - data.attrs = self.idex_attrs.get_global_attributes( - f"imap_idex_{level}_evt" - ) - data["epoch"] = calculate_idex_event_time( - data["shcoarse"].data, data["shfine"].data - ) - data["epoch"].attrs = epoch_attrs - self.data.append(data) + processed_data = self._create_evt_msg_data(data) + processed_data["epoch"].attrs = epoch_attrs + self.data.append(processed_data) if IDEXAPID.IDEX_CATLST in dataset: - logger.info(f"Processing IDEX {level} Catalog List Summary data.") + logger.info(f"Processing IDEX {level} CATLST data") data = dataset[IDEXAPID.IDEX_CATLST] data.attrs = self.idex_attrs.get_global_attributes( f"imap_idex_{level}_catlst" @@ -115,6 +114,112 @@ def __init__(self, packet_file: str | Path) -> None: logger.info("IDEX L1A data processing completed.") + def _create_evt_msg_data(self, data: xr.Dataset) -> xr.Dataset: + """ + Process IDEX message data into a more usable format. + + Parameters + ---------- + data : xarray.Dataset + The raw message data to process. + + Returns + ------- + xarray.Dataset + The processed message data. + """ + # Convert the time to epoch time in nanoseconds since J2000 in the TT timescale + epoch = calculate_idex_event_time(data["shcoarse"].data, data["shfine"].data) + # initialize dataset with time variables + l1a_msg_ds = xr.Dataset( + data_vars={ + "epoch": xr.DataArray(epoch, name="epoch", dims=["epoch"]), + "shfine": xr.DataArray( + data["shfine"].data, + dims=["epoch"], + attrs=self.idex_attrs.get_variable_attributes("shfine"), + ), + "shcoarse": xr.DataArray( + data["shcoarse"].data, + dims=["epoch"], + attrs=self.idex_attrs.get_variable_attributes("shcoarse"), + ), + }, + attrs=self.idex_attrs.get_global_attributes("imap_idex_l1a_msg"), + ) + # Load the event decoding dictionaries + with open( + f"{imap_module_directory}/idex/idex_evt_msg_parsing_dictionaries.json" + ) as f: + msg_dicts = json.load(f) + + # restore integer keys since JSON stringifies them + msg_json_data = { + dict_name: {int(k): v for k, v in pairs.items()} + for dict_name, pairs in msg_dicts.items() + } + # Get the event message templates and log entry name dictionaries + # These are used to decode the raw event messages into human-readable formats + # during rendering. + event_description_templates = msg_json_data.get("eventMsgDictionary", {}) + log_entry_names = msg_json_data.get("logEntryIdDictionary", {}) + + # Get the event id - this will tell us what event happened. + # The following parameter values will tell us additional details about the event + # For example the event may be a science state change and the parameters will + # tell us what state it changed to (e.g. on or off). + event_ids = data["elid_evtpkt"].data + # Stack the parameter bytes into a single array of shape (num_events, 4) for + # easier access during rendering. + params_bytes = np.stack( + [ + data["el1par_evtpkt"].data, + data["el2par_evtpkt"].data, + data["el3par_evtpkt"].data, + data["el4par_evtpkt"].data, + ], + axis=-1, + ) + + # initialize an empty list for messages + messages = [] + for idx in range(len(event_ids)): + # Look up the string format using the event_id. + event_id = event_ids[idx] + current_desc_template = event_description_templates.get(event_id) + current_param_bytes = params_bytes[idx].tolist() + event_name = log_entry_names.get(event_id, f"EVENT_0x{event_id:02X}") + # Render the event message using the template if available. + if current_desc_template: + try: + message = render_event_template( + current_desc_template, current_param_bytes, msg_json_data + ) + except Exception as exc: + message = ( + f"{event_name} [template_render_error={exc}] " + f"params=" + f"({', '.join(f'0x{x:02X}' for x in current_param_bytes)})" + ) + else: + # If no template exists for an event ID, fall back to a message + # that still preserves the event name and raw parameter bytes. + phex = ", ".join(f"0x{x:02X}" for x in current_param_bytes) + message = f"{event_name} ({phex})" + + messages.append(message) + + l1a_msg_ds["messages"] = xr.DataArray( + messages, + name="messages", + dims=["epoch"], + attrs=self.idex_attrs.get_variable_attributes( + "messages", check_schema=False + ), + ) + l1a_msg_ds.attrs = self.idex_attrs.get_global_attributes("imap_idex_l1a_msg") + return l1a_msg_ds + def _create_science_dataset(self, science_decom_packet_list: list) -> xr.Dataset: """ Process IDEX science packets into an xarray Dataset. @@ -458,14 +563,36 @@ def _set_sample_trigger_times( """ # Retrieve the number of samples for high gain delay - # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value, with the last 10 bits - # representing the high gain sample delay and the first 2 bits used for padding. - # To extract the high gain bits, the bitwise right shift (>> 20) moves the bits - # 20 positions to the right, and the mask (0b1111111111) keeps only the least - # significant 10 bits. - # TODO use the delay corresponding to the trigger - high_gain_delay = (packet["IDX__TXHDRSAMPDELAY"] >> 22) & 0b1111111111 + # packet['IDX__TXHDRSAMPDELAY'] is a 32-bit value: + # bits0-9: high-gain delay, + # bits10-19: mid-gain delay, + # bits20-29: low-gain delay. + # bits30-31 are padding/reserved. + # Each delay is extracted by right-shifting to align the field, + # then masking with #0b1111111111 (10 bits). + n_blocks = packet["IDX__TXHDRBLOCKS"] + trigger_item = packet["IDX__TXHDRTRIGID"] + + tof_delay = packet["IDX__TXHDRSAMPDELAY"] # last two bits are padding + + # mask to extract 10-bit values + tof_mask = 0b1111111111 + + # Determine the delay based on the trigger id. + hg_delay = tof_delay & tof_mask # first 10 bits (0-9) + mg_delay = (tof_delay >> 10) & tof_mask # next 10 bits (10-19) + lg_delay = (tof_delay >> 20) & tof_mask # next 10 bits (20-29) + + u10 = trigger_item & 0x3FF + if (u10 >> 0) & 1: + delay = hg_delay + elif (u10 >> 1) & 1: + delay = lg_delay + elif (u10 >> 2) & 1: + delay = mg_delay + else: + delay = hg_delay # Retrieve number of low/high sample pre-trigger blocks @@ -485,11 +612,10 @@ def _set_sample_trigger_times( * (num_low_sample_pretrigger_blocks + 1) * self.NUMBER_SAMPLES_PER_LOW_SAMPLE_BLOCK ) - self.high_sample_trigger_time = ( - self.HIGH_SAMPLE_RATE - * (num_high_sample_pretrigger_blocks + 1) - * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - - self.HIGH_SAMPLE_RATE * high_gain_delay + self.high_sample_trigger_time = self.HIGH_SAMPLE_RATE * ( + num_high_sample_pretrigger_blocks + 1 + ) * self.NUMBER_SAMPLES_PER_HIGH_SAMPLE_BLOCK - self.HIGH_SAMPLE_RATE * ( + delay - 1 ) def _parse_high_sample_waveform(self, waveform_raw: str) -> list[int]: @@ -563,7 +689,7 @@ def _calc_low_sample_resolution(self, num_samples: int) -> npt.NDArray: time_low_sample_rate_data : numpy.ndarray Low time sample data array. """ - time_low_sample_rate_init = np.linspace(0, num_samples, num_samples) + time_low_sample_rate_init: np.ndarray = np.arange(num_samples, dtype=np.float64) time_low_sample_rate_data = ( self.LOW_SAMPLE_RATE * time_low_sample_rate_init - self.low_sample_trigger_time @@ -590,7 +716,9 @@ def _calc_high_sample_resolution(self, num_samples: int) -> npt.NDArray: time_high_sample_rate_data : numpy.ndarray High sample time data array. """ - time_high_sample_rate_init = np.linspace(0, num_samples, num_samples) + time_high_sample_rate_init: np.ndarray = np.arange( + num_samples, dtype=np.float64 + ) time_high_sample_rate_data = ( self.HIGH_SAMPLE_RATE * time_high_sample_rate_init - self.high_sample_trigger_time @@ -630,8 +758,15 @@ def process(self) -> Dataset | None: # Gather the huge amount of metadata info trigger_vars = {} for var, value in self.telemetry_items.items(): - trigger_vars[var] = xr.DataArray( - name=var, + # SCI0AID is not updated properly. To this end, TXHDRFSWAIDCOPY must be + # used as the proper AID. + if var == "idx__sci0aid": + continue + # rename idx__txhdrfswaidcopy to aid for better readability in the final + # dataset + var_name = "aid" if var == "idx__txhdrfswaidcopy" else var + trigger_vars[var_name] = xr.DataArray( + name=var_name, data=[value], dims=("epoch"), attrs=idex_attrs.get_variable_attributes(var), diff --git a/imap_processing/idex/idex_l1b.py b/imap_processing/idex/idex_l1b.py index a11cfb9ecb..2b71a570a6 100644 --- a/imap_processing/idex/idex_l1b.py +++ b/imap_processing/idex/idex_l1b.py @@ -15,10 +15,13 @@ """ import logging -from enum import Enum +from enum import Enum, IntEnum +import numpy as np import pandas as pd import xarray as xr +from numpy.typing import NDArray +from xarray import DataArray from imap_processing import imap_module_directory from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes @@ -42,6 +45,40 @@ logger = logging.getLogger(__name__) +class EventMessage(Enum): + """Enum class for event messages.""" + + PULSER_ON = "SEQ success (len=0x0580, opCodeLCDictionary(enstim))" + PULSER_OFF = "SEQ success (len=0x0580, opCodeLCDictionary(susprel))" + SCIENCE_ON = ( + "SCI state change: sciState16Dictionary(ACQSETUP) ==> sciState16Dictionary(ACQ)" + ) + SCIENCE_OFF = ( + "SCI state change: sciState16Dictionary(ACQ) ==> sciState16Dictionary(CHILL)" + ) + + +class TriggerOrigin(IntEnum): + """Enum class for event trigger origins.""" + + HS_ADC0I_TOF_HG = 0 + HS_ADC0Q_TOF_LG = 1 + HS_ADC1Q_TOF_MG = 2 + LS_ADC1_TARGET_HG = 3 + SW_TRIGGER = 4 + EXTERNAL_TRIGGER = 5 + + +TRIGGER_LABELS = { + TriggerOrigin.HS_ADC0I_TOF_HG: "HS ADC0I trigger (TOF HG)", + TriggerOrigin.HS_ADC0Q_TOF_LG: "HS ADC0Q trigger (TOF LG)", + TriggerOrigin.HS_ADC1Q_TOF_MG: "HS ADC1Q trigger (TOF MG)", + TriggerOrigin.LS_ADC1_TARGET_HG: "LS ADC1 trigger (Target HG / low range)", + TriggerOrigin.SW_TRIGGER: "SW trigger", + TriggerOrigin.EXTERNAL_TRIGGER: "external trigger", +} + + class TriggerMode(Enum): """ Enum class for data collection trigger Modes. @@ -80,9 +117,106 @@ def get_mode_label(mode: int, channel: str) -> str: return f"{channel.upper()}{TriggerMode(mode).name}" -def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset: +def idex_l1b(l1a_dataset: xr.Dataset, descriptor: str) -> xr.Dataset | None: """ - Will process IDEX l1a data to create l1b data products. + Process IDEX l1a data to create l1b data products based on the descriptor. + + Parameters + ---------- + l1a_dataset : xarray.Dataset + IDEX L1a dataset to process. + descriptor : str + Descriptor to determine the type of l1b processing to perform. E.g. "sci-1week" + or "msg". + + Returns + ------- + l1b_dataset : xarray.Dataset + The``xarray`` dataset containing the processed data and supporting metadata. + """ + if descriptor.startswith("sci"): + return idex_l1b_science(l1a_dataset) + elif descriptor.startswith("msg"): + return idex_l1b_msg(l1a_dataset) + else: + raise ValueError(f"Unsupported descriptor: {descriptor}") + + +def idex_l1b_msg(l1a_dataset: xr.Dataset) -> xr.Dataset | None: + """ + Will process IDEX l1a msg data. + + Parameters + ---------- + l1a_dataset : xarray.Dataset + IDEX L1a dataset to process. + + Returns + ------- + l1b_dataset : xarray.Dataset + The``xarray`` dataset containing the msg housekeeping data and + supporting metadata. + """ + logger.info( + f"Running IDEX L1B MSG processing on dataset: " + f"{l1a_dataset.attrs['Logical_source']}" + ) + # create the attribute manager for this data level + idex_attrs = get_idex_attrs("l1b") + # set up a dataset with only epoch. + l1b_dataset = setup_dataset(l1a_dataset, [], idex_attrs, data_vars=None) + l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_msg") + # Compute science_on and pulser_on variables based on the event message. The + # "science_on" variable indicates when the science data collection is turned on or + # off and the "pulser_on" variable indicates when the pulser is turned on or off. + # The following logic is applied to determine the pulser_on status. + # enstim → set pulser_on = 1 + # susprel AND the previous message was enstim → set pulser_on = 0 + # susprel but previous message was NOT enstim → pulser_on stays whatever it was + l1a_messages = l1a_dataset.messages.values + # Set science_on to 1 when science is on and 0 when it is off. 255 otherwise. + science_on = np.where(l1a_messages == EventMessage.SCIENCE_ON.value, 1, 255) + science_on[l1a_messages == EventMessage.SCIENCE_OFF.value] = 0 + # Find indices where there are consecutive PULSER_ON followed by PULSER_OFF + # messages. These are the only cases where we should set pulser_on to 1 and 0. + # Compare the messages by shifting the pulser off messages back by one and looking + # for matching overlaps. + consecutive_pulser_on_off = np.where( + (l1a_messages[:-1] == EventMessage.PULSER_ON.value) + & (l1a_messages[1:] == EventMessage.PULSER_OFF.value) + )[0] + pulser_on = np.full(len(l1a_messages), 255) # initialize with 255 (unknown) + pulser_on[consecutive_pulser_on_off] = 1 + pulser_on[consecutive_pulser_on_off + 1] = 0 + l1b_dataset["pulser_on"] = xr.DataArray( + data=pulser_on, + dims="epoch", + name="pulser_on", + attrs=idex_attrs.get_variable_attributes("pulser_on"), + ) + l1b_dataset["science_on"] = xr.DataArray( + data=science_on, + dims="epoch", + name="science_on", + attrs=idex_attrs.get_variable_attributes("science_on"), + ) + + # Filter dataset to only include rows where there is an event + # (either science or pulser) + null_event = (pulser_on == 255) & (science_on == 255) + l1b_dataset = l1b_dataset.isel(epoch=~null_event) + if len(l1b_dataset["epoch"]) == 0: + logger.warning( + "No science or pulser events found. No l1b dataset will be created." + ) + return None + logger.info("IDEX L1B MSG data processing completed.") + return l1b_dataset + + +def idex_l1b_science(l1a_dataset: xr.Dataset) -> xr.Dataset: + """ + Will process IDEX l1a science data. Parameters ---------- @@ -117,21 +251,21 @@ def idex_l1b(l1a_dataset: xr.Dataset) -> xr.Dataset: # used for calculations yet but are saved in the CDF for reference. spice_data = get_spice_data(l1a_dataset, idex_attrs) - trigger_settings = get_trigger_mode_and_level(l1a_dataset) - if trigger_settings: - trigger_settings["triggerlevel"].attrs = idex_attrs.get_variable_attributes( - "trigger_level" - ) - trigger_settings["triggermode"].attrs = idex_attrs.get_variable_attributes( - "trigger_mode" - ) - + trigger_settings = get_trigger_mode_and_level(l1a_dataset, idex_attrs) + trigger_origin = get_trigger_origin( + l1a_dataset["idx__txhdrtrigid"].data, idex_attrs + ) # Create l1b Dataset - prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample"] - data_vars = processed_vars | waveforms_converted | trigger_settings | spice_data + prefixes = ["shcoarse", "shfine", "time_high_sample", "time_low_sample", "aid"] + data_vars = ( + processed_vars + | waveforms_converted + | trigger_settings + | spice_data + | trigger_origin + ) l1b_dataset = setup_dataset(l1a_dataset, prefixes, idex_attrs, data_vars) l1b_dataset.attrs = idex_attrs.get_global_attributes("imap_idex_l1b_sci") - # Convert variables l1b_dataset = convert_raw_to_eu( l1b_dataset, @@ -225,6 +359,7 @@ def convert_waveforms( def get_trigger_mode_and_level( l1a_dataset: xr.Dataset, + idex_attrs: ImapCdfAttributes, ) -> dict[str, xr.DataArray] | dict: """ Determine the trigger mode and threshold level for each event. @@ -233,6 +368,8 @@ def get_trigger_mode_and_level( ---------- l1a_dataset : xarray.Dataset IDEX L1a dataset containing the six waveform arrays and instrument settings. + idex_attrs : ImapCdfAttributes + CDF attribute manager object. Returns ------- @@ -243,8 +380,8 @@ def get_trigger_mode_and_level( channels = ["lg", "mg", "hg"] # 10 bit mask mask = 0b1111111111 - trigger_modes = [] - trigger_levels = [] + # Initialize a dict to hold the mode labels and threshold levels for each channel + data_dict = {} def compute_trigger_values( trigger_mode: int, trigger_controls: int, gain_channel: str @@ -302,28 +439,63 @@ def compute_trigger_values( vectorize=True, output_dtypes=[object, float], ) - trigger_modes.append(mode_array.rename("trigger_mode")) - trigger_levels.append(level_array.rename("trigger_level")) - - try: # There should be an array of modes and threshold levels for each channel. - # At each index (event) only one of the three arrays should have a value that is - # not 'None' because each event can only be triggered by one channel. - # By merging the three arrays, we get value for each event. - merged_modes = xr.merge([trigger_modes[0], xr.merge(trigger_modes[1:])]) - merged_levels = xr.merge([trigger_levels[0], xr.merge(trigger_levels[1:])]) - - return { - "triggermode": merged_modes.trigger_mode, - "triggerlevel": merged_levels.trigger_level, - } - - except xr.MergeError as e: - raise ValueError( - f"Only one channel can trigger a dust event. Please make sure " - f"there is only one valid trigger value per event. This " - f"caused Merge Error: {e}" - ) from e + # write each of them out as separate variables because there may be + # multiple channels that can trigger an event. The trigger origin variable + # can be used to determine which channel(s) triggered the event. + mode_array.attrs = idex_attrs.get_variable_attributes(f"trigger_mode_{channel}") + data_dict[f"trigger_mode_{channel}"] = mode_array + level_array.attrs = idex_attrs.get_variable_attributes( + f"trigger_level_{channel}" + ) + data_dict[f"trigger_level_{channel}"] = level_array + + return data_dict + + +def get_trigger_origin( + trigger_id: NDArray, idex_attrs: ImapCdfAttributes +) -> dict[str, DataArray]: + """ + Determine the trigger origin for each event. + + Parameters + ---------- + trigger_id : numpy.ndarray + Array of raw trigger ID values from the l1a dataset. The trigger ID is a 32-bit + integer where the lower 10 bits contain information about the trigger origin. + idex_attrs : ImapCdfAttributes + CDF attribute manager object. + + Returns + ------- + dict[str, xarray.DataArray] + A dictionary containing the trigger_origin DataArray with the trigger + origin info for each event. + """ + # extract the lower 10 bits of the trigger ID to get the trigger origin information + trigger_bits = trigger_id & 0x3FF + # For each event, determine which bits are set and get the corresponding trigger + # origin labels + origin_labels = np.array( + [ + ", ".join( + [TRIGGER_LABELS[TriggerOrigin(i)] for i in range(6) if (bits >> i) & 1] + ) + for bits in trigger_bits + ], + dtype=object, + ) + # Update any events with no trigger bits set to "unknown trigger origin" + origin_labels[origin_labels == ""] = "Unknown trigger origin" + return { + "trigger_origin": xr.DataArray( + name="trigger_origin", + data=np.squeeze(origin_labels), + dims="epoch", + attrs=idex_attrs.get_variable_attributes("trigger_origin"), + ) + } def get_spice_data( diff --git a/imap_processing/idex/idex_l2b.py b/imap_processing/idex/idex_l2b.py index 7712549642..0fb1939225 100644 --- a/imap_processing/idex/idex_l2b.py +++ b/imap_processing/idex/idex_l2b.py @@ -12,12 +12,13 @@ l0_file = "imap_processing/tests/idex/imap_idex_l0_raw_20231218_v001.pkts" l0_file_hk = "imap_processing/tests/idex/imap_idex_l0_raw_20250108_v001.pkts" - l1a_data = PacketParser(l0_file).data[0] - evt_data = PacketParser(l0_file_hk).data[0] - l1a_data, l1a_evt_data, l1b_evt_data = PacketParser(l0_file) - l1b_data = idex_l1b(l1a_data) + l1a_data, _ = PacketParser(l0_file) + _, l1a_msg_data = PacketParser(l0_file_hk) + msg_data_l1b = idex_l1b(msg_data_l1a, "msg") + l1b_data = idex_l1b(l1a_data, "sci-1week") + l1a_data = idex_l2a(l1b_data) - l2b_and_l2c_datasets = idex_l2b(l2a_data, [evt_data]) + l2b_and_l2c_datasets = idex_l2b(l2a_data, [msg_data_l1b]) write_cdf(l2b_and_l2c_datasets[0]) write_cdf(l2b_and_l2c_datasets[1]) """ @@ -38,7 +39,6 @@ IDEX_EVENT_REFERENCE_FRAME, IDEX_SPACING_DEG, SECONDS_IN_DAY, - IDEXEvtAcquireCodes, ) from imap_processing.idex.idex_utils import get_idex_attrs from imap_processing.spice.time import epoch_to_doy, et_to_datetime64, ttj2000ns_to_et @@ -84,7 +84,7 @@ def idex_l2b( - l2a_datasets: list[xr.Dataset], evt_datasets: list[xr.Dataset] + l2a_datasets: list[xr.Dataset], msg_data_l1b: list[xr.Dataset] ) -> list[xr.Dataset]: """ Will process IDEX l2a data to create l2b and l2c data products. @@ -96,8 +96,8 @@ def idex_l2b( ---------- l2a_datasets : list[xarray.Dataset] IDEX L2a datasets to process. - evt_datasets : list[xarray.Dataset] - List of IDEX housekeeping event message datasets. + msg_data_l1b : list[xarray.Dataset] + List of IDEX L1B event message datasets. Returns ------- @@ -113,8 +113,9 @@ def idex_l2b( # create the attribute manager for this data level idex_l2b_attrs = get_idex_attrs("l2b") idex_l2c_attrs = get_idex_attrs("l2c") - evt_dataset = xr.concat(evt_datasets, dim="epoch") - + msg_ds = ( + xr.concat(msg_data_l1b, dim="epoch").sortby("epoch").drop_duplicates("epoch") + ) # Concat all the l2a datasets together l2a_dataset = xr.concat(l2a_datasets, dim="epoch") epoch_doy = epoch_to_doy(l2a_dataset["epoch"].data) @@ -130,10 +131,14 @@ def idex_l2b( counts_by_mass_map, daily_epoch, ) = compute_counts_by_charge_and_mass(l2a_dataset, epoch_doy_unique) - # Get science acquisition start and stop times - _, evt_time, evt_values = get_science_acquisition_timestamps(evt_dataset) + # Filter the message dataset to only include science acquisition on/off events. + # (ignore fill vals) + science_on_msg_ds = msg_ds.isel(epoch=np.isin(msg_ds.science_on, [0, 1])) + msg_time = science_on_msg_ds["epoch"].data + msg_values = science_on_msg_ds["science_on"].data + # Get science acquisition percentage for each day - daily_on_percentage = get_science_acquisition_on_percentage(evt_time, evt_values) + daily_on_percentage = get_science_acquisition_on_percentage(msg_time, msg_values) ( rate_by_charge, rate_by_mass, @@ -164,7 +169,7 @@ def idex_l2b( common_vars = { "on_off_times": xr.DataArray( name="on_off_times", - data=evt_time, + data=msg_time, dims="on_off_times", attrs=idex_l2b_attrs.get_variable_attributes( "on_off_times", check_schema=False @@ -172,7 +177,7 @@ def idex_l2b( ), "on_off_events": xr.DataArray( name="on_off_events", - data=np.asarray(evt_values, dtype=np.uint8), + data=np.asarray(msg_values, dtype=np.uint8), dims="on_off_times", attrs=idex_l2b_attrs.get_variable_attributes( "on_off_events", check_schema=False @@ -390,7 +395,7 @@ def compute_counts_by_charge_and_mass( counts_by_mass = [] counts_by_charge_map = [] counts_by_mass_map = [] - daily_epoch = np.zeros(len(epoch_doy_unique), dtype=np.float64) + daily_epoch: np.ndarray = np.zeros(len(epoch_doy_unique), dtype=np.float64) for i in range(len(epoch_doy_unique)): doy = epoch_doy_unique[i] # Get the indices for the current day @@ -589,79 +594,17 @@ def bin_spin_phases(spin_phases: xr.DataArray) -> np.ndarray: return np.asarray(bin_indices) -def get_science_acquisition_timestamps( - evt_dataset: xr.Dataset, -) -> tuple[np.ndarray, np.ndarray, np.ndarray]: - """ - Get the science acquisition start and stop times and messages from the event data. - - Parameters - ---------- - evt_dataset : xarray.Dataset - Contains IDEX event message data. - - Returns - ------- - event_logs : np.ndarray - Array containing science acquisition start and stop events messages. - event_timestamps : np.ndarray - Array containing science acquisition start and stop timestamps. - event_values : np.ndarray - Array containing values indicating if the event is a start (1) or - stop (0). - """ - # Sort the event dataset by the epoch time. Drop duplicates - evt_dataset = evt_dataset.sortby("epoch").drop_duplicates("epoch") - # First find indices of the state change events - sc_indices = np.where(evt_dataset["elid_evtpkt"].data == "SCI_STE")[0] - event_logs = [] - event_timestamps = [] - event_values = [] - # Get the values of the state change events - val1 = ( - evt_dataset["el1par_evtpkt"].data[sc_indices] << 8 - | evt_dataset["el2par_evtpkt"].data[sc_indices] - ) - val2 = ( - evt_dataset["el3par_evtpkt"].data[sc_indices] << 8 - | evt_dataset["el4par_evtpkt"].data[sc_indices] - ) - epochs = evt_dataset["epoch"][sc_indices].data - # Now the state change values and check if it is either a science - # acquisition start or science acquisition stop event. - for v1, v2, epoch in zip(val1, val2, epochs, strict=False): - # An "acquire" start will have val1=ACQSETUP and val2=ACQ - # An "acquire" stop will have val1=ACQ and val2=CHILL - if (v1, v2) == (IDEXEvtAcquireCodes.ACQSETUP, IDEXEvtAcquireCodes.ACQ): - event_logs.append("SCI state change: ACQSETUP to ACQ") - event_timestamps.append(epoch) - event_values.append(1) - elif (v1, v2) == (IDEXEvtAcquireCodes.ACQ, IDEXEvtAcquireCodes.CHILL): - event_logs.append("SCI state change: ACQ to CHILL") - event_timestamps.append(epoch) - event_values.append(0) - - logger.info( - f"Found science acquisition events: {event_logs} at times: {event_timestamps}" - ) - return ( - np.asarray(event_logs), - np.asarray(event_timestamps), - np.asarray(event_values), - ) - - def get_science_acquisition_on_percentage( - evt_time: NDArray, evt_values: NDArray + msg_time: NDArray, msg_values: NDArray ) -> dict: """ Calculate the percentage of time science acquisition was occurring for each day. Parameters ---------- - evt_time : np.ndarray + msg_time : np.ndarray Array of timestamps for science acquisition start and stop events. - evt_values : np.ndarray + msg_values : np.ndarray Array of values indicating if the event is a start (1) or stop (0). Returns @@ -670,7 +613,7 @@ def get_science_acquisition_on_percentage( Percentages of time the instrument was in science acquisition mode for each day of year. """ - if len(evt_time) == 0: + if len(msg_time) == 0: logger.warning( "No science acquisition events found in event dataset. Returning empty " "uptime percentages. All rate variables will be set to -1." @@ -680,17 +623,17 @@ def get_science_acquisition_on_percentage( daily_totals: collections.defaultdict = defaultdict(timedelta) daily_on: collections.defaultdict = defaultdict(timedelta) # Convert epoch event times to datetime - dates = et_to_datetime64(ttj2000ns_to_et(evt_time)).astype(datetime) + dates = et_to_datetime64(ttj2000ns_to_et(msg_time)).astype(datetime) # Simulate an event at the start of the first day. start_of_first_day = dates[0].replace(hour=0, minute=0, second=0, microsecond=0) # Assume that the state at the start of the day is the opposite of what the first # state is. - state_at_start = 0 if evt_values[0] == 1 else 1 + state_at_start = 0 if msg_values[0] == 1 else 1 dates = np.insert(dates, 0, start_of_first_day) - evt_values = np.insert(evt_values, 0, state_at_start) + msg_values = np.insert(msg_values, 0, state_at_start) for i in range(len(dates)): start = dates[i] - state = evt_values[i] + state = msg_values[i] if i == len(dates) - 1: # If this is the last event, set the "end" value the end of the day. end = (start + timedelta(days=1)).replace( diff --git a/imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv b/imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv index 793897ef64..80927ecbe2 100644 --- a/imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv +++ b/imap_processing/idex/idex_variable_unpacking_and_eu_conversion.csv @@ -10,8 +10,8 @@ 9,detector_voltage,idx__txhdrhvpshkch01,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI 10,sensor_voltage,idx__txhdrhvpshkch01,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI 11,target_voltage,idx__txhdrhvpshkch23,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI -12,reflectron_voltage,idx__txhdrhvpshkch23,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI -13,rejection_voltage,idx__txhdrhvpshkch45,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI +12,rejection_voltage,idx__txhdrhvpshkch23,20,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI +13,reflectron_voltage,idx__txhdrhvpshkch45,4,4,12,V,0,1.4652,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI 14,current_hvps_sensor,idx__txhdrhvpshkch45,20,4,12,mA,0,0.000007326,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI 15,positive_current_hvps,idx__txhdrhvpshkch67,4,4,12,mA,0,0.000024339,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI 16,negative_current_hvps,idx__txhdrhvpshkch67,20,4,12,mA,0,0.000024339,0,0,0,0,0,0,UNSEGMENTED_POLY,IDEX_SCI diff --git a/imap_processing/lo/l1b/lo_l1b.py b/imap_processing/lo/l1b/lo_l1b.py index 9dff5341ed..b6ce01d517 100644 --- a/imap_processing/lo/l1b/lo_l1b.py +++ b/imap_processing/lo/l1b/lo_l1b.py @@ -169,8 +169,29 @@ "exposure_time_6deg", "spin_cycle", ] + +# Fields to include in the split background rates/goodtimes datasets +BACKGROUND_RATE_FIELDS = [ + "start_met", + "end_met", + "bin_start", + "bin_end", + "h_background_rates", + "h_background_variance", + "o_background_rates", + "o_background_variance", +] +GOODTIMES_FIELDS = [ + "gt_start_met", + "gt_end_met", + "bin_start", + "bin_end", + "esa_goodtime_flags", +] + # ------------------------------------------------------------------- DE_CLOCK_TICK_S = 4.096e-3 # seconds per DE clock tick +NUM_ESA_STEPS = 7 def lo_l1b( @@ -234,6 +255,11 @@ def lo_l1b( ds = l1b_star(sci_dependencies, attr_mgr_l1b) datasets_to_return.append(ds) + elif descriptor == "goodtimes": + logger.info("\nProcessing IMAP-Lo L1B Background Rates and Goodtimes...") + ds = l1b_bgrates_and_goodtimes(sci_dependencies, attr_mgr_l1b) + datasets_to_return.extend(ds) + else: logger.warning(f"Unexpected descriptor: {descriptor!r}") @@ -482,7 +508,7 @@ def set_esa_mode( # Get the ESA mode for the pointing esa_mode = sweep_df["esa_mode"].values[0] # Repeat the ESA mode for each direct event in the pointing - esa_mode_array = np.repeat(esa_mode, len(l1b_science["epoch"])) + esa_mode_array: np.ndarray = np.repeat(esa_mode, len(l1b_science["epoch"])) else: raise ValueError("Multiple ESA modes found in sweep table for pointing.") @@ -1198,8 +1224,12 @@ def set_bad_or_goodtimes( # Combined mask for epochs that fall within the time and bin ranges combined_mask = time_mask & bin_mask + # TODO: Handle the case where no matching rows are found, because + # otherwise, the bacgkround rates will be set to 0 for those epochs, + # which is not correct. + # Get the time flags for each epoch's esa_step from matching rows - time_flags = np.zeros(len(epochs), dtype=int) + time_flags: np.ndarray = np.zeros(len(epochs), dtype=int) for epoch_idx in range(len(epochs)): matching_rows = np.where(combined_mask[epoch_idx])[0] if len(matching_rows) > 0: @@ -1811,7 +1841,7 @@ def calculate_de_rates( ) # exposure time shape: (num_asc, num_esa_steps) - exposure_time = np.zeros((num_asc, 7), dtype=float) + exposure_time: np.ndarray = np.zeros((num_asc, 7), dtype=float) # exposure_time_6deg = 4 * avg_spin_per_asc / 60 # 4 sweeps per ASC (28 / 7) in 60 bins asc_avg_spin_durations = 4 * l1b_de["avg_spin_durations"].data[unique_idx] / 60 @@ -2016,7 +2046,7 @@ def _get_esa_level_indices(epochs: np.ndarray, anc_dependencies: list) -> np.nda # Can we just take the last 7 entries of the sweep table for that # date and use those values instead of this extra work with the # separate LUT ancillary file? - energy_step_mapping = np.zeros(7, dtype=int) + energy_step_mapping: np.ndarray = np.zeros(7, dtype=int) # Loop through the LUT entries and populate the mapping for _, row in lut_entries.iterrows(): # Original ESA step index is 1-based, convert to 0-based @@ -2172,7 +2202,7 @@ def calculate_star_sensor_profile_for_group( count_array = valid_bin_mask.sum(axis=0).astype(np.int32) # Compute average amplitude per bin - avg_amplitude = np.full(720, np.nan, dtype=np.float64) + avg_amplitude: np.ndarray = np.full(720, np.nan, dtype=np.float64) mask = count_array > 0 avg_amplitude[mask] = sum_array[mask] / count_array[mask] @@ -2261,15 +2291,15 @@ def calculate_star_sensor_profiles_by_group( logger.debug(f"Last group contains {last_group_size} records (partial group)") # Assign group labels to the dataset for xarray groupby operations - group_labels = np.repeat(np.arange(n_groups), group_size)[:n_valid] + group_labels: np.ndarray = np.repeat(np.arange(n_groups), group_size)[:n_valid] l1a_star = l1a_star.assign_coords(group=("epoch", group_labels)) # Extract first MET for each group using xarray groupby group_mets = l1a_star["shcoarse"].groupby("group").first().values.astype(np.int64) # Initialize output arrays - avg_amplitudes = np.zeros((n_groups, 720), dtype=np.float64) - counts_per_bin = np.zeros((n_groups, 720), dtype=np.int32) + avg_amplitudes: np.ndarray = np.zeros((n_groups, 720), dtype=np.float64) + counts_per_bin: np.ndarray = np.zeros((n_groups, 720), dtype=np.int32) # Process each group using xarray groupby for group_label, group_data in l1a_star.groupby("group"): @@ -2466,3 +2496,367 @@ def l1b_star( ) return l1b_star_ds + + +def l1b_bgrates_and_goodtimes( # noqa: PLR0912 + sci_dependencies: dict, + attr_mgr_l1b: ImapCdfAttributes, + cycle_count: int = 10, + delay_max: int = 840, +) -> xr.Dataset: + """ + Create the IMAP-Lo L1B Background dataset. + + Creates a Background dataset from the L1B Histogram Rates dataset. + + Parameters + ---------- + sci_dependencies : dict + Dictionary of datasets needed for L1B data product creation in xarray Datasets. + attr_mgr_l1b : ImapCdfAttributes + Attribute manager for L1B dataset metadata. + cycle_count : int + Maximum number of ASCs to group together (default: 10). + delay_max : int + Maximum allowed delay between entries in seconds (default: 840). + + Returns + ------- + l1b_bgrates_ds : xr.Dataset + L1B bgrates dataset with ESA flags per epoch and bin. + Each dataset also includes a background rate. + """ + l1b_histrates = sci_dependencies["imap_lo_l1b_histrates"] + # l1b_nhk = sci_dependencies["imap_lo_l1b_nhk"] + + # Initialize the dataset + l1b_backgrounds_and_goodtimes_ds = xr.Dataset() + datasets_to_return = [] + + # Set the expected background rate based on the pivot angle + # This assumes a static pivot_angle for the entire pointing + # pivot_angle = _get_nearest_pivot_angle(l1b_histrates["epoch"].values[0], l1b_nhk) + # if (pivot_angle < 95.0) & (pivot_angle > 85.0): + # h_bg_rate_nom = 0.0028 + # else: + # h_bg_rate_nom = 0.0033 + h_bg_rate_nom = 0.0028 + o_bg_rate_nom = h_bg_rate_nom / 100 + + interval_nom = 420 * cycle_count # seconds + exposure = interval_nom * 0.5 # 50% duty cycle + + h_intensity = np.sum( + l1b_histrates["h_counts"][:, 0:NUM_ESA_STEPS, 20:50], axis=(1, 2) + ) + o_intensity = np.sum( + l1b_histrates["o_counts"][:, 0:NUM_ESA_STEPS, 20:50], axis=(1, 2) + ) + + # Use proper SPICE-based time conversion with current kernels + # Note: The reference script adds +9 seconds because they use an + # "older time kernel (pre 2012)" + # We use current SPICE kernels, so we should NOT add that offset + met = ttj2000ns_to_met(l1b_histrates["epoch"].values) + + max_row_count = np.shape(h_intensity)[0] + bg_start_met = xr.DataArray([0.0]) + bg_end_met = xr.DataArray([0.0]) + epochs = l1b_histrates["epoch"].values.copy() + epochs = xr.DataArray(epochs, dims=["epoch"]) + goodtimes = xr.DataArray(np.zeros((max_row_count, 2), dtype=np.int64)) + h_background_rate = xr.DataArray(np.zeros((1, NUM_ESA_STEPS), dtype=np.float32)) + h_background_rate_variance = xr.DataArray( + np.zeros((1, NUM_ESA_STEPS), dtype=np.float32) + ) + o_background_rate = xr.DataArray(np.zeros((1, NUM_ESA_STEPS), dtype=np.float32)) + o_background_rate_variance = xr.DataArray( + np.zeros((1, NUM_ESA_STEPS), dtype=np.float32) + ) + + # Walk through the histrate data in chunks of cycle_count (10) + # and identify goodtime intervals and calculate background rates + row_count = 0 + sum_h_bg_counts = 0.0 + sum_h_bg_exposure = 0.0 + sum_o_bg_counts = 0.0 + begin = 0.0 + end = 0.0 + logger.debug( + f"Starting goodtimes calculation with {max_row_count} epochs, " + f"cycle_count={cycle_count}, delay_max={delay_max}" + ) + logger.debug(f"h_bg_rate_nom={h_bg_rate_nom}, exposure={exposure}") + for index in range(0, max_row_count, cycle_count): + # Calculate the interval for this chunk + if (index + cycle_count - 1) < max_row_count: + interval = met[index + cycle_count - 1] - met[index] + else: + interval = interval_nom * max_row_count + + logger.debug( + f"\n Index {index}: met[{index}]=" + f"{met[index] if index < max_row_count else 'N/A'}, " + f"interval={interval}, begin={begin}" + ) + + # Skip this chunk if the interval is too large (indicates a gap) + if interval > (interval_nom + delay_max): + logger.debug( + f" Skipping chunk due to large interval ({interval} > " + f"{interval_nom + delay_max})" + ) + # If we were tracking a goodtime interval, close it before the gap + if begin > 0.0: + end = met[index - 1] + logger.debug(f" Closing interval before gap: {begin} -> {end}") + + epochs[row_count] = l1b_histrates["epoch"][index - 1].values.item() + goodtimes[row_count, :] = [int(begin - 620), int(end + 320)] + logger.debug( + f" STORED interval {row_count} (large interval): " + f"{int(begin - 620)} -> {int(end + 320)} (raw: {begin} -> {end})" + ) + + row_count += 1 + begin = 0.0 + end = 0.0 + + # Skip this chunk after closing interval + continue + + # Check for time gap from previous chunk + delta_time = 0.0 + if index > 0: + delta_time = met[index] - (met[index - 1] + 420) + logger.debug( + f" Delta time from previous: {delta_time} (max: {delay_max})" + ) + + # If there's a gap and we have an active interval, close it + if (delta_time > delay_max) & (begin > 0.0): + end = met[index - 1] + logger.debug(f" Closing interval due to time gap: {begin} -> {end}") + + epochs[row_count] = l1b_histrates["epoch"][index - 1].values.item() + goodtimes[row_count, :] = [int(begin - 620), int(end + 320)] + logger.debug( + f" STORED interval {row_count} (time gap): " + f"{int(begin - 620)} -> {int(end + 320)} (raw: {begin} -> {end})" + ) + + row_count += 1 + begin = 0.0 + end = 0.0 + + # Calculate counts and rate for this chunk + antiram_h_counts = float(np.sum(h_intensity[index : index + cycle_count])) + antiram_o_counts = float(np.sum(o_intensity[index : index + cycle_count])) + antiram_h_rate = antiram_h_counts / exposure + + logger.debug( + f" Rate: {antiram_h_rate:.6f}, threshold: {h_bg_rate_nom:.6f}, " + f"counts: {antiram_h_counts}" + ) + + # If rate is below threshold, accumulate for background + if antiram_h_rate < h_bg_rate_nom: + if begin == 0.0: + begin = met[index] + logger.debug(f" Starting new interval at {begin}") + + sum_h_bg_counts = sum_h_bg_counts + antiram_h_counts + sum_o_bg_counts = sum_o_bg_counts + antiram_o_counts + sum_h_bg_exposure = sum_h_bg_exposure + exposure + + # If rate exceeds threshold, close the interval if one is active + if antiram_h_rate >= h_bg_rate_nom: + if begin > 0.0: + end = met[index - 1] + logger.debug( + f" Closing interval due to rate threshold: {begin} -> {end}" + ) + print(" antiram_h_rate: ", antiram_h_rate, " at index ", index) + print("l1b_histrates epoch: ", l1b_histrates["epoch"][index - 1].values) + epochs[row_count] = l1b_histrates["epoch"][index - 1].values.item() + goodtimes[row_count, :] = [int(begin - 620), int(end + 320)] + logger.debug( + f" STORED interval {row_count} (rate threshold): " + f"{int(begin - 620)} -> {int(end + 320)} (raw: {begin} -> {end})" + ) + + row_count += 1 + begin = 0.0 + end = 0.0 + + # Handle the final interval if one is still open + if (end == 0.0) & (begin > 0.0): + end = met[max_row_count - 1] + if end > begin: + epochs[row_count] = l1b_histrates["epoch"][max_row_count - 1] + goodtimes[row_count, :] = [int(begin - 620), int(end + 320)] + logger.debug( + f" STORED interval {row_count} (final): " + f"{int(begin - 620)} -> {int(end + 320)} (raw: {begin} -> {end})" + ) + + row_count += 1 + begin = 0.0 + end = 0.0 + + # Record the background rates for the entire pointing + if sum_h_bg_exposure > 0.0: + h_bg_rate = sum_h_bg_counts / sum_h_bg_exposure + h_bg_rate_variance = np.sqrt(sum_h_bg_counts) / sum_h_bg_exposure + o_bg_rate = sum_o_bg_counts / sum_h_bg_exposure + o_bg_rate_variance = np.sqrt(sum_o_bg_counts) / sum_h_bg_exposure + + if h_bg_rate_variance <= 0.0: + h_bg_rate_variance = h_bg_rate + + if o_bg_rate_variance <= 0.0: + o_bg_rate_variance = o_bg_rate + + if h_bg_rate <= 0.0: + h_bg_rate = h_bg_rate_nom / 50.0 + h_bg_rate_variance = h_bg_rate + + if o_bg_rate <= 0.0: + o_bg_rate = o_bg_rate_nom * 0.3 + o_bg_rate_variance = o_bg_rate + + h_background_rate[0, :] = np.full(NUM_ESA_STEPS, h_bg_rate) + h_background_rate_variance[0, :] = np.full(NUM_ESA_STEPS, h_bg_rate_variance) + o_background_rate[0, :] = np.full(NUM_ESA_STEPS, o_bg_rate) + o_background_rate_variance[0, :] = np.full(NUM_ESA_STEPS, o_bg_rate_variance) + bg_start_met[0] = met[0] + bg_end_met[0] = met[max_row_count - 1] + + # Handle case where no goodtimes were found -- produce a + # single record with invalid times (the defaults above) + if row_count == 0: + row_count = 1 + + # Trim arrays to actual size + epoch = epochs.isel(epoch=slice(0, row_count)) + goodtimes = goodtimes.isel(dim_0=slice(0, row_count)) + + l1b_backgrounds_and_goodtimes_ds["epoch"] = xr.DataArray( + data=epoch, + name="epoch", + dims=["epoch"], + attrs=attr_mgr_l1b.get_variable_attributes("epoch"), + ) + l1b_backgrounds_and_goodtimes_ds["epoch"].attrs["DEPEND_0"] = "epoch" + l1b_backgrounds_and_goodtimes_ds["start_met"] = xr.DataArray( + data=bg_start_met, + name="start_met", + dims=["met"], + attrs=attr_mgr_l1b.get_variable_attributes("met"), + ) + l1b_backgrounds_and_goodtimes_ds["end_met"] = xr.DataArray( + data=bg_end_met, + name="end_met", + dims=["met"], + attrs=attr_mgr_l1b.get_variable_attributes("met"), + ) + l1b_backgrounds_and_goodtimes_ds["gt_start_met"] = xr.DataArray( + data=goodtimes[:, 0], + name="Goodtime_start", + dims=["epoch"], + # attrs=attr_mgr_l1b.get_variable_attributes("epoch"), + ) + l1b_backgrounds_and_goodtimes_ds["gt_end_met"] = xr.DataArray( + data=goodtimes[:, 1], + name="Goodtime_end", + dims=["epoch"], + # attrs=attr_mgr_l1b.get_variable_attributes("epoch"), + ) + l1b_backgrounds_and_goodtimes_ds["h_background_rates"] = xr.DataArray( + data=h_background_rate, + name="h_bg_rate", + dims=["met", "esa_step"], + # attrs=attr_mgr_l1b.get_variable_attributes("esa_background_rates"), + ) + l1b_backgrounds_and_goodtimes_ds["h_background_variance"] = xr.DataArray( + data=h_background_rate_variance, + name="h_bg_rate_variance", + dims=["met", "esa_step"], + ) + l1b_backgrounds_and_goodtimes_ds["o_background_rates"] = xr.DataArray( + data=o_background_rate, + name="o_bg_rate", + dims=["met", "esa_step"], + # attrs=attr_mgr_l1b.get_variable_attributes("esa_background_rates"), + ) + l1b_backgrounds_and_goodtimes_ds["o_background_variance"] = xr.DataArray( + data=o_background_rate_variance, + name="o_bg_rate_variance", + dims=["met", "esa_step"], + ) + + # We're only creating one record for all bins for now + # Note that this is true for both GoodTimes and background rates, + # so we cheat here by just using one record. + l1b_backgrounds_and_goodtimes_ds["bin_start"] = xr.DataArray( + data=np.zeros(row_count, dtype=int), + name="bin_start", + dims=["epoch"], + # attrs=attr_mgr_l1b.get_variable_attributes("bin_start"), + ) + l1b_backgrounds_and_goodtimes_ds["bin_end"] = xr.DataArray( + data=np.zeros(row_count, dtype=int) + 59, + name="bin_end", + dims=["epoch"], + # attrs=attr_mgr_l1b.get_variable_attributes("bin_end"), + ) + + # For now, set all ESA flags to 1 (good) since we don't have + # an algorithm for this yet + l1b_backgrounds_and_goodtimes_ds["esa_goodtime_flags"] = xr.DataArray( + data=np.zeros((row_count, NUM_ESA_STEPS), dtype=int) + 1, + name="E-step", + dims=["epoch", "esa_step"], + # attrs=attr_mgr_l1b.get_variable_attributes("esa_goodtime_flags"), + ) + + logger.info("L1B Background Rates and Goodtimes created successfully") + + l1b_bgrates_ds, l1b_goodtimes_ds = split_backgrounds_and_goodtimes_dataset( + l1b_backgrounds_and_goodtimes_ds, attr_mgr_l1b + ) + datasets_to_return.extend([l1b_bgrates_ds, l1b_goodtimes_ds]) + print("epoch bgrates meta", l1b_bgrates_ds["epoch"].attrs) + print("epoch goodtimes meta", l1b_goodtimes_ds["epoch"].attrs) + return datasets_to_return + + +def split_backgrounds_and_goodtimes_dataset( + l1b_backgrounds_and_goodtimes_ds: xr.Dataset, attr_mgr_l1b: ImapCdfAttributes +) -> tuple[xr.Dataset, xr.Dataset]: + """ + Separate the L1B backgrounds and goodtimes dataset. + + Parameters + ---------- + l1b_backgrounds_and_goodtimes_ds : xr.Dataset + The L1B all backgrounds and goodtimes dataset containing + both background rates and goodtimes. + attr_mgr_l1b : ImapCdfAttributes + Attribute manager used to get the L1B background rates and + goodtimes dataset attributes. + + Returns + ------- + l1b_bgrates : xr.Dataset + The L1B background rates dataset. + l1b_goodtimes_rates : xr.Dataset + The L1B goodtimes rates dataset. + """ + # Use centralized lists for fields to include in split datasets + l1b_goodtimes_ds = l1b_backgrounds_and_goodtimes_ds[GOODTIMES_FIELDS] + l1b_goodtimes_ds.attrs = attr_mgr_l1b.get_global_attributes("imap_lo_l1b_goodtimes") + lib_bgrates_ds = l1b_backgrounds_and_goodtimes_ds[BACKGROUND_RATE_FIELDS] + lib_bgrates_ds.attrs = attr_mgr_l1b.get_global_attributes("imap_lo_l1b_bgrates") + + return lib_bgrates_ds, l1b_goodtimes_ds diff --git a/imap_processing/lo/l1b/tof_conversions.py b/imap_processing/lo/l1b/tof_conversions.py index 420fe8225a..9f2751e134 100644 --- a/imap_processing/lo/l1b/tof_conversions.py +++ b/imap_processing/lo/l1b/tof_conversions.py @@ -4,8 +4,9 @@ tof_conv = namedtuple("tof_conv", ["C0", "C1"]) # TOF conversion coefficients from Lo's TOF Conversion_annotated.docx +# TOF3_CONV was updated in March 2026 per email from Nathan # TODO: Ask Lo to put these in the algorithm document for better reference TOF0_CONV = tof_conv(C0=5.52524e-01, C1=1.68374e-01) TOF1_CONV = tof_conv(C0=-7.20181e-01, C1=1.65124e-01) TOF2_CONV = tof_conv(C0=3.74422e-01, C1=1.66409e-01) -TOF3_CONV = tof_conv(C0=4.41970e-01, C1=1.72024e-01) +TOF3_CONV = tof_conv(C0=4.6726e-01, C1=1.7144e-01) diff --git a/imap_processing/lo/l1c/lo_l1c.py b/imap_processing/lo/l1c/lo_l1c.py index 1204fe51f9..470ec01bc8 100644 --- a/imap_processing/lo/l1c/lo_l1c.py +++ b/imap_processing/lo/l1c/lo_l1c.py @@ -31,7 +31,9 @@ # 1 time, 7 energy steps, 3600 spin angle bins, and 40 off angle bins PSET_SHAPE = (1, N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS) PSET_DIMS = ["epoch", "esa_energy_step", "spin_angle", "off_angle"] -ESA_ENERGY_STEPS = np.arange(N_ESA_ENERGY_STEPS) + 1 # 1 to 7 inclusive +ESA_ENERGY_STEPS: np.ndarray = ( + np.arange(N_ESA_ENERGY_STEPS) + 1 # 1 to 7 inclusive +) SPIN_ANGLE_BIN_EDGES = np.linspace(0, 360, N_SPIN_ANGLE_BINS + 1) SPIN_ANGLE_BIN_CENTERS = (SPIN_ANGLE_BIN_EDGES[:-1] + SPIN_ANGLE_BIN_EDGES[1:]) / 2 OFF_ANGLE_BIN_EDGES = np.linspace(-2, 2, N_OFF_ANGLE_BINS + 1) @@ -689,7 +691,7 @@ def create_goodtimes_fraction( total_pointing_duration = pointing_end_met - pointing_start_met # Initialize as all zeros (no good time) - goodtimes_fraction = np.zeros( + goodtimes_fraction: np.ndarray = np.zeros( (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS), dtype=np.float32 ) @@ -799,7 +801,7 @@ def calculate_exposure_times( "Pointing duration is zero or negative. Exposure times will be zero." ) # Return zero exposure times with correct shape and dimensions - zero_exposure = np.zeros(PSET_SHAPE, dtype=np.float32) + zero_exposure: np.ndarray = np.zeros(PSET_SHAPE, dtype=np.float32) return xr.DataArray( data=zero_exposure, dims=PSET_DIMS, @@ -1067,14 +1069,17 @@ def set_background_rates( if species not in {FilterType.HYDROGEN, FilterType.OXYGEN}: raise ValueError(f"Species must be 'h' or 'o', but got {species.value}.") - bg_rates = np.zeros( - (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), dtype=np.float16 + bg_rates: np.ndarray = np.zeros( + (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), + dtype=np.float16, ) - bg_stat_uncert = np.zeros( - (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), dtype=np.float16 + bg_stat_uncert: np.ndarray = np.zeros( + (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), + dtype=np.float16, ) - bg_sys_err = np.zeros( - (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), dtype=np.float16 + bg_sys_err: np.ndarray = np.zeros( + (N_ESA_ENERGY_STEPS, N_SPIN_ANGLE_BINS, N_OFF_ANGLE_BINS), + dtype=np.float16, ) # read in the background rates from ancillary file diff --git a/imap_processing/mag/l0/decom_mag.py b/imap_processing/mag/l0/decom_mag.py index eb1973c299..b1e432948e 100644 --- a/imap_processing/mag/l0/decom_mag.py +++ b/imap_processing/mag/l0/decom_mag.py @@ -84,7 +84,7 @@ def generate_dataset( # TODO: Correct CDF attributes from email vector_data = np.zeros((len(l0_data), len(l0_data[0].VECTORS))) - shcoarse_data = np.zeros(len(l0_data), dtype="datetime64[ns]") + shcoarse_data: np.ndarray = np.zeros(len(l0_data), dtype="datetime64[ns]") support_data = defaultdict(list) diff --git a/imap_processing/mag/l1a/mag_l1a_data.py b/imap_processing/mag/l1a/mag_l1a_data.py index f4bfa22f10..ca360490fc 100644 --- a/imap_processing/mag/l1a/mag_l1a_data.py +++ b/imap_processing/mag/l1a/mag_l1a_data.py @@ -378,7 +378,7 @@ def update_compression_array( Length of new array to add or append to the compression_flags attribute. This is expected to be the length of the vector array. """ - new_flags = np.full( + new_flags: np.ndarray = np.full( (length, 2), [packet_properties.compression, packet_properties.compression_width], dtype=np.int8, @@ -1066,7 +1066,7 @@ def unpack_one_vector( f"{width * AXIS_COUNT} or {width * AXIS_COUNT + RANGE_BIT_WIDTH} if " f"has_range." ) - padding = np.zeros(8 - (width % 8), dtype=np.uint8) + padding: np.ndarray = np.zeros(8 - (width % 8), dtype=np.uint8) # take slices of the input data and pack from an array of bits to an array of # uint8 bytes diff --git a/imap_processing/mag/l1d/mag_l1d_data.py b/imap_processing/mag/l1d/mag_l1d_data.py index add61b8475..fac808f5dd 100644 --- a/imap_processing/mag/l1d/mag_l1d_data.py +++ b/imap_processing/mag/l1d/mag_l1d_data.py @@ -174,7 +174,10 @@ def __post_init__(self, day: np.datetime64) -> None: self.frame = ValidFrames.MAGO # set the magnitude before truncating - self.magnitude = np.zeros(self.vectors.shape[0], dtype=np.float64) # type: ignore[has-type] + self.magnitude: np.ndarray = np.zeros( # type: ignore[var-annotated] + self.vectors.shape[0], # type: ignore[has-type] + dtype=np.float64, + ) self.truncate_to_24h(day) self.vectors, self.magi_vectors = self._calibrate_and_offset_vectors( @@ -295,26 +298,40 @@ def rotate_frame(self, end_frame: ValidFrames) -> None: self.epoch_et: np.ndarray = ttj2000ns_to_et(self.epoch) self.magi_epoch_et: np.ndarray = ttj2000ns_to_et(self.magi_epoch) - self.vectors = frame_transform( + new_vectors = frame_transform( self.epoch_et, self.vectors, from_frame=start_frame.spice_frame, to_frame=end_frame.spice_frame, allow_spice_noframeconnect=True, ) + if np.isnan(self.vectors).any() or (self.vectors == FILLVAL).any(): + new_vectors = np.where( + np.isnan(self.vectors) | (self.vectors == FILLVAL), + FILLVAL, + new_vectors, + ) + self.vectors = new_vectors # If we were in MAGO frame, we need to rotate MAGI vectors from MAGI to # end_frame if start_frame == ValidFrames.MAGO: start_frame = ValidFrames.MAGI - self.magi_vectors = frame_transform( + new_magi_vectors = frame_transform( self.magi_epoch_et, self.magi_vectors, from_frame=start_frame.spice_frame, to_frame=end_frame.spice_frame, allow_spice_noframeconnect=True, ) + if np.isnan(self.magi_vectors).any() or (self.magi_vectors == FILLVAL).any(): + new_magi_vectors = np.where( + np.isnan(self.magi_vectors) | (self.magi_vectors == FILLVAL), + FILLVAL, + new_magi_vectors, + ) + self.magi_vectors = new_magi_vectors self.frame = end_frame @@ -441,7 +458,7 @@ def calculate_spin_offsets(self) -> xr.Dataset: epoch_met = ttj2000ns_to_met(self.epoch) sc_spin_phase = spin.get_spacecraft_spin_phase(epoch_met) # mark vectors as nan where they are nan in sc_spin_phase - vectors = self.vectors.copy().astype(np.float64) + vectors: np.ndarray = self.vectors.copy().astype(np.float64) vectors[np.isnan(sc_spin_phase), :] = np.nan @@ -528,8 +545,8 @@ def calculate_spin_offsets(self) -> xr.Dataset: if not np.isnan(avg_x) and not np.isnan(avg_y): offset_epochs.append(chunk_epoch[0]) - x_avg_calcs.append(avg_x) - y_avg_calcs.append(avg_y) + x_avg_calcs.append(np.float64(avg_x)) + y_avg_calcs.append(np.float64(avg_y)) # Add validity time range for this chunk validity_start_times.append(chunk_epoch[0]) diff --git a/imap_processing/mag/l2/mag_l2.py b/imap_processing/mag/l2/mag_l2.py index 2c47dd5026..65632ae1a5 100644 --- a/imap_processing/mag/l2/mag_l2.py +++ b/imap_processing/mag/l2/mag_l2.py @@ -12,6 +12,14 @@ logger = logging.getLogger(__name__) +DEFAULT_L2_FRAMES = [ + ValidFrames.SRF, + ValidFrames.GSE, + ValidFrames.GSM, + ValidFrames.RTN, + ValidFrames.DSRF, # should be last as some vectors may become NaN +] + def mag_l2( calibration_dataset: xr.Dataset, @@ -19,6 +27,7 @@ def mag_l2( input_data: xr.Dataset, day_to_process: np.datetime64, mode: DataMode = DataMode.NORM, + frames: list[ValidFrames] = DEFAULT_L2_FRAMES, ) -> list[xr.Dataset]: """ Complete MAG L2 processing. @@ -70,6 +79,9 @@ def mag_l2( mode : DataMode The data mode to process. Default is DataMode.NORM (normal mode). Can also be DataMode.BURST for burst mode processing. + frames : list[ValidFrames] + List of frames to output. DEFAULT_L2_FRAMES is [SRF, GSE, GSM, RTN, DSRF] + Note that DSRF should be last as some vectors may become NaN after rotation. Returns ------- @@ -79,6 +91,9 @@ def mag_l2( """ always_output_mago = configuration.ALWAYS_OUTPUT_MAGO + if not frames: + frames = DEFAULT_L2_FRAMES + # TODO Check that the input file matches the offsets file if not np.array_equal(input_data["epoch"].data, offsets_dataset["epoch"].data): raise ValueError("Input file and offsets file must have the same timestamps.") @@ -118,19 +133,13 @@ def mag_l2( attributes.add_instrument_variable_attrs("mag", "l2") # Rotate from the MAG frame into the SRF frame - frames: list[xr.Dataset] = [] - - for frame in [ - ValidFrames.SRF, - ValidFrames.GSE, - ValidFrames.GSM, - ValidFrames.RTN, - ValidFrames.DSRF, # should be last as some vectors may become NaN - ]: + datasets: list[xr.Dataset] = [] + + for frame in frames: l2_data.rotate_frame(frame) - frames.append(l2_data.generate_dataset(attributes, day)) + datasets.append(l2_data.generate_dataset(attributes, day)) - return frames + return datasets def retrieve_matrix_from_l2_calibration( diff --git a/imap_processing/mag/l2/mag_l2_data.py b/imap_processing/mag/l2/mag_l2_data.py index f7e1aea61a..dc60d4c863 100644 --- a/imap_processing/mag/l2/mag_l2_data.py +++ b/imap_processing/mag/l2/mag_l2_data.py @@ -7,6 +7,7 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes +from imap_processing.mag import constants from imap_processing.mag.constants import FILLVAL, DataMode from imap_processing.mag.l1b.mag_l1b import calibrate_vector from imap_processing.spice.geometry import SpiceFrame, frame_transform @@ -417,13 +418,21 @@ def rotate_frame(self, end_frame: ValidFrames) -> None: """ if self.epoch_et is None: self.epoch_et = ttj2000ns_to_et(self.epoch) - self.vectors = frame_transform( + new_vectors = frame_transform( self.epoch_et, self.vectors, from_frame=self.frame.spice_frame, to_frame=end_frame.spice_frame, allow_spice_noframeconnect=True, ) + if np.isnan(self.vectors).any() or (self.vectors == constants.FILLVAL).any(): + new_vectors = np.where( + np.isnan(self.vectors) | (self.vectors == constants.FILLVAL), + constants.FILLVAL, + new_vectors, + ) + + self.vectors = new_vectors self.frame = end_frame diff --git a/imap_processing/spice/pointing_frame.py b/imap_processing/spice/pointing_frame.py index 5c3837c9f3..9d3990fe40 100644 --- a/imap_processing/spice/pointing_frame.py +++ b/imap_processing/spice/pointing_frame.py @@ -23,7 +23,7 @@ logger = logging.getLogger(__name__) -POINTING_SEGMENT_DTYPE = np.dtype( +POINTING_SEGMENT_DTYPE: np.dtype = np.dtype( [ # sclk ticks are a double precision number of SCLK ticks since the # start of the mission (e.g. MET_seconds / TICK_DURATION) diff --git a/imap_processing/swapi/l1/swapi_l1.py b/imap_processing/swapi/l1/swapi_l1.py index 6b6ad85bc5..b552caa27f 100644 --- a/imap_processing/swapi/l1/swapi_l1.py +++ b/imap_processing/swapi/l1/swapi_l1.py @@ -126,7 +126,7 @@ def decompress_count( # Decompress counts based on compression indicators # If 0, value is already decompressed. If 1, value is compressed. # If 1 and count is 0xFFFF, value is overflow. - new_count = copy.deepcopy(count_data).astype(np.int32) + new_count: np.ndarray = copy.deepcopy(count_data).astype(np.int32) # If data is compressed, decompress it compressed_indices = compression_flag == 1 @@ -492,7 +492,7 @@ def process_swapi_science( # =================================================================== # Quality flags # =================================================================== - quality_flags_data = np.zeros( + quality_flags_data: np.ndarray = np.zeros( (total_full_sweeps, NUM_ENERGY_STEPS), dtype=np.uint16 ) @@ -550,9 +550,9 @@ def process_swapi_science( ] for flag_name in hk_flags_name: - current_flag = np.repeat(good_sweep_hk_data[flag_name.lower()].data, 6).reshape( - -1, NUM_ENERGY_STEPS - ) + current_flag: np.ndarray = np.repeat( + good_sweep_hk_data[flag_name.lower()].data, 6 + ).reshape(-1, NUM_ENERGY_STEPS) # Use getattr to dynamically access the flag in SWAPIFlags class flag_to_set = getattr(SWAPIFlags, flag_name) # set the quality flag for each data diff --git a/imap_processing/swapi/l2/swapi_l2.py b/imap_processing/swapi/l2/swapi_l2.py index 906f38a7bb..6065102af6 100644 --- a/imap_processing/swapi/l2/swapi_l2.py +++ b/imap_processing/swapi/l2/swapi_l2.py @@ -70,7 +70,9 @@ def solve_full_sweep_energy( # The first 63 energies are coarse steps, then followed by 9 fine steps. # The 9 fine steps may be defined in the main table (fixed steps), or "solve" # which requires a separate lookup in the lut-notes table. - energy_data = np.empty((len(sweep_table), NUM_ENERGY_STEPS), dtype=float) + energy_data: np.ndarray = np.empty( + (len(sweep_table), NUM_ENERGY_STEPS), dtype=float + ) for i_sweep, (time, sweep_id, esa_lvl5_val) in enumerate( zip(data_time, sweep_table, esa_lvl5_data, strict=True) diff --git a/imap_processing/swe/l1b/swe_l1b.py b/imap_processing/swe/l1b/swe_l1b.py index 046d4db833..84b9817901 100644 --- a/imap_processing/swe/l1b/swe_l1b.py +++ b/imap_processing/swe/l1b/swe_l1b.py @@ -385,7 +385,9 @@ def get_esa_energy_pattern(esa_lut_file: Path, esa_table_num: int = 0) -> npt.ND # Now define variable to store pattern for the first two columns # because that pattern is repeated in the rest of the columns. - first_two_columns = np.zeros((swe_constants.N_ESA_STEPS, 2), dtype=np.float64) + first_two_columns: np.ndarray = np.zeros( + (swe_constants.N_ESA_STEPS, 2), dtype=np.float64 + ) # Get row indices of all four quarter cycles. Then minus 1 to get # the row indices in 0-23 instead of 1-24. cycle_row_indices = esa_table_df["v_index"].values - 1 @@ -443,7 +445,7 @@ def get_checker_board_pattern( # Now define variable to store pattern for the first two columns # because that pattern is repeated in the rest of the columns. - first_two_columns = np.zeros((24, 2), dtype=np.int64) + first_two_columns: np.ndarray = np.zeros((24, 2), dtype=np.int64) # Get row indices of all four quarter cycles. Then minus 1 to get # the row indices in 0-23 instead of 1-24. cycle_row_indices = esa_table_df["v_index"].values - 1 @@ -475,7 +477,7 @@ def get_checker_board_pattern( # Generate increment offsets: [0, 0, 12, 12, ..., 168, 168] - # shape: (30,) - column_offsets = np.repeat(np.arange(15) * 12, 2) + column_offsets: np.ndarray = np.repeat(np.arange(15) * 12, 2) increment_by = np.tile(column_offsets, (24, 1)) # Final checkerboard pattern with index offsets applied diff --git a/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py b/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py index fb45059fab..36b9b13434 100644 --- a/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py +++ b/imap_processing/tests/ancillary/test_ancillary_dataset_combiner.py @@ -236,6 +236,19 @@ def test_glows_excluded_regions_combiner(glows_ancillary_filepath): assert dataset["ecliptic_latitude_deg"].dims == ("region",) +def test_glows_excluded_regions_combiner_empty_file(tmp_path): + file_path = tmp_path / "imap_glows_l1b-map-of-excluded-regions_20251112_v001.dat" + file_path.write_text("# header only\n") + + combiner = GlowsAncillaryCombiner([], "20251115") + dataset = combiner.convert_file_to_dataset(file_path) + + assert "ecliptic_longitude_deg" in dataset.data_vars + assert "ecliptic_latitude_deg" in dataset.data_vars + assert len(dataset["ecliptic_longitude_deg"]) == 0 + assert len(dataset["ecliptic_latitude_deg"]) == 0 + + def test_glows_uv_sources_combiner(glows_ancillary_filepath): file_path = ( glows_ancillary_filepath / "imap_glows_map-of-uv-sources_20250923_v002.dat" @@ -300,6 +313,26 @@ def test_glows_exclusions_by_instr_team_combiner(glows_ancillary_filepath): assert combiner.timestamped_data[0].version == "v002" +def test_glows_l2_calibration_combiner(tmp_path): + file_path = tmp_path / "imap_glows_l2-calibration_20251112_v001.dat" + file_path.write_text( + "# header\n2025-11-13T18:12:48 1.020\n" + "2025-11-14T09:58:04 0.849\n" + "2025-11-14T20:58:04 0.649\n" + ) + + combiner = GlowsAncillaryCombiner([], "20251115") + dataset = combiner.convert_file_to_dataset(file_path) + + assert "start_time_utc" in dataset.data_vars + assert ( + np.diff(dataset.start_time_utc.values.astype("datetime64")) >= np.timedelta64(0) + ).all() + assert "cps_per_r" in dataset.data_vars + assert len(dataset["cps_per_r"]) == 3 + assert dataset["cps_per_r"].values[0] == pytest.approx(1.020) + + def test_ancillary_combiner_empty_input(): """Test AncillaryCombiner with empty input list.""" combiner = AncillaryCombiner([], "20251031") diff --git a/imap_processing/tests/codice/test_codice_l1a.py b/imap_processing/tests/codice/test_codice_l1a.py index 5c27efd637..646058fedb 100644 --- a/imap_processing/tests/codice/test_codice_l1a.py +++ b/imap_processing/tests/codice/test_codice_l1a.py @@ -641,6 +641,7 @@ def test_hi_omni(mock_get_file_paths, codice_lut_path): assert cdf_file.name == f"imap_codice_l1a_hi-omni_{VALIDATION_FILE_DATE}_v001.cdf" +@pytest.mark.xfail(reason="Need to revisit in future PR") @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_hi_sectored(mock_get_file_paths, codice_lut_path): """Tests hi-sectored.""" diff --git a/imap_processing/tests/codice/test_codice_l1b.py b/imap_processing/tests/codice/test_codice_l1b.py index 1873603e3a..77aa35d0cc 100644 --- a/imap_processing/tests/codice/test_codice_l1b.py +++ b/imap_processing/tests/codice/test_codice_l1b.py @@ -309,6 +309,7 @@ def test_l1b_hi_omni(mock_get_file_paths, codice_lut_path): assert cdf_file.name == f"imap_codice_l1b_hi-omni_{VALIDATION_FILE_DATE}_v999.cdf" +@pytest.mark.xfail(reason="Need to revisit in future PR") @patch("imap_data_access.processing_input.ProcessingInputCollection.get_file_paths") def test_l1b_hi_sectored(mock_get_file_paths, codice_lut_path): mock_get_file_paths.side_effect = [ diff --git a/imap_processing/tests/ena_maps/conftest.py b/imap_processing/tests/ena_maps/conftest.py index 905f0b7934..e6391c34ae 100644 --- a/imap_processing/tests/ena_maps/conftest.py +++ b/imap_processing/tests/ena_maps/conftest.py @@ -12,12 +12,14 @@ @pytest.fixture(scope="module") def ultra_l1c_pset_datasets(): """Make fake L1C Ultra PSET products on a HEALPix tiling for testing""" - l1c_nside = 32 + l1c_nside = 16 + counts_nside = 32 return { "nside": l1c_nside, "products": [ mock_l1c_pset_product_healpix( nside=l1c_nside, + counts_nside=counts_nside, stripe_center_lat=mid_latitude, width_scale=5, counts_scaling_params=(50, 0.5), diff --git a/imap_processing/tests/ena_maps/test_ena_maps.py b/imap_processing/tests/ena_maps/test_ena_maps.py index 617aa1035b..379a1fa59a 100644 --- a/imap_processing/tests/ena_maps/test_ena_maps.py +++ b/imap_processing/tests/ena_maps/test_ena_maps.py @@ -8,6 +8,7 @@ from copy import deepcopy from pathlib import Path from unittest import mock +from unittest.mock import patch import astropy_healpix.healpy as hp import numpy as np @@ -20,6 +21,7 @@ from imap_processing.ena_maps.utils import spatial_utils from imap_processing.ena_maps.utils.coordinates import CoordNames from imap_processing.spice import geometry +from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et @pytest.fixture(autouse=True, scope="module") @@ -76,7 +78,10 @@ def test_instantiate(self): ultra_pset.num_points, hp.nside2npix(self.nside), ) - + # check that the midpoint_j2000_et property is equal to the expected value + assert ultra_pset.midpoint_j2000_et == ttj2000ns_to_et( + ultra_pset.epoch + self.l1c_pset_products[0].epoch_delta[0] / 2 + ) # Check the repr exists assert "UltraPointingSet" in repr(ultra_pset) @@ -93,6 +98,16 @@ def test_instantiate(self): "energy_bin_geometric_mean", ) + # TODO remove this test when the TODO in the ultra downsample_counts function is + # removed. + def test_old_pset(self): + pset = self.l1c_pset_products[0] + pset = pset.drop_dims("counts_pixel_index") + pset = ena_maps.UltraPointingSet( + pset, + spice_reference_frame=geometry.SpiceFrame.IMAP_DPS, + ) + @pytest.mark.usefixtures("_setup_ultra_l1c_pset_products") def test_init_cdf( self, @@ -101,10 +116,14 @@ def test_init_cdf( cdf_filepath = write_cdf(ultra_pset, istp=False) - ultra_pset_from_dataset = ena_maps.UltraPointingSet(ultra_pset) - - ultra_pset_from_str = ena_maps.UltraPointingSet(cdf_filepath) - ultra_pset_from_path = ena_maps.UltraPointingSet(Path(cdf_filepath)) + # Mock the downsample_counts method to avoid dimension bugs + # since this is a dummy cdf, the dimensions are not present. + with patch.object( + ena_maps.UltraPointingSet, "downsample_counts", lambda self: None + ): + ultra_pset_from_str = ena_maps.UltraPointingSet(cdf_filepath) + ultra_pset_from_path = ena_maps.UltraPointingSet(Path(cdf_filepath)) + ultra_pset_from_dataset = ena_maps.UltraPointingSet(ultra_pset) np.testing.assert_allclose( ultra_pset_from_dataset.data["counts"].values, @@ -118,7 +137,6 @@ def test_init_cdf( rtol=1e-6, ) - @pytest.mark.usefixtures("_setup_ultra_l1c_pset_products") @pytest.mark.usefixtures("_setup_ultra_l1c_pset_products") def test_different_spacing_raises_error(self): """Test that different spaced az/el from the L1C dataset raises ValueError""" @@ -135,6 +153,23 @@ def test_different_spacing_raises_error(self): spice_reference_frame=geometry.SpiceFrame.IMAP_DPS, ) + def test_downsample_counts(self): + ultra_pset = self.l1c_pset_products[0] + + # First check that counts are at a finer resolution than exposure factor + counts_npix_before = ultra_pset["counts"].shape[-1] + assert counts_npix_before != ultra_pset["exposure_factor"].shape[-1] + pset = ena_maps.UltraPointingSet(ultra_pset) + + # Verify counts are now at the same resolution as pset + counts_nside_after = hp.npix2nside(pset.data["counts"].shape[-1]) + assert counts_nside_after == pset.nside + + # Verify counts are the same after downsampling. + np.testing.assert_allclose( + pset.data["counts"].values.sum(), ultra_pset["counts"].values.sum() + ) + @pytest.fixture(scope="module") def hi_pset_cdf_path(imap_tests_path): @@ -148,12 +183,15 @@ class TestHiPointingSet: def test_init(self, hi_pset_cdf_path): """Test coverage for __init__ method.""" pset_ds = load_cdf(hi_pset_cdf_path) + delta = 10 + pset_ds["epoch_delta"] = ("epoch", np.array([delta])) # Add dummy epoch delta hi_pset = ena_maps.HiPointingSet(pset_ds) assert isinstance(hi_pset, ena_maps.HiPointingSet) assert hi_pset.spice_reference_frame == geometry.SpiceFrame.IMAP_HAE assert hi_pset.num_points == 3600 np.testing.assert_array_equal(hi_pset.az_el_points.shape, (3600, 2)) - + # check that the midpoint_j2000_et property is equal to the expected value + assert hi_pset.midpoint_j2000_et == ttj2000ns_to_et(hi_pset.epoch + delta / 2) for var_name in ["exposure_factor", "bg_rate", "bg_rate_sys_err"]: assert var_name in hi_pset.data @@ -164,7 +202,9 @@ def test_from_cdf(self, hi_pset_cdf_path): def test_plays_nice_with_rectangular_sky_map(self, hi_pset_cdf_path): """Test that HiPointingSet works with RectangularSkyMap""" - hi_pset = ena_maps.HiPointingSet(hi_pset_cdf_path) + hi_ds = load_cdf(hi_pset_cdf_path) + hi_ds["epoch_delta"] = ("epoch", np.array([0])) # Add dummy epoch delta + hi_pset = ena_maps.HiPointingSet(hi_ds) rect_map = ena_maps.RectangularSkyMap( spacing_deg=2, spice_frame=geometry.SpiceFrame.IMAP_HAE ) @@ -213,6 +253,16 @@ def lo_pset_ds(): dims=["epoch"], name="epoch", ) + dataset.coords["pointing_start_met"] = xr.DataArray( + [1], + dims=["epoch"], + name="epoch", + ) + dataset.coords["pointing_end_met"] = xr.DataArray( + [10], + dims=["epoch"], + name="epoch", + ) dataset.coords["spin_angle"] = xr.DataArray( [i for i in range(3600)], dims=["spin_angle"], @@ -254,6 +304,15 @@ def test_init(self, lo_pset_ds): assert lo_pset.num_points == 144000 np.testing.assert_array_equal(lo_pset.az_el_points.shape, (144000, 2)) + # check that the midpoint_j2000_et property is equal to the expected value + assert lo_pset.midpoint_j2000_et == ttj2000ns_to_et( + lo_pset.epoch + + ( + met_to_ttj2000ns(lo_pset_ds.pointing_end_met) + - met_to_ttj2000ns(lo_pset_ds.pointing_start_met) + ) + / 2 + ) for var_name in ["exposure_time", "h_counts"]: assert var_name in lo_pset.data @@ -295,6 +354,7 @@ def create_hi_pset_with_multidim_coords( HiPointingSet with multi-dimensional az_el_points. """ pset_ds = load_cdf(hi_pset_cdf_path) + pset_ds["epoch_delta"] = ("epoch", np.array([0])) # Add dummy epoch delta hi_pset = ena_maps.HiPointingSet(pset_ds) hi_pset.data["hae_longitude"] = xr.DataArray( np.random.uniform(0, 360, shape), diff --git a/imap_processing/tests/external_test_data_config.py b/imap_processing/tests/external_test_data_config.py index 6aa8238bcf..736e506d40 100644 --- a/imap_processing/tests/external_test_data_config.py +++ b/imap_processing/tests/external_test_data_config.py @@ -53,6 +53,7 @@ (f"imap_codice_l1a_lo-counters-singles_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-direct-events_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-ialirt_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), + ("imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-nsw-priority_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-nsw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), (f"imap_codice_l1a_lo-sw-angular_{VALIDATION_FILE_DATE}_{VALIDATION_FILE_VERSION}.cdf", "codice/data/l1a_validation"), @@ -109,6 +110,7 @@ ("imap_hi_l1b_90sensor-hk_20241105-repoint00099_v001.cdf", "hi/data/l1/"), ("imap_hi_l1a_90sensor-de_20241105-repoint00099_v001.cdf", "hi/data/l1/"), ("imap_hi_l1c_45sensor-pset_20250415_v999.cdf", "hi/data/l1/"), + ("imap_hi_l1b_45sensor-goodtimes_20250415_v999.cdf", "hi/data/l1/"), # I-ALiRT ("apid_478.bin", "ialirt/data/l0/"), @@ -129,12 +131,17 @@ ("iois_1_packets_2025_284_05_54_39", "ialirt/data/l0/"), ("iois_1_packets_2025_344_05_57_56", "ialirt/data/l0/"), ("iois_1_packets_2025_344_05_59_58", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_03_05", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_04_06", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_05_07", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_06_08", "ialirt/data/l0/"), + ("iois_1_packets_2026_090_05_07_09", "ialirt/data/l0/"), ("imap_recon_od005_20250925_20251014_v01.bsp", "spice/test_data/"), ("imap_2025_283_2025_284_001.ah.bc", "spice/test_data/"), # IDEX ("idex_l1a_validation_file.h5", "idex/test_data/"), - ("idex_l1b_validation_file.h5", "idex/test_data/"), + ("imap_idex_l1b_sci_20231218_v002.h5", "idex/test_data/"), ("imap_idex_l2a-calibration-curve-yield-params_20250101_v001.csv", "idex/test_data/"), ("imap_idex_l2a-calibration-curve-t-rise_20250101_v001.csv", "idex/test_data/"), @@ -154,6 +161,8 @@ ("voltage_culling_results_repoint00047.csv", "ultra/data/l1/"), ("validate_high_energy_culling_results_repoint00047_v2.csv", "ultra/data/l1/"), ("validate_stat_culling_results_repoint00047_v2.csv", "ultra/data/l1/"), + ("validate_upstream_ion_1_culling_results_repoint00047_v1.csv", "ultra/data/l1/"), + ("validate_spectral_culling_results_repoint00047_v1.csv", "ultra/data/l1/"), ("de_test_data_repoint00047.csv", "ultra/data/l1/"), ("FM45_UltraFM45Extra_TV_Tests_2024-01-22T0930_20240122T093008.CCSDS", "ultra/data/l0/"), ("ultra45_raw_sc_rawnrgevnt_19840122_00.csv", "ultra/data/l0/"), diff --git a/imap_processing/tests/glows/conftest.py b/imap_processing/tests/glows/conftest.py index c2d573f735..fcf055e001 100644 --- a/imap_processing/tests/glows/conftest.py +++ b/imap_processing/tests/glows/conftest.py @@ -13,6 +13,7 @@ AncillaryParameters, ) from imap_processing.glows.l2.glows_l2 import glows_l2 +from imap_processing.glows.l2.glows_l2_data import DailyLightcurve @pytest.fixture @@ -235,6 +236,43 @@ def mock_conversion_table_dict(): return mock_dict +@pytest.fixture +def mock_calibration_dataset(): + """Create a mock CalibrationDataset object for testing.""" + + # Both cps_per_r and start_time_utc are 2D: (epoch, *_dim_0). + return xr.Dataset( + { + "cps_per_r": xr.DataArray( + [[0.849, 1.020, 1.500], [0.849, 1.020, 1.500]], + dims=["epoch", "cps_per_r_dim_0"], + ), + "start_time_utc": xr.DataArray( + np.array( + [ + [ + "2011-09-19T09:58:04", + "2011-09-20T18:12:48", + "2011-09-21T18:15:50", + ], + [ + "2011-09-19T09:58:04", + "2011-09-20T18:12:48", + "2011-09-21T18:15:50", + ], + ], + ), + dims=["epoch", "start_time_utc_dim_0"], + ), + }, + coords={ + "epoch": np.array( + ["2011-09-19T00:00:00", "2011-09-20T00:00:00"], dtype="datetime64[s]" + ) + }, + ) + + @pytest.fixture def mock_pipeline_settings(): """Create a mock PipelineSettings dataset for testing.""" @@ -278,6 +316,23 @@ def mock_pipeline_settings(): return mock_pipeline_dataset +@pytest.fixture +def mock_ecliptic_bin_centers(monkeypatch): + """Mock ecliptic coordinates for bin centers.""" + + def _mock_compute_coords( + _data_start_time_et: float, spin_angle: np.ndarray + ) -> tuple[np.ndarray, np.ndarray]: + n_bins = len(spin_angle) + return np.zeros(n_bins, dtype=float), np.zeros(n_bins, dtype=float) + + monkeypatch.setattr( + DailyLightcurve, + "compute_ecliptic_coords_of_bin_centers", + staticmethod(_mock_compute_coords), + ) + + def mock_update_spice_parameters(self, *args, **kwargs): self.spin_period_ground_average = np.float64(0.0) self.spin_period_ground_std_dev = np.float64(0.0) diff --git a/imap_processing/tests/glows/test_glows_l1b.py b/imap_processing/tests/glows/test_glows_l1b.py index d43c862024..e529db24a4 100644 --- a/imap_processing/tests/glows/test_glows_l1b.py +++ b/imap_processing/tests/glows/test_glows_l1b.py @@ -298,19 +298,19 @@ def test_process_histogram( 0, 0, 0, - 0, - 0, + 64, # flags_set_onboard: bit 6 (is_night) set + 1, # is_generated_on_ground 0, 3600, 0, encoded_val, + np.single(30.0), # filter_temperature_variance: exceeds 2.03°C threshold encoded_val, + np.single(3500.0), # hv_voltage_variance: exceeds 50.0V threshold encoded_val, + np.single(11000.0), # spin_period_variance: exceeds 0.033333s threshold encoded_val, - encoded_val, - encoded_val, - encoded_val, - encoded_val, + np.single(2.0), # pulse_length_variance: exceeds 1.0μs threshold time_val, time_val, time_val, @@ -328,6 +328,19 @@ def test_process_histogram( ) assert len(output) == len(dataclasses.asdict(test_l1b)) + # flags[0:10] = onboard flags (1=good, 0=bad), one per bit of flags_set_onboard + # flags[10] = is_generated_on_ground (1=onboard, 0=ground) + # flags[11] = is_beyond_daily_statistical_error (placeholder, always 1) + # flags[12:16] = std_dev threshold flags + # flags[16] = is_beyond_background + assert test_l1b.flags[6] == 0 # is_night + assert test_l1b.flags[10] == 0 # is_generated_on_ground + assert test_l1b.flags[12] == 0 # is_temp_ok + assert test_l1b.flags[13] == 0 # is_hv_ok + assert test_l1b.flags[14] == 0 # is_spin_std_ok + assert test_l1b.flags[15] == 0 # is_pulse_ok + assert test_l1b.flags[16] == 1 # is_beyond_background + @patch.object( HistogramL1B, diff --git a/imap_processing/tests/glows/test_glows_l1b_data.py b/imap_processing/tests/glows/test_glows_l1b_data.py index f52e462cf5..108d866425 100644 --- a/imap_processing/tests/glows/test_glows_l1b_data.py +++ b/imap_processing/tests/glows/test_glows_l1b_data.py @@ -3,6 +3,7 @@ from unittest.mock import patch import numpy as np +import pandas as pd import pytest import xarray as xr @@ -293,3 +294,144 @@ def test_pipeline_settings_from_flattened_json(): assert len(settings.active_bad_angle_flags) == 4 assert settings.active_bad_angle_flags[3] is False # is_suspected_transient + + +def test_get_threshold(): + "Test PipelineSettings.get_threshold method." + + test_data = { + "n_sigma_threshold_lower": 3.0, + "n_sigma_threshold_upper": 3.0, + "relative_difference_threshold": 7e-05, + "std_dev_threshold__celsius_deg": 2.03, + "std_dev_threshold__volt": 50.0, + "std_dev_threshold__sec": 0.033333, + "std_dev_threshold__usec": 1.0, + } + pipeline_dataset = xr.Dataset({k: xr.DataArray(v) for k, v in test_data.items()}) + settings = PipelineSettings(pipeline_dataset) + + expected = [2.03, 50.0, 0.033333, 1.0, 7e-5] + description = [ + "std_dev_threshold__celsius_deg", + "std_dev_threshold__volt", + "std_dev_threshold__sec", + "std_dev_threshold__usec", + "relative_difference_threshold", + ] + + for name, exp in zip(description, expected, strict=False): + threshold = settings.get_threshold(name) + assert threshold == exp + + +@patch("imap_processing.glows.l1b.glows_l1b_data.geometry.imap_state") +@patch("imap_processing.glows.l1b.glows_l1b_data.get_instrument_spin_phase") +@patch("imap_processing.glows.l1b.glows_l1b_data.get_spin_data") +@patch("imap_processing.glows.l1b.glows_l1b_data.geometry.frame_transform") +@patch("imap_processing.glows.l1b.glows_l1b_data.sct_to_et") +@patch("imap_processing.glows.l1b.glows_l1b_data.met_to_sclkticks") +def test_update_spice_parameters_spin_axis_near_wrapping_point( + mock_met_to_sclkticks, + mock_sct_to_et, + mock_frame_transform, + mock_get_spin_data, + mock_get_instrument_spin_phase, + mock_imap_state, +): + """Test spin axis orientation calculation near the longitude wrapping point. + + This test verifies that cartesian_to_latitudinal is called with degrees=False + so that circmean/circstd receive values in radians. The bug was that without + degrees=False, values in degrees would be passed to circmean/circstd which + expect radians with low=-pi, high=pi. + + Test conditions: + - Longitude values straddling +/-pi (wrapping point at 180 degrees) + - Latitude near equator (-4 degrees) + + If the bug existed (degrees=True or default), circmean would receive values + like 179 or -179 degrees when it expects radians in [-pi, pi]. This would + produce nonsensical results because 179 >> pi. + """ + # Mock time conversions - creates a time range of 5 seconds + mock_met_to_sclkticks.return_value = 1000 + mock_sct_to_et.side_effect = lambda x: 100.0 if x == 1000 else 105.0 + + # Mock spin data + mock_spin_df = { + "spin_start_met": np.array([99.0, 100.0, 101.0, 102.0, 103.0, 104.0]), + "spin_period_sec": np.array([15.0, 15.0, 15.0, 15.0, 15.0, 15.0]), + } + mock_get_spin_data.return_value = pd.DataFrame(mock_spin_df) + + # Mock instrument spin phase + mock_get_instrument_spin_phase.return_value = 0.5 + + # Create cartesian vectors that straddle the longitude wrapping point. + # Some points at +179 degrees and some at -179 degrees (which should + # average to ~180 degrees when using proper circular mean). + # Latitude near equator at -4 degrees. + # + # For spherical to cartesian (r=1): + # x = cos(lat) * cos(lon) + # y = cos(lat) * sin(lon) + # z = sin(lat) + lat_rad = np.deg2rad(-4.0) # Near equator + + # Create 5 time steps: [+179, -179, +178, -178, +180] degrees longitude + longitudes_deg = np.array([179.0, -179.0, 178.0, -178.0, 180.0]) + longitudes_rad = np.deg2rad(longitudes_deg) + + # Build cartesian vectors for each time step + n_times = len(longitudes_rad) + cartesian_vecs = np.zeros((n_times, 3)) + for i, lon_rad in enumerate(longitudes_rad): + cartesian_vecs[i, 0] = np.cos(lat_rad) * np.cos(lon_rad) # x + cartesian_vecs[i, 1] = np.cos(lat_rad) * np.sin(lon_rad) # y + cartesian_vecs[i, 2] = np.sin(lat_rad) # z + + mock_frame_transform.return_value = cartesian_vecs + + # Mock imap_state to return position and velocity for each time step + mock_imap_state.return_value = np.tile( + [[1e8, 2e8, 3e7, 10.0, 20.0, 5.0]], (n_times, 1) + ) + + # Create a minimal HistogramL1B-like object to test update_spice_parameters + class MockHistogram: + def __init__(self): + self.imap_start_time = 100.0 + self.glows_time_offset = 5.0 # 5 second duration + + mock_hist = MockHistogram() + + # Call the actual update_spice_parameters method + HistogramL1B.update_spice_parameters(mock_hist) + + # Verify the spin axis orientation values + lon_result = mock_hist.spin_axis_orientation_average[0] + lat_result = mock_hist.spin_axis_orientation_average[1] + lon_std = mock_hist.spin_axis_orientation_std_dev[0] + lat_std = mock_hist.spin_axis_orientation_std_dev[1] + + # The circular mean of [179, -179, 178, -178, 180] should be ~180 degrees + # (or equivalently -180 degrees). The key test is that the result is NOT + # near 0 degrees, which would happen if the values weren't properly handled + # as circular data near the wrapping point. + # + # With the bug (degrees=True), circmean would receive [179, -179, ...] and + # interpret these as radians, giving completely wrong results. + # + # Check that longitude is near 180 degrees (could be reported as -180) + assert abs(abs(lon_result) - 180.0) < 5.0, ( + f"Longitude {lon_result} should be near +/-180 degrees. " + "If near 0, the circular mean failed at the wrapping point." + ) + + # Latitude should be near -4 degrees + np.testing.assert_allclose(lat_result, -4.0, atol=1.0) + + # Standard deviations should be small (all points are within a few degrees) + assert lon_std < 5.0, f"Longitude std dev {lon_std} should be small" + assert lat_std < 1.0, f"Latitude std dev {lat_std} should be small" diff --git a/imap_processing/tests/glows/test_glows_l2.py b/imap_processing/tests/glows/test_glows_l2.py index a647397e42..508f47c14d 100644 --- a/imap_processing/tests/glows/test_glows_l2.py +++ b/imap_processing/tests/glows/test_glows_l2.py @@ -9,12 +9,13 @@ HistogramL1B, PipelineSettings, ) -from imap_processing.glows.l2.glows_l2 import ( - glows_l2, -) +from imap_processing.glows.l2.glows_l2 import glows_l2 from imap_processing.glows.l2.glows_l2_data import DailyLightcurve, HistogramL2 +from imap_processing.glows.utils.constants import GlowsConstants from imap_processing.spice.time import et_to_datetime64, ttj2000ns_to_et -from imap_processing.tests.glows.conftest import mock_update_spice_parameters +from imap_processing.tests.glows.conftest import ( + mock_update_spice_parameters, +) @pytest.fixture @@ -24,10 +25,10 @@ def l1b_hists(): hist = xr.DataArray( np.ones((4, 5)), dims=["epoch", "bins"], coords={"epoch": epoch, "bins": bins} ) - hist[1, 0] = -1 - hist[2, 0] = -1 - hist[1, 1] = -1 - hist[2, 3] = -1 + hist[1, 0] = GlowsConstants.HISTOGRAM_FILLVAL + hist[2, 0] = GlowsConstants.HISTOGRAM_FILLVAL + hist[1, 1] = GlowsConstants.HISTOGRAM_FILLVAL + hist[2, 3] = GlowsConstants.HISTOGRAM_FILLVAL input = xr.Dataset(coords={"epoch": epoch, "bins": bins}) input["histogram"] = hist @@ -35,6 +36,7 @@ def l1b_hists(): return input +@patch.object(HistogramL2, "compute_position_angle", return_value=42.0) @patch.object( HistogramL1B, "flag_uv_and_excluded", @@ -44,10 +46,13 @@ def l1b_hists(): def test_glows_l2( mock_spice_function, mock_flag_uv_and_excluded, + mock_compute_position_angle, l1a_dataset, mock_ancillary_exclusions, mock_pipeline_settings, mock_conversion_table_dict, + mock_ecliptic_bin_centers, + mock_calibration_dataset, caplog, ): mock_spice_function.side_effect = mock_update_spice_parameters @@ -63,18 +68,35 @@ def test_glows_l2( ) # Test case 1: L1B dataset has good times - l2 = glows_l2(l1b_hist_dataset, mock_pipeline_settings)[0] + l2 = glows_l2(l1b_hist_dataset, mock_pipeline_settings, mock_calibration_dataset)[0] assert l2.attrs["Logical_source"] == "imap_glows_l2_hist" assert np.allclose(l2["filter_temperature_average"].values, [57.6], rtol=0.1) # Test case 2: L1B dataset has no good times (all flags 0) - l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) + l1b_hist_dataset_no_good_times = l1b_hist_dataset.copy(deep=True) + l1b_hist_dataset_no_good_times["flags"].values = np.zeros( + l1b_hist_dataset_no_good_times.flags.shape + ) caplog.set_level("WARNING") - result = glows_l2(l1b_hist_dataset, mock_pipeline_settings) + result = glows_l2( + l1b_hist_dataset_no_good_times, mock_pipeline_settings, mock_calibration_dataset + ) assert result == [] assert any(record.levelname == "WARNING" for record in caplog.records) + # Test case 3: Dataset has zero exposure and flux values + l1b_hist_dataset_zero_values = l1b_hist_dataset.copy(deep=True) + l1b_hist_dataset_zero_values["spin_period_average"].data[:] = 0 + l1b_hist_dataset_zero_values["number_of_spins_per_block"].data[:] = 0 + caplog.set_level("WARNING") + result = glows_l2( + l1b_hist_dataset_zero_values, mock_pipeline_settings, mock_calibration_dataset + ) + assert result == [] + assert any(record.levelname == "WARNING" for record in caplog.records) + +@patch.object(HistogramL2, "compute_position_angle", return_value=42.0) @patch.object( HistogramL1B, "flag_uv_and_excluded", @@ -84,10 +106,13 @@ def test_glows_l2( def test_generate_l2( mock_spice_function, mock_flag_uv_and_excluded, + mock_compute_position_angle, l1a_dataset, mock_ancillary_exclusions, mock_pipeline_settings, mock_conversion_table_dict, + mock_ecliptic_bin_centers, + mock_calibration_dataset, ): mock_spice_function.side_effect = mock_update_spice_parameters @@ -106,37 +131,38 @@ def test_generate_l2( ) # Test case 1: L1B dataset has good times - l2 = HistogramL2(l1b_hist_dataset, pipeline_settings) - - expected_values = { - "filter_temperature_average": [57.59], - "filter_temperature_std_dev": [0.21], - "hv_voltage_average": [1715.4], - "hv_voltage_std_dev": [0.0], - } - - assert np.isclose( - l2.filter_temperature_average, - expected_values["filter_temperature_average"], - 0.01, - ) - assert np.isclose( - l2.filter_temperature_std_dev, - expected_values["filter_temperature_std_dev"], - 0.01, - ) - assert np.isclose( - l2.hv_voltage_average, expected_values["hv_voltage_average"], 0.01 - ) - assert np.isclose( - l2.hv_voltage_std_dev, expected_values["hv_voltage_std_dev"], 0.01 - ) - - # Test case 2: L1B dataset has no good times (all flags 0) - l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) - ds = HistogramL2(l1b_hist_dataset, pipeline_settings) - expected_number_of_good_l1b_inputs = 0 - assert ds.number_of_good_l1b_inputs == expected_number_of_good_l1b_inputs + with patch.object(HistogramL2, "get_calibration_factor", return_value=1): + l2 = HistogramL2(l1b_hist_dataset, pipeline_settings, mock_calibration_dataset) + + expected_values = { + "filter_temperature_average": [57.59], + "filter_temperature_std_dev": [0.21], + "hv_voltage_average": [1715.4], + "hv_voltage_std_dev": [0.0], + } + + assert np.isclose( + l2.filter_temperature_average, + expected_values["filter_temperature_average"], + 0.01, + ) + assert np.isclose( + l2.filter_temperature_std_dev, + expected_values["filter_temperature_std_dev"], + 0.01, + ) + assert np.isclose( + l2.hv_voltage_average, expected_values["hv_voltage_average"], 0.01 + ) + assert np.isclose( + l2.hv_voltage_std_dev, expected_values["hv_voltage_std_dev"], 0.01 + ) + + # Test case 2: L1B dataset has no good times (all flags 0) + l1b_hist_dataset["flags"].values = np.zeros(l1b_hist_dataset.flags.shape) + ds = HistogramL2(l1b_hist_dataset, pipeline_settings, mock_calibration_dataset) + expected_number_of_good_l1b_inputs = 0 + assert ds.number_of_good_l1b_inputs == expected_number_of_good_l1b_inputs def test_bin_exclusions(l1b_hists): diff --git a/imap_processing/tests/glows/test_glows_l2_data.py b/imap_processing/tests/glows/test_glows_l2_data.py index b485e85626..7937b46e93 100644 --- a/imap_processing/tests/glows/test_glows_l2_data.py +++ b/imap_processing/tests/glows/test_glows_l2_data.py @@ -1,9 +1,13 @@ +from unittest.mock import patch + import numpy as np import pytest import xarray as xr from imap_processing.glows.l1b.glows_l1b_data import PipelineSettings from imap_processing.glows.l2.glows_l2_data import DailyLightcurve, HistogramL2 +from imap_processing.glows.utils.constants import GlowsConstants +from imap_processing.spice.time import met_to_sclkticks, sct_to_et @pytest.fixture @@ -41,8 +45,12 @@ def pipeline_settings(): ] pipeline_dataset = xr.Dataset( { - "active_bad_time_flags": xr.DataArray(active_bad_time_flags), - "active_bad_angle_flags": xr.DataArray(active_bad_angle_flags), + "active_bad_time_flags": xr.DataArray( + active_bad_time_flags, dims=["time_flag_index"] + ), + "active_bad_angle_flags": xr.DataArray( + active_bad_angle_flags, dims=["angle_flag_index"] + ), } ) return PipelineSettings(pipeline_dataset) @@ -53,14 +61,15 @@ def l1b_dataset(): """Minimal L1B dataset for testing DailyLightcurve. Two timestamps, four bins. - Bin 3 is masked (-1) at timestamp 0. + Bin 3 is masked (HISTOGRAM_FILLVAL) at timestamp 0. histogram_flag_array has shape (epoch, bad_angle_flags, bins) with all zeros. """ n_epochs, n_bins, n_flags = 2, 4, 4 + fillval = GlowsConstants.HISTOGRAM_FILLVAL epoch = xr.DataArray(np.arange(n_epochs), dims=["epoch"]) bins = xr.DataArray(np.arange(n_bins), dims=["bins"]) - histogram = np.array([[10, 20, 30, -1], [10, 20, 30, 40]], dtype=float) + histogram = np.array([[10, 20, 30, fillval], [10, 20, 30, 40]], dtype=float) spin_angle = np.tile(np.linspace(0, 270, n_bins), (n_epochs, 1)) histogram_flag_array = np.zeros((n_epochs, n_flags, n_bins), dtype=np.uint8) @@ -70,19 +79,108 @@ def l1b_dataset(): "spin_period_average": (["epoch"], [15.0, 15.0]), "number_of_spins_per_block": (["epoch"], [5, 5]), "imap_spin_angle_bin_cntr": (["epoch", "bins"], spin_angle), + "imap_start_time": (["epoch"], [0.0, 1.0]), "histogram_flag_array": ( ["epoch", "bad_angle_flags", "bins"], histogram_flag_array, ), + "number_of_bins_per_histogram": (["epoch"], [n_bins, n_bins]), }, coords={"epoch": epoch, "bins": bins}, ) return ds -def test_photon_flux(l1b_dataset): - """Flux = sum(histograms) / sum(exposure_times) per bin (Eq. 50).""" - lc = DailyLightcurve(l1b_dataset) +def test_get_calibration_factor(mock_calibration_dataset): + """Test selecting correct calibration factor. + + Mock calibration data: + start_time_utc (dims epoch × start_time_utc_dim_0, same per epoch): + ["2011-09-19T09:58:04", "2011-09-20T18:12:48", "2011-09-21T18:15:50"] + cps_per_r (dims epoch × cps_per_r_dim_0, same per epoch): + index 0 → 0.849, index 1 → 1.020, index 2 → 1.500 + """ + # Case 1: The mid-epoch ('2011-09-22T10:30:55.015') falls after the + # start_time_utc entries, so the last entry (index 2) is selected → 1.500. + + # ["2011-09-22T07:45:55.015", "2011-09-22T10:30:55.015", "2011-09-22T13:15:55.015"] + later_epoch = np.array([369949621199000000, 369959521199000000, 369969421199000000]) + assert HistogramL2.get_calibration_factor( + later_epoch, mock_calibration_dataset + ) == pytest.approx(1.500) + + # Case 2: The mid-epoch ('2011-09-21T00:52:15.000') falls between the 2nd and + # 3rd start_time_utc entries, so the 2nd entry (index 1) is selected → 1.020. + + # ['2011-09-21T00:50:15.000', '2011-09-21T00:52:15.000', '2011-09-21T00:54:15.000'] + between_epoch = np.array( + [369838281184000000, 369838401184000000, 369838521184000000] + ) + assert HistogramL2.get_calibration_factor( + between_epoch, mock_calibration_dataset + ) == pytest.approx(1.020) + + # Case 3: The mid-epoch is before all start_time_utc entries, + # so a KeyError is raised by xarray's "pad" selection method. + + # ['2011-09-18T19:59:08.816', '2011-09-18T20:01:08.816', '2011-09-18T20:03:08.816'] + early_epoch = np.array([369648015000000000, 369648135000000000, 369648255000000000]) + with pytest.raises(KeyError): + HistogramL2.get_calibration_factor(early_epoch, mock_calibration_dataset) + + +@pytest.mark.external_kernel +def test_ecliptic_coords_computation(furnish_kernels): + """Test method that computes ecliptic coordinates.""" + + # Use a met value within the SPICE kernel coverage (2026-01-01). + data_start_time_et = sct_to_et(met_to_sclkticks(504975603.125)) + n_bins = 4 + spin_angle = np.linspace(0, 270, n_bins) + + kernels = [ + "naif0012.tls", + "imap_sclk_0000.tsc", + "imap_130.tf", + "imap_science_120.tf", + "sim_1yr_imap_pointing_frame.bc", + ] + + with furnish_kernels(kernels): + ecliptic_lon, ecliptic_lat = ( + DailyLightcurve.compute_ecliptic_coords_of_bin_centers( + data_start_time_et, spin_angle + ) + ) + + # ecliptic_lon and ecliptic_lat must have one entry per bin + assert len(ecliptic_lon) == n_bins + assert len(ecliptic_lat) == n_bins + + # ecliptic longitude must be in [0, 360) + assert np.all(ecliptic_lon >= 0.0) + assert np.all(ecliptic_lon < 360.0) + + # ecliptic latitude must be in [-90, 90] + assert np.all(ecliptic_lat >= -90.0) + assert np.all(ecliptic_lat <= 90.0) + + # values must be finite (no NaN / Inf from SPICE) + assert np.all(np.isfinite(ecliptic_lon)) + assert np.all(np.isfinite(ecliptic_lat)) + + +def test_photon_flux(l1b_dataset, mock_ecliptic_bin_centers): + """ + Flux = (sum(histograms) / sum(exposure_times)) / + Rayleigh calibration factor + + per bin (Eq. 50-53) + """ + mock_cal_factor = 2 + lc = DailyLightcurve( + l1b_dataset, position_angle=0.0, calibration_factor=mock_cal_factor + ) # l1b_exposure_time_per_bin = spin_period_average * # number_of_spins_per_block / number_of_bins_per_histogram @@ -92,94 +190,106 @@ def test_photon_flux(l1b_dataset): expected_exposure = np.array( [2 * exposure_per, 2 * exposure_per, 2 * exposure_per, 2 * exposure_per] ) - expected_flux = expected_raw / expected_exposure + expected_flux = (expected_raw / expected_exposure) / mock_cal_factor assert np.allclose(lc.raw_histograms, expected_raw) assert np.allclose(lc.exposure_times, expected_exposure) assert np.allclose(lc.photon_flux, expected_flux) -def test_flux_uncertainty(l1b_dataset): - """Uncertainty = sqrt(sum_hist) / exposure per bin (Eq. 54).""" - lc = DailyLightcurve(l1b_dataset) +def test_flux_uncertainty(l1b_dataset, mock_ecliptic_bin_centers): + """ + Uncertainty = sqrt(sum_hist) / exposure / + Rayleigh calibration factor + + per bin (Eq. 54-55).""" + mock_cal_factor = 2 + lc = DailyLightcurve( + l1b_dataset, position_angle=0.0, calibration_factor=mock_cal_factor + ) - expected_uncertainty = np.sqrt(lc.raw_histograms) / lc.exposure_times + expected_uncertainty = ( + np.sqrt(lc.raw_histograms) / lc.exposure_times + ) / mock_cal_factor assert np.allclose(lc.flux_uncertainties, expected_uncertainty) -def test_zero_exposure_bins(): +def test_zero_exposure_bins(l1b_dataset, mock_ecliptic_bin_centers): """Bins with all-masked histograms get zero flux and uncertainty. Exposure time still accumulates uniformly from each good-time file even - when all histogram values are masked (-1). Flux and uncertainty are zero - because the raw histogram sums are zero. + when all histogram values are masked (HISTOGRAM_FILLVAL). Flux and + uncertainty are zero because the raw histogram sums are zero. """ - n_epochs, n_bins, n_flags = 2, 3, 4 - histogram = np.full((n_epochs, n_bins), -1, dtype=float) - spin_angle = np.tile(np.linspace(0, 240, n_bins), (n_epochs, 1)) - histogram_flag_array = np.zeros((n_epochs, n_flags, n_bins), dtype=np.uint8) - - ds = xr.Dataset( - { - "histogram": (["epoch", "bins"], histogram), - "spin_period_average": (["epoch"], [15.0, 15.0]), - "number_of_spins_per_block": (["epoch"], [5, 5]), - "imap_spin_angle_bin_cntr": (["epoch", "bins"], spin_angle), - "histogram_flag_array": ( - ["epoch", "bad_angle_flags", "bins"], - histogram_flag_array, - ), - }, - coords={"epoch": xr.DataArray(np.arange(n_epochs), dims=["epoch"])}, + mock_cal_factor = 1 + l1b_dataset["histogram"].values[:] = GlowsConstants.HISTOGRAM_FILLVAL + lc = DailyLightcurve( + l1b_dataset, position_angle=0.0, calibration_factor=mock_cal_factor ) - lc = DailyLightcurve(ds) - expected_exposure = 2 * 15.0 * 5 / 3 + expected_exposure = 2 * 15.0 * 5 / 4 assert np.all(lc.photon_flux == 0) assert np.all(lc.flux_uncertainties == 0) assert np.allclose(lc.exposure_times, expected_exposure) -def test_number_of_bins(l1b_dataset): - lc = DailyLightcurve(l1b_dataset) +def test_zero_exposure_values(l1b_dataset, mock_ecliptic_bin_centers): + """Zero exposure yields zero flux and zero uncertainty per bin.""" + + # Note: all bins have the same exposure time, so if one is zero all are zero. + + # Update values used to calculate exposure times to + # ensure a zero exposure result. + l1b_dataset["spin_period_average"].data[:] = 0 + l1b_dataset["number_of_spins_per_block"].data[:] = 0 + + mock_cal_factor = 1 + + with np.errstate(divide="raise", invalid="raise"): + lc = DailyLightcurve( + l1b_dataset, position_angle=0.0, calibration_factor=mock_cal_factor + ) + + expected = np.zeros(l1b_dataset.sizes["bins"], dtype=float) + assert lc.exposure_times.shape == expected.shape + assert len(np.unique(lc.exposure_times)) == 1 + assert np.array_equal(lc.exposure_times, expected) + assert np.array_equal(lc.photon_flux, expected) + assert np.array_equal(lc.flux_uncertainties, expected) + assert np.all(np.isfinite(lc.photon_flux)) + assert np.all(np.isfinite(lc.flux_uncertainties)) + + +def test_number_of_bins(l1b_dataset, mock_ecliptic_bin_centers): + mock_cal_factor = 1 + lc = DailyLightcurve( + l1b_dataset, position_angle=0.0, calibration_factor=mock_cal_factor + ) assert lc.number_of_bins == 4 assert len(lc.spin_angle) == 4 assert len(lc.photon_flux) == 4 assert len(lc.flux_uncertainties) == 4 assert len(lc.exposure_times) == 4 + assert len(lc.ecliptic_lon) == 4 + assert len(lc.ecliptic_lat) == 4 -def test_histogram_flag_array_or_propagation(): +def test_histogram_flag_array_or_propagation(l1b_dataset, mock_ecliptic_bin_centers): """histogram_flag_array is OR'd across all L1B epochs and flag rows per bin. Per Section 12.3.4: a flag is True in L2 if it is True in any L1B block. """ - n_epochs, n_bins, n_flags = 3, 4, 4 - histogram = np.ones((n_epochs, n_bins), dtype=float) - spin_angle = np.tile(np.linspace(0, 270, n_bins), (n_epochs, 1)) - # epoch 0, flag row 0 (IS_CLOSE_TO_UV_SOURCE=1): bin 0 flagged # epoch 1, flag row 1 (IS_INSIDE_EXCLUDED_REGION=2): bin 2 flagged - # epoch 2: no flags set - histogram_flag_array = np.zeros((n_epochs, n_flags, n_bins), dtype=np.uint8) - histogram_flag_array[0, 0, 0] = 1 # IS_CLOSE_TO_UV_SOURCE on bin 0 - histogram_flag_array[1, 1, 2] = 2 # IS_INSIDE_EXCLUDED_REGION on bin 0, 2 - histogram_flag_array[1, 0, 0] = 2 - - ds = xr.Dataset( - { - "histogram": (["epoch", "bins"], histogram), - "spin_period_average": (["epoch"], [15.0, 15.0, 15.0]), - "number_of_spins_per_block": (["epoch"], [5, 5, 5]), - "imap_spin_angle_bin_cntr": (["epoch", "bins"], spin_angle), - "histogram_flag_array": ( - ["epoch", "bad_angle_flags", "bins"], - histogram_flag_array, - ), - }, - coords={"epoch": xr.DataArray(np.arange(n_epochs), dims=["epoch"])}, + # epoch 1, flag row 0 (IS_CLOSE_TO_UV_SOURCE=2): bin 0 flagged + l1b_dataset["histogram_flag_array"].values[0, 0, 0] = 1 + l1b_dataset["histogram_flag_array"].values[1, 1, 2] = 2 + l1b_dataset["histogram_flag_array"].values[1, 0, 0] = 2 + + mock_cal_factor = 1 + lc = DailyLightcurve( + l1b_dataset, position_angle=0.0, calibration_factor=mock_cal_factor ) - lc = DailyLightcurve(ds) assert ( lc.histogram_flag_array[0] == 3 @@ -189,8 +299,11 @@ def test_histogram_flag_array_or_propagation(): assert lc.histogram_flag_array[3] == 0 # no flags on bin 3 -def test_histogram_flag_array_zero_epochs(): - """histogram_flag_array is all zeros when the input dataset is empty.""" +def test_histogram_flag_array_zero_epochs(mock_ecliptic_bin_centers): + """histogram_flag_array is all zeros when the input dataset is empty. + + Note: this is NEVER expected to happen in production + """ n_bins, n_flags = 4, 4 histogram = np.empty((0, n_bins), dtype=float) spin_angle = np.empty((0, n_bins), dtype=float) @@ -206,12 +319,15 @@ def test_histogram_flag_array_zero_epochs(): ["epoch", "bad_angle_flags", "bins"], histogram_flag_array, ), + "number_of_bins_per_histogram": (["epoch"], []), }, coords={"epoch": xr.DataArray(np.arange(0), dims=["epoch"])}, ) - lc = DailyLightcurve(ds) + mock_cal_factor = 1 + lc = DailyLightcurve(ds, position_angle=0.0, calibration_factor=mock_cal_factor) - assert len(lc.histogram_flag_array) == n_bins + # if the dataset is empty, there is no way to infer the number_of_bins + assert len(lc.histogram_flag_array) == 0 assert np.all(lc.histogram_flag_array == 0) @@ -228,3 +344,167 @@ def test_filter_good_times(): expected_good_times = [0, 2, 3] assert np.array_equal(good_times, expected_good_times) + + +@pytest.mark.parametrize( + "sunrise_offset, sunset_offset, expected_is_night", + [ + # sunrise>0 extends at sunrise; sunset>0 shortens at sunset + (1, 1, [1, 1, 1, 1, 0, 0, 0, 1]), + # sunrise<0 shortens at sunrise; sunset>0 shortens at sunset + (-1, 1, [1, 1, 1, 1, 0, 1, 1, 1]), + # sunrise>0 extends at sunrise; sunset<0 extends at sunset + (1, -1, [1, 1, 0, 0, 0, 0, 0, 1]), + # sunrise<0 shortens at sunrise; sunset<0 extends at sunset + (-1, -1, [1, 1, 0, 0, 0, 1, 1, 1]), + # zero offsets: no change + (0, 0, [1, 1, 1, 0, 0, 0, 1, 1]), + ], +) +def test_apply_is_night_offsets(sunrise_offset, sunset_offset, expected_is_night): + """Test apply_is_night_offsets function.""" + + # Setup: epochs 0-2 day, 3-5 night, 6-7 day (processed flags: 0=night, 1=day). + flags = np.ones((8, 17), dtype=float) + flags[3:6, 6] = 0 # epochs 3-5 are night + original_flags = flags.copy() + + result = HistogramL2.apply_is_night_offsets( + flags, + is_night_idx=6, + sunrise_offset=sunrise_offset, + sunset_offset=sunset_offset, + ) + + assert np.array_equal(result[:, 6], np.array(expected_is_night, dtype=float)) + + if sunrise_offset == 0 and sunset_offset == 0: + # No offsets: original array returned as-is (no copy) + assert result is flags + else: + # Offsets applied: result is a copy, original flags are unchanged + assert result is not flags + assert np.array_equal(flags, original_flags) + + +# ── spin_angle tests ────────────────────────────────────────────────────────── + + +def test_spin_angle_offset_formula(l1b_dataset, mock_ecliptic_bin_centers): + """spin_angle = (imap_spin_angle_bin_cntr - position_angle + 360) % 360. + + Fixture spin_angle_bin_cntr = [0, 90, 180, 270], position_angle = 90. + Expected before roll: [270, 0, 90, 180]. + Minimum is at index 1, so roll = -1 -> [0, 90, 180, 270]. + """ + mock_cal_factor = 1 + lc = DailyLightcurve( + l1b_dataset, position_angle=90.0, calibration_factor=mock_cal_factor + ) + expected = np.array([0.0, 90.0, 180.0, 270.0]) + assert np.allclose(lc.spin_angle, expected) + + +def test_spin_angle_starts_at_minimum(l1b_dataset, mock_ecliptic_bin_centers): + """After rolling, lc.spin_angle[0] is the minimum value. + + Fixture spin_angle_bin_cntr = [0, 90, 180, 270], position_angle = 45. + Before roll: [315, 45, 135, 225]; minimum 45 is at index 1 -> roll = -1 + -> [45, 135, 225, 315]. + """ + mock_cal_factor = 1 + lc = DailyLightcurve( + l1b_dataset, position_angle=45.0, calibration_factor=mock_cal_factor + ) + assert lc.spin_angle[0] == np.min(lc.spin_angle) + assert np.allclose(lc.spin_angle, np.array([45.0, 135.0, 225.0, 315.0])) + + +# ── position_angle_offset_average tests ────────────────────────────────────── + + +def test_compute_position_angle(): + """compute_position_angle returns (360 - azimuth) % 360 (Eq. 30).""" + target_module = ( + "imap_processing.glows.l2.glows_l2_data.get_instrument_mounting_az_el" + ) + with patch(target_module, return_value=(270.0, 0.0)): + result = HistogramL2.compute_position_angle(None) + assert result == pytest.approx(90.0) + + +@pytest.fixture +def l1b_dataset_full(): + """Minimal L1B dataset with all variables required by HistogramL2. + + Two epochs, four bins, 17 flags. Both epochs are daytime (is_night=1). + All other flags are 1 (good). + """ + n_epochs, n_bins, n_angle_flags, n_time_flags = 2, 4, 4, 17 + fillval = GlowsConstants.HISTOGRAM_FILLVAL + epoch = np.arange(n_epochs, dtype=np.float64) + + histogram = np.array([[10, 20, 30, fillval], [10, 20, 30, 40]], dtype=float) + spin_angle = np.tile(np.linspace(0, 270, n_bins), (n_epochs, 1)) + histogram_flag_array = np.zeros((n_epochs, n_angle_flags, n_bins), dtype=np.uint8) + + # All flags good (1). Index 6 is is_night: 1 = daytime (good). + flags = np.ones((n_epochs, n_time_flags), dtype=float) + + return xr.Dataset( + { + "histogram": (["epoch", "bins"], histogram), + "spin_period_average": (["epoch"], [15.0, 15.0]), + "number_of_spins_per_block": (["epoch"], [5, 5]), + "imap_spin_angle_bin_cntr": (["epoch", "bins"], spin_angle), + "histogram_flag_array": ( + ["epoch", "bad_angle_flags", "bins"], + histogram_flag_array, + ), + "number_of_bins_per_histogram": (["epoch"], [n_bins, n_bins]), + "flags": (["epoch", "flag_index"], flags), + "filter_temperature_average": (["epoch"], [20.0, 21.0]), + "hv_voltage_average": (["epoch"], [1000.0, 1000.0]), + "pulse_length_average": (["epoch"], [5.0, 5.0]), + "spin_period_ground_average": (["epoch"], [15.0, 15.0]), + "spacecraft_location_average": ( + ["epoch", "xyz"], + np.ones((n_epochs, 3)), + ), + "spacecraft_velocity_average": ( + ["epoch", "xyz"], + np.ones((n_epochs, 3)), + ), + "spin_axis_orientation_average": ( + ["epoch", "lonlat"], + np.ones((n_epochs, 2)), + ), + "imap_start_time": (["epoch"], [0.0, 1.0]), + "imap_time_offset": (["epoch"], [60.0, 60.0]), + }, + coords={"epoch": xr.DataArray(epoch, dims=["epoch"])}, + ) + + +def test_position_angle_offset_average( + l1b_dataset_full, + pipeline_settings, + mock_ecliptic_bin_centers, + mock_calibration_dataset, +): + """position_angle_offset_average is a scalar equal to the result of + compute_position_angle (Eq. 30, Section 10.6). It is constant across the + observational day since it depends only on instrument mounting geometry. + """ + mock_pa = 42.5 + mock_cal_factor = 1 + + with ( + patch.object(HistogramL2, "compute_position_angle", return_value=mock_pa), + patch.object( + HistogramL2, "get_calibration_factor", return_value=mock_cal_factor + ), + ): + l2 = HistogramL2(l1b_dataset_full, pipeline_settings, mock_calibration_dataset) + + assert l2.position_angle_offset_average == pytest.approx(mock_pa) diff --git a/imap_processing/tests/hi/conftest.py b/imap_processing/tests/hi/conftest.py index d66f328099..a63872ee07 100644 --- a/imap_processing/tests/hi/conftest.py +++ b/imap_processing/tests/hi/conftest.py @@ -23,6 +23,11 @@ def hi_test_cal_prod_config_path(hi_l1_test_data_path): return hi_l1_test_data_path / "imap_hi_90sensor-cal-prod_20240101_v001.csv" +@pytest.fixture(scope="session") +def hi_test_background_config_path(hi_l1_test_data_path): + return hi_l1_test_data_path / "imap_hi_90sensor-backgrounds_20240101_v001.csv" + + def create_metaevent(esa_step, met_subseconds, met_seconds): start_bitmask_data = 0 # META return ( diff --git a/imap_processing/tests/hi/data/l1/imap_hi_90sensor-backgrounds_20240101_v001.csv b/imap_processing/tests/hi/data/l1/imap_hi_90sensor-backgrounds_20240101_v001.csv new file mode 100644 index 0000000000..b67ed1935a --- /dev/null +++ b/imap_processing/tests/hi/data/l1/imap_hi_90sensor-backgrounds_20240101_v001.csv @@ -0,0 +1,19 @@ +# THIS IS A TEST FILE AND SHOULD NOT BE USED IN PRODUCTION PROCESSING. +# +# Backgrounds Determination Configuration File +# Valid start date: 2024-01-01 (from filename) +# This file will be used in processing until a new calibration file is uploaded +# to the SDC with either a higher version number or new date in the filename. +# For details on how the SDC selects ancillary files for processing, see: +# https://imap-processing.readthedocs.io/en/latest/data-access/calibration-files.html +# +# When creating PSET products, the following table will be used to determine per +# calibration product backgrounds. The backgrounds for each calibration product are +# computed individually and the sum is subtracted from the calibration product. Background +# uncertainties are summed in quadrature and reported in background uncertainties. +# +calibration_prod,background_index,coincidence_type_list,tof_ab_low,tof_ab_high,tof_ac1_low,tof_ac1_high,tof_bc1_low,tof_bc1_high,tof_c1c2_low,tof_c1c2_high,scaling_factor,uncertainty +0,0,ABC1C2|ABC1,-20,16,-46,-15,-511,511,0,1023,0.00306,0.00030 +0,1,AB,-20,16,-46,-15,-511,511,0,1023,0.0189,0.0020 +1,0,BC1C2|AC1,-20,16,-46,-15,-511,511,0,1023,0.00306,0.00030 +1,1,AB,-20,16,-46,-15,-511,511,0,1023,0.0189,0.0020 \ No newline at end of file diff --git a/imap_processing/tests/hi/test_hi_goodtimes.py b/imap_processing/tests/hi/test_hi_goodtimes.py index 56dbcff22a..545c8c3afc 100644 --- a/imap_processing/tests/hi/test_hi_goodtimes.py +++ b/imap_processing/tests/hi/test_hi_goodtimes.py @@ -23,6 +23,7 @@ _identify_cull_pattern, create_goodtimes_dataset, hi_goodtimes, + mark_bad_tdc_cal, mark_drf_times, mark_incomplete_spin_sets, mark_overflow_packets, @@ -76,14 +77,29 @@ class TestCullCode: """Test suite for CullCode IntEnum.""" def test_cull_code_values(self): - """Test CullCode enum values.""" + """Test CullCode enum values are bit flags (powers of 2).""" assert CullCode.GOOD == 0 - assert CullCode.LOOSE == 1 + assert CullCode.INCOMPLETE_SPIN == 1 + assert CullCode.DRF == 2 + assert CullCode.BAD_TDC_CAL == 4 + assert CullCode.OVERFLOW == 8 + assert CullCode.STAT_FILTER_0 == 16 + assert CullCode.STAT_FILTER_1 == 32 + assert CullCode.STAT_FILTER_2 == 64 def test_cull_code_is_int(self): """Test that CullCode values are integers.""" assert isinstance(CullCode.GOOD, int) - assert isinstance(CullCode.LOOSE, int) + assert isinstance(CullCode.INCOMPLETE_SPIN, int) + + def test_cull_codes_can_be_combined(self): + """Test that cull codes can be combined with bitwise OR.""" + combined = CullCode.INCOMPLETE_SPIN | CullCode.DRF + assert combined == 3 + # Check individual flags can be extracted with bitwise AND + assert combined & CullCode.INCOMPLETE_SPIN == CullCode.INCOMPLETE_SPIN + assert combined & CullCode.DRF == CullCode.DRF + assert combined & CullCode.BAD_TDC_CAL == 0 class TestGoodtimesFromL1bDe: @@ -106,7 +122,7 @@ def test_from_l1b_de_dimensions(self, goodtimes_instance): """Test that dimensions are correct.""" assert "met" in goodtimes_instance.dims assert "spin_bin" in goodtimes_instance.dims - assert goodtimes_instance.dims["spin_bin"] == 90 + assert goodtimes_instance.sizes["spin_bin"] == 90 def test_from_l1b_de_coordinates(self, goodtimes_instance): """Test that coordinates are set correctly.""" @@ -145,8 +161,8 @@ def test_from_l1b_de_esa_step_preserved(self, mock_l1b_de, goodtimes_instance): def test_from_l1b_de_attributes(self, goodtimes_instance): """Test that attributes are set correctly.""" - assert goodtimes_instance.attrs["sensor"] == "45sensor" - assert goodtimes_instance.attrs["pointing"] == 42 + assert goodtimes_instance.attrs["Sensor"] == "45sensor" + assert goodtimes_instance.attrs["Repointing"] == "repoint00042" class TestRemoveTimes: @@ -156,11 +172,13 @@ def test_mark_bad_times_single_met_all_bins(self, goodtimes_instance): """Test flagging a single MET with all bins.""" met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=None, cull=CullCode.LOOSE + met=met_val, bins=None, cull=CullCode.INCOMPLETE_SPIN ) # Check that all bins for the first MET are flagged - assert np.all(goodtimes_instance["cull_flags"].values[0, :] == CullCode.LOOSE) + assert np.all( + goodtimes_instance["cull_flags"].values[0, :] == CullCode.INCOMPLETE_SPIN + ) # Check that other METs are still good assert np.all(goodtimes_instance["cull_flags"].values[1:, :] == CullCode.GOOD) @@ -170,12 +188,13 @@ def test_mark_bad_times_single_met_specific_bins(self, goodtimes_instance): met_val = goodtimes_instance.coords["met"].values[0] bins_to_flag = np.array([0, 1, 2, 10]) goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=bins_to_flag, cull=CullCode.LOOSE + met=met_val, bins=bins_to_flag, cull=CullCode.INCOMPLETE_SPIN ) # Check that specified bins are flagged assert np.all( - goodtimes_instance["cull_flags"].values[0, bins_to_flag] == CullCode.LOOSE + goodtimes_instance["cull_flags"].values[0, bins_to_flag] + == CullCode.INCOMPLETE_SPIN ) # Check that other bins are still good @@ -188,11 +207,13 @@ def test_mark_bad_times_multiple_mets(self, goodtimes_instance): """Test flagging multiple METs.""" met_vals = goodtimes_instance.coords["met"].values[:3] goodtimes_instance.goodtimes.mark_bad_times( - met=met_vals, bins=None, cull=CullCode.LOOSE + met=met_vals, bins=None, cull=CullCode.INCOMPLETE_SPIN ) # Check that first 3 METs are flagged - assert np.all(goodtimes_instance["cull_flags"].values[:3, :] == CullCode.LOOSE) + assert np.all( + goodtimes_instance["cull_flags"].values[:3, :] == CullCode.INCOMPLETE_SPIN + ) # Check that other METs are still good assert np.all(goodtimes_instance["cull_flags"].values[3:, :] == CullCode.GOOD) @@ -204,11 +225,13 @@ def test_mark_bad_times_time_range(self, goodtimes_instance): met_end = met_vals[5] goodtimes_instance.goodtimes.mark_bad_times( - met=(met_start, met_end), bins=None, cull=CullCode.LOOSE + met=(met_start, met_end), bins=None, cull=CullCode.INCOMPLETE_SPIN ) # Check that METs 2-5 are flagged - assert np.all(goodtimes_instance["cull_flags"].values[2:6, :] == CullCode.LOOSE) + assert np.all( + goodtimes_instance["cull_flags"].values[2:6, :] == CullCode.INCOMPLETE_SPIN + ) # Check that other METs are still good assert np.all(goodtimes_instance["cull_flags"].values[:2, :] == CullCode.GOOD) @@ -246,19 +269,24 @@ def test_mark_bad_times_met_out_of_range(self, goodtimes_instance): with pytest.raises(ValueError, match="MET value\\(s\\) "): goodtimes_instance.goodtimes.mark_bad_times(met=met_out_of_range) - def test_mark_bad_times_overwrites_existing_cull(self, goodtimes_instance): - """Test that new cull code overwrites existing one.""" + def test_mark_bad_times_combines_cull_codes(self, goodtimes_instance): + """Test that cull codes are combined using bitwise OR.""" met_val = goodtimes_instance.coords["met"].values[0] - # Flag with LOOSE + # Flag with INCOMPLETE_SPIN (1) goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=None, cull=CullCode.LOOSE + met=met_val, bins=None, cull=CullCode.INCOMPLETE_SPIN + ) + assert np.all( + goodtimes_instance["cull_flags"].values[0, :] == CullCode.INCOMPLETE_SPIN ) - assert np.all(goodtimes_instance["cull_flags"].values[0, :] == CullCode.LOOSE) - # Overwrite with a different cull code - goodtimes_instance.goodtimes.mark_bad_times(met=met_val, bins=None, cull=2) - assert np.all(goodtimes_instance["cull_flags"].values[0, :] == 2) + # Add another cull code with bitwise OR (1 | 2 = 3) + goodtimes_instance.goodtimes.mark_bad_times( + met=met_val, bins=None, cull=CullCode.DRF + ) + expected = CullCode.INCOMPLETE_SPIN | CullCode.DRF # 1 | 2 = 3 + assert np.all(goodtimes_instance["cull_flags"].values[0, :] == expected) class TestGetGoodIntervals: @@ -268,89 +296,136 @@ def test_get_good_intervals_all_good(self, goodtimes_instance): """Test getting intervals when all times are good.""" intervals = goodtimes_instance.goodtimes.get_good_intervals() - # Should have one interval per MET - n_met = len(goodtimes_instance.coords["met"]) - assert len(intervals) == n_met - - # Check interval structure + # With sweep-based grouping, consecutive sweeps with identical patterns + # are merged. The number of intervals depends on sweep structure. + assert len(intervals) >= 1 assert intervals.dtype == INTERVAL_DTYPE + # All intervals should be good (cull_value == 0) + for interval in intervals: + assert interval["cull_value"] == 0 + # All-good intervals have all ESAs marked in bitmask + assert interval["esa_step_mask"] > 0 + def test_get_good_intervals_structure(self, goodtimes_instance): - """Test interval structure and field names.""" + """Test interval structure and attributes.""" intervals = goodtimes_instance.goodtimes.get_good_intervals() - # Check that all fields exist - assert "met_start" in intervals.dtype.names - assert "met_end" in intervals.dtype.names - assert "spin_bin_low" in intervals.dtype.names - assert "spin_bin_high" in intervals.dtype.names - assert "n_good_bins" in intervals.dtype.names - assert "esa_step" in intervals.dtype.names + # Check that intervals have the correct dtype + assert intervals.dtype == INTERVAL_DTYPE + + # Check that all required fields exist + required_fields = [ + "met_start", + "met_end", + "spin_bin_low", + "spin_bin_high", + "n_bins", + "esa_step_mask", + "cull_value", + ] + for field in required_fields: + assert field in intervals.dtype.names def test_get_good_intervals_all_good_values(self, goodtimes_instance): """Test interval values when all bins are good.""" intervals = goodtimes_instance.goodtimes.get_good_intervals() - # When all bins are good, should have bins 0-89 + # With sweep-based grouping, we may have multiple intervals + assert len(intervals) >= 1 + + # All intervals should be all-good (cull_value == 0) for interval in intervals: - assert interval["spin_bin_low"] == 0 - assert interval["spin_bin_high"] == 89 - assert interval["n_good_bins"] == 90 - assert interval["met_start"] == interval["met_end"] + assert interval["esa_step_mask"] > 0 + assert interval["cull_value"] == 0 + + # First interval should start at first MET + met_values = goodtimes_instance.coords["met"].values + assert intervals[0]["met_start"] == met_values[0] + + # Last interval's met_end should be the last MET + assert intervals[-1]["met_end"] == met_values[-1] + + # met_end of each interval (except last) should be met_start of next + for i in range(len(intervals) - 1): + assert intervals[i]["met_end"] == intervals[i + 1]["met_start"] def test_get_good_intervals_with_culled_bins(self, goodtimes_instance): """Test intervals when some bins are culled.""" - # Flag bins 0-20 for first MET + # Flag bins 0-20 for first MET only met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=np.arange(21), cull=CullCode.LOOSE + met=met_val, bins=np.arange(21), cull=CullCode.INCOMPLETE_SPIN ) intervals = goodtimes_instance.goodtimes.get_good_intervals() - # First interval should only have bins 21-89 - assert intervals[0]["spin_bin_low"] == 21 - assert intervals[0]["spin_bin_high"] == 89 - assert intervals[0]["n_good_bins"] == 69 + # Only good intervals are output: + # - First sweep has one ESA step with partial cull (bins 21-89 good) + # - Remaining sweeps are fully good (all bins) + # The number of intervals depends on sweep grouping + assert len(intervals) >= 2 + + # Check for the partial interval (bins 21-89 good for the culled ESA step) + has_partial = any( + interval["spin_bin_low"] == 21 and interval["spin_bin_high"] == 89 + for interval in intervals + ) + assert has_partial, "Should have at least one partial region with bins 21-89" def test_get_good_intervals_with_gaps(self, goodtimes_instance): - """Test intervals when good bins have gaps (wraparound).""" + """Test intervals when bins have gaps in cull values.""" # Flag bins 20-70 for first MET, leaving bins 0-19 and 71-89 as good met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=np.arange(20, 71), cull=CullCode.LOOSE + met=met_val, bins=np.arange(20, 71), cull=CullCode.INCOMPLETE_SPIN ) intervals = goodtimes_instance.goodtimes.get_good_intervals() - # Should create 2 intervals for the first MET (bins split by gap) - # Plus 11 more intervals for the remaining METs (12 total METs) - assert len(intervals) == 13 + # Only good intervals are output: + # - First sweep has one ESA step with partial cull (bins 0-19 and 71-89 good) + # - Remaining sweeps are fully good + # Bad intervals (bins 20-70) are not output + assert len(intervals) >= 3 - # First two intervals should be for the same MET - assert intervals[0]["met_start"] == intervals[1]["met_start"] + # Check that we have good bin regions for the partial ESA step + # bins 0-19 good + low_good = [ + i for i in intervals if i["spin_bin_low"] == 0 and i["spin_bin_high"] == 19 + ] + assert len(low_good) >= 1 - # Check the two segments - assert intervals[0]["spin_bin_low"] == 0 - assert intervals[0]["spin_bin_high"] == 19 - assert intervals[1]["spin_bin_low"] == 71 - assert intervals[1]["spin_bin_high"] == 89 + # bins 71-89 good + high_good = [ + i for i in intervals if i["spin_bin_low"] == 71 and i["spin_bin_high"] == 89 + ] + assert len(high_good) >= 1 def test_get_good_intervals_all_bins_culled(self, goodtimes_instance): """Test intervals when all bins are culled for a MET.""" # Flag all bins for first MET met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=None, cull=CullCode.LOOSE + met=met_val, bins=None, cull=CullCode.INCOMPLETE_SPIN ) intervals = goodtimes_instance.goodtimes.get_good_intervals() - # Should have 11 intervals (one per good MET, excluding the first, 12-1=11) - assert len(intervals) == 11 + # Only good intervals are output - the fully-culled ESA step is not output + # The remaining good ESA steps should be output + assert len(intervals) >= 1 + + # All output intervals should have good bins (cull_value indicates what + # was culled). Check that we have a fully-good interval (bins 0-89) for + # the good ESA steps + full_good = [ + i for i in intervals if i["spin_bin_low"] == 0 and i["spin_bin_high"] == 89 + ] + assert len(full_good) >= 1 - # First interval should be for the second MET - assert intervals[0]["met_start"] == goodtimes_instance.coords["met"].values[1] + # The cull_value should indicate the cull code for the culled ESA step + assert full_good[0]["cull_value"] == CullCode.INCOMPLETE_SPIN def test_get_good_intervals_empty(self): """Test intervals with empty goodtimes dataset.""" @@ -369,14 +444,24 @@ def test_get_good_intervals_empty(self): intervals = gt.goodtimes.get_good_intervals() assert len(intervals) == 0 - def test_get_good_intervals_esa_step_included(self, goodtimes_instance): - """Test that ESA step is included in intervals.""" + def test_get_good_intervals_esa_step_mask(self, goodtimes_instance): + """Test that ESA step mask includes ESA steps in each interval.""" intervals = goodtimes_instance.goodtimes.get_good_intervals() - # Check that each interval has an ESA step - for i, interval in enumerate(intervals): - expected_esa_step = goodtimes_instance["esa_step"].values[i] - assert interval["esa_step"] == expected_esa_step + # With sweep-based grouping, each interval has its own ESA step mask + assert len(intervals) >= 1 + + # Collect all ESA steps across all intervals + all_esa_steps_in_intervals = set() + for interval in intervals: + esa_step_mask = interval["esa_step_mask"] + for bit_position in range(10): # ESA steps 1-10 + if (esa_step_mask >> bit_position) & 1: + all_esa_steps_in_intervals.add(bit_position + 1) + + # All unique ESA steps should be represented across all intervals + unique_esa_steps = set(goodtimes_instance["esa_step"].values) + assert all_esa_steps_in_intervals == unique_esa_steps class TestGetCullStatistics: @@ -398,7 +483,7 @@ def test_get_cull_statistics_with_culls(self, goodtimes_instance): # Flag first MET, all bins met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=None, cull=CullCode.LOOSE + met=met_val, bins=None, cull=CullCode.INCOMPLETE_SPIN ) stats = goodtimes_instance.goodtimes.get_cull_statistics() @@ -408,7 +493,7 @@ def test_get_cull_statistics_with_culls(self, goodtimes_instance): assert stats["good_bins"] == total_bins - 90 assert stats["culled_bins"] == 90 assert stats["fraction_good"] == (total_bins - 90) / total_bins - assert stats["cull_code_counts"][CullCode.LOOSE] == 90 + assert stats["cull_code_counts"][CullCode.INCOMPLETE_SPIN] == 90 def test_get_cull_statistics_multiple_cull_codes(self, goodtimes_instance): """Test statistics with multiple cull codes.""" @@ -416,7 +501,7 @@ def test_get_cull_statistics_multiple_cull_codes(self, goodtimes_instance): # Flag first MET with LOOSE goodtimes_instance.goodtimes.mark_bad_times( - met=met_vals[0], bins=None, cull=CullCode.LOOSE + met=met_vals[0], bins=None, cull=CullCode.INCOMPLETE_SPIN ) # Flag second MET with code 2 @@ -425,7 +510,7 @@ def test_get_cull_statistics_multiple_cull_codes(self, goodtimes_instance): stats = goodtimes_instance.goodtimes.get_cull_statistics() assert stats["culled_bins"] == 180 - assert stats["cull_code_counts"][CullCode.LOOSE] == 90 + assert stats["cull_code_counts"][CullCode.INCOMPLETE_SPIN] == 90 assert stats["cull_code_counts"][2] == 90 @@ -448,14 +533,17 @@ def test_to_txt_format(self, goodtimes_instance, tmp_path): with open(output_path) as f: lines = f.readlines() - # Should have one line per interval (12 METs, all good) - assert len(lines) == 12 + # With sweep-based grouping, may have multiple intervals + assert len(lines) >= 1 # Check format of first line + # Format: pointing met_start met_end bin_low bin_high sensor + # esa_steps[10] cull_value parts = lines[0].strip().split() - assert len(parts) == 7 + assert len(parts) == 17 # 6 base fields + 10 ESA step flags + cull_value assert parts[0] == "00042" # pointing - assert parts[5] == "45sensor" # sensor + assert parts[5] == "45" # sensor + assert parts[16] == "0" # cull_value (all good, no culled ESA steps) def test_to_txt_values(self, goodtimes_instance, tmp_path): """Test the values in the output file.""" @@ -463,47 +551,87 @@ def test_to_txt_values(self, goodtimes_instance, tmp_path): goodtimes_instance.goodtimes.write_txt(output_path) with open(output_path) as f: - line = f.readline() + lines = f.readlines() - parts = line.strip().split() - pointing, met_start, met_end, bin_low, bin_high, sensor, esa_step = parts + # With sweep-based grouping, may have multiple intervals + assert len(lines) >= 1 + + # Check first line format + parts = lines[0].strip().split() + # Format: pointing met_start met_end bin_low bin_high sensor + # esa_steps[10] cull_value + pointing = parts[0] + met_start = parts[1] + bin_low = parts[3] + bin_high = parts[4] + sensor = parts[5] + cull_value = parts[16] assert pointing == "00042" - assert int(met_start) == int(goodtimes_instance.coords["met"].values[0]) - assert int(met_end) == int(goodtimes_instance.coords["met"].values[0]) + # First interval should start at first MET + assert float(met_start) == goodtimes_instance.coords["met"].values[0] assert int(bin_low) == 0 assert int(bin_high) == 89 - assert sensor == "45sensor" - assert int(esa_step) == goodtimes_instance["esa_step"].values[0] + assert sensor == "45" + assert cull_value == "0" # All good, no culled ESA steps + + # Collect all ESA steps across all intervals + all_esa_steps = set() + for line in lines: + parts = line.strip().split() + esa_step_flags = parts[6:16] + for i, flag in enumerate(esa_step_flags): + if flag == "1": + all_esa_steps.add(i + 1) + + # All unique ESA steps should be represented + unique_esa_steps = set(goodtimes_instance["esa_step"].values) + assert all_esa_steps == unique_esa_steps def test_to_txt_with_culled_bins(self, goodtimes_instance, tmp_path): """Test output when some bins are culled.""" # Flag bins 0-20 for first MET met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=np.arange(21), cull=CullCode.LOOSE + met=met_val, bins=np.arange(21), cull=CullCode.INCOMPLETE_SPIN ) output_path = tmp_path / "goodtimes.txt" goodtimes_instance.goodtimes.write_txt(output_path) with open(output_path) as f: - first_line = f.readline() - - parts = first_line.strip().split() - bin_low = int(parts[3]) - bin_high = int(parts[4]) + lines = f.readlines() - # First interval should only include bins 21-89 - assert bin_low == 21 - assert bin_high == 89 + # Only good intervals are output + # Should have intervals for: + # - Fully good ESA steps (all bins) + # - Partially good ESA step (bins 21-89) + assert len(lines) >= 2 + + # Check for interval with good bins 21-89 (partial) + partial_lines = [ + line + for line in lines + if line.strip().split()[3] == "21" and line.strip().split()[4] == "89" + ] + assert len(partial_lines) >= 1 + # cull_value should indicate what was culled + assert partial_lines[0].strip().split()[16] == "1" + + # Check for fully good intervals (bins 0-89) + full_lines = [ + line + for line in lines + if line.strip().split()[3] == "0" and line.strip().split()[4] == "89" + ] + assert len(full_lines) >= 1 def test_to_txt_with_gaps(self, goodtimes_instance, tmp_path): """Test output when bins have gaps.""" # Flag bins 20-70, leaving 0-19 and 71-89 as good met_val = goodtimes_instance.coords["met"].values[0] goodtimes_instance.goodtimes.mark_bad_times( - met=met_val, bins=np.arange(20, 71), cull=CullCode.LOOSE + met=met_val, bins=np.arange(20, 71), cull=CullCode.INCOMPLETE_SPIN ) output_path = tmp_path / "goodtimes.txt" @@ -512,19 +640,38 @@ def test_to_txt_with_gaps(self, goodtimes_instance, tmp_path): with open(output_path) as f: lines = f.readlines() - # Should have 13 lines (2 for first MET, 1 for each of 11 remaining METs) - assert len(lines) == 13 - - # First two lines should be for same MET - parts1 = lines[0].strip().split() - parts2 = lines[1].strip().split() - assert parts1[1] == parts2[1] # Same met_start - - # Check bin ranges - assert int(parts1[3]) == 0 - assert int(parts1[4]) == 19 - assert int(parts2[3]) == 71 - assert int(parts2[4]) == 89 + # Only good intervals are output (no culled intervals) + # Should have intervals for: + # - Fully good ESA steps (all bins) + # - Partially good ESA step (bins 0-19 and 71-89) + assert len(lines) >= 3 + + # Check for good region bins 0-19 + low_good = [ + line + for line in lines + if line.strip().split()[3] == "0" and line.strip().split()[4] == "19" + ] + assert len(low_good) >= 1 + # cull_value should indicate what was culled + assert low_good[0].strip().split()[16] == "1" + + # Check for good region bins 71-89 + high_good = [ + line + for line in lines + if line.strip().split()[3] == "71" and line.strip().split()[4] == "89" + ] + assert len(high_good) >= 1 + assert high_good[0].strip().split()[16] == "1" + + # Check for fully good intervals (bins 0-89) for other ESA steps + full_good = [ + line + for line in lines + if line.strip().split()[3] == "0" and line.strip().split()[4] == "89" + ] + assert len(full_good) >= 1 class TestFinalizeDataset: @@ -615,7 +762,7 @@ def test_finalize_preserves_cull_flags_data(self, goodtimes_instance): goodtimes_instance.goodtimes.mark_bad_times( met=goodtimes_instance.coords["met"].values[0], bins=np.arange(10), - cull=CullCode.LOOSE, + cull=CullCode.INCOMPLETE_SPIN, ) original_flags = goodtimes_instance["cull_flags"].values.copy() @@ -693,7 +840,7 @@ def test_finalize_formats_logical_source(self, goodtimes_instance): def test_finalize_preserves_original_dataset(self, goodtimes_instance): """Test that finalize doesn't modify the original dataset.""" - original_dims = set(goodtimes_instance.dims.keys()) + original_dims = set(goodtimes_instance.sizes.keys()) original_coords = set(goodtimes_instance.coords.keys()) with patch("imap_processing.hi.hi_goodtimes.met_to_ttj2000ns") as mock_convert: @@ -705,7 +852,7 @@ def test_finalize_preserves_original_dataset(self, goodtimes_instance): goodtimes_instance.goodtimes.finalize_dataset() # Original should be unchanged - assert set(goodtimes_instance.dims.keys()) == original_dims + assert set(goodtimes_instance.sizes.keys()) == original_dims assert set(goodtimes_instance.coords.keys()) == original_coords assert "epoch" not in goodtimes_instance.coords @@ -752,7 +899,7 @@ def test_finalize_with_empty_dataset(self): "esa_step": xr.DataArray(np.array([], dtype=np.uint8), dims=["met"]), }, coords={"met": np.array([]), "spin_bin": np.arange(90)}, - attrs={"sensor": "45sensor", "pointing": 1}, + attrs={"Sensor": "45sensor", "Pointing": 1}, ) with patch("imap_processing.hi.hi_goodtimes.met_to_ttj2000ns") as mock_convert: @@ -774,8 +921,9 @@ def test_interval_dtype_fields(self): assert "met_end" in field_names assert "spin_bin_low" in field_names assert "spin_bin_high" in field_names - assert "n_good_bins" in field_names - assert "esa_step" in field_names + assert "n_bins" in field_names + assert "esa_step_mask" in field_names + assert "cull_value" in field_names def test_interval_dtype_types(self): """Test that INTERVAL_DTYPE has correct field types.""" @@ -783,8 +931,9 @@ def test_interval_dtype_types(self): assert INTERVAL_DTYPE["met_end"] == np.float64 assert INTERVAL_DTYPE["spin_bin_low"] == np.uint8 assert INTERVAL_DTYPE["spin_bin_high"] == np.uint8 - assert INTERVAL_DTYPE["n_good_bins"] == np.uint8 - assert INTERVAL_DTYPE["esa_step"] == np.uint8 + assert INTERVAL_DTYPE["n_bins"] == np.uint8 + assert INTERVAL_DTYPE["esa_step_mask"] == np.uint16 + assert INTERVAL_DTYPE["cull_value"] == np.uint8 def _create_l1b_de_dataset( @@ -1000,8 +1149,8 @@ def test_mark_incomplete_spin_sets_incomplete(self, l1b_de_incomplete): # First 2 METs should be good, last 2 should be culled assert np.all(gt["cull_flags"].values[0, :] == CullCode.GOOD) assert np.all(gt["cull_flags"].values[1, :] == CullCode.GOOD) - assert np.all(gt["cull_flags"].values[2, :] == CullCode.LOOSE) - assert np.all(gt["cull_flags"].values[3, :] == CullCode.LOOSE) + assert np.all(gt["cull_flags"].values[2, :] == CullCode.INCOMPLETE_SPIN) + assert np.all(gt["cull_flags"].values[3, :] == CullCode.INCOMPLETE_SPIN) def test_mark_incomplete_spin_sets_with_invalid_spins( self, l1b_de_with_invalid_spins @@ -1011,7 +1160,7 @@ def test_mark_incomplete_spin_sets_with_invalid_spins( mark_incomplete_spin_sets(gt, l1b_de_with_invalid_spins) # First MET should be culled (has spin invalid flag), second should be good - assert np.all(gt["cull_flags"].values[0, :] == CullCode.LOOSE) + assert np.all(gt["cull_flags"].values[0, :] == CullCode.INCOMPLETE_SPIN) assert np.all(gt["cull_flags"].values[1, :] == CullCode.GOOD) def test_mark_incomplete_spin_sets_no_de_packets(self): @@ -1048,7 +1197,9 @@ def test_mark_incomplete_spin_sets_no_de_packets(self): # First and last METs should be good, middle one should be culled assert np.all(gt["cull_flags"].values[0, :] == CullCode.GOOD) - assert np.all(gt["cull_flags"].values[1, :] == CullCode.LOOSE) # No packets + assert np.all( + gt["cull_flags"].values[1, :] == CullCode.INCOMPLETE_SPIN + ) # No packets assert np.all(gt["cull_flags"].values[2, :] == CullCode.GOOD) def test_mark_incomplete_spin_sets_mixed_cadence(self): @@ -1065,7 +1216,7 @@ def test_mark_incomplete_spin_sets_mixed_cadence(self): mark_incomplete_spin_sets(gt, l1b_de) # Should be culled (invalid pattern) - assert np.all(gt["cull_flags"].values[0, :] == CullCode.LOOSE) + assert np.all(gt["cull_flags"].values[0, :] == CullCode.INCOMPLETE_SPIN) def test_mark_incomplete_spin_sets_duplicate_spin_num(self): """Test that duplicate last_spin_num values are culled.""" @@ -1081,7 +1232,7 @@ def test_mark_incomplete_spin_sets_duplicate_spin_num(self): mark_incomplete_spin_sets(gt, l1b_de) # Should be culled (duplicate spin numbers) - assert np.all(gt["cull_flags"].values[0, :] == CullCode.LOOSE) + assert np.all(gt["cull_flags"].values[0, :] == CullCode.INCOMPLETE_SPIN) def test_mark_incomplete_spin_sets_custom_cull_code(self, l1b_de_incomplete): """Test that custom cull code is used.""" @@ -1225,7 +1376,7 @@ def test_mark_drf_times_single_transition( # Check that METs in the window are culled (indices 0-30) for i in range(31): assert np.all( - goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.LOOSE + goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.DRF ), ( f"MET at index {i} (value " f"{goodtimes_for_drf.coords['met'].values[i]}) should be culled" @@ -1252,7 +1403,7 @@ def test_mark_drf_times_multiple_transitions( # Check first window (indices 0-30) for i in range(31): assert np.all( - goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.LOOSE + goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.DRF ), f"MET at index {i} should be culled (first window)" # Check between windows (indices 31-59, should be good) @@ -1264,7 +1415,7 @@ def test_mark_drf_times_multiple_transitions( # Check second window (indices 60-90) for i in range(60, 91): assert np.all( - goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.LOOSE + goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.DRF ), f"MET at index {i} should be culled (second window)" # Check after second window (indices 91+, should be good) @@ -1319,11 +1470,9 @@ def test_mark_drf_times_overwrites_existing_culls( mark_drf_times(goodtimes_for_drf, hk_single_drf_transition) - # First 5 METs should now be LOOSE (overwritten), not 2 + # First 5 METs should now be DRF (overwritten via bitwise OR with existing 2) for i in range(5): - assert np.all( - goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.LOOSE - ) + assert np.all(goodtimes_for_drf["cull_flags"].values[i, :] == CullCode.DRF) def test_mark_drf_times_transition_at_start(self): """Test DRF transition near the start - window exactly at data start.""" @@ -1362,7 +1511,7 @@ def test_mark_drf_times_transition_at_start(self): # Window: 3800 - 1800 = 2000 to 3800 # This includes METs from 2000 to 3800 (indices 0-30) for i in range(31): - assert np.all(gt["cull_flags"].values[i, :] == CullCode.LOOSE), ( + assert np.all(gt["cull_flags"].values[i, :] == CullCode.DRF), ( f"MET at index {i} should be culled" ) @@ -1408,11 +1557,253 @@ def test_mark_drf_times_transition_at_end(self): # Transition at last index (MET ~2940) # Should remove 30-minute window before it # Most METs should still be good except the last ~30 - n_culled = np.sum(gt["cull_flags"].values[:, 0] == CullCode.LOOSE) + n_culled = np.sum(gt["cull_flags"].values[:, 0] == CullCode.DRF) assert n_culled > 0 # Some should be culled assert n_culled <= 31 # But not all (only last ~30 minutes) +class TestMarkBadTdcCal: + """Test suite for mark_bad_tdc_cal() function.""" + + @pytest.fixture + def goodtimes_for_tdc(self): + """Create a goodtimes dataset with METs spanning a range.""" + # Create METs every 50 seconds for 200 seconds (5 METs) + n_mets = 5 + met_values = np.arange(1000.0, 1000.0 + n_mets * 50, 50) + + gt = xr.Dataset( + { + "cull_flags": xr.DataArray( + np.zeros((n_mets, 90), dtype=np.uint8), dims=["met", "spin_bin"] + ), + "esa_step": xr.DataArray(np.ones(n_mets, dtype=np.uint8), dims=["met"]), + }, + coords={"met": met_values, "spin_bin": np.arange(90)}, + attrs={"sensor": "45sensor", "pointing": 1}, + ) + return gt + + @pytest.fixture + def diagfee_all_good(self): + """Create DIAG_FEE dataset where all TDC calibrations pass.""" + # 4 DIAG_FEE packets, all with bit 1 set (=2, meaning calibration good) + return xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000, 1050, 1100, 1150])), + "tdc1_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc2_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc3_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + } + ) + + @pytest.fixture + def diagfee_tdc1_fails(self): + """Create DIAG_FEE dataset where TDC1 fails at packet index 2.""" + # TDC1 fails at index 2 (bit 1 not set, so value 0) + return xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000, 1050, 1100, 1150])), + "tdc1_cal_ctrl_stat": ( + ["epoch"], + np.array([2, 2, 0, 2]), + ), # fails at idx 2 + "tdc2_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc3_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + } + ) + + @pytest.fixture + def diagfee_with_duplicate(self): + """Create DIAG_FEE dataset with duplicate packets within 10 seconds.""" + # First two packets are within 10 seconds (should skip first) + return xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000, 1005, 1100, 1150])), + "tdc1_cal_ctrl_stat": ( + ["epoch"], + np.array([0, 2, 2, 2]), + ), # First would fail but is skipped + "tdc2_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc3_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + } + ) + + def test_mark_bad_tdc_cal_all_good(self, goodtimes_for_tdc, diagfee_all_good): + """Test that no times are marked when all TDC calibrations pass.""" + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_all_good) + + # All times should remain good + assert np.all(goodtimes_for_tdc["cull_flags"].values == CullCode.GOOD) + + def test_mark_bad_tdc_cal_tdc1_fails(self, goodtimes_for_tdc, diagfee_tdc1_fails): + """Test that times are marked when TDC1 fails.""" + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_tdc1_fails) + + # TDC1 fails at packet 2 (MET 1100), should mark times from 1100 to 1150 + # goodtimes METs are [1000, 1050, 1100, 1150, 1200] + # MET 1100 falls in window [1100, 1150), so MET 1100 should be culled + met_values = goodtimes_for_tdc.coords["met"].values + + # MET 1100 (index 2) should be culled + idx_1100 = np.where(met_values == 1100.0)[0][0] + assert np.all( + goodtimes_for_tdc["cull_flags"].values[idx_1100, :] == CullCode.BAD_TDC_CAL + ) + + # METs before 1100 should still be good + assert np.all( + goodtimes_for_tdc["cull_flags"].values[0, :] == CullCode.GOOD + ) # 1000 + assert np.all( + goodtimes_for_tdc["cull_flags"].values[1, :] == CullCode.GOOD + ) # 1050 + + def test_mark_bad_tdc_cal_skip_duplicate_packets( + self, goodtimes_for_tdc, diagfee_with_duplicate + ): + """Test that duplicate DIAG_FEE packets within 10 seconds are skipped.""" + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_with_duplicate) + + # First packet (MET 1000) has TDC1 fail but should be skipped + # because it's within 10 seconds of the next packet (MET 1005) + # So all times should remain good + assert np.all(goodtimes_for_tdc["cull_flags"].values == CullCode.GOOD) + + def test_mark_bad_tdc_cal_insufficient_packets(self, goodtimes_for_tdc): + """Test that less than 2 packets logs warning and returns early.""" + # Create DIAG_FEE with only 1 packet + diagfee_single = xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000])), + "tdc1_cal_ctrl_stat": (["epoch"], np.array([0])), # Fails but ignored + "tdc2_cal_ctrl_stat": (["epoch"], np.array([2])), + "tdc3_cal_ctrl_stat": (["epoch"], np.array([2])), + } + ) + + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_single) + + # All times should remain good (no culling due to insufficient packets) + assert np.all(goodtimes_for_tdc["cull_flags"].values == CullCode.GOOD) + + def test_mark_bad_tdc_cal_tdc2_fails(self, goodtimes_for_tdc): + """Test that times are marked when TDC2 fails.""" + diagfee_tdc2_fails = xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000, 1050, 1100, 1150])), + "tdc1_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc2_cal_ctrl_stat": ( + ["epoch"], + np.array([2, 0, 2, 2]), + ), # fails at idx 1 + "tdc3_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + } + ) + + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_tdc2_fails) + + # TDC2 fails at packet 1 (MET 1050), should mark times from 1050 to 1100 + met_values = goodtimes_for_tdc.coords["met"].values + + # MET 1050 (index 1) should be culled + idx_1050 = np.where(met_values == 1050.0)[0][0] + assert np.all( + goodtimes_for_tdc["cull_flags"].values[idx_1050, :] == CullCode.BAD_TDC_CAL + ) + + def test_mark_bad_tdc_cal_tdc3_fails(self, goodtimes_for_tdc): + """Test that times are marked when TDC3 fails.""" + diagfee_tdc3_fails = xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000, 1050, 1100, 1150])), + "tdc1_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc2_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc3_cal_ctrl_stat": ( + ["epoch"], + np.array([0, 2, 2, 2]), + ), # fails at idx 0 + } + ) + + # Check that setting check_tdc_3=False results in all good values + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_tdc3_fails, check_tdc_3=False) + assert np.all(goodtimes_for_tdc["cull_flags"].values[0, :] == CullCode.GOOD) + + # Now run with check_tdc_3=True, should mark times from 1050 to 1100 + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_tdc3_fails, check_tdc_3=True) + + # TDC3 fails at packet 0 (MET 1000), should mark times from 1000 to 1050 + # MET 1000 (index 0) should be culled + assert np.all( + goodtimes_for_tdc["cull_flags"].values[0, :] == CullCode.BAD_TDC_CAL + ) # 1000 + + # MET 1050 should be good (next DIAG_FEE packet starts good window) + assert np.all( + goodtimes_for_tdc["cull_flags"].values[1, :] == CullCode.GOOD + ) # 1050 + + def test_mark_bad_tdc_cal_custom_cull_code( + self, goodtimes_for_tdc, diagfee_tdc1_fails + ): + """Test that custom cull code is used.""" + custom_cull_code = 5 + mark_bad_tdc_cal( + goodtimes_for_tdc, diagfee_tdc1_fails, cull_code=custom_cull_code + ) + + # Check that culled times use custom code + assert np.any(goodtimes_for_tdc["cull_flags"].values == custom_cull_code) + + def test_mark_bad_tdc_cal_last_packet_fails(self, goodtimes_for_tdc): + """Test behavior when the last DIAG_FEE packet has TDC failure.""" + diagfee_last_fails = xr.Dataset( + { + "shcoarse": (["epoch"], np.array([1000, 1050, 1100, 1150])), + "tdc1_cal_ctrl_stat": ( + ["epoch"], + np.array([2, 2, 2, 0]), + ), # fails at last + "tdc2_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + "tdc3_cal_ctrl_stat": (["epoch"], np.array([2, 2, 2, 2])), + } + ) + + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_last_fails) + + # TDC1 fails at last packet (MET 1150), should mark all times >= 1150 + met_values = goodtimes_for_tdc.coords["met"].values + + # METs >= 1150 should be culled + for i, met in enumerate(met_values): + if met >= 1150: + assert np.all( + goodtimes_for_tdc["cull_flags"].values[i, :] == CullCode.BAD_TDC_CAL + ) + else: + assert np.all( + goodtimes_for_tdc["cull_flags"].values[i, :] == CullCode.GOOD + ) + + def test_mark_bad_tdc_cal_empty_diagfee(self, goodtimes_for_tdc): + """Test that function handles empty DIAG_FEE data gracefully.""" + diagfee_empty = xr.Dataset( + { + "shcoarse": (["epoch"], np.array([], dtype=np.float64)), + "tdc1_cal_ctrl_stat": (["epoch"], np.array([], dtype=np.uint8)), + "tdc2_cal_ctrl_stat": (["epoch"], np.array([], dtype=np.uint8)), + "tdc3_cal_ctrl_stat": (["epoch"], np.array([], dtype=np.uint8)), + } + ) + + # Should log warning and return without error + mark_bad_tdc_cal(goodtimes_for_tdc, diagfee_empty) + + # All times should remain good + assert np.all(goodtimes_for_tdc["cull_flags"].values == CullCode.GOOD) + + class TestMarkOverflowPackets: """Test suite for mark_overflow_packets function.""" @@ -1501,8 +1892,8 @@ def test_full_packet_with_qualified_event(self, mock_goodtimes, mock_config_df): mark_overflow_packets(mock_goodtimes, l1b_de, mock_config_df) # MET ~1006 should be culled (maps to goodtimes MET 1000) - # The MET 1000 bin should have all spin bins culled - assert mock_goodtimes["cull_flags"].values[0, :].sum() == 90 + # The MET 1000 bin should have all spin bins culled with OVERFLOW flag + assert np.all(mock_goodtimes["cull_flags"].values[0, :] == CullCode.OVERFLOW) def test_full_packet_with_unqualified_event(self, mock_goodtimes, mock_config_df): """Test that full packet with unqualified final event is NOT culled.""" @@ -1906,7 +2297,7 @@ def test_multiple_sweeps(self): result = _compute_normalized_counts_per_sweep(ds, tof_ab_limit_ns=15) assert len(result["normalized_count"]) == 5 - assert result.dims["esa_sweep"] == 5 + assert result.sizes["esa_sweep"] == 5 class TestStatisticalFilter0: @@ -2026,7 +2417,9 @@ def test_fails_anomalous_sweep(self, goodtimes_for_filter): # Current sweeps have 5x the events, should be culled # Check that at least some METs are culled - assert np.any(goodtimes_for_filter["cull_flags"].values == CullCode.LOOSE) + assert np.any( + goodtimes_for_filter["cull_flags"].values == CullCode.STAT_FILTER_0 + ) def test_insufficient_pointings(self, goodtimes_for_filter): """Test that fewer than min_pointings raises ValueError.""" @@ -2112,7 +2505,7 @@ def test_partial_sweep_culling(self, goodtimes_for_filter): second_sweep_flags = goodtimes_for_filter["cull_flags"].values[9:, :] assert np.all(first_sweep_flags == CullCode.GOOD) - assert np.all(second_sweep_flags == CullCode.LOOSE) + assert np.all(second_sweep_flags == CullCode.STAT_FILTER_0) class TestIdentifyCullPattern: @@ -2363,9 +2756,11 @@ def test_sums_counts_per_eight_spin(self): # 10 packets, 2 packets per ESA step = 5 unique (esa_sweep, esa_step) combos # All in same sweep (no high-to-low transition), ESA steps 1-5 ds = self._create_test_dataset(n_packets=10, events_per_packet=10) - qualified_types = {12} - result = _compute_qualified_counts_per_sweep(ds, qualified_types) + # Create qualified mask based on coincidence type 12 + qualified_mask = np.isin(ds["coincidence_type"].values, [12]) + + result = _compute_qualified_counts_per_sweep(ds, qualified_mask) assert "qualified_count" in result.data_vars assert "esa_sweep" in result.dims @@ -2392,8 +2787,11 @@ def test_raises_without_coordinate(self): coords={"event_met": np.arange(2), "epoch": np.arange(1)}, ) + # Create qualified mask for coincidence type 12 + qualified_mask = np.isin(ds["coincidence_type"].values, [12]) + with pytest.raises(ValueError, match="must have esa_sweep coordinate"): - _compute_qualified_counts_per_sweep(ds, {12}) + _compute_qualified_counts_per_sweep(ds, qualified_mask) class TestBuildPerSweepDatasets: @@ -2437,9 +2835,14 @@ def _create_test_dataset( def test_builds_per_sweep_datasets(self): """Test that per-sweep datasets are built correctly.""" ds = self._create_test_dataset() - qualified_types = {12} - per_sweep_datasets = _build_per_sweep_datasets([ds], qualified_types) + # Add qualified mask based on coincidence type 12 directly to dataset + ds["qualified_mask"] = xr.DataArray( + np.isin(ds["coincidence_type"].values, [12]), + dims=["event_met"], + ) + + per_sweep_datasets = _build_per_sweep_datasets([ds]) # Should have per-sweep dataset for index 0 with 2D structure assert 0 in per_sweep_datasets @@ -2462,9 +2865,18 @@ def test_multiple_datasets(self): """Test with multiple datasets.""" ds1 = self._create_test_dataset(base_met=1000.0) ds2 = self._create_test_dataset(base_met=2000.0) - qualified_types = {12} - per_sweep_datasets = _build_per_sweep_datasets([ds1, ds2], qualified_types) + # Add qualified masks directly to datasets + ds1["qualified_mask"] = xr.DataArray( + np.isin(ds1["coincidence_type"].values, [12]), + dims=["event_met"], + ) + ds2["qualified_mask"] = xr.DataArray( + np.isin(ds2["coincidence_type"].values, [12]), + dims=["event_met"], + ) + + per_sweep_datasets = _build_per_sweep_datasets([ds1, ds2]) # Should have per-sweep datasets for both indices assert 0 in per_sweep_datasets @@ -2630,13 +3042,18 @@ def test_passes_normal_data(self, goodtimes_for_filter1): self._create_l1b_de_dataset(events_per_packet=10, base_met=2500.0), self._create_l1b_de_dataset(events_per_packet=10, base_met=3500.0), ] - qualified_types = {12} + + # Add qualified masks directly to datasets + for ds in l1b_de_datasets: + ds["qualified_mask"] = xr.DataArray( + np.isin(ds["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_1( goodtimes_for_filter1, l1b_de_datasets, current_index=2, - qualified_coincidence_types=qualified_types, ) # All times should still be good @@ -2688,17 +3105,23 @@ def test_fails_extreme_outlier(self, goodtimes_for_filter1): }, ) - qualified_types = {12} + # Add qualified masks directly to datasets + for ds in l1b_de_datasets: + ds["qualified_mask"] = xr.DataArray( + np.isin(ds["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_1( goodtimes_for_filter1, l1b_de_datasets, current_index=2, - qualified_coincidence_types=qualified_types, ) # At least the first MET should be marked bad (extreme outlier) - assert np.any(goodtimes_for_filter1["cull_flags"].values == CullCode.LOOSE) + assert np.any( + goodtimes_for_filter1["cull_flags"].values == CullCode.STAT_FILTER_1 + ) def test_insufficient_pointings(self, goodtimes_for_filter1): """Test that fewer than min_pointings raises ValueError.""" @@ -2707,27 +3130,37 @@ def test_insufficient_pointings(self, goodtimes_for_filter1): self._create_l1b_de_dataset(), self._create_l1b_de_dataset(), ] - qualified_types = {12} + + # Add qualified masks directly to datasets + for ds in l1b_de_datasets: + ds["qualified_mask"] = xr.DataArray( + np.isin(ds["coincidence_type"].values, [12]), + dims=["event_met"], + ) with pytest.raises(ValueError, match="At least 4 valid Pointings required"): mark_statistical_filter_1( goodtimes_for_filter1, l1b_de_datasets, current_index=1, - qualified_coincidence_types=qualified_types, ) def test_current_index_out_of_range(self, goodtimes_for_filter1): """Test that current_index out of range raises ValueError.""" - l1b_de_datasets = [self._create_l1b_de_dataset()] * 5 - qualified_types = {12} + l1b_de_datasets = [self._create_l1b_de_dataset() for _ in range(5)] + + # Add qualified masks directly to datasets + for ds in l1b_de_datasets: + ds["qualified_mask"] = xr.DataArray( + np.isin(ds["coincidence_type"].values, [12]), + dims=["event_met"], + ) with pytest.raises(ValueError, match="current_index.*out of range"): mark_statistical_filter_1( goodtimes_for_filter1, l1b_de_datasets, current_index=10, - qualified_coincidence_types=qualified_types, ) @@ -2920,18 +3353,17 @@ def _create_l1b_de_for_filter2( return xr.Dataset( { - # Event-level variables (event dimension) - "ccsds_index": (["event"], ccsds_index), - "event_met": (["event"], event_met_values), - "coincidence_type": (["event"], coincidence_type), - "nominal_bin": (["event"], nominal_bin), # Packet-level variables (epoch dimension) "ccsds_met": (["epoch"], packet_mets), "esa_step": (["epoch"], packet_esa_steps), + # Event-level variables (event dimension) + "ccsds_index": (["event_met"], ccsds_index), + "coincidence_type": (["event_met"], coincidence_type), + "nominal_bin": (["even_met"], nominal_bin), }, coords={ - "event": np.arange(n_events), "epoch": np.arange(n_packets), + "event_met": event_met_values, }, ) @@ -2940,16 +3372,19 @@ def test_no_qualified_events(self, goodtimes_for_filter2): l1b_de = self._create_l1b_de_for_filter2() # Change all events to unqualified type l1b_de["coincidence_type"] = xr.DataArray( - np.full(len(l1b_de["event"]), 4, dtype=np.uint8), - dims=["event"], + np.full(len(l1b_de["event_met"]), 4, dtype=np.uint8), + dims=["event_met"], ) - qualified_types = {12} # Type 12 is qualified, but no events have it + # Add qualified mask directly to dataset - no events match type 12 + l1b_de["qualified_mask"] = xr.DataArray( + np.isin(l1b_de["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_2( goodtimes_for_filter2, l1b_de, - qualified_types, min_events=6, max_time_delta=10.0, ) @@ -2988,25 +3423,27 @@ def test_no_clusters(self, goodtimes_for_filter2): l1b_de = xr.Dataset( { - "ccsds_index": (["event"], ccsds_index), - "event_met": (["event"], event_met_values), - "coincidence_type": (["event"], coincidence_type), - "nominal_bin": (["event"], nominal_bin), "ccsds_met": (["epoch"], packet_mets), "esa_step": (["epoch"], packet_esa_steps), + "ccsds_index": (["event_met"], ccsds_index), + "coincidence_type": (["event_met"], coincidence_type), + "nominal_bin": (["event_met"], nominal_bin), }, coords={ - "event": np.arange(n_events), "epoch": np.arange(n_packets), + "event_met": event_met_values, }, ) - qualified_types = {12} + # Add qualified mask directly to dataset + l1b_de["qualified_mask"] = xr.DataArray( + np.isin(l1b_de["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_2( goodtimes_for_filter2, l1b_de, - qualified_types, min_events=6, max_time_delta=0.2, ) @@ -3037,19 +3474,22 @@ def test_cluster_detected(self, goodtimes_for_filter2): ], dtype=np.float64, ), - dims=["event"], + dims=["event_met"], ) l1b_de["nominal_bin"] = xr.DataArray( np.array([40, 41, 42, 43, 44, 45, 10, 20, 30, 50], dtype=np.uint8), - dims=["event"], + dims=["event_met"], ) - qualified_types = {12} + # Add qualified mask directly to dataset + l1b_de["qualified_mask"] = xr.DataArray( + np.isin(l1b_de["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_2( goodtimes_for_filter2, l1b_de, - qualified_types, min_events=6, max_time_delta=0.1, bin_padding=1, @@ -3057,7 +3497,7 @@ def test_cluster_detected(self, goodtimes_for_filter2): # Bins 39-46 should be marked for MET 1000.0 (first MET) cull_flags = goodtimes_for_filter2["cull_flags"].sel(met=1000.0).values - assert np.all(cull_flags[39:47] == CullCode.LOOSE) + assert np.all(cull_flags[39:47] == CullCode.STAT_FILTER_2) # Other bins should be unmarked assert np.all(cull_flags[:39] == 0) assert np.all(cull_flags[47:] == 0) @@ -3085,19 +3525,22 @@ def test_multiple_clusters_same_packet(self, goodtimes_for_filter2): ], dtype=np.float64, ), - dims=["event"], + dims=["event_met"], ) l1b_de["nominal_bin"] = xr.DataArray( np.array([10, 11, 12, 13, 14, 15, 70, 71, 72, 73, 74, 75], dtype=np.uint8), - dims=["event"], + dims=["event_met"], ) - qualified_types = {12} + # Add qualified mask directly to dataset + l1b_de["qualified_mask"] = xr.DataArray( + np.isin(l1b_de["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_2( goodtimes_for_filter2, l1b_de, - qualified_types, min_events=6, max_time_delta=0.1, bin_padding=1, @@ -3105,9 +3548,9 @@ def test_multiple_clusters_same_packet(self, goodtimes_for_filter2): cull_flags = goodtimes_for_filter2["cull_flags"].sel(met=1000.0).values # First cluster: bins 9-16 - assert np.all(cull_flags[9:17] == CullCode.LOOSE) + assert np.all(cull_flags[9:17] == CullCode.STAT_FILTER_2) # Second cluster: bins 69-76 - assert np.all(cull_flags[69:77] == CullCode.LOOSE) + assert np.all(cull_flags[69:77] == CullCode.STAT_FILTER_2) # Middle bins should be unmarked assert np.all(cull_flags[17:69] == 0) @@ -3122,19 +3565,22 @@ def test_bin_padding_with_wrapping(self, goodtimes_for_filter2): np.array( [1000.01, 1000.02, 1000.03, 1000.04, 1000.05, 1000.06], dtype=np.float64 ), - dims=["event"], + dims=["event_met"], ) l1b_de["nominal_bin"] = xr.DataArray( np.array([0, 0, 1, 1, 2, 2], dtype=np.uint8), - dims=["event"], + dims=["event_met"], ) - qualified_types = {12} + # Add qualified mask directly to dataset + l1b_de["qualified_mask"] = xr.DataArray( + np.isin(l1b_de["coincidence_type"].values, [12]), + dims=["event_met"], + ) mark_statistical_filter_2( goodtimes_for_filter2, l1b_de, - qualified_types, min_events=6, max_time_delta=0.1, bin_padding=2, @@ -3142,9 +3588,9 @@ def test_bin_padding_with_wrapping(self, goodtimes_for_filter2): cull_flags = goodtimes_for_filter2["cull_flags"].sel(met=1000.0).values # Bins 0-4 should be marked (cluster at 0-2 + padding of 2) - assert np.all(cull_flags[0:5] == CullCode.LOOSE) + assert np.all(cull_flags[0:5] == CullCode.STAT_FILTER_2) # Bins 88-89 should also be marked due to wrapping (bin -2 and -1) - assert np.all(cull_flags[88:90] == CullCode.LOOSE) + assert np.all(cull_flags[88:90] == CullCode.STAT_FILTER_2) # Middle bins should be unmarked assert np.all(cull_flags[5:88] == 0) # Check that no cull_flags were set on any other METs @@ -3173,27 +3619,112 @@ def test_custom_parameters(self, goodtimes_for_filter2): ], dtype=np.float64, ), - dims=["event"], + dims=["event_met"], ) l1b_de["nominal_bin"] = xr.DataArray( np.array([40, 41, 42, 43, 10, 20, 30, 50, 60, 70], dtype=np.uint8), - dims=["event"], + dims=["event_met"], ) - qualified_types = {12} + # Add qualified mask directly to dataset + l1b_de["qualified_mask"] = xr.DataArray( + np.isin(l1b_de["coincidence_type"].values, [12]), + dims=["event_met"], + ) # With min_events=4, should detect cluster mark_statistical_filter_2( goodtimes_for_filter2, l1b_de, - qualified_types, min_events=4, max_time_delta=0.1, bin_padding=1, ) cull_flags = goodtimes_for_filter2["cull_flags"].sel(met=1000.0).values - assert np.all(cull_flags[39:45] == CullCode.LOOSE) + assert np.all(cull_flags[39:45] == CullCode.STAT_FILTER_2) + + def test_only_qualified_events_contribute_to_clusters(self, goodtimes_for_filter2): + """Test that only qualified events are used for cluster detection. + + This test verifies the filtering behavior by creating a scenario where: + - Unqualified events (type 4) form a cluster if incorrectly included + - Qualified events (type 12) are spread out and don't form a cluster + - No cluster should be detected because only qualified events should be used + """ + n_events = 12 + # Create base dataset structure with correct event_met dimension + event_met_values = np.array( + [ + # 6 unqualified events clustered together + 1000.01, + 1000.02, + 1000.03, + 1000.04, + 1000.05, + 1000.06, + # 6 qualified events spread out (no cluster) + 1010.0, + 1020.0, + 1030.0, + 1040.0, + 1050.0, + 1060.0, + ], + dtype=np.float64, + ) + + # First 6 events are unqualified (type 4), last 6 are qualified (type 12) + coincidence_type = np.array( + [4, 4, 4, 4, 4, 4, 12, 12, 12, 12, 12, 12], dtype=np.uint8 + ) + + # All events at similar bins (so cluster would be detected if all included) + nominal_bin = np.array( + [40, 41, 42, 43, 44, 45, 40, 41, 42, 43, 44, 45], dtype=np.uint8 + ) + + # Create ccsds_index - all events in same packet + ccsds_index = np.zeros(n_events, dtype=np.uint16) + + l1b_de = xr.Dataset( + { + "ccsds_index": (["event_met"], ccsds_index), + "coincidence_type": (["event_met"], coincidence_type), + "nominal_bin": (["event_met"], nominal_bin), + "ccsds_met": (["epoch"], np.array([1000.0])), + "esa_step": (["epoch"], np.array([1], dtype=np.uint8)), + }, + coords={ + "event_met": event_met_values, + "epoch": np.arange(1), + }, + ) + + # Add qualified mask directly to dataset - only type 12 events are qualified + qualified_mask = np.isin(l1b_de["coincidence_type"].values, [12]) + l1b_de["qualified_mask"] = xr.DataArray(qualified_mask, dims=["event_met"]) + + # Verify our test setup: 6 unqualified, 6 qualified + assert np.sum(~qualified_mask) == 6 # 6 unqualified + assert np.sum(qualified_mask) == 6 # 6 qualified + + mark_statistical_filter_2( + goodtimes_for_filter2, + l1b_de, + min_events=6, + max_time_delta=0.1, + bin_padding=1, + ) + + # No bins should be marked because: + # - The 6 unqualified events form a cluster but should be filtered out + # - The 6 qualified events are spread out and don't form a cluster + cull_flags = goodtimes_for_filter2["cull_flags"].sel(met=1000.0).values + assert np.all(cull_flags == 0), ( + "Bins were incorrectly marked - unqualified events may have been " + "included in cluster detection" + ) class TestFindCurrentPointingIndex: @@ -3260,6 +3791,7 @@ def test_loads_cal_config(self, tmp_path): patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), patch("imap_processing.hi.hi_goodtimes.mark_drf_times"), patch("imap_processing.hi.hi_goodtimes.mark_overflow_packets"), + patch("imap_processing.hi.hi_goodtimes.mark_bad_tdc_cal"), patch("imap_processing.hi.hi_goodtimes.mark_statistical_filter_0"), patch("imap_processing.hi.hi_goodtimes.mark_statistical_filter_1"), patch("imap_processing.hi.hi_goodtimes.mark_statistical_filter_2"), @@ -3271,13 +3803,14 @@ def test_loads_cal_config(self, tmp_path): [mock_l1b_de], current_index=0, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=cal_path, ) mock_cal_load.assert_called_once_with(cal_path) def test_calls_all_filters(self, tmp_path): - """Test that all 6 filters are called.""" + """Test that all 7 filters are called.""" mock_goodtimes = MagicMock() mock_goodtimes.goodtimes.get_cull_statistics.return_value = { "good_bins": 100, @@ -3296,22 +3829,24 @@ def test_calls_all_filters(self, tmp_path): "imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets" ) as mock_f1, patch("imap_processing.hi.hi_goodtimes.mark_drf_times") as mock_f2, - patch("imap_processing.hi.hi_goodtimes.mark_overflow_packets") as mock_f3, + patch("imap_processing.hi.hi_goodtimes.mark_bad_tdc_cal") as mock_f3, + patch("imap_processing.hi.hi_goodtimes.mark_overflow_packets") as mock_f4, patch( "imap_processing.hi.hi_goodtimes.mark_statistical_filter_0" - ) as mock_f4, + ) as mock_f5, patch( "imap_processing.hi.hi_goodtimes.mark_statistical_filter_1" - ) as mock_f5, + ) as mock_f6, patch( "imap_processing.hi.hi_goodtimes.mark_statistical_filter_2" - ) as mock_f6, + ) as mock_f7, ): _apply_goodtimes_filters( mock_goodtimes, [mock_l1b_de], current_index=0, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3321,6 +3856,7 @@ def test_calls_all_filters(self, tmp_path): mock_f4.assert_called_once() mock_f5.assert_called_once() mock_f6.assert_called_once() + mock_f7.assert_called_once() def test_raises_statistical_filter_0_errors(self, tmp_path): """Test that ValueError from statistical filter 0 is raised.""" @@ -3340,6 +3876,7 @@ def test_raises_statistical_filter_0_errors(self, tmp_path): ), patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), patch("imap_processing.hi.hi_goodtimes.mark_drf_times"), + patch("imap_processing.hi.hi_goodtimes.mark_bad_tdc_cal"), patch("imap_processing.hi.hi_goodtimes.mark_overflow_packets"), patch( "imap_processing.hi.hi_goodtimes.mark_statistical_filter_0", @@ -3352,6 +3889,7 @@ def test_raises_statistical_filter_0_errors(self, tmp_path): [mock_l1b_de], current_index=0, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3373,6 +3911,7 @@ def test_raises_statistical_filter_1_errors(self, tmp_path): ), patch("imap_processing.hi.hi_goodtimes.mark_incomplete_spin_sets"), patch("imap_processing.hi.hi_goodtimes.mark_drf_times"), + patch("imap_processing.hi.hi_goodtimes.mark_bad_tdc_cal"), patch("imap_processing.hi.hi_goodtimes.mark_overflow_packets"), patch("imap_processing.hi.hi_goodtimes.mark_statistical_filter_0"), patch( @@ -3386,6 +3925,7 @@ def test_raises_statistical_filter_1_errors(self, tmp_path): [mock_l1b_de], current_index=0, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3411,9 +3951,10 @@ def test_raises_value_error_when_repoint_not_complete(self, tmp_path): ValueError, match="Goodtimes cannot yet be processed for repoint00001" ): _ = hi_goodtimes( - l1b_de_datasets=[mock_de], current_repointing="repoint00001", + l1b_de_datasets=[mock_de], l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3451,9 +3992,10 @@ def test_calls_find_current_index_when_repoint_complete(self, tmp_path): patch("imap_processing.hi.hi_goodtimes._apply_goodtimes_filters"), ): hi_goodtimes( - l1b_de_datasets=mock_datasets, current_repointing="repoint00004", + l1b_de_datasets=mock_datasets, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3493,9 +4035,10 @@ def test_marks_all_bad_when_incomplete_de_set(self, tmp_path): ), ): hi_goodtimes( - l1b_de_datasets=mock_datasets, current_repointing="repoint00001", + l1b_de_datasets=mock_datasets, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3537,9 +4080,10 @@ def test_calls_apply_filters_when_full_de_set(self, tmp_path): ) as mock_apply, ): hi_goodtimes( - l1b_de_datasets=mock_datasets, current_repointing="repoint00004", + l1b_de_datasets=mock_datasets, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) @@ -3579,9 +4123,10 @@ def test_returns_datasets(self, tmp_path): patch("imap_processing.hi.hi_goodtimes._apply_goodtimes_filters"), ): result = hi_goodtimes( - l1b_de_datasets=mock_datasets, current_repointing="repoint00004", + l1b_de_datasets=mock_datasets, l1b_hk=mock_hk, + l1a_diagfee=MagicMock(), cal_product_config_path=tmp_path / "cal.csv", ) diff --git a/imap_processing/tests/hi/test_hi_l1c.py b/imap_processing/tests/hi/test_hi_l1c.py index f0fad87072..8f7b2f2b8a 100644 --- a/imap_processing/tests/hi/test_hi_l1c.py +++ b/imap_processing/tests/hi/test_hi_l1c.py @@ -1,7 +1,6 @@ """Test coverage for imap_processing.hi.l1c.hi_l1c.py""" import io -from collections import namedtuple from unittest import mock from unittest.mock import MagicMock @@ -10,19 +9,43 @@ import pytest import xarray as xr -import imap_processing.hi.utils from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import load_cdf, write_cdf -from imap_processing.hi import hi_l1c +from imap_processing.hi import hi_l1c, utils from imap_processing.hi.utils import HIAPID, HiConstants from imap_processing.spice.time import met_to_ttj2000ns, ttj2000ns_to_et +@pytest.fixture(scope="module") +def hi_l1b_de_dataset(hi_l1_test_data_path): + """Load the Hi L1B DE test dataset.""" + l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf" + return load_cdf(l1b_de_path) + + +@pytest.fixture(scope="module") +def hi_goodtimes_dataset(hi_l1_test_data_path): + """Load the Hi goodtimes test dataset.""" + goodtimes_path = ( + hi_l1_test_data_path / "imap_hi_l1b_45sensor-goodtimes_20250415_v999.cdf" + ) + return load_cdf(goodtimes_path) + + @mock.patch("imap_processing.hi.hi_l1c.generate_pset_dataset") -def test_hi_l1c(mock_generate_pset_dataset, hi_test_cal_prod_config_path): +def test_hi_l1c( + mock_generate_pset_dataset, + hi_test_cal_prod_config_path, + hi_test_background_config_path, +): """Test coverage for hi_l1c function""" mock_generate_pset_dataset.return_value = xr.Dataset() - pset = hi_l1c.hi_l1c(xr.Dataset(), hi_test_cal_prod_config_path)[0] + pset = hi_l1c.hi_l1c( + xr.Dataset(), + hi_test_cal_prod_config_path, + xr.Dataset(), + hi_test_background_config_path, + )[0] # Empty attributes, global values get added in post-processing assert pset.attrs == {} @@ -30,16 +53,17 @@ def test_hi_l1c(mock_generate_pset_dataset, hi_test_cal_prod_config_path): @pytest.mark.external_kernel @pytest.mark.external_test_data def test_generate_pset_dataset( - hi_l1_test_data_path, + hi_l1b_de_dataset, + hi_goodtimes_dataset, hi_test_cal_prod_config_path, + hi_test_background_config_path, use_fake_spin_data_for_time, use_fake_repoint_data_for_time, imap_ena_sim_metakernel, ): """Test coverage for generate_pset_dataset function""" use_fake_spin_data_for_time(482372987.999) - l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf" - l1b_dataset = load_cdf(l1b_de_path) + l1b_dataset = hi_l1b_de_dataset l1b_met = l1b_dataset["ccsds_met"].values[0] # Set repoint start and end times. seconds_per_day = 24 * 60 * 60 @@ -47,8 +71,13 @@ def test_generate_pset_dataset( np.asarray([l1b_met - 15 * 60, l1b_met + seconds_per_day]), np.asarray([l1b_met, l1b_met + seconds_per_day + 1]), ) + goodtimes = hi_goodtimes_dataset + l1c_dataset = hi_l1c.generate_pset_dataset( - l1b_dataset, hi_test_cal_prod_config_path + l1b_dataset, + hi_test_cal_prod_config_path, + goodtimes, + hi_test_background_config_path, ) assert l1c_dataset.epoch.data[0] == l1b_dataset.epoch.data[0].astype(np.int64) @@ -81,6 +110,7 @@ def test_generate_pset_dataset_uses_midpoint_time( mock_pset_exposure, mock_pset_backgrounds, hi_test_cal_prod_config_path, + hi_test_background_config_path, ): """Test that generate_pset_dataset uses midpoint ET for pset_geometry.""" # Create a mock L1B dataset @@ -113,11 +143,21 @@ def test_generate_pset_dataset_uses_midpoint_time( # Mock the return values for the sub-functions mock_pset_geometry.return_value = {} mock_pset_counts.return_value = {} - mock_pset_exposure.return_value = {} + # pset_exposure must return exposure_times for pset_backgrounds to use + mock_exposure_times = xr.DataArray( + np.ones((1, n_energy_steps, 3600), dtype=np.float32), + dims=["epoch", "esa_energy_step", "spin_angle_bin"], + ) + mock_pset_exposure.return_value = {"exposure_times": mock_exposure_times} mock_pset_backgrounds.return_value = {} # Call generate_pset_dataset - _ = hi_l1c.generate_pset_dataset(mock_l1b_dataset, hi_test_cal_prod_config_path) + _ = hi_l1c.generate_pset_dataset( + mock_l1b_dataset, + hi_test_cal_prod_config_path, + xr.Dataset(), + hi_test_background_config_path, + ) # Calculate expected midpoint ET # The PSET dataset should have epoch and epoch_delta based on pointing times @@ -219,22 +259,24 @@ def test_pset_geometry(mock_frame_transform, mock_geom_frame_transform, sensor_s @mock.patch("imap_processing.hi.hi_l1c.get_pointing_times", return_value=(100, 200)) def test_pset_counts( mock_pointing_times, - hi_l1_test_data_path, + hi_l1b_de_dataset, + hi_goodtimes_dataset, hi_test_cal_prod_config_path, + hi_test_background_config_path, ): """Test coverage for pset_counts function.""" - l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf" - l1b_dataset = load_cdf(l1b_de_path) - cal_config_df = imap_processing.hi.utils.CalibrationProductConfig.from_csv( + cal_config_df = utils.CalibrationProductConfig.from_csv( hi_test_cal_prod_config_path ) empty_pset = hi_l1c.empty_pset_dataset( 100, - l1b_dataset.esa_energy_step, + hi_l1b_de_dataset.esa_energy_step, cal_config_df.cal_prod_config.calibration_product_numbers, HIAPID.H90_SCI_DE.sensor, ) - counts_var = hi_l1c.pset_counts(empty_pset.coords, cal_config_df, l1b_dataset) + counts_var = hi_l1c.pset_counts( + empty_pset.coords, cal_config_df, hi_l1b_de_dataset, hi_goodtimes_dataset + ) assert "counts" in counts_var @@ -242,16 +284,17 @@ def test_pset_counts( @mock.patch("imap_processing.hi.hi_l1c.get_pointing_times", return_value=(100, 200)) def test_pset_counts_empty_l1b( mock_pointing_times, - hi_l1_test_data_path, + hi_l1b_de_dataset, + hi_goodtimes_dataset, hi_test_cal_prod_config_path, + hi_test_background_config_path, ): """Test coverage for pset_counts function when the input L1b contains no counts.""" - l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf" - l1b_dataset = load_cdf(l1b_de_path) + # Make a copy and modify it - # remove all but one event and set its trigger_id to zero - l1b_dataset = l1b_dataset.isel(event_met=[0]) + l1b_dataset = hi_l1b_de_dataset.isel(event_met=[0]).copy(deep=True) l1b_dataset["trigger_id"].data[0] = 0 - cal_config_df = imap_processing.hi.utils.CalibrationProductConfig.from_csv( + cal_config_df = utils.CalibrationProductConfig.from_csv( hi_test_cal_prod_config_path ) empty_pset = hi_l1c.empty_pset_dataset( @@ -260,7 +303,9 @@ def test_pset_counts_empty_l1b( cal_config_df.cal_prod_config.calibration_product_numbers, HIAPID.H90_SCI_DE.sensor, ) - counts_var = hi_l1c.pset_counts(empty_pset.coords, cal_config_df, l1b_dataset) + counts_var = hi_l1c.pset_counts( + empty_pset.coords, cal_config_df, l1b_dataset, hi_goodtimes_dataset + ) assert counts_var["counts"].data.sum() == 0 @@ -274,21 +319,13 @@ def test_get_tof_window_mask(): "tof_bc1": -13, "tof_c1c2": -14, } - Row = namedtuple( - "Row", - [ - "Index", - "tof_ab_low", - "tof_ab_high", - "tof_ac1_low", - "tof_ac1_high", - "tof_bc1_low", - "tof_bc1_high", - "tof_c1c2_low", - "tof_c1c2_high", - ], - ) - prod_config_row = Row((1, 0), 0, 1, -1, 2, 1, 5, 4, 6) + # Use dict-based tof_windows instead of named tuple + tof_windows = { + "tof_ab": (0, 1), + "tof_ac1": (-1, 2), + "tof_bc1": (1, 5), + "tof_c1c2": (4, 6), + } synth_df = xr.Dataset( coords={ "event_met": xr.DataArray( @@ -323,7 +360,7 @@ def test_get_tof_window_mask(): }, ) expected_mask = np.array([True, False, False, False, False, False, True]) - window_mask = hi_l1c.get_tof_window_mask(synth_df, prod_config_row, fill_vals) + window_mask = utils.get_tof_window_mask(synth_df, tof_windows, fill_vals) np.testing.assert_array_equal(expected_mask, window_mask) @@ -356,7 +393,7 @@ def test_empty_pset_dataset_arbitrary_cal_prod_numbers(use_fake_repoint_data_for @pytest.mark.external_test_data def test_pset_counts_arbitrary_cal_prod_numbers( - hi_l1_test_data_path, use_fake_repoint_data_for_time + hi_l1b_de_dataset, hi_goodtimes_dataset, use_fake_repoint_data_for_time ): """Test pset_counts with non-sequential calibration product numbers.""" # Create a test calibration product config with non-sequential numbers @@ -368,12 +405,7 @@ def test_pset_counts_arbitrary_cal_prod_numbers( 10,2,0.00085,BC1C2,0,1023,-1023,1023,-1023,1023,0,1023 """ - l1b_de_path = hi_l1_test_data_path / "imap_hi_l1b_45sensor-de_20250415_v999.cdf" - l1b_dataset = load_cdf(l1b_de_path) - - cal_config_df = imap_processing.hi.utils.CalibrationProductConfig.from_csv( - io.StringIO(csv_content) - ) + cal_config_df = utils.CalibrationProductConfig.from_csv(io.StringIO(csv_content)) # Create PSET with non-sequential calibration product numbers l1b_met = 482373065 @@ -383,7 +415,7 @@ def test_pset_counts_arbitrary_cal_prod_numbers( empty_pset = hi_l1c.empty_pset_dataset( l1b_met, - l1b_dataset.esa_energy_step, + hi_l1b_de_dataset.esa_energy_step, cal_config_df.cal_prod_config.calibration_product_numbers, HIAPID.H90_SCI_DE.sensor, ) @@ -395,7 +427,9 @@ def test_pset_counts_arbitrary_cal_prod_numbers( with mock.patch( "imap_processing.hi.hi_l1c.get_pointing_times", return_value=(100, 200) ): - counts_var = hi_l1c.pset_counts(empty_pset.coords, cal_config_df, l1b_dataset) + counts_var = hi_l1c.pset_counts( + empty_pset.coords, cal_config_df, hi_l1b_de_dataset, hi_goodtimes_dataset + ) # Verify counts array has correct shape based on coordinates assert "counts" in counts_var @@ -410,57 +444,365 @@ def test_pset_counts_arbitrary_cal_prod_numbers( assert counts_var["counts"].data.shape == expected_shape # Check that total number of expected counts is correct # ABC1C2 is coincidence type 15 - esa_1_2_mask = (l1b_dataset["esa_step"][l1b_dataset["ccsds_index"]] < 3).values - coincidence_15_mask = (l1b_dataset["coincidence_type"] == 15).values + esa_1_2_mask = ( + hi_l1b_de_dataset["esa_step"][hi_l1b_de_dataset["ccsds_index"]] < 3 + ).values + coincidence_15_mask = (hi_l1b_de_dataset["coincidence_type"] == 15).values np.testing.assert_equal( np.sum(counts_var["counts"].data[:, :, 0]), np.sum(coincidence_15_mask & esa_1_2_mask), ) # BC1C2 is coincidence type 7 - coincidence_7_mask = (l1b_dataset["coincidence_type"] == 7).values + coincidence_7_mask = (hi_l1b_de_dataset["coincidence_type"] == 7).values np.testing.assert_equal( np.sum(counts_var["counts"].data[:, :, 1]), np.sum(coincidence_7_mask & esa_1_2_mask), ) -def test_pset_backgrounds(): +@mock.patch("imap_processing.hi.hi_l1c.get_pointing_times", return_value=(100, 200)) +@mock.patch("imap_processing.hi.hi_l1c.iter_qualified_events_by_config") +def test_pset_counts_goodtimes_filtering( + mock_iter_qualified, + mock_pointing_times, +): + """Test that pset_counts properly filters events based on goodtimes.""" + # Create 10 events: METs 100-109, nominal_bins 0-9, all at spin_phase=0.5 + # (spin_phase 0.5 -> spin_angle_bin 1800) + n_events = 10 + event_mets = np.arange(100.0, 100.0 + n_events) + nominal_bins = np.arange(n_events, dtype=np.uint8) + + l1b_dataset = xr.Dataset( + coords={ + "epoch": xr.DataArray(np.arange(2), dims=["epoch"]), + "event_met": xr.DataArray(event_mets, dims=["event_met"]), + }, + data_vars={ + "trigger_id": xr.DataArray( + np.ones(n_events, dtype=np.uint16), + dims=["event_met"], + attrs={"FILLVAL": 65535}, + ), + "nominal_bin": xr.DataArray(nominal_bins, dims=["event_met"]), + "spin_phase": xr.DataArray(np.full(n_events, 0.5), dims=["event_met"]), + "ccsds_index": xr.DataArray( + np.zeros(n_events, dtype=np.int32), dims=["event_met"] + ), + "esa_energy_step": xr.DataArray( + np.array([1, 1], dtype=np.uint8), + dims=["epoch"], + attrs={"FILLVAL": 255}, + ), + }, + attrs={"Logical_source": "imap_hi_l1b_90sensor-de"}, + ) + + # Goodtimes: METs 100-104 good, METs 105-109 bad + goodtimes_ds = xr.Dataset( + { + "cull_flags": xr.DataArray( + np.zeros((2, 90), dtype=np.uint8), + dims=["met", "spin_bin"], + ), + }, + coords={"met": [100.0, 105.0], "spin_bin": np.arange(90)}, + ) + goodtimes_ds["cull_flags"].values[1, :] = 1 # All bins bad for MET >= 105 + + # Create empty pset with single ESA step and single calibration product + empty_pset = hi_l1c.empty_pset_dataset( + 100, + l1b_dataset.esa_energy_step, + np.array([0]), + HIAPID.H90_SCI_DE.sensor, + ) + + # Mock iter_qualified_events_by_config to mark all events as qualified + # and return a single (esa_energy, config_row, mask) tuple + mock_config_row = MagicMock() + mock_config_row.Index = (0, 1) # (calibration_prod, esa_energy_step) + + def mock_iter(de_ds, config_df, esa_energy_steps): + n_remaining = len(de_ds["event_met"]) + yield 1, mock_config_row, np.ones(n_remaining, dtype=bool) + + mock_iter_qualified.side_effect = mock_iter + + # Use MagicMock for cal_config since it's not used with our mock + mock_cal_config = MagicMock() + + counts_var = hi_l1c.pset_counts( + empty_pset.coords, mock_cal_config, l1b_dataset, goodtimes_ds + ) + + # Only 5 events (METs 100-104) should pass goodtimes filtering + # All 5 events have spin_phase=0.5 -> spin_angle_bin 1800 + total_counts = counts_var["counts"].data.sum() + assert total_counts == 5, f"Expected 5 counts, got {total_counts}" + # Verify all counts are in the expected spin bin (1800) + assert counts_var["counts"].data[0, 0, 0, 1800] == 5 + + +@pytest.mark.external_test_data +def test_pset_backgrounds( + hi_test_background_config_path, + hi_test_cal_prod_config_path, + hi_l1b_de_dataset, + hi_goodtimes_dataset, + use_fake_spin_data_for_time, + use_fake_repoint_data_for_time, +): """Test coverage for pset_backgrounds function.""" - # Create some fake coordinates to use + # Setup required SPICE data + use_fake_spin_data_for_time(482372987.999) + l1b_met = hi_l1b_de_dataset["ccsds_met"].values[0] + seconds_per_day = 24 * 60 * 60 + use_fake_repoint_data_for_time( + np.asarray([l1b_met - 15 * 60, l1b_met + seconds_per_day]), + np.asarray([l1b_met, l1b_met + seconds_per_day + 1]), + ) + + # Load the background config + background_df = utils.BackgroundConfig.from_csv(hi_test_background_config_path) + + # Create empty pset dataset to get coordinates + cal_config_df = utils.CalibrationProductConfig.from_csv( + hi_test_cal_prod_config_path + ) + empty_pset = hi_l1c.empty_pset_dataset( + l1b_met, + hi_l1b_de_dataset.esa_energy_step, + cal_config_df.cal_prod_config.calibration_product_numbers, + HIAPID.H90_SCI_DE.sensor, + ) + + # Create exposure_times for the test + exposure_times_data = np.full( + ( + len(empty_pset.coords["epoch"]), + len(empty_pset.coords["esa_energy_step"]), + len(empty_pset.coords["spin_angle_bin"]), + ), + 1.0, + dtype=np.float32, + ) + exposure_times = xr.DataArray( + exposure_times_data, + dims=["epoch", "esa_energy_step", "spin_angle_bin"], + coords={ + "epoch": empty_pset.coords["epoch"], + "esa_energy_step": empty_pset.coords["esa_energy_step"], + "spin_angle_bin": empty_pset.coords["spin_angle_bin"], + }, + ) + + # Call pset_backgrounds with the new signature + backgrounds_vars = hi_l1c.pset_backgrounds( + empty_pset.coords, + background_df, + hi_l1b_de_dataset, + hi_goodtimes_dataset, + exposure_times, + ) + + assert "background_rates" in backgrounds_vars + assert backgrounds_vars["background_rates"].data.shape == ( + len(empty_pset.coords["epoch"]), + len(empty_pset.coords["esa_energy_step"]), + len(empty_pset.coords["calibration_prod"]), + len(empty_pset.coords["spin_angle_bin"]), + ) + + assert "background_rates_uncertainty" in backgrounds_vars + assert backgrounds_vars["background_rates_uncertainty"].data.shape == ( + len(empty_pset.coords["epoch"]), + len(empty_pset.coords["esa_energy_step"]), + len(empty_pset.coords["calibration_prod"]), + len(empty_pset.coords["spin_angle_bin"]), + ) + + +@mock.patch("imap_processing.hi.hi_l1c.good_time_and_phase_mask") +def test_compute_background_counts_missing_cal_prod_raises_error( + mock_good_time_and_phase_mask, + hi_test_background_config_path, +): + """Test _compute_background_counts raises ValueError with invalid bkgnd config.""" + # Mock good_time_and_phase_mask to return all True + mock_good_time_and_phase_mask.side_effect = lambda a, b, c: np.ones( + a.shape, dtype=bool + ) + # Load the background config (has cal prods 0 and 1) + background_df = utils.BackgroundConfig.from_csv(hi_test_background_config_path) + + # Create minimal pset_coords with a calibration product (999) that's + # NOT in the background config + missing_cal_prod = 999 + pset_coords = { + "epoch": xr.DataArray(np.array([0], dtype=np.int64), dims=["epoch"]), + "calibration_prod": xr.DataArray( + np.array([0, 1, missing_cal_prod], dtype=np.int32), + dims=["calibration_prod"], + ), + } + + hi_l1b_de_dataset = xr.Dataset( + { + "coincidence_type": xr.DataArray( + np.array([15], dtype=np.uint8), dims=["event_met"] + ), + "trigger_id": xr.DataArray( + np.array([0], dtype=np.float64), + dims=["event_met"], + attrs={"FILLVAL": 65535}, + ), + "nominal_bin": xr.DataArray( + np.array([0], dtype=np.uint8), dims=["event_met"] + ), + "tof_ab": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + "tof_ac1": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + "tof_bc1": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + "tof_c1c2": xr.DataArray( + np.array([50], dtype=np.float32), dims=["event_met"] + ), + }, + coords={ + "epoch": xr.DataArray(np.array([0], dtype=np.int64), dims=["epoch"]), + "event_met": xr.DataArray( + np.array([0], dtype=np.float64), dims=["event_met"] + ), + }, + ) + + # Verify that calling _compute_background_counts raises ValueError + # with expected message + with pytest.raises( + ValueError, + match=f"Calibration product {missing_cal_prod} not found " + f"in background configuration", + ): + hi_l1c._compute_background_counts( + pset_coords, + background_df, + hi_l1b_de_dataset, + xr.Dataset(), + ) + + +@mock.patch("imap_processing.hi.hi_l1c._compute_background_counts") +def test_pset_backgrounds_cal_prod_mismatch_raises_error( + mock_compute_background_counts, +): + """Test pset_backgrounds raises ValueError when cal prods don't match. + + This tests the validation at lines 634-639 of hi_l1c.py that checks + if calibration products in pset_coords match those in background_config_df. + """ + # Create pset_coords with calibration products [0, 1] n_epoch = 1 - n_energy = 9 - n_cal_prod = 2 + n_energy = 2 n_spin_bins = 3600 pset_coords = { - "epoch": xr.DataArray(np.arange(n_epoch)), - "esa_energy_step": xr.DataArray(np.arange(n_energy) + 1), - "calibration_prod": xr.DataArray(np.arange(n_cal_prod)), - "spin_angle_bin": xr.DataArray(np.arange(n_spin_bins)), + "epoch": xr.DataArray(np.array([0], dtype=np.int64), dims=["epoch"]), + "esa_energy_step": xr.DataArray( + np.arange(n_energy) + 1, dims=["esa_energy_step"] + ), + "calibration_prod": xr.DataArray( + np.array([0, 1], dtype=np.int64), + dims=["calibration_prod"], + ), + "spin_angle_bin": xr.DataArray(np.arange(n_spin_bins), dims=["spin_angle_bin"]), } - backgrounds_vars = hi_l1c.pset_backgrounds(pset_coords) - assert "background_rates" in backgrounds_vars - np.testing.assert_array_equal( - backgrounds_vars["background_rates"].data, - np.zeros((n_epoch, n_energy, n_cal_prod, n_spin_bins)), + + # Create a background config DataFrame with DIFFERENT calibration products [5, 6] + # This simulates a mismatch between pset_coords and background_config_df + background_config_data = { + "coincidence_type_list": ["ABC1C2", "ABC1C2"], + "coincidence_type_values": [[15], [15]], + "tof_ab_low": [0, 0], + "tof_ab_high": [100, 100], + "tof_ac1_low": [0, 0], + "tof_ac1_high": [100, 100], + "tof_bc1_low": [0, 0], + "tof_bc1_high": [100, 100], + "tof_c1c2_low": [0, 0], + "tof_c1c2_high": [100, 100], + "scaling_factor": [1.0, 1.0], + "uncertainty": [0.1, 0.1], + } + # Use calibration products [5, 6] which don't match pset_coords [0, 1] + mismatched_cal_prods = [5, 6] + background_indices = [0, 0] + multi_index = pd.MultiIndex.from_arrays( + [mismatched_cal_prods, background_indices], + names=["calibration_prod", "background_index"], ) - assert "background_rates_uncertainty" in backgrounds_vars - np.testing.assert_array_equal( - backgrounds_vars["background_rates_uncertainty"].data, - np.ones((n_epoch, n_energy, n_cal_prod, n_spin_bins)), + background_df = pd.DataFrame(background_config_data, index=multi_index) + + # Create mock exposure_times + exposure_times = xr.DataArray( + np.ones((n_epoch, n_energy, n_spin_bins), dtype=np.float32), + dims=["epoch", "esa_energy_step", "spin_angle_bin"], ) + # Mock _compute_background_counts to return a DataArray with the mismatched + # calibration products (simulating what would happen if the earlier check + # didn't catch the mismatch) + mock_background_counts = xr.DataArray( + np.zeros((n_epoch, len(mismatched_cal_prods), 1)), + dims=["epoch", "calibration_prod", "background_index"], + coords={ + "epoch": pset_coords["epoch"], + "calibration_prod": mismatched_cal_prods, + "background_index": [0], + }, + ) + mock_compute_background_counts.return_value = mock_background_counts + + # Create minimal l1b dataset and goodtimes (not used due to mock) + l1b_de_dataset = xr.Dataset() + goodtimes_ds = xr.Dataset() + + # Verify that pset_backgrounds raises ValueError with expected message + with pytest.raises( + ValueError, + match="Calibration products in pset_coords and " + "background_config_df do not match", + ): + hi_l1c.pset_backgrounds( + pset_coords, + background_df, + l1b_de_dataset, + goodtimes_ds, + exposure_times, + ) + +@mock.patch("imap_processing.hi.hi_l1c.good_time_and_phase_mask") @mock.patch("imap_processing.hi.hi_l1c.get_pointing_times", return_value=(100, 200)) @mock.patch("imap_processing.hi.hi_l1c.get_spin_data", return_value=None) -@mock.patch("imap_processing.hi.hi_l1c.get_instrument_spin_phase") +@mock.patch( + "imap_processing.hi.hi_l1c.get_spacecraft_to_instrument_spin_phase_offset", + return_value=0.0, +) +@mock.patch("imap_processing.hi.hi_l1c.get_spacecraft_spin_phase") @mock.patch("imap_processing.hi.hi_l1c.get_de_clock_ticks_for_esa_step") @mock.patch("imap_processing.hi.hi_l1c.find_last_de_packet_data") def test_pset_exposure( mock_find_last_de_packet_data, mock_de_clock_ticks, - mock_spin_phase, + mock_sc_spin_phase, + mock_phase_offset, mock_spin_data, mock_pointing_times, + mock_good_time_and_phase_mask, ): """Test coverage for pset_exposure function""" l1b_energy_steps = xr.DataArray( @@ -484,7 +826,7 @@ def test_pset_exposure( # deterministic histogram values. # ESA step 1 should have repeating values of 3, 1. # ESA step 2 should have repeating values of 6, 2 - mock_spin_phase.return_value = np.concat( + mock_sc_spin_phase.return_value = np.concat( [hi_l1c.SPIN_PHASE_BIN_CENTERS, hi_l1c.SPIN_PHASE_BIN_CENTERS[::2]] ) mock_de_clock_ticks.return_value = ( @@ -497,8 +839,13 @@ def test_pset_exposure( l1b_dataset = MagicMock() l1b_dataset.attrs = {"Logical_source": "90sensor"} + # Mock goodtime to return all true + mock_good_time_and_phase_mask.side_effect = lambda x, y, z: np.ones( + x.shape, dtype=bool + ) + # All the setup is done, call the pset_exposure function - exposure_dict = hi_l1c.pset_exposure(empty_pset.coords, l1b_dataset) + exposure_dict = hi_l1c.pset_exposure(empty_pset.coords, l1b_dataset, xr.Dataset()) # Based on the spin phase and clock_tick mocks, the expected clock ticks are: # - Repeated values of 3, 1 for the first half of the spin bins @@ -518,6 +865,79 @@ def test_pset_exposure( ) +@mock.patch("imap_processing.hi.hi_l1c.get_pointing_times", return_value=(100, 200)) +@mock.patch("imap_processing.hi.hi_l1c.get_spin_data", return_value=None) +@mock.patch( + "imap_processing.hi.hi_l1c.get_spacecraft_to_instrument_spin_phase_offset", + return_value=0.0, +) +@mock.patch("imap_processing.hi.hi_l1c.get_spacecraft_spin_phase") +@mock.patch("imap_processing.hi.hi_l1c.get_de_clock_ticks_for_esa_step") +@mock.patch("imap_processing.hi.hi_l1c.find_last_de_packet_data") +def test_pset_exposure_goodtimes_filtering( + mock_find_last_de_packet_data, + mock_de_clock_ticks, + mock_sc_spin_phase, + mock_phase_offset, + mock_spin_data, + mock_pointing_times, +): + """Test that pset_exposure properly filters clock ticks based on goodtimes.""" + l1b_energy_steps = xr.DataArray( + np.arange(1) + 1, # Single ESA step for simplicity + attrs={"FILLVAL": 255}, + ) + empty_pset = hi_l1c.empty_pset_dataset( + 100, l1b_energy_steps, np.array([0]), HIAPID.H90_SCI_DE.sensor + ) + + # Mock find_last_de_packet_data to return a single ESA step + mock_find_last_de_packet_data.return_value = xr.Dataset( + coords={"epoch": xr.DataArray(np.arange(1), dims=["epoch"])}, + data_vars={ + "ccsds_met": xr.DataArray(np.array([150.0]), dims=["epoch"]), + "esa_energy_step": xr.DataArray(np.array([1]), dims=["epoch"]), + }, + ) + + # Create 10 clock ticks at METs 100-109 with uniform spin phases + n_ticks = 10 + clock_tick_mets = np.arange(100.0, 100.0 + n_ticks) + mock_de_clock_ticks.return_value = (clock_tick_mets, np.ones(n_ticks)) + + # Mock spacecraft spin phase - each tick maps to a different spin bin + # Spin phases 0.0, 0.1, 0.2, ... -> nominal_bins 0, 9, 18, ... + spin_phases = np.arange(n_ticks) / n_ticks + mock_sc_spin_phase.return_value = spin_phases + + # Create a goodtimes dataset that marks half the clock ticks as bad + # METs 100-104 are good (cull_flags=0), METs 105-109 are bad (cull_flags=1) + goodtimes_ds = xr.Dataset( + { + "cull_flags": xr.DataArray( + np.zeros((2, 90), dtype=np.uint8), + dims=["met", "spin_bin"], + ), + }, + coords={"met": [100.0, 105.0], "spin_bin": np.arange(90)}, + ) + # Mark all spin bins as bad for METs >= 105 + goodtimes_ds["cull_flags"].values[1, :] = 1 + + # Mock l1b_dataset + l1b_dataset = MagicMock() + l1b_dataset.attrs = {"Logical_source": "90sensor"} + + # Call pset_exposure with the goodtimes dataset + exposure_dict = hi_l1c.pset_exposure(empty_pset.coords, l1b_dataset, goodtimes_ds) + + # Only the first 5 clock ticks (METs 100-104) should contribute + # Their spin phases are 0.0, 0.1, 0.2, 0.3, 0.4 -> spin_angle_bins 0, 360, 720, ... + total_exposure_ticks = exposure_dict["exposure_times"].data.sum() + expected_ticks = 5.0 * HiConstants.DE_CLOCK_TICK_S + np.testing.assert_allclose(total_exposure_ticks, expected_ticks, rtol=0.01) + + def test_find_second_de_packet_data(): """Test coverage for find_second_de_packet_data function""" # Create a test l1b_dataset @@ -620,3 +1040,72 @@ def test_get_de_clock_ticks_for_esa_step_exceptions(fake_spin_df): ValueError, match="Error determining start/end time for exposure time" ): hi_l1c.get_de_clock_ticks_for_esa_step(bad_ccsds_met, fake_spin_df) + + +class TestGoodTimeAndPhaseMask: + """Tests for good_time_and_phase_mask function.""" + + def test_filters_bad_times_with_nominal_bins(self): + """Events in bad times are filtered out using nominal_bins.""" + # Create mock goodtimes with some bad times + gt_ds = xr.Dataset( + { + "cull_flags": xr.DataArray( + np.zeros((3, 90), dtype=np.uint8), + dims=["met", "spin_bin"], + ) + }, + coords={"met": [100.0, 200.0, 300.0], "spin_bin": np.arange(90)}, + ) + # Mark spin_bin 10 as bad at MET 200 + gt_ds["cull_flags"].values[1, 10] = 1 + + mets = np.array([150.0, 250.0, 250.0]) + nominal_bins = np.array([10, 10, 20]) + + mask = hi_l1c.good_time_and_phase_mask(mets, nominal_bins, gt_ds) + # Event at 150 maps to MET index 0, bin 10 → good (cull_flags[0,10]=0) + # Event at 250 maps to MET index 1, bin 10 → bad (cull_flags[1,10]=1) + # Event at 250 maps to MET index 1, bin 20 → good (cull_flags[1,20]=0) + expected = np.array([True, False, True]) + np.testing.assert_array_equal(mask, expected) + + def test_met_before_goodtimes_range(self): + """Events before goodtimes range are clipped to first interval.""" + gt_ds = xr.Dataset( + { + "cull_flags": xr.DataArray( + np.zeros((2, 90), dtype=np.uint8), + dims=["met", "spin_bin"], + ) + }, + coords={"met": [100.0, 200.0], "spin_bin": np.arange(90)}, + ) + # Mark spin_bin 0 as bad at first MET + gt_ds["cull_flags"].values[0, 0] = 1 + + # Event at MET 50 (before goodtimes range) should use index 0 + mets = np.array([50.0]) + nominal_bins = np.array([0]) + + mask = hi_l1c.good_time_and_phase_mask(mets, nominal_bins, gt_ds) + # Clipped to index 0, bin 0 is bad + assert not mask[0] + + def test_all_bins_bad_for_interval(self): + """When all bins are bad for an interval, all events are filtered.""" + gt_ds = xr.Dataset( + { + "cull_flags": xr.DataArray( + np.ones((1, 90), dtype=np.uint8), # All bad + dims=["met", "spin_bin"], + ) + }, + coords={"met": [100.0], "spin_bin": np.arange(90)}, + ) + + mets = np.array([100.0, 150.0, 200.0]) + nominal_bins = np.array([0, 45, 89]) + + mask = hi_l1c.good_time_and_phase_mask(mets, nominal_bins, gt_ds) + assert not np.any(mask) diff --git a/imap_processing/tests/hi/test_hi_l2.py b/imap_processing/tests/hi/test_hi_l2.py index 0d0ac24276..81db20e041 100644 --- a/imap_processing/tests/hi/test_hi_l2.py +++ b/imap_processing/tests/hi/test_hi_l2.py @@ -1426,3 +1426,57 @@ def test_combine_maps_invalid_length(): with pytest.raises(ValueError, match="Expected 1 or 2 sky maps"): combine_maps({"a": sky_map, "b": sky_map, "c": sky_map}) + + +def test_combine_maps_handles_nan_intensity(mock_sky_map_for_combine): + """Test that combine_maps handles NaN values in ena_intensity correctly. + + When one map has NaN values and the other has valid data, the combined + result should use the valid data rather than propagating NaN. + This tests the fix using .fillna(0) in the weighted sum. + """ + ram_map = mock_sky_map_for_combine() + anti_map = mock_sky_map_for_combine(intensity_offset=20) + + # Set some positions in ram to NaN for ena_intensity + # Use a slice to set NaN at specific positions + ram_intensity = ram_map.data_1d["ena_intensity"].values.copy() + ram_intensity[0, 0, 0, 0] = np.nan # Set first position to NaN + ram_intensity[0, 1, 2, 1] = np.nan # Set another position to NaN + ram_map.data_1d["ena_intensity"] = xr.DataArray( + ram_intensity, dims=ram_map.data_1d["ena_intensity"].dims + ) + + # Anti map has valid values at all positions (intensity = 70 from offset=20) + sky_maps = {"ram": ram_map, "anti": anti_map} + result = combine_maps(sky_maps) + + # At positions where ram had NaN, the result should use anti's value + # Anti has intensity=70, uncertainty=5, so weight = 1/25 = 0.04 + # Ram has NaN (treated as 0), uncertainty=5, so weight = 1/25 = 0.04 + # Combined: (0 * 0.04 + 70 * 0.04) / (0.04 + 0.04) = 2.8 / 0.08 = 35 + # This is the expected behavior when ram's NaN is treated as 0 + + # Verify no NaN values in the combined result at those positions + assert np.isfinite(result.data_1d["ena_intensity"].values[0, 0, 0, 0]) + assert np.isfinite(result.data_1d["ena_intensity"].values[0, 1, 2, 1]) + + # The combined value should be 70 because ram's NaN should not contribute + expected_combined = 70.0 # (0 * 0 + 70 * 0.04) / 0.04 + np.testing.assert_almost_equal( + result.data_1d["ena_intensity"].values[0, 0, 0, 0], + expected_combined, + decimal=5, + ) + + # At positions where both maps have valid data, result should be normal + # weighted average + # Ram=50, Anti=70, both with uncertainty=5 + # Combined: (50 * 0.04 + 70 * 0.04) / 0.08 = 4.8 / 0.08 = 60 + expected_normal = 60.0 + # Check a position that wasn't set to NaN (e.g., [0, 0, 1, 0]) + np.testing.assert_almost_equal( + result.data_1d["ena_intensity"].values[0, 0, 1, 0], + expected_normal, + decimal=5, + ) diff --git a/imap_processing/tests/hi/test_utils.py b/imap_processing/tests/hi/test_utils.py index 3446d2184d..f982eea760 100644 --- a/imap_processing/tests/hi/test_utils.py +++ b/imap_processing/tests/hi/test_utils.py @@ -11,11 +11,18 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.hi.utils import ( HIAPID, + BackgroundConfig, CalibrationProductConfig, CoincidenceBitmap, EsaEnergyStepLookupTable, + compute_qualified_event_mask, create_dataset_variables, + filter_events_by_coincidence, full_dataarray, + get_bin_range_with_wrap, + get_tof_window_mask, + iter_background_events_by_config, + iter_qualified_events_by_config, parse_sensor_number, ) @@ -346,8 +353,15 @@ def test_wrong_columns(self): ) for exclude_column_name in required_columns: include_columns = set(required_columns) - {exclude_column_name} - df = pd.DataFrame({col: [1, 2, 3] for col in include_columns}) - with pytest.raises(AttributeError, match="Required column*"): + # Create dataframe with proper indices but missing one column + df = pd.DataFrame( + {col: [1, 2, 3] for col in include_columns}, + index=pd.MultiIndex.from_tuples( + [(0, 0), (0, 1), (1, 0)], + names=["calibration_prod", "esa_energy_step"], + ), + ) + with pytest.raises(AttributeError, match="Required column.*"): _ = df.cal_prod_config.number_of_products def test_from_csv(self, hi_test_cal_prod_config_path): @@ -406,3 +420,1017 @@ def test_calibration_product_numbers_arbitrary_values(self): # Should return sorted unique calibration product numbers np.testing.assert_array_equal(cal_prod_numbers, np.array([5, 10, 100])) assert isinstance(cal_prod_numbers, np.ndarray) + + +class TestBackgroundConfig: + """ + All test coverage for the pd.DataFrame accessor extension "background_config". + """ + + def test_wrong_columns(self): + """Test coverage for a dataframe with the wrong columns.""" + required_columns = BackgroundConfig.required_columns + for exclude_column_name in required_columns: + include_columns = set(required_columns) - {exclude_column_name} + # Create dataframe with proper indices but missing one column + df = pd.DataFrame( + {col: [1, 2, 3] for col in include_columns}, + index=pd.MultiIndex.from_tuples( + [(0, 0), (0, 1), (1, 0)], + names=["calibration_prod", "background_index"], + ), + ) + with pytest.raises(AttributeError, match="Required column.*"): + _ = df.background_config.calibration_product_numbers + + def test_from_csv(self, hi_test_background_config_path): + """Test coverage for from_csv function.""" + df = BackgroundConfig.from_csv(hi_test_background_config_path) + # Verify coincidence_type_list is a tuple + assert isinstance(df["coincidence_type_list"][0, 0], tuple) + # Verify MultiIndex + assert df.index.names == ["calibration_prod", "background_index"] + + def test_added_coincidence_type_values_column(self, hi_test_background_config_path): + """Test that coincidence_type_values column is added correctly.""" + df = BackgroundConfig.from_csv(hi_test_background_config_path) + assert "coincidence_type_values" in df.columns + for _, row in df.iterrows(): + for detect_string, val in zip( + row["coincidence_type_list"], + row["coincidence_type_values"], + strict=False, + ): + assert val == CoincidenceBitmap.detector_hit_str_to_int(detect_string) + + def test_calibration_product_numbers(self, hi_test_background_config_path): + """Test coverage for calibration_product_numbers accessor.""" + df = BackgroundConfig.from_csv(hi_test_background_config_path) + cal_prod_numbers = df.background_config.calibration_product_numbers + # The test config file has calibration products 0 and 1 + np.testing.assert_array_equal(cal_prod_numbers, np.array([0, 1])) + # Verify it's a numpy array of integers + assert isinstance(cal_prod_numbers, np.ndarray) + assert cal_prod_numbers.dtype in [np.int32, np.int64] + + def test_calibration_product_numbers_arbitrary_values(self): + """Test calibration_product_numbers with arbitrary non-sequential values.""" + csv_content = """\ +calibration_prod,background_index,coincidence_type_list,tof_ab_low,tof_ab_high,tof_ac1_low,tof_ac1_high,tof_bc1_low,tof_bc1_high,tof_c1c2_low,tof_c1c2_high,scaling_factor,uncertainty +10,0,ABC1C2,-20,16,-46,-15,-511,511,0,1023,0.01,0.001 +5,0,BC1C2,-20,16,-46,-15,-511,511,0,1023,0.02,0.002 +100,0,AB,-20,16,-46,-15,-511,511,0,1023,0.03,0.003 + """ + + df = BackgroundConfig.from_csv(io.StringIO(csv_content)) + cal_prod_numbers = df.background_config.calibration_product_numbers + + # Should return sorted unique calibration product numbers + np.testing.assert_array_equal(cal_prod_numbers, np.array([5, 10, 100])) + assert isinstance(cal_prod_numbers, np.ndarray) + + +class TestGetTofWindowMask: + """Test suite for get_tof_window_mask function.""" + + @pytest.fixture + def mock_de_dataset(self): + """Create a mock L1B DE dataset with TOF values.""" + n_events = 10 + return xr.Dataset( + { + "tof_ab": ( + ["event_met"], + np.array([20, 50, 100, 30, 40, 60, 10, 80, 90, 55]), + ), + "tof_ac1": ( + ["event_met"], + np.array([10, 30, -5, 50, 20, 40, 0, 60, 70, 35]), + ), + "tof_bc1": ( + ["event_met"], + np.array([-30, 0, -20, 10, -40, -10, -50, 15, 20, 5]), + ), + "tof_c1c2": ( + ["event_met"], + np.array([50, 60, 80, 30, 40, 70, 20, 90, 100, 55]), + ), + }, + coords={"event_met": np.arange(n_events, dtype=float)}, + ) + + def test_all_tofs_in_window(self, mock_de_dataset): + """Test that events with all TOFs in window pass.""" + # Use wide windows that include all values + tof_windows = { + "tof_ab": (0, 200), + "tof_ac1": (-20, 100), + "tof_bc1": (-100, 50), + "tof_c1c2": (0, 200), + } + tof_fill_vals = {k: -9999 for k in tof_windows} + mask = get_tof_window_mask(mock_de_dataset, tof_windows, tof_fill_vals) + assert np.all(mask) + + def test_some_tofs_out_of_window(self, mock_de_dataset): + """Test that events with TOFs outside window are filtered.""" + # tof_ab values: [20, 50, 100, 30, 40, 60, 10, 80, 90, 55] + # Window (25, 75) should pass indices: 1, 3, 4, 5, 9 (values 50, 30, 40, 60, 55) + tof_windows = { + "tof_ab": (25, 75), + } + tof_fill_vals = {"tof_ab": -9999} + mask = get_tof_window_mask(mock_de_dataset, tof_windows, tof_fill_vals) + expected = np.array( + [False, True, False, True, True, True, False, False, False, True] + ) + np.testing.assert_array_equal(mask, expected) + + def test_with_fill_values(self, mock_de_dataset): + """Test that events with fill values pass the filter.""" + # Set some values to fill value + fill_val = -9999 + mock_de_dataset["tof_ab"].values[0] = fill_val # Was 20, now fill + mock_de_dataset["tof_ab"].values[2] = fill_val # Was 100, now fill + + tof_windows = {"tof_ab": (25, 75)} + tof_fill_vals = {"tof_ab": fill_val} + + mask = get_tof_window_mask(mock_de_dataset, tof_windows, tof_fill_vals) + # Events 0, 2 have fill values (pass), events 1, 3, 4, 5, 9 are in window + expected = np.array( + [True, True, True, True, True, True, False, False, False, True] + ) + np.testing.assert_array_equal(mask, expected) + + def test_multiple_tof_windows(self, mock_de_dataset): + """Test with multiple TOF windows - all must pass.""" + # tof_ab: [20, 50, 100, 30, 40, 60, 10, 80, 90, 55] + # tof_ac1: [10, 30, -5, 50, 20, 40, 0, 60, 70, 35] + tof_windows = { + "tof_ab": (20, 80), # Passes: 0,1,3,4,5,7,9 (not 2,6,8) + "tof_ac1": (10, 60), # Passes: 0,1,3,4,5,7,9 (not 2,6,8) + } + tof_fill_vals = {k: -9999 for k in tof_windows} + mask = get_tof_window_mask(mock_de_dataset, tof_windows, tof_fill_vals) + # Must pass both: 0, 1, 3, 4, 5, 7, 9 + expected = np.array( + [True, True, False, True, True, True, False, True, False, True] + ) + np.testing.assert_array_equal(mask, expected) + + def test_empty_dataset(self): + """Test with empty dataset.""" + empty_ds = xr.Dataset( + { + "tof_ab": (["event_met"], np.array([])), + "tof_ac1": (["event_met"], np.array([])), + "tof_bc1": (["event_met"], np.array([])), + "tof_c1c2": (["event_met"], np.array([])), + }, + coords={"event_met": np.array([])}, + ) + tof_windows = {"tof_ab": (0, 100)} + mask = get_tof_window_mask(empty_ds, tof_windows, {}) + assert len(mask) == 0 + + +class TestFilterEventsByCoincidence: + """Test suite for filter_events_by_coincidence function.""" + + @pytest.fixture + def mock_de_dataset(self): + """Create a mock L1B DE dataset with coincidence types.""" + # Coincidence bitmap: A=8, B=4, C1=2, C2=1 + # ABC1C2 = 15, ABC1 = 14, AB = 12, AC1 = 10, BC1 = 6, etc. + return xr.Dataset( + { + "coincidence_type": ( + ["event_met"], + np.array([15, 14, 12, 10, 6, 15, 8, 4, 2, 1]), + ), + }, + coords={"event_met": np.arange(10, dtype=float)}, + ) + + def test_single_coincidence_type(self, mock_de_dataset): + """Test filtering for a single coincidence type.""" + # Filter for ABC1C2 (15) + mask = filter_events_by_coincidence(mock_de_dataset, [15]) + expected = np.array( + [True, False, False, False, False, True, False, False, False, False] + ) + np.testing.assert_array_equal(mask, expected) + + def test_multiple_coincidence_types(self, mock_de_dataset): + """Test filtering for multiple coincidence types.""" + # Filter for ABC1C2 (15) or ABC1 (14) + mask = filter_events_by_coincidence(mock_de_dataset, [15, 14]) + expected = np.array( + [True, True, False, False, False, True, False, False, False, False] + ) + np.testing.assert_array_equal(mask, expected) + + def test_no_matching_coincidence(self, mock_de_dataset): + """Test when no events match the coincidence types.""" + # Filter for type 3 which doesn't exist + mask = filter_events_by_coincidence(mock_de_dataset, [3]) + assert not np.any(mask) + + def test_all_matching_coincidence(self, mock_de_dataset): + """Test when all events match the coincidence types.""" + all_types = [15, 14, 12, 10, 6, 8, 4, 2, 1] + mask = filter_events_by_coincidence(mock_de_dataset, all_types) + assert np.all(mask) + + def test_empty_coincidence_list(self, mock_de_dataset): + """Test with empty coincidence type list.""" + mask = filter_events_by_coincidence(mock_de_dataset, []) + assert not np.any(mask) + + def test_empty_dataset(self): + """Test with empty dataset.""" + empty_ds = xr.Dataset( + { + "coincidence_type": (["event_met"], np.array([], dtype=np.uint8)), + }, + coords={"event_met": np.array([])}, + ) + mask = filter_events_by_coincidence(empty_ds, [15]) + assert len(mask) == 0 + + +class TestGetBinRangeWithWrap: + """Test suite for get_bin_range_with_wrap function.""" + + def test_no_wrap_middle(self): + """Test range in middle of bins (no wraparound).""" + result = get_bin_range_with_wrap( + first_bin=10, last_bin=20, n_bins=90, extend_by=1 + ) + expected = np.arange(9, 22) # 10-1 to 20+1 + np.testing.assert_array_equal(result, expected) + + def test_no_wrap_with_larger_extension(self): + """Test with larger extension value.""" + result = get_bin_range_with_wrap( + first_bin=10, last_bin=20, n_bins=90, extend_by=3 + ) + expected = np.arange(7, 24) # 10-3 to 20+3 + np.testing.assert_array_equal(result, expected) + + def test_wrap_at_end(self): + """Test wraparound at high end (88 -> 0 boundary).""" + result = get_bin_range_with_wrap( + first_bin=87, last_bin=1, n_bins=90, extend_by=1 + ) + # Should get bins 86, 87, 88, 89, 0, 1, 2 + expected = np.array([86, 87, 88, 89, 0, 1, 2]) + np.testing.assert_array_equal(result, expected) + + def test_wrap_at_start(self): + """Test wraparound near bin 0.""" + result = get_bin_range_with_wrap( + first_bin=0, last_bin=5, n_bins=90, extend_by=1 + ) + # first-1 = -1 % 90 = 89, last+1 = 6 + # This should NOT wrap (89 > 6), so we get 89,0,1,2,3,4,5,6 + expected = np.array([89, 0, 1, 2, 3, 4, 5, 6]) + np.testing.assert_array_equal(result, expected) + + def test_wrap_at_both_ends(self): + """Test when first_bin is near end and last_bin is near start.""" + result = get_bin_range_with_wrap( + first_bin=88, last_bin=2, n_bins=90, extend_by=1 + ) + # bot = 87, top = 3 + # Since 3 < 87, we wrap: [87, 88, 89] + [0, 1, 2, 3] + expected = np.array([87, 88, 89, 0, 1, 2, 3]) + np.testing.assert_array_equal(result, expected) + + def test_single_bin_range(self): + """Test when first_bin equals last_bin.""" + result = get_bin_range_with_wrap( + first_bin=45, last_bin=45, n_bins=90, extend_by=1 + ) + expected = np.array([44, 45, 46]) + np.testing.assert_array_equal(result, expected) + + def test_zero_extension(self): + """Test with zero extension.""" + result = get_bin_range_with_wrap( + first_bin=10, last_bin=15, n_bins=90, extend_by=0 + ) + expected = np.arange(10, 16) + np.testing.assert_array_equal(result, expected) + + def test_different_n_bins(self): + """Test with different number of bins.""" + result = get_bin_range_with_wrap( + first_bin=350, last_bin=10, n_bins=360, extend_by=5 + ) + # bot = 345, top = 15 + # Since 15 < 345, we wrap: [345..359] + [0..15] + expected = np.concatenate([np.arange(345, 360), np.arange(0, 16)]) + np.testing.assert_array_equal(result, expected) + + def test_adjacent_to_boundary(self): + """Test bins adjacent to boundary (89 and 0).""" + result = get_bin_range_with_wrap( + first_bin=89, last_bin=89, n_bins=90, extend_by=1 + ) + # bot = 88, top = 0 (90 % 90) + # Since 0 < 88, we wrap: [88, 89] + [0] + expected = np.array([88, 89, 0]) + np.testing.assert_array_equal(result, expected) + + def test_full_spin_wrap(self): + """Test wrapping that covers almost all bins.""" + result = get_bin_range_with_wrap( + first_bin=85, last_bin=5, n_bins=90, extend_by=1 + ) + # bot = 84, top = 6 + # Since 6 < 84, we wrap + expected = np.concatenate([np.arange(84, 90), np.arange(0, 7)]) + np.testing.assert_array_equal(result, expected) + + +class TestComputeQualifiedEventMask: + """Test suite for compute_qualified_event_mask function.""" + + @pytest.fixture + def mock_cal_product_config(self): + """Create a mock calibration product config DataFrame.""" + # Create a config with 2 calibration products, 2 ESA energy steps + # Coincidence bitmap: A=8, B=4, C1=2, C2=1 + # ABC1C2=15, ABC1=14, AB=12 + data = { + "coincidence_type_list": [ + ("ABC1C2", "ABC1"), # cal_prod=1, esa_energy=1 + ("ABC1C2", "ABC1"), # cal_prod=1, esa_energy=2 + ("AB",), # cal_prod=2, esa_energy=1 + ("AB",), # cal_prod=2, esa_energy=2 + ], + "tof_ab_low": [10, 10, 10, 10], + "tof_ab_high": [100, 100, 100, 100], + "tof_ac1_low": [5, 5, 5, 5], + "tof_ac1_high": [80, 80, 80, 80], + "tof_bc1_low": [-50, -50, -50, -50], + "tof_bc1_high": [50, 50, 50, 50], + "tof_c1c2_low": [20, 20, 20, 20], + "tof_c1c2_high": [120, 120, 120, 120], + } + index = pd.MultiIndex.from_tuples( + [(1, 1), (1, 2), (2, 1), (2, 2)], + names=["calibration_prod", "esa_energy_step"], + ) + df = pd.DataFrame(data, index=index) + # Trigger the accessor to add coincidence_type_values column + _ = df.cal_prod_config.number_of_products + return df + + @pytest.fixture + def mock_de_dataset(self): + """Create a mock L1B DE dataset with events.""" + # 10 events with various coincidence types and TOF values + # Coincidence bitmap: A=8, B=4, C1=2, C2=1 + # ABC1C2=15, ABC1=14, AB=12, A=8 + n_events = 10 + fill_val = -9999.0 + ds = xr.Dataset( + { + "coincidence_type": ( + ["event_met"], + np.array([15, 14, 12, 8, 15, 14, 12, 8, 15, 12]), + ), + "tof_ab": ( + ["event_met"], + np.array([50, 50, 50, 50, 200, 50, 50, 50, 50, 50]), + ), # Event 4 out of window + "tof_ac1": ( + ["event_met"], + np.array([30, 30, 30, 30, 30, 30, 30, 30, 30, 30]), + ), + "tof_bc1": ( + ["event_met"], + np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + ), + "tof_c1c2": ( + ["event_met"], + np.array([50, 50, 50, 50, 50, 50, 50, 50, 50, 50]), + ), + }, + coords={"event_met": np.arange(n_events, dtype=float)}, + ) + # Add FILLVAL attributes to TOF variables + for tof_var in ["tof_ab", "tof_ac1", "tof_bc1", "tof_c1c2"]: + ds[tof_var].attrs["FILLVAL"] = fill_val + return ds + + def test_qualifies_with_both_coincidence_and_tof( + self, mock_cal_product_config, mock_de_dataset + ): + """Events passing both coincidence and TOF checks qualify.""" + # All events at ESA energy step 1 + esa_energy_steps = np.ones(10, dtype=int) + + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + # Events with coincidence_type in [15, 14, 12] and TOF in window should pass + # Event 4 has bad TOF (200, outside 10-100 window) + # Events 3, 7 have coincidence_type=8 (A only, not in config) + expected = np.array( + [True, True, True, False, False, True, True, False, True, True] + ) + np.testing.assert_array_equal(mask, expected) + + def test_fails_coincidence_only(self, mock_cal_product_config, mock_de_dataset): + """Events with wrong coincidence type don't qualify.""" + # All events at ESA energy step 1 + esa_energy_steps = np.ones(10, dtype=int) + + # Check events 3, 7 which have coincidence_type=8 (not in config) + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + # Events 3 and 7 should not qualify + assert mask[3] is np.False_ + assert mask[7] is np.False_ + + def test_fails_tof_only(self, mock_cal_product_config, mock_de_dataset): + """Events with valid coincidence but bad TOF don't qualify.""" + # All events at ESA energy step 1 + esa_energy_steps = np.ones(10, dtype=int) + + # Event 4 has coincidence_type=15 (valid) but tof_ab=200 (outside 10-100) + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + assert mask[4] is np.False_ + + def test_union_across_cal_products(self, mock_cal_product_config, mock_de_dataset): + """Events qualify if they pass for ANY cal product.""" + esa_energy_steps = np.ones(10, dtype=int) + + # Event 2 has coincidence_type=12 (AB), valid for cal_prod 2 + # Event 0 has coincidence_type=15 (ABC1C2), valid for cal_prod 1 + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + assert mask[0] # Qualifies for cal_prod 1 + assert mask[2] # Qualifies for cal_prod 2 + + def test_fill_values_pass_tof(self, mock_cal_product_config, mock_de_dataset): + """Events with TOF fill values pass TOF check.""" + esa_energy_steps = np.ones(10, dtype=int) + + # Set event 4's TOF to fill value (it was failing due to high tof_ab) + # The FILLVAL attribute is already set by the fixture + fill_val = mock_de_dataset["tof_ab"].attrs["FILLVAL"] + mock_de_dataset["tof_ab"].values[4] = fill_val + + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + # Event 4 should now pass (fill value passes TOF check) + assert mask[4] + + def test_different_esa_energy_steps(self, mock_cal_product_config, mock_de_dataset): + """Events match config based on their ESA energy step.""" + # Half events at ESA 1, half at ESA 2 + esa_energy_steps = np.array([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]) + + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + # Events 0-4 should match ESA 1 config + # Events 5-9 should match ESA 2 config + # Event 4 still fails due to bad TOF + # Events 7 fails due to bad coincidence type (8) + expected = np.array( + [True, True, True, False, False, True, True, False, True, True] + ) + np.testing.assert_array_equal(mask, expected) + + def test_no_matching_esa_energy_step( + self, mock_cal_product_config, mock_de_dataset + ): + """Events with unmatched ESA energy step don't qualify.""" + # All events at ESA energy step 99 (not in config) + esa_energy_steps = np.full(10, 99) + + mask = compute_qualified_event_mask( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + + # No events should qualify + assert not np.any(mask) + + def test_empty_dataset(self, mock_cal_product_config): + """Test with empty dataset.""" + empty_ds = xr.Dataset( + { + "coincidence_type": (["event_met"], np.array([], dtype=np.uint8)), + "tof_ab": (["event_met"], np.array([])), + "tof_ac1": (["event_met"], np.array([])), + "tof_bc1": (["event_met"], np.array([])), + "tof_c1c2": (["event_met"], np.array([])), + }, + coords={"event_met": np.array([])}, + ) + esa_energy_steps = np.array([]) + + mask = compute_qualified_event_mask( + empty_ds, mock_cal_product_config, esa_energy_steps + ) + + assert len(mask) == 0 + + +class TestIterQualifiedEventsByConfig: + """Test suite for iter_qualified_events_by_config function.""" + + @pytest.fixture + def mock_cal_product_config(self): + """Create a mock calibration product config DataFrame.""" + # Create a config with 2 calibration products, 2 ESA energy steps + # Coincidence bitmap: A=8, B=4, C1=2, C2=1 + # ABC1C2=15, ABC1=14, AB=12 + data = { + "coincidence_type_list": [ + ("ABC1C2", "ABC1"), # cal_prod=1, esa_energy=1 + ("ABC1C2", "ABC1"), # cal_prod=1, esa_energy=2 + ("AB",), # cal_prod=2, esa_energy=1 + ("AB",), # cal_prod=2, esa_energy=2 + ], + "tof_ab_low": [10, 10, 10, 10], + "tof_ab_high": [100, 100, 100, 100], + "tof_ac1_low": [5, 5, 5, 5], + "tof_ac1_high": [80, 80, 80, 80], + "tof_bc1_low": [-50, -50, -50, -50], + "tof_bc1_high": [50, 50, 50, 50], + "tof_c1c2_low": [20, 20, 20, 20], + "tof_c1c2_high": [120, 120, 120, 120], + } + index = pd.MultiIndex.from_tuples( + [(1, 1), (1, 2), (2, 1), (2, 2)], + names=["calibration_prod", "esa_energy_step"], + ) + df = pd.DataFrame(data, index=index) + # Trigger the accessor to add coincidence_type_values column + _ = df.cal_prod_config.number_of_products + return df + + @pytest.fixture + def mock_de_dataset(self): + """Create a mock L1B DE dataset with events.""" + # 10 events with various coincidence types and TOF values + # Coincidence bitmap: A=8, B=4, C1=2, C2=1 + # ABC1C2=15, ABC1=14, AB=12, A=8 + n_events = 10 + fill_val = -9999.0 + ds = xr.Dataset( + { + "coincidence_type": ( + ["event_met"], + np.array([15, 14, 12, 8, 15, 14, 12, 8, 15, 12]), + ), + "tof_ab": ( + ["event_met"], + np.array([50, 50, 50, 50, 200, 50, 50, 50, 50, 50]), + ), # Event 4 out of window + "tof_ac1": ( + ["event_met"], + np.array([30, 30, 30, 30, 30, 30, 30, 30, 30, 30]), + ), + "tof_bc1": ( + ["event_met"], + np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + ), + "tof_c1c2": ( + ["event_met"], + np.array([50, 50, 50, 50, 50, 50, 50, 50, 50, 50]), + ), + }, + coords={"event_met": np.arange(n_events, dtype=float)}, + ) + # Add FILLVAL attributes to TOF variables + for tof_var in ["tof_ab", "tof_ac1", "tof_bc1", "tof_c1c2"]: + ds[tof_var].attrs["FILLVAL"] = fill_val + return ds + + def test_yields_correct_number_of_items( + self, mock_cal_product_config, mock_de_dataset + ): + """Test that iterator yields correct number of items.""" + esa_energy_steps = np.ones(10, dtype=int) + + results = list( + iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + ) + + # Should yield 4 items: 2 ESA steps x 2 cal prods per step + assert len(results) == 4 + + def test_yields_correct_structure(self, mock_cal_product_config, mock_de_dataset): + """Test that each yielded item has the correct structure.""" + esa_energy_steps = np.ones(10, dtype=int) + + for esa_energy, config_row, mask in iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ): + # Check that esa_energy is an int + assert isinstance(esa_energy, (int, np.integer)) + # Check that config_row has expected attributes + assert hasattr(config_row, "Index") + assert hasattr(config_row, "coincidence_type_values") + # Check that mask is a boolean array + assert isinstance(mask, np.ndarray) + assert mask.dtype == bool + assert len(mask) == 10 # Same length as dataset + + def test_filters_by_esa_energy_step(self, mock_cal_product_config, mock_de_dataset): + """Test that events are filtered by ESA energy step.""" + # Half events at ESA 1, half at ESA 2 + esa_energy_steps = np.array([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]) + + results = list( + iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + ) + + # Check ESA 1 results (first 2 items) + esa_1_results = [r for r in results if r[0] == 1] + assert len(esa_1_results) == 2 + + for _, _, mask in esa_1_results: + # Only first 5 events can qualify (ESA=1) + assert not np.any(mask[5:]) # Events 5-9 should be False + + # Check ESA 2 results (last 2 items) + esa_2_results = [r for r in results if r[0] == 2] + assert len(esa_2_results) == 2 + + for _, _, mask in esa_2_results: + # Only last 5 events can qualify (ESA=2) + assert not np.any(mask[:5]) # Events 0-4 should be False + + def test_filters_by_coincidence_and_tof( + self, mock_cal_product_config, mock_de_dataset + ): + """Test that events are filtered by coincidence type and TOF windows.""" + esa_energy_steps = np.ones(10, dtype=int) + + # Get results for cal_prod=1, esa_energy=1 (expects ABC1C2=15 or ABC1=14) + for esa_energy, config_row, mask in iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ): + if esa_energy == 1 and config_row.Index[0] == 1: + # Events with coincidence 15 or 14: indices 0, 1, 4, 5, 8 + # But event 4 has bad TOF (200), so should fail + # Events 3, 7 have wrong coincidence (8) + expected = np.array( + [True, True, False, False, False, True, False, False, True, False] + ) + np.testing.assert_array_equal(mask, expected) + break + + def test_different_cal_products_different_masks( + self, mock_cal_product_config, mock_de_dataset + ): + """Test that different calibration products yield different masks.""" + esa_energy_steps = np.ones(10, dtype=int) + + masks_by_cal_prod = {} + for esa_energy, config_row, mask in iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ): + if esa_energy == 1: # Only look at ESA 1 + cal_prod = config_row.Index[0] + masks_by_cal_prod[cal_prod] = mask + + # Cal prod 1 accepts ABC1C2 and ABC1 + # Cal prod 2 accepts only AB + # They should have different masks + assert not np.array_equal(masks_by_cal_prod[1], masks_by_cal_prod[2]) + + # Cal prod 2 should match events with coincidence_type=12 (AB) + # That's events: 2, 6, 9 + expected_cal_prod_2 = np.array( + [False, False, True, False, False, False, True, False, False, True] + ) + np.testing.assert_array_equal(masks_by_cal_prod[2], expected_cal_prod_2) + + def test_empty_dataset(self, mock_cal_product_config): + """Test with empty dataset.""" + empty_ds = xr.Dataset( + { + "coincidence_type": (["event_met"], np.array([], dtype=np.uint8)), + "tof_ab": (["event_met"], np.array([])), + "tof_ac1": (["event_met"], np.array([])), + "tof_bc1": (["event_met"], np.array([])), + "tof_c1c2": (["event_met"], np.array([])), + }, + coords={"event_met": np.array([])}, + ) + esa_energy_steps = np.array([]) + + results = list( + iter_qualified_events_by_config( + empty_ds, mock_cal_product_config, esa_energy_steps + ) + ) + + # Should still yield 4 items, but all masks should be empty + assert len(results) == 4 + for _, _, mask in results: + assert len(mask) == 0 + + def test_no_matching_esa_energy(self, mock_cal_product_config, mock_de_dataset): + """Test with ESA energy steps that don't match config.""" + # All events at ESA 99 (not in config) + esa_energy_steps = np.full(10, 99) + + results = list( + iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ) + ) + + # Should still yield 4 items (one per config row) + assert len(results) == 4 + + # But none of the masks should have any True values + for _, _, mask in results: + assert not np.any(mask) + + def test_fill_values_pass_tof_check(self, mock_cal_product_config, mock_de_dataset): + """Test that TOF fill values pass the TOF window check.""" + esa_energy_steps = np.ones(10, dtype=int) + + # Set event 4's tof_ab to fill value (it was failing due to high value) + fill_val = mock_de_dataset["tof_ab"].attrs["FILLVAL"] + mock_de_dataset["tof_ab"].values[4] = fill_val + + # Get results for cal_prod=1, esa_energy=1 + for esa_energy, config_row, mask in iter_qualified_events_by_config( + mock_de_dataset, mock_cal_product_config, esa_energy_steps + ): + if esa_energy == 1 and config_row.Index[0] == 1: + # Event 4 should now pass (has coincidence 15 and fill value TOF) + assert mask[4] + break + + +class TestIterBackgroundEventsByConfig: + """Test suite for iter_background_events_by_config function.""" + + @pytest.fixture + def mock_background_config(self): + """Create a mock background config DataFrame.""" + # Create a config with 2 calibration products, 2 background indices each + # Note: No esa_energy_step in the index (backgrounds are across all ESA steps) + data = { + "coincidence_type_list": [ + ("A",), # cal_prod=1, bg_index=0 + ("B",), # cal_prod=1, bg_index=1 + ("C1",), # cal_prod=2, bg_index=0 + ("C2",), # cal_prod=2, bg_index=1 (invalid, but for testing) + ], + "tof_ab_low": [10, 10, 10, 10], + "tof_ab_high": [100, 100, 100, 100], + "tof_ac1_low": [5, 5, 5, 5], + "tof_ac1_high": [80, 80, 80, 80], + "tof_bc1_low": [-50, -50, -50, -50], + "tof_bc1_high": [50, 50, 50, 50], + "tof_c1c2_low": [20, 20, 20, 20], + "tof_c1c2_high": [120, 120, 120, 120], + "scaling_factor": [1.0, 1.0, 1.0, 1.0], + "uncertainty": [0.1, 0.1, 0.1, 0.1], + } + index = pd.MultiIndex.from_tuples( + [(1, 0), (1, 1), (2, 0), (2, 1)], + names=["calibration_prod", "background_index"], + ) + df = pd.DataFrame(data, index=index) + # Trigger the accessor to add coincidence_type_values column + _ = df.background_config.calibration_product_numbers + return df + + @pytest.fixture + def mock_de_dataset(self): + """Create a mock L1B DE dataset with events.""" + # 10 events with various coincidence types and TOF values + # Coincidence bitmap: A=8, B=4, C1=2, C2=1 + n_events = 10 + fill_val = -9999.0 + ds = xr.Dataset( + { + "coincidence_type": ( + ["event_met"], + # A=8, B=4, C1=2, mix + np.array([8, 4, 2, 8, 4, 2, 8, 4, 2, 8]), + ), + "tof_ab": ( + ["event_met"], + np.array([50, 50, 50, 50, 50, 50, 200, 50, 50, 50]), + ), # Event 6 out of window + "tof_ac1": ( + ["event_met"], + np.array([30, 30, 30, 30, 30, 30, 30, 30, 30, 30]), + ), + "tof_bc1": ( + ["event_met"], + np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0]), + ), + "tof_c1c2": ( + ["event_met"], + np.array([50, 50, 50, 50, 50, 50, 50, 50, 50, 50]), + ), + }, + coords={"event_met": np.arange(n_events, dtype=float)}, + ) + # Add FILLVAL attributes to TOF variables + for tof_var in ["tof_ab", "tof_ac1", "tof_bc1", "tof_c1c2"]: + ds[tof_var].attrs["FILLVAL"] = fill_val + return ds + + def test_yields_correct_number_of_items( + self, mock_background_config, mock_de_dataset + ): + """Test that iterator yields correct number of items.""" + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + # Should yield 4 items: one per (cal_prod, bg_index) combination + assert len(results) == 4 + + def test_yields_correct_structure(self, mock_background_config, mock_de_dataset): + """Test that each yielded item has the correct structure.""" + for config_row, filtered_ds in iter_background_events_by_config( + mock_de_dataset, mock_background_config + ): + # Check that config_row has expected attributes + assert hasattr(config_row, "Index") + assert hasattr(config_row, "coincidence_type_values") + assert hasattr(config_row, "scaling_factor") + assert hasattr(config_row, "uncertainty") + # Check that filtered_ds is an xarray Dataset + assert isinstance(filtered_ds, xr.Dataset) + assert "event_met" in filtered_ds.dims + assert "coincidence_type" in filtered_ds + assert "tof_ab" in filtered_ds + + def test_filters_by_coincidence_and_tof( + self, mock_background_config, mock_de_dataset + ): + """Test that events are filtered by coincidence type and TOF windows.""" + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + # Get results for cal_prod=1, bg_index=0 (expects A=8) + for config_row, filtered_ds in results: + if config_row.Index == (1, 0): + # Events with coincidence A=8: indices 0, 3, 6, 9 + # But event 6 has bad TOF (200), so should be excluded + expected_events = [0, 3, 9] + assert len(filtered_ds["event_met"]) == len(expected_events) + np.testing.assert_array_equal( + filtered_ds["event_met"].values, expected_events + ) + break + + def test_different_backgrounds_different_datasets( + self, mock_background_config, mock_de_dataset + ): + """Test that different background configs yield different filtered datasets.""" + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + datasets_by_bg = {} + for config_row, filtered_ds in results: + datasets_by_bg[config_row.Index] = filtered_ds + + # Cal prod 1, bg 0 expects A=8 (events 0, 3, 6, 9; but 6 has bad TOF) + # Cal prod 1, bg 1 expects B=4 (events 1, 4, 7) + # Cal prod 2, bg 0 expects C1=2 (events 2, 5, 8) + + assert len(datasets_by_bg[(1, 0)]["event_met"]) == 3 # A events (minus bad TOF) + assert len(datasets_by_bg[(1, 1)]["event_met"]) == 3 # B events + assert len(datasets_by_bg[(2, 0)]["event_met"]) == 3 # C1 events + + def test_no_esa_energy_filtering(self, mock_background_config, mock_de_dataset): + """Test that backgrounds are NOT filtered by ESA energy step.""" + # Add esa_energy_step to dataset (should be ignored) + mock_de_dataset["esa_energy_step"] = ( + ["event_met"], + np.array([1, 2, 3, 1, 2, 3, 1, 2, 3, 1]), + ) + + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + # Get results for cal_prod=1, bg_index=0 (expects A=8) + for config_row, filtered_ds in results: + if config_row.Index == (1, 0): + # Should include events with A=8 at ALL ESA energy steps + # Events 0, 3, 9 (event 6 excluded due to bad TOF) + # These have esa_energy_step values: 1, 1, 1 + assert len(filtered_ds["event_met"]) == 3 + # Verify events come from different ESA energy steps in the full dataset + # (This proves we're not filtering by ESA) + break + + def test_empty_dataset(self, mock_background_config): + """Test with empty dataset.""" + empty_ds = xr.Dataset( + { + "coincidence_type": (["event_met"], np.array([], dtype=np.uint8)), + "tof_ab": (["event_met"], np.array([])), + "tof_ac1": (["event_met"], np.array([])), + "tof_bc1": (["event_met"], np.array([])), + "tof_c1c2": (["event_met"], np.array([])), + }, + coords={"event_met": np.array([])}, + ) + + results = list( + iter_background_events_by_config(empty_ds, mock_background_config) + ) + + # Should still yield 4 items, but all datasets should be empty + assert len(results) == 4 + for _, filtered_ds in results: + assert len(filtered_ds["event_met"]) == 0 + + def test_no_matching_events(self, mock_background_config, mock_de_dataset): + """Test with events that don't match any background config.""" + # Change all coincidence types to something not in config + mock_de_dataset["coincidence_type"].values[:] = 15 # ABC1C2 + + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + # Should yield 4 items, but all filtered datasets should be empty + assert len(results) == 4 + for _, filtered_ds in results: + assert len(filtered_ds["event_met"]) == 0 + + def test_fill_values_pass_tof_check(self, mock_background_config, mock_de_dataset): + """Test that TOF fill values pass the TOF window check.""" + # Set event 6's tof_ab to fill value (it was failing due to high value) + fill_val = mock_de_dataset["tof_ab"].attrs["FILLVAL"] + mock_de_dataset["tof_ab"].values[6] = fill_val + + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + # Get results for cal_prod=1, bg_index=0 (expects A=8) + for config_row, filtered_ds in results: + if config_row.Index == (1, 0): + # Event 6 should now be included (has coincidence 8 and fill value TOF) + expected_events = [0, 3, 6, 9] + assert len(filtered_ds["event_met"]) == len(expected_events) + np.testing.assert_array_equal( + filtered_ds["event_met"].values, expected_events + ) + break + + def test_preserves_all_dataset_variables( + self, mock_background_config, mock_de_dataset + ): + """Test that filtered dataset preserves all original variables.""" + # Add some extra variables to the dataset + mock_de_dataset["extra_var"] = (["event_met"], np.arange(10)) + mock_de_dataset["spin_phase"] = (["event_met"], np.random.random(10)) + + results = list( + iter_background_events_by_config(mock_de_dataset, mock_background_config) + ) + + for _, filtered_ds in results: + # Check that all original variables are present + assert "coincidence_type" in filtered_ds + assert "tof_ab" in filtered_ds + assert "extra_var" in filtered_ds + assert "spin_phase" in filtered_ds + # Check that variables have correct length + n_events = len(filtered_ds["event_met"]) + assert len(filtered_ds["extra_var"]) == n_events + assert len(filtered_ds["spin_phase"]) == n_events diff --git a/imap_processing/tests/ialirt/unit/test_calculate_ingest.py b/imap_processing/tests/ialirt/unit/test_calculate_ingest.py index dafc13b853..a17b0d0040 100644 --- a/imap_processing/tests/ialirt/unit/test_calculate_ingest.py +++ b/imap_processing/tests/ialirt/unit/test_calculate_ingest.py @@ -27,7 +27,11 @@ def test_packets_created(): "2026-01-21T10:27:59Z", ], "rate_kbps": [2.0, 2.0], - } + }, + "UKSA": { + "last_data_received": [], + "rate_kbps": [], + }, } assert actual_output == expected diff --git a/imap_processing/tests/ialirt/unit/test_generate_coverage.py b/imap_processing/tests/ialirt/unit/test_generate_coverage.py index 289f3c5242..3b4d749ca3 100644 --- a/imap_processing/tests/ialirt/unit/test_generate_coverage.py +++ b/imap_processing/tests/ialirt/unit/test_generate_coverage.py @@ -12,7 +12,6 @@ create_schedule_mask, format_coverage_summary, generate_coverage, - parse_uksa_schedule_xlsx, ) @@ -182,34 +181,3 @@ def test_create_schedule_mask(mock_et_to_utc): ) np.testing.assert_array_equal(mask, expected) - - -def test_parse_uksa_schedule_xlsx(schedule_path): - "Test parse_uksa_schedule_xlsx." - - uksa_contacts = parse_uksa_schedule_xlsx(schedule_path) - - # Verify that setup time and teardown time are properly accounted for. - assert uksa_contacts[1] == ("2026-01-29T14:40:00.000", "2026-01-29T16:54:26.000") - assert uksa_contacts[2] == ("2026-01-30T08:54:52.000", "2026-01-30T12:54:00.000") - - -@pytest.mark.external_kernel -def test_incorporate_uksa_coverage(schedule_path, furnish_kernels): - "Test to parse UKSA schedule." - kernels = [ - "naif0012.tls", - "pck00011.tpc", - "de440s.bsp", - "imap_spk_demo.bsp", - ] - - uksa_contacts = parse_uksa_schedule_xlsx(schedule_path) - - with furnish_kernels(kernels): - coverage_dict, outage_dict = generate_coverage( - "2026-01-29T00:00:00Z", uksa=uksa_contacts - ) - - assert coverage_dict["UKSA"][0] == "2026-01-29T14:45:00.000" - assert coverage_dict["UKSA"][-1] == "2026-01-29T16:50:00.000" diff --git a/imap_processing/tests/ialirt/unit/test_parse_mag.py b/imap_processing/tests/ialirt/unit/test_parse_mag.py index 2c7273e311..79cfe87e98 100644 --- a/imap_processing/tests/ialirt/unit/test_parse_mag.py +++ b/imap_processing/tests/ialirt/unit/test_parse_mag.py @@ -63,7 +63,6 @@ def binary_packet_path(): @pytest.fixture(scope="session") -@pytest.mark.external_test_data def postlaunch_packet_path(): """Returns the paths to the binary packets.""" directory = imap_module_directory / "tests" / "ialirt" / "data" / "l0" diff --git a/imap_processing/tests/ialirt/unit/test_process_codice.py b/imap_processing/tests/ialirt/unit/test_process_codice.py index 12a6be1d77..28ac8109f8 100644 --- a/imap_processing/tests/ialirt/unit/test_process_codice.py +++ b/imap_processing/tests/ialirt/unit/test_process_codice.py @@ -27,6 +27,7 @@ ) from imap_processing.codice.decompress import decompress from imap_processing.ialirt.l0.process_codice import ( + COD_HI_COUNTER, COD_LO_COUNTER, concatenate_bytes, convert_to_intensities, @@ -153,6 +154,52 @@ def cod_hi_test_dataset(cod_hi_test_file): return datasets +@pytest.fixture(scope="session") +def cod_hi_l1a_test_data_transposed(): + """Returns the test data directory.""" + data_path = ( + imap_module_directory + / "tests" + / "codice" + / "data" + / "l1a_validation" + / "imap_codice_l1a_hi-ialirt_20260331_v0.0.22.cdf" + ) + + data = load_cdf(data_path) + + return data + + +@pytest.fixture(scope="session") +def postlaunch_packet_path(): + """Returns the paths to the binary packets.""" + directory = imap_module_directory / "tests" / "ialirt" / "data" / "l0" + filenames = [ + "iois_1_packets_2026_090_05_03_05", + "iois_1_packets_2026_090_05_04_06", + "iois_1_packets_2026_090_05_05_07", + "iois_1_packets_2026_090_05_06_08", + "iois_1_packets_2026_090_05_07_09", + ] + return tuple(directory / fname for fname in filenames) + + +@pytest.fixture +def postlaunch_xarray_data(postlaunch_packet_path, sc_packet_path): + """Create xarray data for multiple packets.""" + apid = 478 + _, xtce_ialirt_path = sc_packet_path + + xarray_data = tuple( + packet_file_to_datasets(packet, xtce_ialirt_path, use_derived_value=False)[apid] + for packet in postlaunch_packet_path + ) + + merged_xarray_data = xr.concat(xarray_data, dim="epoch") + return merged_xarray_data + + @pytest.fixture def codice_test_data(test_datasets): return test_datasets[478] @@ -804,34 +851,38 @@ def test_process_codice_lo( @pytest.mark.external_test_data -@patch("imap_processing.ialirt.l0.process_codice.COD_HI_COUNTER", 197) -@patch( - "imap_processing.codice.constants.IAL_BIT_STRUCTURE", - OLD_IAL_BIT_STRUCTURE, -) -def test_process_codice_hi( - cod_hi_test_dataset, l1a_lut_path, l2_lut_path, cod_hi_l2_test_data -): +def test_process_codice_hi(postlaunch_xarray_data, cod_hi_l1a_test_data_transposed): """Test process_codice for hi.""" - test_data = cod_hi_l2_test_data["h"] - - n = cod_hi_test_dataset.dims["epoch"] - cod_hi_test_dataset = cod_hi_test_dataset.assign( - sc_sclk_sec=("epoch", np.zeros(n, dtype=np.int64)), - sc_sclk_sub_sec=("epoch", np.zeros(n, dtype=np.int64)), + grouped_cod_hi_data = find_groups( + postlaunch_xarray_data, (0, COD_HI_COUNTER), "cod_hi_counter", "cod_hi_acq" ) + unique_cod_hi_groups = np.unique(grouped_cod_hi_data["group"]) - _, cod_hi_data = process_codice( - cod_hi_test_dataset, l1a_lut_path, l2_lut_path, "codice_hi" - ) - samples_per_group = test_data.shape[0] // len(cod_hi_data) - grouped_test_data = test_data.reshape( - len(cod_hi_data), - samples_per_group, - *test_data.shape[1:], - ) + for group in unique_cod_hi_groups: + cod_hi_data_stream = concatenate_bytes(grouped_cod_hi_data, group, "hi") + cod_hi_science_values, cod_hi_metadata_values = process_ialirt_data_streams( + [cod_hi_data_stream] + ) + if not cod_hi_science_values: + continue + cod_hi_dataset = create_xarray_dataset( + cod_hi_science_values, cod_hi_metadata_values, "hi" + ) + l1a_lut_path = ( + imap_module_directory + / "tests" + / "codice" + / "data" + / "l1a_lut" + / "imap_codice_l1a-sci-lut_20260129_v002.json" + ) + l1a_hi = l1a_ialirt_hi(cod_hi_dataset, l1a_lut_path) - for i, group in enumerate(cod_hi_data): - arr = np.array(group["codice_hi_h"], dtype=float) + expected = cod_hi_l1a_test_data_transposed.sel( + epoch=l1a_hi["epoch"], method="nearest" + ) - np.testing.assert_allclose(arr, grouped_test_data[i], atol=3e-2, rtol=1e-5) + np.testing.assert_array_equal( + l1a_hi["h"].values, + expected["h"].data, + ) diff --git a/imap_processing/tests/ialirt/unit/test_process_hit.py b/imap_processing/tests/ialirt/unit/test_process_hit.py index e187843b79..25bea11a0b 100644 --- a/imap_processing/tests/ialirt/unit/test_process_hit.py +++ b/imap_processing/tests/ialirt/unit/test_process_hit.py @@ -172,13 +172,13 @@ def test_process_hit(xarray_data, caplog): # Tests that it functions normally hit_product = process_hit(xarray_data) - assert len(hit_product) == 1 + assert len(hit_product) == 15 assert hit_product[0]["hit_e_a_side_low_en"] == 0 assert hit_product[0]["hit_e_a_side_med_en"] == 0 assert hit_product[0]["hit_e_b_side_low_en"] == 0 assert hit_product[0]["hit_e_b_side_high_en"] == 0 - assert hit_product[0]["hit_e_b_side_med_en"] == 1 + assert hit_product[0]["hit_e_b_side_med_en"] == 0 assert hit_product[0]["hit_he_omni_high_en"] == 0 diff --git a/imap_processing/tests/ialirt/unit/test_process_status.py b/imap_processing/tests/ialirt/unit/test_process_status.py new file mode 100644 index 0000000000..b86bf7020c --- /dev/null +++ b/imap_processing/tests/ialirt/unit/test_process_status.py @@ -0,0 +1,52 @@ +"""Tests for the process_status module.""" + +import pytest +import xarray as xr + +from imap_processing import imap_module_directory +from imap_processing.ialirt.l0.process_status import process_status +from imap_processing.utils import packet_file_to_datasets + + +@pytest.fixture(scope="session") +def postlaunch_packet_path(): + """Returns the paths to the binary packets.""" + directory = imap_module_directory / "tests" / "ialirt" / "data" / "l0" + filenames = [ + "iois_1_packets_2026_090_05_03_05", + "iois_1_packets_2026_090_05_04_06", + "iois_1_packets_2026_090_05_05_07", + "iois_1_packets_2026_090_05_06_08", + "iois_1_packets_2026_090_05_07_09", + ] + return tuple(directory / fname for fname in filenames) + + +@pytest.fixture +def postlaunch_xarray_data(postlaunch_packet_path, sc_packet_path): + """Create xarray data for multiple packets.""" + apid = 478 + _, xtce_ialirt_path = sc_packet_path + + xarray_data = tuple( + packet_file_to_datasets(packet, xtce_ialirt_path, use_derived_value=False)[apid] + for packet in postlaunch_packet_path + ) + + merged_xarray_data = xr.concat(xarray_data, dim="epoch") + return merged_xarray_data + + +@pytest.mark.external_test_data +def test_process_status(postlaunch_xarray_data): + """Test the process_status function.""" + + status_data = process_status(postlaunch_xarray_data) + + for i in range(len(status_data)): + assert status_data[i]["sc_swapi_status"] == 1 + assert status_data[i]["sc_mag_status"] == 1 + assert status_data[i]["sc_hit_status"] == 1 + assert status_data[i]["sc_codice_status"] == 1 + assert status_data[i]["sc_lo_status"] == 1 + assert status_data[i]["sc_autonomy_status"] == 1 diff --git a/imap_processing/tests/idex/conftest.py b/imap_processing/tests/idex/conftest.py index 86a199e30b..7e22a3f829 100644 --- a/imap_processing/tests/idex/conftest.py +++ b/imap_processing/tests/idex/conftest.py @@ -13,14 +13,14 @@ TEST_DATA_PATH = imap_module_directory / "tests" / "idex" / "test_data" TEST_L0_FILE_SCI = TEST_DATA_PATH / "imap_idex_l0_raw_20231218_v001.pkts" -TEST_L0_FILE_EVT = TEST_DATA_PATH / "imap_idex_l0_raw_20250108_v001.pkts" # 1418 +TEST_L0_FILE_MSG = TEST_DATA_PATH / "imap_idex_l0_raw_20250108_v001.pkts" # 1418 TEST_L0_FILE_CATLST = TEST_DATA_PATH / "imap_idex_l0_raw_20241206_v001.pkts" # 1419 L1A_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1a_validation_file.h5" -L1B_EXAMPLE_FILE = TEST_DATA_PATH / "idex_l1b_validation_file.h5" +L1B_EXAMPLE_FILE = TEST_DATA_PATH / "imap_idex_l1b_sci_20231218_v002.h5" L2A_CDF = TEST_DATA_PATH / "imap_idex_l2a_sci-1week_20251017_v001.cdf" -L1B_EVT_CDF = TEST_DATA_PATH / "imap_idex_l1b_evt_20250108_v001.cdf" +L1B_MSG_CDF = TEST_DATA_PATH / "imap_idex_l1b_msg_20250108_v001.cdf" pytestmark = pytest.mark.external_test_data @@ -50,15 +50,27 @@ def decom_test_data_catlst() -> xr.Dataset: @pytest.fixture -def decom_test_data_evt() -> xr.Dataset: - """List of ``xarray`` datasets containing the raw and derived event log data. +def decom_test_data_msg() -> xr.Dataset: + """``xarray`` dataset containing the raw event message data. Returns ------- - dataset : list[xarray.Dataset] - A list of ``xarray`` datasets containing the event log datasets. + dataset : xarray.Dataset + ``xarray`` dataset containing the event log data. + """ + return PacketParser(TEST_L0_FILE_MSG).data[0] + + +@pytest.fixture +def test_l1b_msg(decom_test_data_msg) -> xr.Dataset: + """``xarray`` dataset containing the l1b msg data. + + Returns + ------- + dataset : xarray.Dataset + ``xarray`` dataset containing the event log data. """ - return PacketParser(TEST_L0_FILE_EVT).data + return idex_l1b(decom_test_data_msg, "msg") @pytest.fixture @@ -112,7 +124,7 @@ def l1b_dataset(mock_get_spice_data, decom_test_data_sci: xr.Dataset) -> xr.Data """ mock_get_spice_data.side_effect = get_spice_data_side_effect_func - dataset = idex_l1b(decom_test_data_sci) + dataset = idex_l1b(decom_test_data_sci, "sci-1week") return dataset diff --git a/imap_processing/tests/idex/test_data/idex_event_messages.csv b/imap_processing/tests/idex/test_data/idex_event_messages.csv new file mode 100644 index 0000000000..07e30e1246 --- /dev/null +++ b/imap_processing/tests/idex/test_data/idex_event_messages.csv @@ -0,0 +1,29 @@ +timestamp,message +2025-01-08 20:40:25.222800 UTC,"MEM FLASH UErr FOUND, PARAM=0x000000fe" +2025-01-08 20:40:51.222740 UTC,"UPK oper fsw says hello, version=02.00.0000" +2025-01-08 20:40:56.222700 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(dualcmd))" +2025-01-08 20:40:57.222700 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(rawwrt))" +2025-01-08 20:41:00.222680 UTC,"SEQ engine has changed state, eng=seqEngineDictionary(0x00) was=seqEngineStateDictionary(ACTIV) is=seqEngineStateDictionary(IDLE) 0x00" +2025-01-08 20:48:39.220040 UTC,AUT hvps state changed to hvStateDictionary(STANDBY) +2025-01-08 20:50:27.219540 UTC,AUT hvps state changed to hvStateDictionary(ACTIVE) +2025-01-08 20:56:32.218180 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(noop))" +2025-01-08 20:56:33.218160 UTC,"SEQ engine has changed state, eng=seqEngineDictionary(0x00) was=seqEngineStateDictionary(ACTIV) is=seqEngineStateDictionary(IDLE) 0x00" +2025-01-08 20:58:15.217920 UTC,SCI state change: sciState16Dictionary(IDLE) ==> sciState16Dictionary(ACQSETUP) +2025-01-08 20:58:20.217900 UTC,SCI state change: sciState16Dictionary(ACQSETUP) ==> sciState16Dictionary(ACQ) +2025-01-08 20:59:15.217800 UTC,SCI state change: sciState16Dictionary(ACQ) ==> sciState16Dictionary(CHILL) +2025-01-08 20:59:20.217800 UTC,SCI state change: sciState16Dictionary(CHILL) ==> sciState16Dictionary(ACQCLEANUP) +2025-01-08 20:59:21.217780 UTC,"MEM cleanup found sci data, blocks empty=0x0000, blocks w/data=0x0001" +2025-01-08 20:59:26.217780 UTC,SCI science activity completed: sciState16Dictionary(ACQ) +2025-01-08 20:59:27.217760 UTC,SCI state change: sciState16Dictionary(ACQCLEANUP) ==> sciState16Dictionary(IDLE) +2025-01-08 21:00:12.217700 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(haltstim))" +2025-01-08 21:00:14.217680 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(haltsci))" +2025-01-08 21:00:15.217680 UTC,"SEQ engine has changed state, eng=seqEngineDictionary(0x00) was=seqEngineStateDictionary(ACTIV) is=seqEngineStateDictionary(IDLE) 0x00" +2025-01-08 21:02:48.217480 UTC,AUT hvps state changed to hvStateDictionary(STANDBY) +2025-01-08 21:04:42.217340 UTC,AUT hvps state changed to hvStateDictionary(OFF) +2025-01-08 21:05:30.217280 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(dualcmd))" +2025-01-08 21:05:33.217280 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(rawwrt))" +2025-01-08 21:05:36.217280 UTC,"SEQ engine has changed state, eng=seqEngineDictionary(0x00) was=seqEngineStateDictionary(ACTIV) is=seqEngineStateDictionary(IDLE) 0x00" +2025-01-08 21:06:36.217180 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(dshtr))" +2025-01-08 21:06:38.217180 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(dualcmd))" +2025-01-08 21:06:39.217180 UTC,"SEQ success (len=0x0580, opCodeLCDictionary(rawwrt))" +2025-01-08 21:06:42.217180 UTC,"SEQ engine has changed state, eng=seqEngineDictionary(0x00) was=seqEngineStateDictionary(ACTIV) is=seqEngineStateDictionary(IDLE) 0x00" diff --git a/imap_processing/tests/idex/test_idex_l0.py b/imap_processing/tests/idex/test_idex_l0.py index 736c0565df..425168c74c 100644 --- a/imap_processing/tests/idex/test_idex_l0.py +++ b/imap_processing/tests/idex/test_idex_l0.py @@ -14,7 +14,7 @@ def test_idex_decom_length(decom_test_data_sci: xr.Dataset): decom_test_data_sci : xarray.Dataset The dataset to test with """ - assert len(decom_test_data_sci) == 110 + assert len(decom_test_data_sci) == 109 def test_idex_decom_event_num(decom_test_data_sci: xr.Dataset): @@ -61,13 +61,12 @@ def test_catlst_event_num(decom_test_data_catlst: list[xr.Dataset]): assert len(ds["epoch"]) == 1 -def test_evt_event_num(decom_test_data_evt: list[xr.Dataset]): +def test_msg_event_num(decom_test_data_msg: xr.Dataset): """Verify that a sample of the data is correct. Parameters ---------- - decom_test_data_evt : list[xarray.Dataset] - The raw and derived (l1a and l1b) datasets to test with. + decom_test_data_msg : xarray.Dataset + Event message data. """ - for ds in decom_test_data_evt: - assert len(ds["epoch"]) == 28 + assert len(decom_test_data_msg["epoch"]) == 28 diff --git a/imap_processing/tests/idex/test_idex_l1a.py b/imap_processing/tests/idex/test_idex_l1a.py index 6dfee849ff..7c038c10cb 100644 --- a/imap_processing/tests/idex/test_idex_l1a.py +++ b/imap_processing/tests/idex/test_idex_l1a.py @@ -4,6 +4,7 @@ from unittest import mock import numpy as np +import pandas as pd import pytest import xarray as xr from cdflib.xarray.xarray_to_cdf import ISTPError @@ -16,6 +17,8 @@ from imap_processing.tests.idex.conftest import TEST_L0_FILE_SCI from imap_processing.utils import packet_generator +TEST_DATA_DIR = f"{imap_module_directory}/tests/idex/test_data" + def test_idex_cdf_file(decom_test_data_sci: xr.Dataset): """Verify the CDF file can be created with no errors. @@ -156,11 +159,22 @@ def test_validate_l1a_idex_data_variables( "Ion Grid": "Ion_Grid", "Time (high sampling)": "time_high_sample_rate", "Time (low sampling)": "time_low_sample_rate", + "idx__txhdrfswaidcopy": "aid", } # The Engineering data is converting to UTC, and the SDC is converting to J2000, # for 'epoch' and 'Timestamp' so this test is using the raw time value 'SCHOARSE' to # validate time - arrays_to_skip = ["Timestamp", "Epoch", "event"] + # TODO remove the low and high time from this list after the IDEX team produces a + # new l1a h5 file. + arrays_to_skip = [ + "Timestamp", + "Epoch", + "event", + "Time (high sampling)", + "Time (low sampling)", + "IDX__SCI0AID", # This is dropped because it is invalid + "IDX__TXHDRFSWAIDCOPY", # this is renamed to aid + ] # loop through all keys from the l1a example dict for var in l1a_example_data.variables: @@ -180,10 +194,9 @@ def test_compressed_packet(): """ Test compressed data decompression against known non-compressed data. """ - test_data_dir = f"{imap_module_directory}/tests/idex/test_data" - compressed = Path(f"{test_data_dir}/compressed_2023_102_14_24_55.pkts") - non_compressed = Path(f"{test_data_dir}/non_compressed_2023_102_14_22_26.pkts") + compressed = Path(f"{TEST_DATA_DIR}/compressed_2023_102_14_24_55.pkts") + non_compressed = Path(f"{TEST_DATA_DIR}/non_compressed_2023_102_14_22_26.pkts") decompressed = PacketParser(compressed).data[0] expected = PacketParser(non_compressed).data[0] @@ -341,25 +354,29 @@ def test_catlst_dataset(decom_test_data_catlst: list[xr.Dataset]): assert filename_l1b.name == "imap_idex_l1b_catlst_20241206_v999.cdf" -def test_evt_dataset(decom_test_data_evt: list[xr.Dataset]): +def test_msg_dataset(decom_test_data_msg: xr.Dataset): """Verify that the dataset contains what we expect and can be written to a cdf. Parameters ---------- - decom_test_data_evt : list[xarray.Dataset] - The raw and derived (l1a and l1b) datasets to test with. + decom_test_data_msg : xarray.Dataset + The raw l1a dataset to test with. """ - for ds in decom_test_data_evt: - assert "shcoarse" in ds - assert "shfine" in ds - # Assert epoch is calculated using fine grained clock ticks - expected_epoch = met_to_ttj2000ns(ds["shcoarse"] + ds["shfine"] * 20e-6) - np.testing.assert_array_equal(ds.epoch, expected_epoch) - assert decom_test_data_evt[0]["elid_evtpkt"][9] == 192 - assert decom_test_data_evt[1]["elid_evtpkt"][9] == "SCI_STE" + assert "shcoarse" in decom_test_data_msg + assert "shfine" in decom_test_data_msg + # Assert epoch is calculated using fine grained clock ticks + expected_epoch = met_to_ttj2000ns( + decom_test_data_msg["shcoarse"] + decom_test_data_msg["shfine"] * 20e-6 + ) + np.testing.assert_array_equal(decom_test_data_msg.epoch, expected_epoch) # Assert that the dataset can be written to a CDF file - filename_l1a = write_cdf(decom_test_data_evt[0]) - assert filename_l1a.name == "imap_idex_l1a_evt_20250108_v999.cdf" + filename_l1a = write_cdf(decom_test_data_msg) + assert filename_l1a.name == "imap_idex_l1a_msg_20250108_v999.cdf" + + # Validate the messages with the IDEX team example data + example_data = pd.read_csv( + f"{TEST_DATA_DIR}/idex_event_messages.csv", skiprows=1, header=None + ) - filename_l1b = write_cdf(decom_test_data_evt[1]) - assert filename_l1b.name == "imap_idex_l1b_evt_20250108_v999.cdf" + messages = example_data.iloc[:, 1].tolist() + np.testing.assert_array_equal(decom_test_data_msg["messages"].data, messages) diff --git a/imap_processing/tests/idex/test_idex_l1b.py b/imap_processing/tests/idex/test_idex_l1b.py index 3cf7a76e22..7387cd8fde 100644 --- a/imap_processing/tests/idex/test_idex_l1b.py +++ b/imap_processing/tests/idex/test_idex_l1b.py @@ -11,10 +11,16 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import write_cdf from imap_processing.idex.idex_l1b import ( + TRIGGER_LABELS, + EventMessage, + TriggerOrigin, get_spice_data, get_trigger_mode_and_level, + get_trigger_origin, + idex_l1b, unpack_instrument_settings, ) +from imap_processing.idex.idex_utils import get_idex_attrs from imap_processing.tests.idex import conftest @@ -149,49 +155,79 @@ def test_get_trigger_settings_success(decom_test_data_sci): # correct when the modes and levels vary from event to event decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 1 decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 0 - + idex_attrs = get_idex_attrs("l1b") n_epochs = len(decom_test_data_sci["epoch"]) - trigger_settings = get_trigger_mode_and_level(decom_test_data_sci) + trigger_settings = get_trigger_mode_and_level(decom_test_data_sci, idex_attrs) + + expected_modes_lg = np.full(n_epochs, None) + expected_modes_hg = expected_modes_lg.copy() + expected_modes_hg[1:] = "HGThreshold" + expected_modes_mg = np.full(n_epochs, None) + expected_modes_mg[0] = "MGThreshold" + expected_levels_lg = np.full(n_epochs, np.nan) + expected_levels_hg = expected_levels_lg.copy() + expected_levels_hg[1:] = 0.16762 + expected_levels_mg = expected_levels_lg.copy() + expected_levels_mg[0] = 1023.0 * 1.13e-2 + + var_names = ["trigger_mode_lg", "trigger_mode_mg", "trigger_mode_hg"] + expected_modes = [expected_modes_lg, expected_modes_mg, expected_modes_hg] + for expected_mode, mode_name in zip(expected_modes, var_names, strict=False): + ( + np.testing.assert_array_equal( + trigger_settings[mode_name].data, + expected_mode, + err_msg=f"The dict entry {mode_name} values did not match the" + f" expected values: {expected_mode}. Found:" + f" {trigger_settings[mode_name].data}", + ), + ) + var_names = ["trigger_level_lg", "trigger_level_mg", "trigger_level_hg"] + expected_levels = [expected_levels_lg, expected_levels_mg, expected_levels_hg] + for expected_level, level_name in zip(expected_levels, var_names, strict=False): + ( + np.testing.assert_array_equal( + trigger_settings[level_name].data, + expected_level, + err_msg=f"The dic entry {level_name} values did not match the" + f" expected values: {expected_level}. Found: " + f"{trigger_settings[level_name].data}", + ), + ) - expected_modes = np.full(n_epochs, "HGThreshold") - expected_modes[0] = "MGThreshold" - expected_levels = np.full(n_epochs, 0.16762) - expected_levels[0] = 1023.0 * 1.13e-2 - assert (trigger_settings["triggermode"].data == expected_modes).all(), ( - f"The dict entry 'triggermode' values did not match the expected values: " - f"{expected_modes}. Found: {trigger_settings['triggermode'].data}" - ) +def test_trigger_origin(): + """Check that the correct labels are produced for trigger origin values""" - assert (trigger_settings["triggerlevel"].data == expected_levels).all(), ( - f"The dict entry 'triggerlevel' values did not match the expected values: " - f"{expected_levels}. Found: {trigger_settings['triggerlevel'].data}" + trigger_bits = np.full(10, 6) + origins = get_trigger_origin(trigger_bits, get_idex_attrs("l1b")) + # Bits 1 and 2 should be set for all events + expected_origin = np.full( + 10, + ", ".join([TRIGGER_LABELS[TriggerOrigin(1)], TRIGGER_LABELS[TriggerOrigin(2)]]), + ) + np.testing.assert_array_equal( + origins["trigger_origin"], + expected_origin, + err_msg=f"The trigger origin values did not match the expected values: " + f"{expected_origin}. Found: {origins}", ) -def test_get_trigger_settings_failure(decom_test_data_sci): - """ - Check that an error is thrown when there are more than one valid trigger for an - event +def test_invalid_trigger_origin(): + """Check the labels when there are invalid trigger origin values""" - Parameters - ---------- - decom_test_data_sci : xarray.Dataset - L1a dataset - """ - decom_test_data_sci["idx__txhdrhgtrigmode"][0] = 1 - decom_test_data_sci["idx__txhdrmgtrigmode"][0] = 2 - - error_ms = ( - "Only one channel can trigger a dust event. Please make sure there is " - "only one valid trigger value per event. This caused Merge Error: " - "conflicting values for variable 'trigger_mode' on objects to be " - "combined. You can skip this check by specifying compat='override'." + trigger_bits = np.full(10, 64) # invalid trigger origin values + origins = get_trigger_origin(trigger_bits, get_idex_attrs("l1b")) + # Bits 1 and 2 should be set for all events + expected_origin = np.full(10, "Unknown trigger origin") + np.testing.assert_array_equal( + origins["trigger_origin"], + expected_origin, + err_msg=f"The trigger origin values did not match the expected values:" + f"{expected_origin}. Found: {origins}", ) - with pytest.raises(ValueError, match=error_ms): - get_trigger_mode_and_level(decom_test_data_sci) - @pytest.mark.usefixtures("use_fake_spin_data_for_time") def test_get_spice_data( @@ -260,6 +296,10 @@ def test_validate_l1b_idex_data_variables( "voltage_3V3_op_ref": "voltage_3p3_op_ref", "voltage_3V3_ref": "voltage_3p3_ref", "voltage_pos3V3_bus": "voltage_pos3p3v_bus", + "HGTriggerLevel": "trigger_level_hg", + "MGTriggerLevel": "trigger_level_mg", + "LGTriggerLevel": "trigger_level_lg", + "TriggerOrigin": "trigger_origin", } # The Engineering data is converting to UTC, and the SDC is converting to J2000, @@ -268,7 +308,7 @@ def test_validate_l1b_idex_data_variables( # SPICE data is mocked. arrays_to_skip = [ "Timestamp", - "Epoch", + "epoch", "Pitch", "Roll", "Yaw", @@ -280,29 +320,102 @@ def test_validate_l1b_idex_data_variables( "VelocityY", "VelocityZ", "RightAscension", + "FIFODelay", + "FIFODelayMicroseconds", + "FIFODelay_H", + "FIFODelay_L", + "FIFODelay_M", + "HSPosttriggerBlocks", ] + # assert that "aid" is in l1b + assert "aid" in l1b_dataset, "The array 'aid' is missing from the l1b dataset." + # select only the first n events + l1b_example_data = l1b_example_data.isel( + event=np.arange(l1b_dataset.sizes["epoch"]) + ) # Compare each corresponding variable for var in l1b_example_data.data_vars: if var not in arrays_to_skip: # Get the corresponding array name cdf_var = match_variables.get(var, var.lower().replace(".", "p")) - warning = ( f"The array '{cdf_var}' does not equal the expected example array " + f"'{var}' produced by the IDEX team" ) - f"'{var}' produced by the IDEX team" - + # TODO remove this block once the IDEX team fixes the l1b validation file. + # They included a lot of extra variables in the current file. + try: + l1b_dataset[cdf_var] + except KeyError: + continue if l1b_dataset[cdf_var].dtype == object: - assert (l1b_dataset[cdf_var].data == l1b_example_data[var]).all(), ( - warning - ) + assert ( + l1b_dataset[cdf_var].data == np.squeeze(l1b_example_data[var]) + ).all(), warning else: - ( - np.testing.assert_array_almost_equal( - l1b_dataset[cdf_var].data, - l1b_example_data[var], - decimal=4, - ), - warning, + np.testing.assert_array_almost_equal( + l1b_dataset[cdf_var].data, + np.squeeze(l1b_example_data[var]), + decimal=4, + err_msg=warning, ) + + +def test_l1b_msg_processing(decom_test_data_msg: xr.Dataset): + """Verify that the MSG data is being processed correctly in the l1b processing. + + Parameters + ---------- + decom_test_data_msg : xr.Dataset + A dataset containing the MSG data produced by the l1a processing. + """ + msg_ds = decom_test_data_msg.copy() + # Set 2 consecutive events to have pulser on and pulser off + msg_ds.messages[2] = EventMessage.PULSER_ON.value + msg_ds.messages[3] = EventMessage.PULSER_OFF.value + # Set 2 to have a non-consecutive pulser on and pulser off to check that + # non-consecutive events are treated as non-valid pulser on and off events + msg_ds.messages[20] = EventMessage.PULSER_ON.value + msg_ds.messages[22] = EventMessage.PULSER_OFF.value + # Process the MSG data with the l1b function + test_l1b_msg = idex_l1b(msg_ds, "msg") + expected_vars = [ + "epoch", + "pulser_on", + "science_on", + ] + for var in expected_vars: + assert var in test_l1b_msg, ( + f"The variable '{var}' is missing from the MSG dataset." + ) + + # Check that the pulser_on variable is correct + expected_pulser_on = np.ones_like(test_l1b_msg["pulser_on"]) * 255 + # The pulser_on variable should be 1 for the 1st and 0 for the 2nd event, and + # 255 for all other events + expected_pulser_on[0] = 1 + expected_pulser_on[1] = 0 + # Check that the science_on variable is correct + expected_science_on = np.ones_like(test_l1b_msg["pulser_on"]) * 255 + # The science_on variable should be 1 for the 3rd event and 0 for the 4th event + expected_science_on[2] = 1 + expected_science_on[3] = 0 + np.testing.assert_array_equal(test_l1b_msg["pulser_on"].data, expected_pulser_on) + np.testing.assert_array_equal(test_l1b_msg["science_on"].data, expected_science_on) + + +def test_no_valid_messages(decom_test_data_msg: xr.Dataset): + """Verify that if there are no valid pulser and science events then no dataset is + returned. + + Parameters + ---------- + decom_test_data_msg : xr.Dataset + A dataset containing the MSG data produced by the l1a processing. + """ + msg_ds = decom_test_data_msg.copy() + # Set all messages to a value that is not a valid pulser on or off event + msg_ds.messages[:] = "Not a science or pulser event" + result = idex_l1b(msg_ds, "msg") + assert result is None diff --git a/imap_processing/tests/idex/test_idex_l2a.py b/imap_processing/tests/idex/test_idex_l2a.py index 5b4517de83..9dd96feeb7 100644 --- a/imap_processing/tests/idex/test_idex_l2a.py +++ b/imap_processing/tests/idex/test_idex_l2a.py @@ -49,7 +49,7 @@ def l2a_dataset( "imap_processing.idex.idex_l1b.get_spice_data", return_value={"spin_phase": spin_phase_angles}, ): - dataset = idex_l2a(idex_l1b(decom_test_data_sci), ancillary_files) + dataset = idex_l2a(idex_l1b(decom_test_data_sci, "sci-1week"), ancillary_files) return dataset diff --git a/imap_processing/tests/idex/test_idex_l2b.py b/imap_processing/tests/idex/test_idex_l2b.py index 0d4692e042..721c274526 100644 --- a/imap_processing/tests/idex/test_idex_l2b.py +++ b/imap_processing/tests/idex/test_idex_l2b.py @@ -1,13 +1,11 @@ """Tests the L2b processing for IDEX data""" -from unittest import mock - import numpy as np import pytest import xarray as xr from numpy.testing import assert_array_equal -from imap_processing.cdf.utils import load_cdf, write_cdf +from imap_processing.cdf.utils import write_cdf from imap_processing.idex.idex_constants import ( FG_TO_KG, IDEX_SPACING_DEG, @@ -23,14 +21,12 @@ compute_counts_by_charge_and_mass, compute_rates_by_charge_and_mass, get_science_acquisition_on_percentage, - get_science_acquisition_timestamps, idex_l2b, ) -from imap_processing.tests.idex.conftest import L1B_EVT_CDF @pytest.fixture -def l2b_and_l2c_datasets(l2a_dataset: xr.Dataset) -> list[xr.Dataset]: +def l2b_and_l2c_datasets(l2a_dataset: xr.Dataset, test_l1b_msg) -> list[xr.Dataset]: """Return a ``xarray`` dataset containing test data. Returns @@ -38,17 +34,16 @@ def l2b_and_l2c_datasets(l2a_dataset: xr.Dataset) -> list[xr.Dataset]: datasets : list[xr.Dataset] A list of ``xarray`` datasets containing the test data for L2B and L2C. """ - l1b_evt_dataset = load_cdf(L1B_EVT_CDF) - l1b_evt_dataset2 = ( - l1b_evt_dataset.copy() + l1b_msg_dataset2 = ( + test_l1b_msg.copy() ) # Add a second dataset with different epoch values for testing l2a_dataset2 = ( l2a_dataset.copy() ) # Add a second dataset with different epoch values for testing - l1b_evt_dataset2["epoch"] = l1b_evt_dataset2["epoch"] + NANOSECONDS_IN_DAY + l1b_msg_dataset2["epoch"] = l1b_msg_dataset2["epoch"] + NANOSECONDS_IN_DAY l2a_dataset2["epoch"] = l2a_dataset2["epoch"] + NANOSECONDS_IN_DAY datasets = idex_l2b( - [l2a_dataset, l2a_dataset2], [l1b_evt_dataset, l1b_evt_dataset2] + [l2a_dataset, l2a_dataset2], [test_l1b_msg.copy(), l1b_msg_dataset2] ) return datasets @@ -177,44 +172,25 @@ def test_bin_spin_phases_warning(caplog): ) in caplog.text -def test_science_acquisition_times(decom_test_data_evt: list[xr.Dataset]): - """Tests that the expected science acquisition times and messages are present. - - Parameters - ---------- - decom_test_data_evt : list[xr.Dataset] - A ``xarray`` dataset containing the test data - """ - logs, times, vals = get_science_acquisition_timestamps(decom_test_data_evt[1]) - # For this example event message dataset we expect science acquisition events. - assert len(logs) == 2 - assert len(times) == 2 - assert len(vals) == 2 - # The first event message is the start of the science acquisition. - assert logs[0] == "SCI state change: ACQSETUP to ACQ" - # The second event message is the end of the science acquisition. - assert logs[1] == "SCI state change: ACQ to CHILL" - - # assert the values are correct - np.testing.assert_array_equal(vals, [1, 0]) - - -def test_get_science_acquisition_on_percentage(decom_test_data_evt: list[xr.Dataset]): +def test_get_science_acquisition_on_percentage(test_l1b_msg: xr.Dataset): """Test the function that calculates the percentage of uptime.""" - _, evt_time, evt_event = get_science_acquisition_timestamps(decom_test_data_evt[1]) - on_percentages = get_science_acquisition_on_percentage(evt_time, evt_event) - # We expect 1 DOY and ~87% uptime for the science acquisition. + test_l1b_msg = test_l1b_msg.isel(epoch=np.isin(test_l1b_msg.science_on, [0, 1])) + msg_time = test_l1b_msg.epoch.data + msg_event = test_l1b_msg.science_on.data + on_percentages = get_science_acquisition_on_percentage(msg_time, msg_event) + # We expect 1 DOY with less than 1% uptime for the science acquisition. assert len(on_percentages) == 1 # The DOY should be 8 for this test dataset. assert on_percentages[8] < 1 - evt_ds = decom_test_data_evt[1].copy() - evt_ds_shifted = evt_ds.copy() - evt_ds_shifted["epoch"] = evt_ds["epoch"] + NANOSECONDS_IN_DAY - combined_ds = xr.concat([evt_ds, evt_ds_shifted], dim="epoch") + msg_ds = test_l1b_msg.copy() + msg_ds_shifted = msg_ds.copy() + msg_ds_shifted["epoch"] = msg_ds["epoch"] + NANOSECONDS_IN_DAY + combined_ds = xr.concat([msg_ds, msg_ds_shifted], dim="epoch") # expect a second DOY. - _, evt_time, evt_event = get_science_acquisition_timestamps(combined_ds) - on_percentages = get_science_acquisition_on_percentage(evt_time, evt_event) + msg_time = combined_ds.epoch.data + msg_event = combined_ds.science_on.data + on_percentages = get_science_acquisition_on_percentage(msg_time, msg_event) # We expect 2 DOYs assert len(on_percentages) == 2 # The uptime should be less than 1% for both @@ -224,13 +200,7 @@ def test_get_science_acquisition_on_percentage(decom_test_data_evt: list[xr.Data def test_get_science_acquisition_on_percentage_no_acquisition(caplog): """Test the function returns an empty dict when there is no science acquisition.""" - with mock.patch( - "imap_processing.idex.idex_l2b.get_science_acquisition_timestamps", - return_value=([], [], []), - ): - on_percentages = get_science_acquisition_on_percentage( - np.array([]), np.array([]) - ) + on_percentages = get_science_acquisition_on_percentage(np.array([]), np.array([])) assert not on_percentages assert "No science acquisition events found" in caplog.text diff --git a/imap_processing/tests/lo/test_lo_l1b.py b/imap_processing/tests/lo/test_lo_l1b.py index 2ff7b947ba..a6fc24b845 100644 --- a/imap_processing/tests/lo/test_lo_l1b.py +++ b/imap_processing/tests/lo/test_lo_l1b.py @@ -28,6 +28,7 @@ get_spin_start_times, identify_species, initialize_l1b_de, + l1b_bgrates_and_goodtimes, l1b_star, lo_l1b, resweep_histogram_data, @@ -42,6 +43,7 @@ set_pointing_direction, set_spin_cycle, set_spin_cycle_from_spin_data, + split_backgrounds_and_goodtimes_dataset, ) from imap_processing.lo.lo_ancillary import read_ancillary_file from imap_processing.spice.spin import get_spin_data @@ -719,7 +721,7 @@ def test_convert_tofs_to_eu(attr_mgr_l1b, attr_mgr_l1a): tof0_expected = np.array([1.394394, 0.889272]) tof1_expected = np.array([0.931059, tof_fill_l1b]) tof2_expected = np.array([2.870557, 1.372876]) - tof3_expected = np.array([3.88245, 1.818162]) + tof3_expected = np.array([3.89606, 1.83878]) # Act l1b_de = convert_tofs_to_eu(l1a_de, l1b_de, attr_mgr_l1a, attr_mgr_l1b) @@ -1048,6 +1050,9 @@ def test_calculate_histogram_rates(l1b_histrates): exposure_factors_60deg = np.zeros((2, 7, 6)) exposure_factors_6deg[0, 0, 0] = 1 exposure_factors_60deg[0, 0, 0] = 1 + exposure_factors_6deg[0, 1, 0] = 0 + exposure_factors_60deg[0, 1, 0] = 0 + exposure_factors = {} exposure_factors["6deg"] = exposure_factors_6deg exposure_factors["60deg"] = exposure_factors_60deg @@ -1613,7 +1618,7 @@ def test_filters_by_count_and_time_window(self, mock_repoint): }, coords={"epoch": [0, 1, 2, 3, 4]}, ) - # Time window: [5s, 25s] - should include epochs 1 and 2 + # # Time window: [5s, 25s] - should include epochs 1 and 2 expected_mask = np.array([False, True, True, False, False]) # Act @@ -1720,10 +1725,11 @@ def test_profile_for_group_end_bins_excluded(self): assert count_per_bin[719] == 0 # All other bins should have count=2 assert np.all(count_per_bin[:718] == 2) - # End bins should have FILLVAL - assert np.all(np.isnan(avg_amplitude[718:])) - # Middle bins should have average value + # Averages should be 100 for all bins except the excluded ones assert np.all(avg_amplitude[:718] == 100.0) + # Excluded bins should be NaN + assert np.isnan(avg_amplitude[718]) + assert np.isnan(avg_amplitude[719]) def test_profile_for_group_empty_data(self): """Test handling of empty data array.""" @@ -1749,6 +1755,7 @@ def test_profiles_by_group_creates_correct_groups(self, mock_repoint): mock_repoint.return_value = pd.DataFrame( {"repoint_in_progress": [False] * n_records} ) + met_times = np.arange(n_records, dtype=np.float64) * 15.0 l1a_star = xr.Dataset( { "count": ("epoch", [720] * n_records), @@ -1762,7 +1769,7 @@ def test_profiles_by_group_creates_correct_groups(self, mock_repoint): ), }, coords={ - "epoch": met_to_ttj2000ns(np.arange(n_records) * 15.0), + "epoch": met_to_ttj2000ns(met_times), "samples": np.arange(720), }, ) @@ -1885,7 +1892,10 @@ def test_initializes_with_spin_data( l1a_star = xr.Dataset( { "count": ("epoch", [720] * n_records), - "shcoarse": ("epoch", met_times), + "shcoarse": ( + "epoch", + np.arange(n_records, dtype=np.float64) * 15.0, + ), "data": ( ("epoch", "samples"), np.random.randint(100, 200, size=(n_records, 720), dtype=np.uint16), @@ -2109,7 +2119,10 @@ def test_multiple_groups_created( l1a_star = xr.Dataset( { "count": ("epoch", [720] * n_records), - "shcoarse": ("epoch", met_times), + "shcoarse": ( + "epoch", + np.arange(n_records, dtype=np.float64) * 15.0, + ), "data": ( ("epoch", "samples"), np.ones((n_records, 720), dtype=np.uint16) * 100, @@ -2176,3 +2189,801 @@ def test_get_pivot_angle_from_nhk(): # Assert assert pivot_angle == expected_pivot_angle + + +def test_l1b_bgrates_and_goodtimes_basic(attr_mgr_l1b): + """Test basic functionality of l1b_bgrates_and_goodtimes.""" + # Arrange - Create a simple L1B histogram rates dataset + # with enough data points to create goodtime intervals + num_epochs = 100 # 10 cycles of 10 epochs each + met_start = 473389200 # Start MET time + met_spacing = 42 # seconds between epochs + + # Create evenly spaced MET times + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Create counts data with low background rates (below threshold) + # h_bg_rate_nom = 0.0028, exposure = 420*10*0.5 = 2100 seconds + # To be below threshold: rate = counts / exposure < 0.0028 + # Summed over 7 ESA steps * 30 azimuth bins * 10 epochs = 2100 values + # Max total counts per chunk: 0.0028 * 2100 = 5.88 counts + # Use 10% of max for safety: 5.88 / 2100 / 10 = 0.00028 per element + h_counts_per_epoch = 0.00028 # Low counts to ensure below threshold + o_counts_per_epoch = 0.000028 # 10x smaller for oxygen + + h_counts = np.ones((num_epochs, 7, 60)) * h_counts_per_epoch + o_counts = np.ones((num_epochs, 7, 60)) * o_counts_per_epoch + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert - Should return a list with two datasets + assert isinstance(result, list) + assert len(result) == 2 + + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Check bgrates dataset structure + assert "h_background_rates" in l1b_bgrates_ds.data_vars + assert "h_background_variance" in l1b_bgrates_ds.data_vars + assert "o_background_rates" in l1b_bgrates_ds.data_vars + assert "o_background_variance" in l1b_bgrates_ds.data_vars + # Note: bgrates uses 'met' dimension, goodtimes has epoch in data vars + + # Check goodtimes dataset structure + assert "gt_start_met" in l1b_goodtimes_ds.data_vars + assert "gt_end_met" in l1b_goodtimes_ds.data_vars + assert "bin_start" in l1b_goodtimes_ds.data_vars + assert "bin_end" in l1b_goodtimes_ds.data_vars + assert "esa_goodtime_flags" in l1b_goodtimes_ds.data_vars + + # Check dimensions + assert l1b_bgrates_ds["h_background_rates"].dims == ("met", "esa_step") + assert l1b_bgrates_ds["h_background_rates"].shape[1] == 7 # 7 ESA steps + + # Check that goodtime intervals were created + assert len(l1b_goodtimes_ds["gt_start_met"]) > 0 + assert len(l1b_goodtimes_ds["gt_end_met"]) > 0 + + # Check that start times are before end times + assert np.all( + l1b_goodtimes_ds["gt_start_met"].values <= l1b_goodtimes_ds["gt_end_met"].values + ) + + # Check bin_start and bin_end values + assert np.all(l1b_goodtimes_ds["bin_start"].values == 0) + assert np.all(l1b_goodtimes_ds["bin_end"].values == 59) + + # Check ESA goodtime flags are all 1 (good) + assert np.all(l1b_goodtimes_ds["esa_goodtime_flags"].values == 1) + + +def test_l1b_bgrates_and_goodtimes_with_gap(attr_mgr_l1b): + """Test l1b_bgrates_and_goodtimes handles data gaps correctly.""" + # Arrange - Create dataset with a large gap in the middle + num_epochs_first = 50 + num_epochs_second = 50 + met_start = 473389200 + met_spacing = 42 + gap_size = 10000 # Large gap (> delay_max + interval_nom) + + # First segment + met_times_first = np.arange( + met_start, met_start + num_epochs_first * met_spacing, met_spacing + ) + # Second segment after gap + met_times_second = np.arange( + met_start + num_epochs_first * met_spacing + gap_size, + met_start + + num_epochs_first * met_spacing + + gap_size + + num_epochs_second * met_spacing, + met_spacing, + ) + + met_times = np.concatenate([met_times_first, met_times_second]) + epoch_times = met_to_ttj2000ns(met_times) + + # Low background counts (below threshold) + h_counts = np.ones((len(met_times), 7, 60)) * 0.00028 + o_counts = np.ones((len(met_times), 7, 60)) * 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Should create at least 2 separate goodtime intervals (before and after gap) + assert len(l1b_goodtimes_ds["gt_start_met"]) >= 2 + + # Check that intervals don't span across the gap + for i in range(len(l1b_goodtimes_ds["gt_start_met"])): + interval_duration = ( + l1b_goodtimes_ds["gt_end_met"].values[i] + - l1b_goodtimes_ds["gt_start_met"].values[i] + ) + # No interval should be as large as the gap + assert interval_duration < gap_size + + +def test_l1b_bgrates_and_goodtimes_high_rate(attr_mgr_l1b): + """Test l1b_bgrates_and_goodtimes handles high count rates correctly.""" + # Arrange - Create dataset with high rates that exceed threshold + num_epochs = 100 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Create high counts (above threshold) + # h_bg_rate_nom = 0.0028, exposure = 420*10*0.5 = 2100 seconds + # To be above threshold: rate > 0.0028 + # Use 10x threshold for high rate periods: 0.028 counts/sec + # That's 0.028 * 2100 / 2100_values = 0.028 per element + h_counts = np.ones((num_epochs, 7, 60)) * 0.028 # High rate (10x threshold) + o_counts = np.ones((num_epochs, 7, 60)) * 0.0028 + + # Make first 20 epochs low (below threshold) + h_counts[:20, :, :] = 0.00028 + o_counts[:20, :, :] = 0.000028 + + # Make middle 60 epochs high (above threshold) + h_counts[20:80, :, :] = 0.028 + o_counts[20:80, :, :] = 0.0028 + + # Make last 20 epochs low again + h_counts[80:, :, :] = 0.00028 + o_counts[80:, :, :] = 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Should create at least 2 intervals (before and after high rate period) + assert len(l1b_goodtimes_ds["gt_start_met"]) >= 2 + + # Check that background rates were calculated + assert np.all(l1b_bgrates_ds["h_background_rates"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_rates"].values > 0) + + +def test_l1b_bgrates_and_goodtimes_no_goodtimes(attr_mgr_l1b): + """When no goodtimes are detected the function should still return datasets.""" + num_epochs = 50 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Make counts high everywhere so no low-rate goodtime intervals are found + h_counts = np.ones((num_epochs, 7, 60)) * 0.1 + o_counts = np.ones((num_epochs, 7, 60)) * 0.01 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + bgrates_ds, goodtimes_ds = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Function should return two datasets + assert "h_background_rates" in bgrates_ds.data_vars + # Goodtimes dataset should exist and contain the gt_* fields + # (defaults when none found) + assert "gt_start_met" in goodtimes_ds.data_vars + assert "gt_end_met" in goodtimes_ds.data_vars + # When no goodtimes were detected the default invalid times are used (zeros) + assert int(goodtimes_ds["gt_start_met"].values[0]) == 0 + assert int(goodtimes_ds["gt_end_met"].values[0]) == 0 + assert int(bgrates_ds["start_met"].values[0]) == 0 + assert int(bgrates_ds["end_met"].values[0]) == 0 + + +def test_l1b_bgrates_and_goodtimes_custom_cycle_count(attr_mgr_l1b): + """Test l1b_bgrates_and_goodtimes with custom cycle_count parameter.""" + # Arrange + num_epochs = 50 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Low counts (below threshold) + h_counts = np.ones((num_epochs, 7, 60)) * 0.00028 + o_counts = np.ones((num_epochs, 7, 60)) * 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act - Use different cycle_count + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=5, delay_max=420 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Should successfully create datasets with custom parameters + assert len(l1b_goodtimes_ds["gt_start_met"]) > 0 + # Background rates should be calculated from the low-count period + assert np.all(l1b_bgrates_ds["h_background_rates"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_rates"].values > 0) + + +def test_l1b_bgrates_and_goodtimes_empty_dataset(attr_mgr_l1b): + """Test l1b_bgrates_and_goodtimes handles edge case with minimal data.""" + # Arrange - Create minimal dataset (just enough for one cycle) + num_epochs = 10 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Low counts (below threshold) + h_counts = np.ones((num_epochs, 7, 60)) * 0.00028 + o_counts = np.ones((num_epochs, 7, 60)) * 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert - Should still create valid datasets even with minimal data + l1b_bgrates_ds, l1b_goodtimes_ds = result + + assert "h_background_rates" in l1b_bgrates_ds.data_vars + assert "gt_start_met" in l1b_goodtimes_ds.data_vars + + +def test_split_backgrounds_and_goodtimes_dataset(attr_mgr_l1b): + """Test split_backgrounds_and_goodtimes_dataset separates fields correctly.""" + # Arrange - Create a combined dataset with both background and goodtime fields + num_records = 5 + epoch_times = met_to_ttj2000ns( + np.arange(473389200, 473389200 + num_records * 420, 420) + ) + + combined_ds = xr.Dataset( + { + # Background rate fields + "epoch": ("epoch", epoch_times), + "h_background_rates": (("met", "esa_step"), np.random.rand(num_records, 7)), + "h_background_variance": ( + ("met", "esa_step"), + np.random.rand(num_records, 7), + ), + "o_background_rates": (("met", "esa_step"), np.random.rand(num_records, 7)), + "o_background_variance": ( + ("met", "esa_step"), + np.random.rand(num_records, 7), + ), + # Goodtime fields + "gt_start_met": ( + "met", + np.arange(473389200, 473389200 + num_records * 420, 420), + ), + "gt_end_met": ( + "met", + np.arange(473389200 + 400, 473389200 + num_records * 420 + 400, 420), + ), + # Also include non-prefixed background start/end fields so + # split_backgrounds_and_goodtimes_dataset can select + "start_met": ( + "met", + np.arange(473389200, 473389200 + num_records * 420, 420), + ), + "end_met": ( + "met", + np.arange(473389200 + 400, 473389200 + num_records * 420 + 400, 420), + ), + "bin_start": ("met", np.zeros(num_records, dtype=int)), + "bin_end": ("met", np.zeros(num_records, dtype=int) + 59), + "esa_goodtime_flags": ( + ("met", "esa_step"), + np.ones((num_records, 7), dtype=int), + ), + }, + coords={ + "met": np.arange(num_records), + "esa_step": np.arange(1, 8), + }, + ) + + # Act + bgrates_ds, goodtimes_ds = split_backgrounds_and_goodtimes_dataset( + combined_ds, attr_mgr_l1b + ) + + # Assert - Check bgrates dataset has background fields + # Note: bgrates includes 'start_met', 'end_met', 'bin_start', 'bin_end' per + # BACKGROUND_RATE_FIELDS + assert "h_background_rates" in bgrates_ds.data_vars + assert "h_background_variance" in bgrates_ds.data_vars + assert "o_background_rates" in bgrates_ds.data_vars + assert "o_background_variance" in bgrates_ds.data_vars + # Note: bgrates uses 'met' dimension, goodtimes has epoch in data vars + + # Check goodtimes dataset structure + assert "gt_start_met" in goodtimes_ds.data_vars + assert "gt_end_met" in goodtimes_ds.data_vars + assert "bin_start" in goodtimes_ds.data_vars + assert "bin_end" in goodtimes_ds.data_vars + assert "esa_goodtime_flags" in goodtimes_ds.data_vars + + # Check dimensions + assert bgrates_ds["h_background_rates"].dims == ("met", "esa_step") + assert bgrates_ds["h_background_rates"].shape[1] == 7 # 7 ESA steps + + # Check that goodtime intervals were created + assert len(goodtimes_ds["gt_start_met"]) > 0 + assert len(goodtimes_ds["gt_end_met"]) > 0 + + # Check that start times are before end times + assert np.all( + goodtimes_ds["gt_start_met"].values <= goodtimes_ds["gt_end_met"].values + ) + + # Check bin_start and bin_end values + assert np.all(goodtimes_ds["bin_start"].values == 0) + assert np.all(goodtimes_ds["bin_end"].values == 59) + + # Check ESA goodtime flags are all 1 (good) + assert np.all(goodtimes_ds["esa_goodtime_flags"].values == 1) + + +def test_l1b_bgrates_and_goodtimes_azimuth_bins(attr_mgr_l1b): + """Test that the function correctly uses azimuth bins 20-50 for calculations.""" + # Arrange - Create dataset with specific counts in different azimuth bins + num_epochs = 30 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Set high counts outside bins 20-50, low counts inside bins 20-50 + h_counts = ( + np.ones((num_epochs, 7, 60)) * 0.028 + ) # High counts (10x threshold) everywhere + o_counts = np.ones((num_epochs, 7, 60)) * 0.0028 + + # Set low counts in the bins that are actually used (20-50) + h_counts[:, :, 20:50] = 0.00028 # Low counts in used bins (below threshold) + o_counts[:, :, 20:50] = 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert - Should create goodtime intervals because bins 20-50 have low counts + l1b_bgrates_ds, l1b_goodtimes_ds = result + + assert len(l1b_goodtimes_ds["gt_start_met"]) > 0 + # Background rates should be calculated from the low-count bins + assert np.all(l1b_bgrates_ds["h_background_rates"].values < 1.0) + + +def test_l1b_bgrates_and_goodtimes_variance_calculation(attr_mgr_l1b): + """Test that variance is calculated correctly and handles edge cases.""" + # Arrange + num_epochs = 30 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Use very low counts to test zero variance handling + h_counts = np.zeros((num_epochs, 7, 60)) + o_counts = np.zeros((num_epochs, 7, 60)) + + # Add some small counts (below threshold) + h_counts[:, :, 20:50] = 0.00001 # Very low but non-zero + o_counts[:, :, 20:50] = 0.000001 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Variance should never be zero (fallback logic should apply) + assert np.all(l1b_bgrates_ds["h_background_variance"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_variance"].values > 0) + + # Background rates should also never be zero (fallback logic should apply) + assert np.all(l1b_bgrates_ds["h_background_rates"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_rates"].values > 0) + + +def test_l1b_bgrates_and_goodtimes_offset_application(attr_mgr_l1b): + """Test that goodtime start/end offsets (-620, +320) are applied correctly.""" + # Arrange + num_epochs = 30 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Low counts (below threshold) + h_counts = np.ones((num_epochs, 7, 60)) * 0.00028 + o_counts = np.ones((num_epochs, 7, 60)) * 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Check that gt_start_met is earlier than gt_end_met (accounting for offsets) + for i in range(len(l1b_goodtimes_ds["gt_start_met"])): + start = l1b_goodtimes_ds["gt_start_met"].values[i] + end = l1b_goodtimes_ds["gt_end_met"].values[i] + + # Start should be before end + assert start < end + + # The difference should be reasonable (not negative due to offset) + assert (end - start) > 0 + + +def test_l1b_bgrates_and_goodtimes_rate_transition_low_to_high(attr_mgr_l1b): + """Test interval closure when transitioning from low to high rate + (covers begin > 0.0 block).""" + # Arrange - Create dataset that transitions from LOW to HIGH rates + # This specifically tests the "if begin > 0.0:" code path at line ~2787 + num_epochs = 50 # Need at least 5 cycles (50 epochs / 10 per cycle) + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Start with LOW rates for first 30 epochs (3 cycles) + # Then switch to HIGH rates for last 20 epochs (2 cycles) + h_counts = np.ones((num_epochs, 7, 60)) * 0.00028 # Low (below threshold) + o_counts = np.ones((num_epochs, 7, 60)) * 0.000028 + + # Make last 20 epochs HIGH (above threshold) to trigger interval closure + h_counts[30:, :, :] = 0.028 # High (10x threshold) + o_counts[30:, :, :] = 0.0028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Should create goodtime interval that gets closed when rate goes high + # The interval should span the first 3 cycles (epochs 0-29) + assert len(l1b_goodtimes_ds["gt_start_met"]) >= 1 + + # First interval should start around epoch 0's time + first_start = l1b_goodtimes_ds["gt_start_met"].values[0] + first_end = l1b_goodtimes_ds["gt_end_met"].values[0] + + # Verify interval was created + assert first_start < first_end + + # Background rates should be calculated from the low-rate period + assert np.all(l1b_bgrates_ds["h_background_rates"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_rates"].values > 0) + + # Variance should also be positive + assert np.all(l1b_bgrates_ds["h_background_variance"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_variance"].values > 0) + + +def test_l1b_bgrates_and_goodtimes_rate_transition_high_to_low_to_high(attr_mgr_l1b): + """Test multiple intervals created by multiple rate transitions.""" + # Arrange - Create dataset with HIGH -> LOW -> HIGH -> LOW pattern + # This tests multiple calls to the "if begin > 0.0:" code path + num_epochs = 80 + met_start = 473389200 + met_spacing = 42 + + met_times = np.arange(met_start, met_start + num_epochs * met_spacing, met_spacing) + epoch_times = met_to_ttj2000ns(met_times) + + # Initialize with HIGH rates + h_counts = np.ones((num_epochs, 7, 60)) * 0.028 + o_counts = np.ones((num_epochs, 7, 60)) * 0.0028 + + # Pattern: HIGH(0-9), LOW(10-29), HIGH(30-39), LOW(40-59), HIGH(60-79) + # Epochs 10-29 (2 cycles): LOW - should create interval 1 + h_counts[10:30, :, :] = 0.00028 + o_counts[10:30, :, :] = 0.000028 + + # Epochs 40-59 (2 cycles): LOW - should create interval 2 + h_counts[40:60, :, :] = 0.00028 + o_counts[40:60, :, :] = 0.000028 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=10, delay_max=840 + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Should create at least 2 goodtime intervals (one for each LOW period) + assert len(l1b_goodtimes_ds["gt_start_met"]) >= 2 + + # All intervals should have valid start < end + for i in range(len(l1b_goodtimes_ds["gt_start_met"])): + assert ( + l1b_goodtimes_ds["gt_start_met"].values[i] + < l1b_goodtimes_ds["gt_end_met"].values[i] + ) + + # Background rates should be positive for all intervals + assert np.all(l1b_bgrates_ds["h_background_rates"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_rates"].values > 0) + + +def test_l1b_bgrates_and_goodtimes_large_interval_with_active_tracking(attr_mgr_l1b): + """ + Test that an active goodtime interval is properly closed when a large interval + gap is encountered. + + This test ensures the code path where: + 1. We're actively tracking an interval (begin > 0.0) + 2. A chunk with interval > (interval_nom + delay_max) is encountered + 3. The active interval is closed before skipping the gap + """ + # Arrange - Create dataset where we start tracking, then hit a large interval + cycle_count = 10 + delay_max = 840 + met_spacing = 42 + + # First: Create enough low-rate chunks to start tracking (begin > 0.0) + num_chunks_before_gap = 2 # 2 chunks of 10 epochs each = 20 epochs + epochs_per_chunk = 10 + num_epochs_first = num_chunks_before_gap * epochs_per_chunk + + met_start = 473389200 + met_times_first = np.arange( + met_start, met_start + num_epochs_first * met_spacing, met_spacing + ) + + # Second: Create a chunk where the interval is too large + # The interval is measured from the first epoch of the chunk to the last + # We need interval > (interval_nom + delay_max) = 4200 + 840 = 5040 + large_gap = 6000 # Larger than threshold + met_times_gap_chunk_start = met_times_first[-1] + met_spacing + + # Create the problematic chunk (10 more epochs) + met_times_gap_chunk = np.arange( + met_times_gap_chunk_start, + met_times_gap_chunk_start + epochs_per_chunk * met_spacing, + met_spacing, + ) + + met_times_gap_chunk_adjusted = met_times_gap_chunk.copy() + met_times_gap_chunk_adjusted[-1] = met_times_gap_chunk[0] + large_gap + + # Third: Add more normal data after the gap + met_times_after = np.arange( + met_times_gap_chunk_adjusted[-1] + met_spacing, + met_times_gap_chunk_adjusted[-1] + met_spacing + 200 * met_spacing, + met_spacing, + ) + + met_times = np.concatenate( + [met_times_first, met_times_gap_chunk_adjusted, met_times_after] + ) + epoch_times = met_to_ttj2000ns(met_times) + + # All counts are low (below h_bg_rate_nom = 0.0028) to ensure we start tracking + h_counts = np.ones((len(met_times), 7, 60)) * 0.00025 + o_counts = np.ones((len(met_times), 7, 60)) * 0.000025 + + l1b_histrates = xr.Dataset( + { + "h_counts": (("epoch", "esa_step", "spin_bin_6"), h_counts), + "o_counts": (("epoch", "esa_step", "spin_bin_6"), o_counts), + }, + coords={ + "epoch": epoch_times, + "esa_step": np.arange(1, 8), + "spin_bin_6": np.arange(60), + }, + ) + + sci_dependencies = {"imap_lo_l1b_histrates": l1b_histrates} + + # Act + result = l1b_bgrates_and_goodtimes( + sci_dependencies, attr_mgr_l1b, cycle_count=cycle_count, delay_max=delay_max + ) + + # Assert + l1b_bgrates_ds, l1b_goodtimes_ds = result + + # Should have created at least 2 intervals: + # 1. The interval that was closed before the gap + # 2. The interval after the gap + assert len(l1b_goodtimes_ds["gt_start_met"]) >= 2 + + # The first interval should end before the gap chunk + # (it should be closed when we detect the large interval) + first_interval_end = l1b_goodtimes_ds["gt_end_met"].values[0] + gap_chunk_start = met_times_gap_chunk_adjusted[0] + + # The first interval should end before the gap chunk starts + # (with the +320 offset applied in the code) + assert first_interval_end < gap_chunk_start + 320 + + # Verify background rates are valid + assert np.all(l1b_bgrates_ds["h_background_rates"].values > 0) + assert np.all(l1b_bgrates_ds["o_background_rates"].values > 0) diff --git a/imap_processing/tests/mag/conftest.py b/imap_processing/tests/mag/conftest.py index 2bb3ceb762..de4aba5ce5 100644 --- a/imap_processing/tests/mag/conftest.py +++ b/imap_processing/tests/mag/conftest.py @@ -175,7 +175,11 @@ def norm_dataset(mag_test_l2_data): dataset.attrs["vectors_per_second"] = vectors_per_second_attr dataset["epoch"] = epoch_vals dataset.attrs["Logical_source"] = "imap_mag_l1c_norm-mago" - vectors = np.array([[i, i, i, 2] for i in range(1, 3505)]) + # Actual dataset is a CDF_FLOAT which is a float32. + vectors = np.array( + [[i, i, i, 2] for i in range(1, 3505)], + dtype=np.float64, + ) dataset["vectors"].data = vectors return dataset diff --git a/imap_processing/tests/mag/test_mag_l1d.py b/imap_processing/tests/mag/test_mag_l1d.py index bd74089b64..9c62196221 100644 --- a/imap_processing/tests/mag/test_mag_l1d.py +++ b/imap_processing/tests/mag/test_mag_l1d.py @@ -8,7 +8,7 @@ from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes from imap_processing.cdf.utils import write_cdf from imap_processing.cli import Mag -from imap_processing.mag.constants import DataMode +from imap_processing.mag.constants import FILLVAL, DataMode from imap_processing.mag.l1d.mag_l1d import mag_l1d from imap_processing.mag.l1d.mag_l1d_data import MagL1d, MagL1dConfiguration from imap_processing.mag.l2.mag_l2_data import ValidFrames @@ -571,6 +571,57 @@ def test_enhanced_gradiometry_with_quality_flags_detailed(): assert np.array_equal(grad_ds["quality_flags"].data, expected_flags) +def test_rotate_frame_preserves_fillval_and_nan(mag_l1d_test_class): + """Test that L1D rotate_frame preserves FILLVAL and NaN vectors.""" + mag_l1d_test_class.frame = ValidFrames.MAGO + + vectors = mag_l1d_test_class.vectors.copy() + magi_vectors = mag_l1d_test_class.magi_vectors.copy() + + # Set some MAGO vectors to FILLVAL and NaN + vectors[0] = [FILLVAL, FILLVAL, FILLVAL] + vectors[2] = [np.nan, np.nan, np.nan] + vectors[4] = [1.0, np.nan, 3.0] + mag_l1d_test_class.vectors = vectors + + # Set some MAGI vectors to FILLVAL and NaN + magi_vectors[1] = [FILLVAL, FILLVAL, FILLVAL] + magi_vectors[3] = [np.nan, np.nan, np.nan] + mag_l1d_test_class.magi_vectors = magi_vectors + + def mock_frame_transform( + epoch_et, + vecs, + from_frame, + to_frame, + allow_spice_noframeconnect, + ): + return np.full(vecs.shape, 99.0) + + with patch( + "imap_processing.mag.l1d.mag_l1d_data.frame_transform", + side_effect=mock_frame_transform, + ): + mag_l1d_test_class.rotate_frame(ValidFrames.SRF) + + assert mag_l1d_test_class.frame == ValidFrames.SRF + + # MAGO: FILLVAL/NaN rows preserved as FILLVAL + assert np.all(mag_l1d_test_class.vectors[0] == FILLVAL) + assert np.all(mag_l1d_test_class.vectors[2] == FILLVAL) + assert mag_l1d_test_class.vectors[4, 1] == FILLVAL + # Normal MAGO vectors get rotated value + assert np.all(mag_l1d_test_class.vectors[1] == 99.0) + assert np.all(mag_l1d_test_class.vectors[3] == 99.0) + + # MAGI: FILLVAL/NaN rows preserved as FILLVAL + assert np.all(mag_l1d_test_class.magi_vectors[1] == FILLVAL) + assert np.all(mag_l1d_test_class.magi_vectors[3] == FILLVAL) + # Normal MAGI vectors get rotated value + assert np.all(mag_l1d_test_class.magi_vectors[0] == 99.0) + assert np.all(mag_l1d_test_class.magi_vectors[2] == 99.0) + + def test_rotate_frames(mag_l1d_test_class): # Reset to initial MAGO frame for this test mag_l1d_test_class.frame = ValidFrames.MAGO diff --git a/imap_processing/tests/mag/test_mag_l2.py b/imap_processing/tests/mag/test_mag_l2.py index 6be5a7150a..2253e4ca05 100644 --- a/imap_processing/tests/mag/test_mag_l2.py +++ b/imap_processing/tests/mag/test_mag_l2.py @@ -5,7 +5,7 @@ import xarray as xr from imap_processing.cdf.imap_cdf_manager import ImapCdfAttributes -from imap_processing.mag.constants import DataMode +from imap_processing.mag.constants import FILLVAL, DataMode from imap_processing.mag.l2.mag_l2 import mag_l2, retrieve_matrix_from_l2_calibration from imap_processing.mag.l2.mag_l2_data import MagL2, ValidFrames from imap_processing.spice.time import ( @@ -18,9 +18,29 @@ from imap_processing.tests.mag.conftest import mag_l1a_dataset_generator -@pytest.mark.parametrize("data_mode", ["norm", "burst"]) -def test_mag_l2_attributes(norm_dataset, mag_test_l2_data, data_mode): - """Test that L2 datasets have correct attributes based on frame and mode.""" +@pytest.mark.parametrize( + "data_mode,frames,expected_frames", + [ + ( + "norm", + [ + ValidFrames.SRF, + ValidFrames.GSE, + ValidFrames.GSM, + ValidFrames.RTN, + ValidFrames.DSRF, + ], + 5, + ), + ("norm", [], 5), + ("burst", [ValidFrames.SRF], 1), + ("burst", [], 5), + ], +) +def test_mag_l2_attributes( + norm_dataset, mag_test_l2_data, data_mode, frames, expected_frames +): + """Test that correct L2 datasets have correct attributes based on frame and mode.""" calibration_dataset = mag_test_l2_data[0] offset_dataset = mag_test_l2_data[1] @@ -35,18 +55,30 @@ def test_mag_l2_attributes(norm_dataset, mag_test_l2_data, data_mode): "imap_processing.mag.l2.mag_l2_data.frame_transform", side_effect=lambda *args, **kwargs: args[1], ): - l2_datasets = mag_l2( - calibration_dataset, - offset_dataset, - test_dataset, - np.datetime64("2025-10-17"), - mode=mode, - ) + if frames: + # ensure when a subset of frames is needed only those are generated + l2_datasets = mag_l2( + calibration_dataset, + offset_dataset, + test_dataset, + np.datetime64("2025-10-17"), + mode=mode, + frames=frames, + ) + else: + # be default all frames are generated + l2_datasets = mag_l2( + calibration_dataset, + offset_dataset, + test_dataset, + np.datetime64("2025-10-17"), + mode=mode, + ) # Verify we have the expected number of datasets # L2 produces 5 frames: SRF, GSE, GSM, RTN, DSRF - assert len(l2_datasets) == 5, ( - f"Expected 5 {data_mode} datasets, got {len(l2_datasets)}" + assert len(l2_datasets) == expected_frames, ( + f"Expected {expected_frames} {data_mode} datasets, got {len(l2_datasets)}" ) for dataset in l2_datasets: @@ -447,6 +479,55 @@ def test_spice_returns(norm_dataset): assert np.array_equal(l2.vectors[0], [-1, -1, -1]) +def test_rotate_frame_preserves_fillval_and_nan(norm_dataset): + """Test that rotate_frame preserves FILLVAL and NaN vectors.""" + + vectors = norm_dataset["vectors"].data[:, :3].copy() + n = len(vectors) + + # Set some vectors to FILLVAL and NaN + vectors[0] = [FILLVAL, FILLVAL, FILLVAL] + vectors[2] = [np.nan, np.nan, np.nan] + # Partial NaN in a row + vectors[4] = [1.0, np.nan, 3.0] + # Partial FILLVAL in a row + vectors[5] = [FILLVAL, 2.0, 3.0] + + l2 = MagL2( + vectors=vectors, + epoch=norm_dataset["epoch"].data, + range=norm_dataset["vectors"].data[:, 3], + global_attributes={}, + quality_flags=np.zeros(n), + quality_bitmask=np.zeros(n), + data_mode=DataMode.NORM, + offsets=np.zeros((n, 3)), + timedelta=np.zeros(n), + ) + + rotated_values = np.full(vectors.shape, 99.0) + with patch( + "imap_processing.mag.l2.mag_l2_data.frame_transform", + return_value=rotated_values, + ): + l2.rotate_frame(ValidFrames.DSRF) + + assert l2.frame == ValidFrames.DSRF + + # Full FILLVAL row -> all components should be FILLVAL + assert np.all(l2.vectors[0] == FILLVAL) + # Full NaN row -> all components should be FILLVAL + assert np.all(l2.vectors[2] == FILLVAL) + # Partial NaN -> affected components should be FILLVAL + assert l2.vectors[4, 1] == FILLVAL + # Partial FILLVAL -> affected components should be FILLVAL + assert l2.vectors[5, 0] == FILLVAL + + # Normal vectors should get the rotated value + assert np.all(l2.vectors[1] == 99.0) + assert np.all(l2.vectors[3] == 99.0) + + def test_qf(norm_dataset): qf = np.zeros(len(norm_dataset["epoch"].data), dtype=int) qf[1:4] = 1 diff --git a/imap_processing/tests/test_cli.py b/imap_processing/tests/test_cli.py index 0bbbe1ecdd..9535307dac 100644 --- a/imap_processing/tests/test_cli.py +++ b/imap_processing/tests/test_cli.py @@ -18,6 +18,7 @@ ProcessingInputCollection, ScienceInput, SPICEInput, + SpinInput, ) from imap_processing.cli import ( @@ -300,8 +301,14 @@ def test_post_processing_returns_empty_list_if_invoked_with_no_data( "l1c", "45sensor-pset", "hi_l1c", - ["imap_hi_l1b_45sensor-de_20250415_v001.cdf"], - ["imap_hi_calibration-prod-config_20240101_v001.csv"], + [ + "imap_hi_l1b_45sensor-de_20250415_v001.cdf", + "imap_hi_l1b_45sensor-goodtimes_20250415_v001.cdf", + ], + [ + "imap_hi_45sensor-cal-prod_20240101_v001.csv", + "imap_hi_45sensor-backgrounds_20240101_v001.csv", + ], 1, ), ( @@ -373,21 +380,8 @@ def test_hi_l1b_goodtimes(mock_hi_goodtimes, mock_instrument_dependencies): mock_hi_goodtimes.return_value = [mock_goodtimes_ds] mocks["mock_write_cdf"].return_value = Path("/path/to/goodtimes_output.cdf") - # Mock load_cdf to return xr.Dataset objects - mock_de_dataset = xr.Dataset() - mock_hk_dataset = xr.Dataset() - # 7 DE files + 1 HK file = 8 total calls to load_cdf - mocks["mock_load_cdf"].side_effect = [ - mock_de_dataset, - mock_de_dataset, - mock_de_dataset, - mock_de_dataset, - mock_de_dataset, - mock_de_dataset, - mock_de_dataset, - mock_hk_dataset, - ] - + # set load_cdf to return empty datasets + mocks["mock_load_cdf"].return_value = xr.Dataset() # Set up the input collection with required dependencies input_collection = ProcessingInputCollection( ScienceInput("imap_hi_l1b_45sensor-de_20250415-repoint00001_v001.cdf"), @@ -398,6 +392,7 @@ def test_hi_l1b_goodtimes(mock_hi_goodtimes, mock_instrument_dependencies): ScienceInput("imap_hi_l1b_45sensor-de_20250415-repoint00006_v001.cdf"), ScienceInput("imap_hi_l1b_45sensor-de_20250415-repoint00007_v001.cdf"), ScienceInput("imap_hi_l1b_45sensor-hk_20250415-repoint00004_v001.cdf"), + ScienceInput("imap_hi_l1a_45sensor-diagfee_20250415-repoint00004_v001.cdf"), AncillaryInput("imap_hi_45sensor-cal-prod_20240101_v001.csv"), ) mocks["mock_pre_processing"].return_value = input_collection @@ -416,17 +411,18 @@ def test_hi_l1b_goodtimes(mock_hi_goodtimes, mock_instrument_dependencies): instrument.process() # Verify load_cdf was called for DE files and HK file - assert mocks["mock_load_cdf"].call_count == 8 # 7 DE + 1 HK + assert mocks["mock_load_cdf"].call_count == 9 # 7 DE + 1 HK + 1 DIAG_FEE # Verify hi_goodtimes was called with correct arguments assert mock_hi_goodtimes.call_count == 1 call_args = mock_hi_goodtimes.call_args # Check that datasets (not paths) were passed for l1b_de_datasets and l1b_hk - assert isinstance(call_args.args[0], list) # l1b_de_datasets is a list - assert len(call_args.args[0]) == 7 # 7 DE datasets + assert call_args.args[0] == "repoint00004" # current_repointing + assert isinstance(call_args.args[1], list) # l1b_de_datasets is a list + assert len(call_args.args[1]) == 7 # 7 DE datasets assert isinstance(call_args.args[2], xr.Dataset) # l1b_hk is a dataset - assert call_args.args[1] == "repoint00004" # current_repointing + assert isinstance(call_args.args[3], xr.Dataset) # l1a_diagfee is a dataset # goodtimes now returns xr.Dataset, so write_cdf should be called assert mocks["mock_write_cdf"].call_count == 1 @@ -622,6 +618,35 @@ def test_ultra_l2(mock_ultra_l2, mock_instrument_dependencies): assert mock_instrument_dependencies["mock_write_cdf"].call_count == 1 +@mock.patch("imap_processing.cli.idex_l1b") +def test_idex_l1b(mock_idex_l1b, mock_instrument_dependencies): + """Test coverage for cli.Idex class with l1b data level""" + mocks = mock_instrument_dependencies + new_ds = xr.Dataset(data_vars={"epoch": [1]}) + old_ds = xr.Dataset(data_vars={"epoch": [0]}) + mocks["mock_load_cdf"].side_effect = [old_ds, new_ds] + input_collection = ProcessingInputCollection( + ScienceInput( + "imap_idex_l1a_sci-1week_20251017_v001.cdf", + "imap_idex_l1a_sci-1week_20251012_v001.cdf", + ), + SPICEInput("naif0012.tls", "imap_sclk_0000.tsc"), + SpinInput("imap_2025_306_2025_307_01.spin"), + ) + mocks["mock_pre_processing"].return_value = input_collection + + dependency_str = input_collection.serialize() + instrument = Idex( + "l1b", "sci-1week", dependency_str, "20251017", "20251017", "v001", False + ) + + instrument.process() + assert mock_idex_l1b.call_count == 1 + # Assert that the dataset with the newer epoch value was passed to idex_l1b for + # processing + xr.testing.assert_equal(mock_idex_l1b.call_args[0][0], new_ds) + + @mock.patch("imap_processing.cli.idex_l2b") def test_idex_l2b(mock_idex_l2b, mock_instrument_dependencies): """Test coverage for cli.Idex class with l2b data level""" diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv new file mode 100644 index 0000000000..f3c31692cf --- /dev/null +++ b/imap_processing/tests/ultra/data/l1/imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv @@ -0,0 +1,2 @@ +repointing_id_start,repointing_id_end,de_product +0,,imap_ultra_l1b_45sensor-de diff --git a/imap_processing/tests/ultra/data/l1/imap_ultra_l1c-45sensor-de-product-lookup_20251001_v001.csv b/imap_processing/tests/ultra/data/l1/imap_ultra_l1c-45sensor-de-product-lookup_20251001_v001.csv new file mode 100644 index 0000000000..f3c31692cf --- /dev/null +++ b/imap_processing/tests/ultra/data/l1/imap_ultra_l1c-45sensor-de-product-lookup_20251001_v001.csv @@ -0,0 +1,2 @@ +repointing_id_start,repointing_id_end,de_product +0,,imap_ultra_l1b_45sensor-de diff --git a/imap_processing/tests/ultra/mock_data.py b/imap_processing/tests/ultra/mock_data.py index 4a4bae4510..ff8cd6b424 100644 --- a/imap_processing/tests/ultra/mock_data.py +++ b/imap_processing/tests/ultra/mock_data.py @@ -167,6 +167,7 @@ def get_binomial_counts(distance_scaling, lat_bin, central_lat_bin): ], sensitivity, ), + "epoch_delta": ([CoordNames.TIME.value], np.array([10], dtype=np.float64)), }, coords={ CoordNames.TIME.value: [ @@ -193,7 +194,8 @@ def get_binomial_counts(distance_scaling, lat_bin, central_lat_bin): # TODO: Add ability to mock with/without energy dim to exposure_factor # The Helio frame L1C will have the energy dimension, but the spacecraft frame will not. def mock_l1c_pset_product_healpix( - nside: int = DEFAULT_HEALPIX_NSIDE_L1C, + nside: int = 32, + counts_nside: int = 128, stripe_center_lat: int = 0, width_scale: float = 10.0, counts_scaling_params: tuple[int, float] = (100, 0.01), @@ -264,19 +266,19 @@ def mock_l1c_pset_product_healpix( energy_bin_delta = np.diff(energy_intervals, axis=1).squeeze() num_energy_bins = len(energy_bin_midpoints) npix = hp.nside2npix(nside) - counts = np.zeros(npix) - exposure_time = np.zeros(npix) - + counts_npix = hp.nside2npix(counts_nside) + # counts are binned at a higher resolution than the other variables. See L1c + # code for more details. # Get latitude for each healpix pixel - pix_indices = np.arange(npix) - lon_pix, lat_pix = hp.pix2ang(nside, pix_indices, lonlat=True) - - counts = np.zeros(shape=(num_energy_bins, npix)) + counts_pix_indices = np.arange(counts_npix) + counts_lon_pix, counts_lat_pix = hp.pix2ang( + counts_nside, counts_pix_indices, lonlat=True + ) # Calculate probability based on distance from target latitude - lat_diff = np.abs(lat_pix - stripe_center_lat) + counts_lat_diff = np.abs(counts_lat_pix - stripe_center_lat) prob_scaling_factor = counts_scaling_params[1] * np.exp( - -(lat_diff**2) / (2 * width_scale**2) + -(counts_lat_diff**2) / (2 * width_scale**2) ) # Generate counts using binomial distribution rng = np.random.default_rng(seed=42) @@ -286,6 +288,12 @@ def mock_l1c_pset_product_healpix( for _ in range(num_energy_bins) ] ) + # Get latitude for each healpix pixel + pix_indices = np.arange(npix) + lon_pix, lat_pix = hp.pix2ang(nside, pix_indices, lonlat=True) + + # Calculate probability based on distance from target latitude + lat_diff = np.abs(lat_pix - stripe_center_lat) # Generate exposure times using gaussian distribution, but wider prob_scaling_factor_exptime = counts_scaling_params[1] * np.exp( @@ -317,7 +325,7 @@ def mock_l1c_pset_product_healpix( counts = counts.astype(int) # add an epoch dimension counts = np.expand_dims(counts, axis=0) - ones_ds = np.ones_like(counts)[0] # pointing independent + ones_ds = np.ones((num_energy_bins, npix)) # pointing independent sensitivity = ones_ds geometric_function = ones_ds efficiency = ones_ds @@ -338,7 +346,7 @@ def mock_l1c_pset_product_healpix( [ CoordNames.TIME.value, CoordNames.ENERGY_ULTRA_L1C.value, - CoordNames.HEALPIX_INDEX.value, + CoordNames.COUNTS_HEALPIX_INDEX.value, ], counts, ), @@ -348,7 +356,7 @@ def mock_l1c_pset_product_healpix( CoordNames.ENERGY_ULTRA_L1C.value, CoordNames.HEALPIX_INDEX.value, ], - np.full_like(counts, 0.05, dtype=float), + np.full((1, num_energy_bins, npix), 0.05, dtype=float), ), "exposure_factor": ( exposure_dims, # special case: optionally energy dependent exposure @@ -405,6 +413,7 @@ def mock_l1c_pset_product_healpix( [CoordNames.TIME.value, CoordNames.HEALPIX_INDEX.value], np.full((1, npix), ImapPSETUltraFlags.NONE.value, dtype=np.uint16), ), + "epoch_delta": ([CoordNames.TIME.value], np.array([10], dtype=np.float64)), }, coords={ CoordNames.TIME.value: [ diff --git a/imap_processing/tests/ultra/unit/conftest.py b/imap_processing/tests/ultra/unit/conftest.py index e3b4f79fbc..bfd8a54990 100644 --- a/imap_processing/tests/ultra/unit/conftest.py +++ b/imap_processing/tests/ultra/unit/conftest.py @@ -8,6 +8,7 @@ import xarray as xr from imap_processing import imap_module_directory +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l0.decom_ultra import ( process_ultra_cmd_echo, process_ultra_energy_rates, @@ -39,6 +40,11 @@ ULTRA_RATES, ) from imap_processing.ultra.l1a.ultra_l1a import ultra_l1a +from imap_processing.ultra.l1b.ultra_l1b_culling import ( + get_binned_energy_ranges, + get_energy_range_flags, +) +from imap_processing.ultra.l1c.l1c_lookup_utils import build_energy_bins from imap_processing.utils import packet_file_to_datasets @@ -483,6 +489,8 @@ def ancillary_files(): return { "l1b-45sensor-logistic-interpolation": path / "imap_ultra_l1b-45sensor-logistic-interpolation_20250101_v000.csv", + "l1b-90sensor-logistic-interpolation": path + / "imap_ultra_l1b-45sensor-logistic-interpolation_20250101_v000.csv", "l1b-sensor-gf-noblades": path / "imap_ultra_l1b-sensor-gf-noblades_20250101_v000.csv", "l1b-sensor-gf-blades": path @@ -551,6 +559,10 @@ def ancillary_files(): / "imap_ultra_l1c-45sensor-static-dead-times_20250101_v000.csv", "l1c-90sensor-static-dead-times": path / "imap_ultra_l1c-90sensor-static-dead-times_20250101_v000.csv", + "l1c-45sensor-de-product-lookup": path + / "imap_ultra_l1c-45sensor-de-product-lookup_20251001_v001.csv", + "l1c-90sensor-de-product-lookup": path + / "imap_ultra_l1c-45sensor-de-product-lookup_20251001_v001.csv", } @@ -644,13 +656,48 @@ def mock_helio_pointing_lookups(): @pytest.fixture def mock_goodtimes_dataset(): """Create a mock goodtimes dataset.""" + # Set up bit flags + intervals, _, _ = build_energy_bins() + energy_ranges = get_binned_energy_ranges(intervals) + energy_flags = get_energy_range_flags(energy_ranges) + + energy_flags_padded = np.zeros(UltraConstants.MAX_ENERGY_RANGES, dtype=np.uint16) + energy_flags_padded[: len(energy_flags)] = energy_flags + + energy_ranges_padded = np.full( + UltraConstants.MAX_ENERGY_RANGE_EDGES, -1.0e31, dtype=np.float32 + ) + energy_ranges_padded[: len(energy_ranges)] = energy_ranges + + nspins = 100 + flags = 2 ** np.arange(9) + quality = np.zeros(nspins, dtype=np.uint16) + quality[0] = flags[0] # Set the first flag for the first spin + quality[1] = flags[1] # Set the second flag for the second + quality[2] = flags[2] # Set the third flag for the third spin return xr.Dataset( { - "spin_number": ("epoch", np.zeros(5)), - "energy_range_flags": ("energy_flags", np.zeros(10, dtype=np.uint16)), - "quality_low_voltage": ("spin_number", np.zeros(5, dtype=np.uint16)), - "quality_high_energy": ("spin_number", np.zeros(5, dtype=np.uint16)), - "quality_statistics": ("spin_number", np.zeros(5, dtype=np.uint16)), - "energy_range_edges": ("energy_ranges", np.zeros(11, dtype=np.uint16)), + "spin_number": ("epoch", np.zeros(nspins)), + "energy_range_flags": ("energy_flags", energy_flags_padded), + "quality_low_voltage": ("spin_number", quality), + "quality_high_energy": ("spin_number", np.zeros(nspins, dtype=np.uint16)), + "quality_statistics": ("spin_number", np.zeros(nspins, dtype=np.uint16)), + "energy_range_edges": ("energy_ranges", energy_ranges_padded), + "spin_period": ( + "spin_number", + np.full(nspins, 15), + ), # nominal spin period of 15 seconds + "quality_upstream_ion_1": ( + "spin_number", + np.zeros(nspins, dtype=np.uint16), + ), + "quality_upstream_ion_2": ( + "spin_number", + np.zeros(nspins, dtype=np.uint16), + ), + "quality_spectral": ( + "spin_number", + np.zeros(nspins, dtype=np.uint16), + ), } ) diff --git a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py index ae1e1bbe47..d2149d4ea2 100644 --- a/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_l1c_lookup_utils.py @@ -4,10 +4,11 @@ import xarray as xr from imap_processing.ultra.l1c.l1c_lookup_utils import ( - calculate_fwhm_spun_scattering, + calculate_accepted_pixels, get_scattering_thresholds_for_energy, get_spacecraft_pointing_lookup_tables, get_static_deadtime_ratios, + in_restricted_fov, mask_below_fwhm_scattering_threshold, ) @@ -31,6 +32,7 @@ def test_get_spacecraft_pointing_lookup_tables(ancillary_files): assert theta_vals.shape == (cols, npix) assert phi_vals.shape == (cols, npix) assert ra_and_dec.shape == (2, npix) + assert boundary_scale_factors.shape == (cols, npix) # Value tests assert for_indices_by_spin_phase.dtype == bool @@ -115,21 +117,31 @@ def test_get_static_deadtime_ratios(ancillary_files): assert np.all((dt_ratio >= 0.0) & (dt_ratio <= 1.0)) -def test_calculate_fwhm_spun_scattering(ancillary_files): - """Test calculate_fwhm_spun_scattering function.""" +def test_in_restricted_fov(): + """Test in_restricted_fov function.""" + # Create mock theta and phi values + # First two values are outside the restricted FOV, the rest are inside + theta_vals = np.array([[-50, 49, 10, 40, 35]]) + # The last value is outside the restricted FOV + phi_vals = np.array([[20, 30, 40, 50, 70]]) # shape (1, 5) + accepted_pixels = in_restricted_fov(theta_vals, phi_vals, 45) + expected_accepted_pixels = np.array([[False, False, True, True, False]]) + np.testing.assert_array_equal(accepted_pixels, expected_accepted_pixels) + + +def test_calculate_accepted_pixels(ancillary_files): + """Test calculate_accepted_pixels function.""" # Make array with ones (we are only testing the shape here) for_pixels = np.ones((50, 10)) theta_vals = np.ones((50, 10)) * 20 # All theta values are 20 phi_vals = np.ones((50, 5)) * 15 # All phi with pytest.raises(ValueError, match="Shape mismatch"): - calculate_fwhm_spun_scattering( - for_pixels, theta_vals, phi_vals, ancillary_files, 45 - ) + calculate_accepted_pixels(for_pixels, theta_vals, phi_vals, ancillary_files, 45) @pytest.mark.external_test_data -def test_calculate_fwhm_spun_scattering_reject(ancillary_files): - """Test calculate_fwhm_spun_scattering function.""" +def test_calculate_accepted_pixels_reject(ancillary_files): + """Test calculate_accepted_pixels function.""" nside = 8 pix = hp.nside2npix(nside) steps = 5 # Reduced for testing @@ -144,16 +156,49 @@ def test_calculate_fwhm_spun_scattering_reject(ancillary_files): # Simulate first 100 pixels are in the FOR for all spin phases inside_inds = 100 for_pixels[:, :, :inside_inds] = True - valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = ( - calculate_fwhm_spun_scattering( - for_pixels, - mock_theta, - mock_phi, - ancillary_files, - 45, - reject_scattering=True, - ) + valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = calculate_accepted_pixels( + for_pixels, + mock_theta, + mock_phi, + ancillary_files, + 45, + reject_scattering=True, ) assert valid_spun_pixels.shape == (steps, energy_dim, pix) # Check that some pixels are rejected assert not np.array_equal(valid_spun_pixels, for_pixels) + + +@pytest.mark.external_test_data +def test_calculate_accepted_pixels_restrict_fov(ancillary_files): + """Test calculate_accepted_pixels function for FOV restrictions.""" + nside = 8 + pix = hp.nside2npix(nside) + steps = 5 # Reduced for testing + energy_dim = 46 + np.random.seed(42) + mock_theta = np.random.uniform(-60, 60, (steps, energy_dim, pix)) + mock_phi = np.random.uniform(-60, 60, (steps, energy_dim, pix)) + # Create for_pixels with all True values to isolate the effect of FOV restrictions + for_pixels = xr.DataArray( + np.ones((steps, energy_dim, pix)).astype(bool), + dims=("spin_phase_step", "energy", "pixel"), + ) + # Set first 30 pixels to be outside of the FOR for all spin phases and energies + outside_inds = 30 + for_pixels[:, :, :outside_inds] = False + valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = calculate_accepted_pixels( + for_pixels, + mock_theta, + mock_phi, + ancillary_files, + 45, + apply_fov_restriction=True, + ) + assert valid_spun_pixels.shape == (steps, energy_dim, pix) + # In this case, the valid spun pixels should be the same as the pixels that are + # in the restricted FOV except the first 30 pixels which are outside the FOR and + # should be False for all spin phases and energies. + expected_accepted_px = in_restricted_fov(mock_theta, mock_phi, 45) + expected_accepted_px[:, :, :outside_inds] = False + assert np.array_equal(expected_accepted_px, valid_spun_pixels) diff --git a/imap_processing/tests/ultra/unit/test_lookup_utils.py b/imap_processing/tests/ultra/unit/test_lookup_utils.py index fabe3a8804..fe82d26668 100644 --- a/imap_processing/tests/ultra/unit/test_lookup_utils.py +++ b/imap_processing/tests/ultra/unit/test_lookup_utils.py @@ -9,6 +9,7 @@ from imap_processing.ultra.l1b.lookup_utils import ( get_angular_profiles, get_back_position, + get_de_product_name, get_ebins, get_energy_efficiencies, get_energy_norm, @@ -113,10 +114,14 @@ def test_get_angular_profiles(): def test_get_energy_efficiencies(ancillary_files): """Tests function get_get_energy_efficiencies.""" - u45_efficiencies = get_energy_efficiencies(ancillary_files) + u45_efficiencies = get_energy_efficiencies(ancillary_files, "ultra45") assert u45_efficiencies.shape == (58081, 157) + # Test that the function can also read the ultra90 efficiencies + u90_efficiencies = get_energy_efficiencies(ancillary_files, "ultra90") + assert u90_efficiencies.shape == (58081, 157) + @pytest.mark.external_test_data def test_get_geometric_function(ancillary_files): @@ -211,3 +216,80 @@ def test_get_scattering_thresholds(ancillary_files): assert thresholds[(8.0, 10.0)] == 8.0 assert thresholds[(10.0, 20.0)] == 6.0 assert thresholds[(20.0, np.inf)] == 4.0 + + +def test_get_de_product_name_no_repoint(): + """Tests function get_de_product_name when the lookup is missing the repoint.""" + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [1, 2], + "repointing_id_end": [3, 4], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + ], + } + ) + with pytest.raises(ValueError, match="No DE product found for repoint ID 0"): + get_de_product_name("repoint00000", 45, "l1b", ancillary_files) + + +def test_get_de_product_name_multiple_products(): + """Tests function get_de_product_name when the lookup is ambiguous.""" + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [2, 2], + "repointing_id_end": [3, 4], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + ], + } + ) + with pytest.raises(ValueError, match="Multiple DE products found"): + get_de_product_name("repoint00002", 45, "l1b", ancillary_files) + + +def test_get_de_product_name(): + """Tests function get_de_product_name.""" + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } + with mock.patch( + "imap_processing.ultra.l1b.lookup_utils.pd.read_csv" + ) as mock_read_csv: + mock_read_csv.return_value = pd.DataFrame( + { + "repointing_id_start": [0, 2, 4], + "repointing_id_end": [1, 4, np.nan], + "de_product": [ + "imap_ultra_l1b_45sensor-de", + "imap_ultra_l1b_45sensor-priority-1-de", + "imap_ultra_l1b_45sensor-priority-2-de", + ], + } + ) + # Test with a repoint in the future. Should return the priority 2 de product + # since the last repoint range does not have an end and should be assumed to + # cover all future repoints. + de_product = get_de_product_name("repoint00100", 45, "l1b", ancillary_files) + assert de_product == "imap_ultra_l1b_45sensor-priority-2-de" + + # Test with valid repoint that falls in the second range. + de_product = get_de_product_name("repoint00003", 45, "l1b", ancillary_files) + assert de_product == "imap_ultra_l1b_45sensor-priority-1-de" diff --git a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py index 5c57b5458e..381e357b28 100644 --- a/imap_processing/tests/ultra/unit/test_spacecraft_pset.py +++ b/imap_processing/tests/ultra/unit/test_spacecraft_pset.py @@ -79,6 +79,8 @@ def test_calculate_spacecraft_pset( ["epoch", "component"], particle_velocity_dps_spacecraft, ), + "theta": (["epoch"], np.zeros(len(species), dtype=np.float32)), + "phi": (["epoch"], np.zeros(len(species), dtype=np.float32)), "energy_spacecraft": (["epoch"], energy_dps_spacecraft), "spin": (["epoch"], df["Spin"].values), "quality_scattering": ( @@ -177,9 +179,13 @@ def test_calculate_spacecraft_pset_with_cdf( de_dict["quality_scattering"] = np.zeros(len(sc_dps_velocity), dtype=np.uint16) de_dict["quality_outliers"] = np.zeros(len(sc_dps_velocity), dtype=np.uint16) de_dict["ebin"] = np.ones(len(sc_dps_velocity), dtype=np.uint8) + de_dict["theta"] = np.zeros(len(df_subset), dtype=np.float32) + de_dict["phi"] = np.zeros(len(df_subset), dtype=np.float32) de_dict["event_times"] = 817561854.185627 + ( df_subset["tdb"].values - df_subset["tdb"].values[0] ) + de_dict["theta"] = np.zeros(len(df_subset), dtype=np.float32) + de_dict["phi"] = np.zeros(len(df_subset), dtype=np.float32) name = "imap_ultra_l1b_45sensor-de" dataset = create_dataset(de_dict, name, "l1b") diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b.py b/imap_processing/tests/ultra/unit/test_ultra_l1b.py index 6900706480..38551dd4fe 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b.py @@ -7,6 +7,7 @@ from imap_processing import imap_module_directory from imap_processing.cdf.utils import load_cdf, write_cdf from imap_processing.quality_flags import ImapDEOutliersUltraFlags +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.de import FILLVAL_FLOAT32 from imap_processing.ultra.l1b.ultra_l1b import ultra_l1b from imap_processing.ultra.utils.ultra_l1_utils import create_dataset @@ -63,12 +64,26 @@ def mock_data_l1b_extendedspin_dict(): ) spin_start_time = np.array([0, 1, 2], dtype="uint64") quality = np.zeros((2, 3), dtype="uint16") + # These should be shape: (3,) + energy_dep_flags = np.zeros(len(spin), dtype="uint16") + energy_range_flags = np.zeros(UltraConstants.MAX_ENERGY_RANGES, dtype=np.uint16) + energy_range_flags[:5] = 1 # Set first 5 to 1 for testing + energy_range_edges = np.ones( + UltraConstants.MAX_ENERGY_RANGE_EDGES, dtype=np.float32 + ) + energy_range_edges[:4] = [3.0, 5.0, 7.0, 10.0] # Example values + energy_range_edges[4:] = -1.0e31 # Fill remaining with fillval data_dict = { "epoch": epoch, "spin_number": spin, "energy_bin_geometric_mean": energy, "spin_start_time": spin_start_time, "quality_ena_rates": quality, + "quality_low_voltage": energy_dep_flags, + "quality_high_energy": energy_dep_flags, + "quality_statistics": energy_dep_flags, + "energy_range_flags": energy_range_flags, + "energy_range_edges": energy_range_edges, } return data_dict @@ -212,7 +227,10 @@ def test_ultra_l1b_extendedspin( data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset - ancillary_files = {} + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) assert len(l1b_extendedspin_dataset) == 1 @@ -244,7 +262,10 @@ def test_cdf_extendedspin( data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset - ancillary_files = {} + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) """Tests that CDF file is created and contains same attributes as xarray.""" l1b_extendedspin_dataset[0].attrs["Data_version"] = "999" @@ -281,7 +302,10 @@ def test_cdf_goodtimes( data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset - ancillary_files = {} + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) goodtimes_dataset = ultra_l1b( @@ -322,7 +346,10 @@ def test_cdf_badtimes( data_dict["imap_ultra_l1a_45sensor-rates"] = rates_dataset data_dict["imap_ultra_l1b_45sensor-status"] = status_dataset - ancillary_files = {} + ancillary_files = { + "l1b-45sensor-de-product-lookup": TEST_PATH + / "imap_ultra_l1b-45sensor-de-product-lookup_20251001_v001.csv" + } l1b_extendedspin_dataset = ultra_l1b(data_dict, ancillary_files) ancillary_files = {} @@ -359,3 +386,34 @@ def test_ultra_l1b_error(mock_data_l1a_rates_dict): ValueError, match="Data dictionary does not contain the expected keys." ): ultra_l1b(mock_data_l1a_rates_dict, ancillary_files) + + +@pytest.mark.external_test_data +def test_ultra_l1b_priority_de( + mock_get_annotated_particle_velocity, + de_dataset, + aux_dataset, + use_fake_spin_data_for_time, + ancillary_files, + use_fake_repoint_data_for_time, +): + """Tests that priority de datasets can be created""" + data_dict = {} + # Create a spin table that cover spin 0-141 + use_fake_spin_data_for_time(443640487, 443642460) + # Use repoint data that will NOT cover the event times to test flag setting + use_fake_repoint_data_for_time(np.arange(0, +86400 * 5, 86400)) + de_dataset.attrs["Repointing"] = "repoint00001" + # Set the logical source to match the priority de key + # Use the de dataset because it should be treated the same as the priority dataset + # and the priority dataset takes a long time to create. + data_dict["imap_ultra_l1a_45sensor-priority-1-de"] = de_dataset + data_dict[aux_dataset.attrs["Logical_source"]] = aux_dataset + + l1b_de_dataset = ultra_l1b(data_dict, ancillary_files) + + assert l1b_de_dataset[0] + assert ( + l1b_de_dataset[0].attrs["Logical_source"] + == "imap_ultra_l1b_45sensor-priority-1-de" + ) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py index 4763d630b6..1df4845720 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_culling.py @@ -27,7 +27,9 @@ flag_low_voltage, flag_rates, flag_scattering, + flag_spectral_events, flag_statistical_outliers, + flag_upstream_ion, get_binned_energy_ranges, get_binned_spins_edges, get_de_rejection_mask, @@ -47,6 +49,30 @@ TEST_PATH = imap_module_directory / "tests" / "ultra" / "data" / "l1" +@pytest.fixture +def setup_repoint_47_data(): + """Fixture to set up data for validation test using repoint 47.""" + de_df = pd.read_csv(TEST_PATH / "de_test_data_repoint00047.csv") + de_ds = xr.Dataset( + { + "de_event_met": ("epoch", de_df.event_times.values), + "energy_spacecraft": ("epoch", de_df.energy_spacecraft.values), + "quality_outliers": ("epoch", de_df.quality_outliers.values), + "quality_scattering": ("epoch", de_df.quality_scattering.values), + "ebin": ("epoch", de_df.ebin.values), + } + ) + xspin = pd.read_csv(TEST_PATH / "extendedspin_test_data_repoint00047.csv") + spin_bin_size = UltraConstants.SPIN_BIN_SIZE + spin_tbin_edges = get_binned_spins_edges( + xspin.spin_number.values, + xspin.spin_period.values, + xspin.spin_start_time.values, + spin_bin_size, + ) + return de_ds, xspin, spin_tbin_edges + + @pytest.fixture def test_data(use_fake_spin_data_for_time): """Fixture to compute and return test data.""" @@ -415,16 +441,26 @@ def test_expand_bin_flags_to_spins(caplog): def test_get_energy_and_spin_dependent_rejection_mask(): """Tests get_energy_and_spin_dependent_rejection_mask function.""" n_spins = 10 + energy_range_flags = np.zeros(16, dtype=np.uint16) + energy_range_flags[:3] = [2**1, 2**2, 2**3] # Example flags for 3 energy bins + energy_range_edges = np.full(17, -1.0e31, dtype=np.float32) + energy_range_edges[:4] = [ + 3, + 5, + 7, + 18, + ] # Example energy bin edges (4 edges = 3 bins) goodtimes_dataset = xr.Dataset( data_vars={ "spin_number": np.arange(n_spins), "quality_low_voltage": np.full(n_spins, 0), "quality_high_energy": np.full(n_spins, 0), "quality_statistics": np.full(n_spins, 0), - "energy_range_flags": np.array( - [2**1, 2**2, 2**3] - ), # Example flags for energy bins - "energy_range_edges": np.array([3, 5, 7, 18]), # Example energy bin edges + "energy_range_flags": energy_range_flags, + "energy_range_edges": energy_range_edges, + "quality_upstream_ion_1": np.full(n_spins, 0), + "quality_upstream_ion_2": np.full(n_spins, 0), + "quality_spectral": np.full(n_spins, 0), } ) # update quality flags to test that events get rejected @@ -694,38 +730,26 @@ def test_flag_high_energy(): assert not np.any(quality_flags[:, 2]) +@mock.patch( + "imap_processing.ultra.l1b.ultra_l1b_culling.UltraConstants.HIGH_ENERGY_CULL_CHANNEL", + 4, +) +@mock.patch( + "imap_processing.ultra.l1b.ultra_l1b_culling.UltraConstants.HIGH_ENERGY_COMBINED_SPIN_BIN_RADIUS", + 3, +) @pytest.mark.external_test_data -def test_validate_high_energy_cull(): +def test_validate_high_energy_cull(setup_repoint_47_data): """Validate that high energy spins are correctly flagged""" # Mock thresholds to match the test data (I used fake ones to create more # complexity) mock_thresholds = np.array([0.05, 1.5, 0.6, 119.2, 0.2]) * 20 - # read test data from csv files - xspin = pd.read_csv(TEST_PATH / "extendedspin_test_data_repoint00047.csv") expected_qf = pd.read_csv( TEST_PATH / "validate_high_energy_culling_results_repoint00047_v2.csv" ).to_numpy() - de_df = pd.read_csv(TEST_PATH / "de_test_data_repoint00047.csv") - de_ds = xr.Dataset( - { - "de_event_met": ("epoch", de_df.event_times.values), - "energy_spacecraft": ("epoch", de_df.energy_spacecraft.values), - "quality_outliers": ("epoch", de_df.quality_outliers.values), - "quality_scattering": ("epoch", de_df.quality_scattering.values), - "ebin": ("epoch", de_df.ebin.values), - } - ) - # Use constants from the code to ensure consistency with the actual culling code - spin_bin_size = UltraConstants.SPIN_BIN_SIZE - spin_tbin_edges = get_binned_spins_edges( - xspin.spin_number.values, - xspin.spin_period.values, - xspin.spin_start_time.values, - spin_bin_size, - ) - intervals, _, _ = build_energy_bins() + de_ds, _, spin_tbin_edges = setup_repoint_47_data # Get the energy ranges - energy_ranges = get_binned_energy_ranges(intervals) + energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) e_flags = flag_high_energy( de_ds, spin_tbin_edges, energy_ranges, None, mock_thresholds ) @@ -845,34 +869,15 @@ def test_get_poisson_stats(): @pytest.mark.external_test_data -def test_validate_stat_cull(): +def test_validate_stat_cull(setup_repoint_47_data): """Validate that statistical-outlier quality flags match expected results.""" # read test data from csv files - xspin = pd.read_csv(TEST_PATH / "extendedspin_test_data_repoint00047.csv") results_df = pd.read_csv( TEST_PATH / "validate_stat_culling_results_repoint00047_v2.csv" ) - de_df = pd.read_csv(TEST_PATH / "de_test_data_repoint00047.csv") - de_ds = xr.Dataset( - { - "de_event_met": ("epoch", de_df.event_times.values), - "energy_spacecraft": ("epoch", de_df.energy_spacecraft.values), - "quality_outliers": ("epoch", de_df.quality_outliers.values), - "quality_scattering": ("epoch", de_df.quality_scattering.values), - "ebin": ("epoch", de_df.ebin.values), - } - ) - # Use constants from the code to ensure consistency with the actual culling code - spin_bin_size = UltraConstants.SPIN_BIN_SIZE - spin_tbin_edges = get_binned_spins_edges( - xspin.spin_number.values, - xspin.spin_period.values, - xspin.spin_start_time.values, - spin_bin_size, - ) - intervals, _, _ = build_energy_bins() + de_ds, _, spin_tbin_edges = setup_repoint_47_data # Get the energy ranges - energy_ranges = get_binned_energy_ranges(intervals) + energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) # Create a mask of flagged events to test that the stat cull algorithm # properly ignores these. The test data was created using this exact mask as well. @@ -903,13 +908,89 @@ def test_get_energy_range_flags(): energy_ranges = get_binned_energy_ranges(intervals) flags = get_energy_range_flags(energy_ranges) - np.testing.assert_array_equal(flags, 2 ** np.arange(5)) + np.testing.assert_array_equal(flags, 2 ** np.arange(6)) def test_get_binned_energy_ranges(): """Tests get_binned_energy_ranges function.""" intervals, _, _ = build_energy_bins() energy_ranges = get_binned_energy_ranges(intervals) - - expected_energy_ranges = np.array([4.2, 9.4425, 21.2116, 47.2388, 105.202, 316.335]) + expected_energy_ranges = np.array( + [3.0, 6.96, 15.71, 34.9866, 77.9161, 116.276, 316.335] + ) np.testing.assert_array_equal(energy_ranges, expected_energy_ranges) + + +@pytest.mark.external_test_data +def test_validate_upstream_ion_cull(setup_repoint_47_data): + """Validate that upstream ion quality flags match expected results.""" + # read test data from csv files + expected_results = pd.read_csv( + TEST_PATH / "validate_upstream_ion_1_culling_results_repoint00047_v1.csv" + ).to_numpy() + de_ds, _, spin_tbin_edges = setup_repoint_47_data + intervals, _, _ = build_energy_bins() + energy_ranges = get_binned_energy_ranges(intervals) + mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) + mask[0:2, 0:2] = ( + True # This will mark the first 2 energy bins and first 2 spin bins as flagged + ) + flags = flag_upstream_ion( + de_ds, + spin_tbin_edges, + energy_ranges, + mask, + UltraConstants.UPSTREAM_ION_ENERGY_CHANNELS_1, + 90, + ) + # Combine the flags with the mask to get the final expected results since the + # masked bins should be flagged as well. + results = flags | mask + np.testing.assert_array_equal(results, ~expected_results.astype(bool)) + + +@pytest.mark.external_test_data +def test_upstream_ion_cull_invalid_channels(setup_repoint_47_data): + """Validate upstream ion error handling.""" + de_ds, _, spin_tbin_edges = setup_repoint_47_data + intervals, _, _ = build_energy_bins() + energy_ranges = get_binned_energy_ranges(intervals) + mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) + with pytest.raises( + ValueError, + match="Channels provided for upstream ion flagging" + " must be within the bounds of the energy ranges.", + ): + flag_upstream_ion( + de_ds, + spin_tbin_edges, + energy_ranges, + mask, + [5, 6, 7], # Invalid channels that are out of bounds + 90, + ) + + +@pytest.mark.external_test_data +def test_validate_spectral_cull(setup_repoint_47_data): + """Validate that spectral flags match expected results.""" + # read test data from csv files + expected_results = pd.read_csv( + TEST_PATH / "validate_spectral_culling_results_repoint00047_v1.csv" + ).to_numpy() + de_ds, _, spin_tbin_edges = setup_repoint_47_data + intervals, _, _ = build_energy_bins() + energy_ranges = get_binned_energy_ranges(intervals) + mask = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=bool) + mask[0:2, 0:2] = ( + True # This will mark the first 2 energy bins and first 2 spin bins as flagged + ) + flags = flag_spectral_events( + de_ds, + spin_tbin_edges, + energy_ranges, + UltraConstants.SPECTRAL_ENERGY_CHANNELS, + 90, + ) + results = flags | mask + np.testing.assert_array_equal(results, ~expected_results.astype(bool)) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py index 0127477c1b..876b6cbea5 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1b_extended.py @@ -694,7 +694,7 @@ def test_get_efficiency(): / "imap_ultra_l1b-45sensor-logistic-interpolation_20250101_v000.csv" } - efficiency = get_efficiency(energy, phi, theta, ancillary_files) + efficiency = get_efficiency(energy, phi, theta, ancillary_files, "ultra45") expected_efficiency = np.array([0.0593281, 0.21803386, 0.0593281, 0.0628940]) np.testing.assert_allclose(efficiency, expected_efficiency, atol=1e-03, rtol=0) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c.py b/imap_processing/tests/ultra/unit/test_ultra_l1c.py index fbe6903e29..bb8b8d1d40 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c.py @@ -179,6 +179,8 @@ def test_calculate_spacecraft_pset_with_cdf( de_dict["spin"] = np.full(len(sc_dps_velocity), 0) de_dict["species"] = np.ones(len(sc_dps_velocity), dtype=np.uint8) de_dict["ebin"] = np.ones(len(sc_dps_velocity), dtype=np.uint8) + de_dict["theta"] = np.zeros(len(df_subset), dtype=np.float32) + de_dict["phi"] = np.zeros(len(df_subset), dtype=np.float32) de_dict["event_times"] = df_subset["tdb"].values name = "imap_ultra_l1b_45sensor-de" @@ -245,6 +247,8 @@ def test_calculate_helio_pset_with_cdf( # Fake SCLK in seconds that matches SPICE. de_dict["event_times"] = np.full(len(df_subset), 2.41187e13) de_dict["ebin"] = np.ones(len(df_subset), dtype=np.uint8) + de_dict["theta"] = np.zeros(len(df_subset), dtype=np.float32) + de_dict["phi"] = np.zeros(len(df_subset), dtype=np.float32) species_bin = np.full(len(df_subset), 1, dtype=np.uint8) # PosYSlit is True for left (start_type = 1) diff --git a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py index 2d6f249571..1751dd6af6 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l1c_pset_bins.py @@ -1,5 +1,6 @@ "Tests pointing sets" +import logging from unittest import mock import astropy_healpix.healpy as hp @@ -10,9 +11,10 @@ from scipy import interpolate from imap_processing import imap_module_directory +from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1c import ultra_l1c_pset_bins from imap_processing.ultra.l1c.spacecraft_pset import ( - calculate_fwhm_spun_scattering, + calculate_accepted_pixels, ) from imap_processing.ultra.l1c.ultra_l1c_pset_bins import ( build_energy_bins, @@ -116,13 +118,9 @@ def test_get_spacecraft_histogram(test_data): energy_bin_edges, _, _ = build_energy_bins() subset_energy_bin_edges = energy_bin_edges[:3] - hist, latitude, longitude, n_pix = get_spacecraft_histogram( - v, energy, subset_energy_bin_edges, nside=1 - ) + hist, n_pix = get_spacecraft_histogram(v, energy, subset_energy_bin_edges, nside=1) assert hist.shape == (len(subset_energy_bin_edges), hp.nside2npix(1)) assert n_pix == hp.nside2npix(1) - assert latitude.shape == (n_pix,) - assert longitude.shape == (n_pix,) # Spot check that 2 counts are in the second energy bin assert np.sum(hist[2, :]) == 2 @@ -133,14 +131,10 @@ def test_get_spacecraft_histogram(test_data): (2.5, 4.137), (3.385, 5.057), ] - hist, latitude, longitude, n_pix = get_spacecraft_histogram( - v, energy, overlapping_bins, nside=1 - ) + hist, n_pix = get_spacecraft_histogram(v, energy, overlapping_bins, nside=1) # Spot check that 3 counts are in the third energy bin assert np.sum(hist[2, :]) == 3 assert n_pix == hp.nside2npix(1) - assert latitude.shape == (n_pix,) - assert longitude.shape == (n_pix,) def mock_imap_state(time, ref_frame): @@ -275,15 +269,13 @@ def test_apply_deadtime_correction( """Tests apply_deadtime_correction function.""" mock_theta, mock_phi, spin_phase_steps, inside_inds, pix, steps = spun_index_data deadtime_ratios = xr.DataArray(np.ones(steps), dims="spin_phase_step") - valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = ( - calculate_fwhm_spun_scattering( - spin_phase_steps, - mock_theta, - mock_phi, - ancillary_files, - 45, - reject_scattering=False, - ) + valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = calculate_accepted_pixels( + spin_phase_steps, + mock_theta, + mock_phi, + ancillary_files, + 45, + reject_scattering=False, ) boundary_sf = xr.DataArray(np.ones((pix, steps)), dims=("pixel", "spin_phase_step")) exposure_pointing_adjusted = calculate_exposure_time( @@ -309,15 +301,13 @@ def test_apply_deadtime_correction_energy_dep( deadtime_ratios = xr.DataArray(np.ones(steps), dims="spin_phase_step") boundary_sf = xr.DataArray(np.ones((steps, pix)), dims=("spin_phase_step", "pixel")) - valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = ( - calculate_fwhm_spun_scattering( - spin_phase_steps, - mock_theta, - mock_phi, - ancillary_files, - 45, - reject_scattering=True, - ) + valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = calculate_accepted_pixels( + spin_phase_steps, + mock_theta, + mock_phi, + ancillary_files, + 45, + reject_scattering=True, ) exposure_pointing_adjusted = calculate_exposure_time( @@ -355,15 +345,13 @@ def test_get_eff_and_gf(imap_ena_sim_metakernel, ancillary_files, spun_index_dat # Simulate first 100 pixels are in the FOR for all spin phases inside_inds = 100 spin_phase_steps[:, :, :inside_inds] = True - valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = ( - calculate_fwhm_spun_scattering( - spin_phase_steps, - mock_theta, - mock_phi, - ancillary_files, - 45, - reject_scattering=False, - ) + valid_spun_pixels, fwhm_theta, fwhm_phi, thresholds = calculate_accepted_pixels( + spin_phase_steps, + mock_theta, + mock_phi, + ancillary_files, + 45, + reject_scattering=False, ) boundary_sf = xr.DataArray( np.ones((steps, energy_dim, pix)), dims=("spin_phase_step", "energy", "pixel") @@ -374,6 +362,7 @@ def test_get_eff_and_gf(imap_ena_sim_metakernel, ancillary_files, spun_index_dat mock_theta, mock_phi, npix=pix, + sensor_id=45, ancillary_files=ancillary_files, apply_bsf=False, ) @@ -395,12 +384,14 @@ def test_get_spacecraft_exposure_times( ancillary_files, use_fake_spin_data_for_time, aux_dataset, + mock_goodtimes_dataset, + caplog, ): """Test get_spacecraft_exposure_times function.""" data_start_time = 445015665.0 data_end_time = 453070000.0 use_fake_spin_data_for_time(data_start_time, data_end_time) - steps = 500 # reduced for testing + steps = 200 # reduced for testing pix = 786 mock_theta = np.random.uniform(-60, 60, (steps, pix)) @@ -412,26 +403,42 @@ def test_get_spacecraft_exposure_times( ) # Spin phase steps, random 0 or 1 pixels_below_threshold, fwhm_theta, fwhm_phi, thresholds = ( - calculate_fwhm_spun_scattering( + calculate_accepted_pixels( spin_phase_steps, mock_theta, mock_phi, ancillary_files, 45 ) ) boundary_sf = xr.DataArray(np.ones((steps, pix)), dims=("spin_phase_step", "pixel")) - exposure_pointing, deadtimes = get_spacecraft_exposure_times( + caplog.set_level(logging.INFO) + ( + exposure_pointing, + deadtimes, + ) = get_spacecraft_exposure_times( rates_dataset, pixels_below_threshold, boundary_sf, aux_dataset, - ( - data_start_time, - data_start_time, - ), - 46, # number of energy bins - pix, + build_energy_bins()[2], + goodtimes_dataset=mock_goodtimes_dataset, ) np.testing.assert_array_equal(exposure_pointing.shape, (46, pix)) np.testing.assert_array_equal(deadtimes.shape, (steps,)) + # Check that the number of good spins per energy bin is logged correctly + # The first 3 energy bins were not used in the goodtimes culling and therefore + # should have all 100 spins. + # Energy range 1,2,3 (which maps to the next UltraConstants.N_CULL_EBINS * 3 bins) + # should have 99 spins because there was a flag set to true for one spin + # in the mock_goodtimes_dataset. The rest of the energy bins should have 100 spins + # because the goodtimes dataset did not flag any spins for those energy bins. + expected_good_spins = np.full(46, 100.0) + expected_good_spins[ + UltraConstants.BASE_CULL_EBIN : UltraConstants.N_CULL_EBINS * 3 + + UltraConstants.BASE_CULL_EBIN + ] = 99.0 + # The goodtimes dataset has flags set to True for energy bin 0-3 and for the + # first three spins. + assert f"Found {expected_good_spins.tolist()} valid spins" in caplog.text + def test_get_spacecraft_background_rates( rates_l1_test_path, use_fake_spin_data_for_time, ancillary_files diff --git a/imap_processing/tests/ultra/unit/test_ultra_l2.py b/imap_processing/tests/ultra/unit/test_ultra_l2.py index f1bb188997..4d496eadd1 100644 --- a/imap_processing/tests/ultra/unit/test_ultra_l2.py +++ b/imap_processing/tests/ultra/unit/test_ultra_l2.py @@ -46,7 +46,8 @@ def _setup_spice_kernels_list(self, spice_test_data_path, furnish_kernels): def _mock_single_pset(self, _setup_spice_kernels_list, furnish_kernels): with furnish_kernels(self.required_kernel_names): self.ultra_pset = mock_l1c_pset_product_healpix( - nside=32, + nside=16, + counts_nside=32, stripe_center_lat=0, timestr="2025-05-15T12:00:00", energy_dependent_exposure=True, @@ -66,6 +67,7 @@ def _mock_multiple_psets(self, _setup_spice_kernels_list, furnish_kernels): self.ultra_psets = [ mock_l1c_pset_product_healpix( nside=16, + counts_nside=32, stripe_center_lat=mid_latitude, width_scale=5, counts_scaling_params=(50, 0.5), @@ -139,7 +141,6 @@ def test_generate_ultra_healpix_skymap_single_pset( pset["energy_bin_delta"] = pset["energy_bin_delta"].expand_dims( {CoordNames.TIME.value: pset["epoch"].values} ) - # Create the Healpix skymap in the desired frame. with furnish_kernels(self.required_kernel_names): hp_skymap, _ = ultra_l2.generate_ultra_healpix_skymap( @@ -725,13 +726,15 @@ def test_ultra_l2_descriptor_hpmap(self, mock_data_dict, furnish_kernels): with furnish_kernels(self.required_kernel_names): output_map = ultra_l2.ultra_l2( data_dict=mock_data_dict, - descriptor="u90-ena-h-sf-nsp-full-hae-nside32-6mo", + descriptor="u90-ena-h-sf-nsp-full-hae-nside32-3mo", )[0] assert "spacecraft frame" in output_map.attrs["Logical_source_description"] + # Check that the logical source contains the expected information from the + # descriptor string assert ( output_map.attrs["Logical_source"] - == "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-nside32-6mo" + == "imap_ultra_l2_u90-ena-h-sf-nsp-full-hae-nside32-3mo" ) assert output_map.attrs["Spice_reference_frame"] == "IMAP_HAE" assert output_map.attrs["HEALPix_nside"] == "32" @@ -741,8 +744,13 @@ def test_ultra_l2_descriptor_hpmap(self, mock_data_dict, furnish_kernels): @pytest.mark.usefixtures("_mock_single_pset") def test_bin_pset_energy_bins_default(self): """Test binning with default bin sizes.""" - # Avoid modifying the original pset - pset = self.ultra_pset.copy(deep=True) + pset = mock_l1c_pset_product_healpix( + nside=16, + counts_nside=16, + stripe_center_lat=0, + timestr="2025-05-15T12:00:00", + energy_dependent_exposure=True, + ) # Set the values in the single input PSET # Create a mock array with known values to test binning # e.g., 0,0,0,0,1,1,1,1,2,2,2,2,...11,11 @@ -752,7 +760,9 @@ def test_bin_pset_energy_bins_default(self): mock_array = ( np.ones_like(pset["exposure_factor"]) * mock_vals[np.newaxis, :, np.newaxis] ) - pset["counts"].values = mock_array + pset["counts"].values = ( + np.ones_like(pset["counts"]) * mock_vals[np.newaxis, :, np.newaxis] + ) pset["exposure_factor"].values = mock_array pset["sensitivity"].values = mock_array[0] pset["geometric_function"].values = mock_array[0] diff --git a/imap_processing/ultra/constants.py b/imap_processing/ultra/constants.py index 811a080ff2..a4a0e0eb79 100644 --- a/imap_processing/ultra/constants.py +++ b/imap_processing/ultra/constants.py @@ -92,6 +92,8 @@ class UltraConstants: 300.0, 1e5, ] + # Counts at l1c are sampled at a finer resolution. + L1C_COUNTS_NSIDE = 128 PSET_ENERGY_BIN_EDGES: ClassVar[list] = [ 3.0, @@ -179,6 +181,13 @@ class UltraConstants: FOV_THETA_OFFSET_DEG = 0.0 FOV_PHI_LIMIT_DEG = 60.0 + # Restricted FOV theta/phi acceptance limits (degrees). + # Samples outside these bounds are excluded from GF, efficiency, exposure, + # and counts maps at L1C (fine energy bin maps only). + RESTRICTED_FOV_THETA_LOW_DEG_45: float = -43.0 + RESTRICTED_FOV_THETA_HIGH_DEG_45: float = 43.0 + RESTRICTED_FOV_THETA_LOW_DEG_90: float = -43.0 + RESTRICTED_FOV_THETA_HIGH_DEG_90: float = 43.0 # For spatiotemporal culling EARTH_RADIUS_KM: float = 6378.1 @@ -191,10 +200,10 @@ class UltraConstants: # Number of energy bins to use in energy dependent culling N_CULL_EBINS = 8 # Bin to start culling at - BASE_CULL_EBIN = 3 + BASE_CULL_EBIN = 0 # Maximum energy threshold in keV. When creating the energy ranges for culling, # merge all energy bins above this threshold into one bin. - MAX_ENERGY_THRESHOLD = 100 + MAX_ENERGY_THRESHOLD = 116.0 # Angle threshold in radians for ULTRA 45 degree culling. # This is only needed for ULTRA 45 since Earth may be in the FOV. EARTH_ANGLE_45_THRESHOLD = np.radians(20) @@ -202,15 +211,46 @@ class UltraConstants: # the number of energy bins used. # n_bins=len(PSET_ENERGY_BIN_EDGES)[BASE_CULL_EBIN:] // N_CULL_EBINS # an error will be raised if this does not match n_bins - HIGH_ENERGY_CULL_THRESHOLDS = np.array([2.0, 1.5, 0.6, 0.2, 0.2]) * SPIN_BIN_SIZE + HIGH_ENERGY_CULL_THRESHOLDS = ( + np.array([4.0, 2.0, 1.20, 0.45, 0.1, 0.1]) * SPIN_BIN_SIZE + ) # Use the channel defined below to determine which spins are contaminated - HIGH_ENERGY_CULL_CHANNEL = 4 + HIGH_ENERGY_CULL_CHANNEL = 5 # For the high energy cull, we want to combine spin bins because an SEP event is # expected to be over a longer time period. Low voltage and statistical culling # will still be done on the original spin bins. The variable below defines the # radius (in number of spin bins) to use when combining for the high energy cull. - HIGH_ENERGY_COMBINED_SPIN_BIN_RADIUS = 3 + HIGH_ENERGY_COMBINED_SPIN_BIN_RADIUS = 5 # Number of iterations to perform for statistical outlier culling. STAT_CULLING_N_ITER = 5 # Sigma threshold to use for statistical outlier culling. STAT_CULLING_STD_THRESHOLD = 0.05 + # Energy channels for the upstream ion cull + # The algorithm will be run twice with the different sets of channels below. + UPSTREAM_ION_ENERGY_CHANNELS_1: ClassVar[list] = [0, 1, 2] + UPSTREAM_ION_ENERGY_CHANNELS_2: ClassVar[list] = [2, 3, 4] + UPSTREAM_SIG_THRESHOLD = 2.5 + # Spectral culling parameters + SPECTRAL_ENERGY_CHANNELS: ClassVar[list] = [0, 1, 2, 3] + SPECTRAL_SIG_THRESHOLD = 1 + # Set dimensions for extended spin/goodtime support variables + # ISTP requires fixed dimensions, so we set these to the maximum we expect to need + # and pad with fill values if we use fewer bins. + MAX_ENERGY_RANGES = 16 + MAX_ENERGY_RANGE_EDGES = MAX_ENERGY_RANGES + 1 + + # L1C PSET constants + + # When True, applies the FOV restrictions defined above to the L1C fine energy bin + # maps (GF, efficiency, exposure, counts). This culls regions of the instrument + # field of view with poor efficiency calibration from inclusion into the map making + # process. + APPLY_FOV_RESTRICTIONS_L1C: bool = True + + # When True, applies the boundary scale factors from the ancillary file to exposure + # time, efficiency, and geometric factor maps. + APPLY_BOUNDARY_SCALE_FACTORS_L1C: bool = False + + # When True, applies the scattering rejection mask based on the FWHM thresholds + # to the L1C fine energy bin maps. + APPLY_SCATTERING_REJECTION_L1C: bool = False diff --git a/imap_processing/ultra/l0/decom_tools.py b/imap_processing/ultra/l0/decom_tools.py index 3b75047910..a39647fd41 100644 --- a/imap_processing/ultra/l0/decom_tools.py +++ b/imap_processing/ultra/l0/decom_tools.py @@ -210,9 +210,9 @@ def decompress_image( pos = 0 # Starting position in the binary string while plane_num < planes_per_packet: # Compressed pixel matrix - p = np.zeros((rows, cols), dtype=np.uint16) + p: np.ndarray = np.zeros((rows, cols), dtype=np.uint16) # Decompressed pixel matrix - p_decom = np.zeros((rows, cols), dtype=np.int16) + p_decom: np.ndarray = np.zeros((rows, cols), dtype=np.int16) for i in range(rows): for j in range(blocks_per_row): @@ -245,10 +245,10 @@ def decompress_image( p[i][column_index] = np.int16(current_pixel0) - delta_f # Perform logarithmic decompression on the pixel value p_decom[i][column_index] = log_decompression( - p[i][column_index], mantissa_bit_length + int(p[i][column_index]), mantissa_bit_length ) - current_pixel0 = p[i][column_index] - current_pixel0 = p[i][0] + current_pixel0 = int(p[i][column_index]) + current_pixel0 = p[i][0] # type: ignore[assignment] planes.append(p_decom) plane_num += 1 # Read P00 for the next plane (if not the last plane) diff --git a/imap_processing/ultra/l0/decom_ultra.py b/imap_processing/ultra/l0/decom_ultra.py index 7c7f5227a6..94c6a84f75 100644 --- a/imap_processing/ultra/l0/decom_ultra.py +++ b/imap_processing/ultra/l0/decom_ultra.py @@ -60,11 +60,11 @@ def extract_initial_items_from_combined_packets( n_packets = len(packets.epoch) # Preallocate arrays - sid = np.zeros(n_packets, dtype=np.uint8) - spin = np.zeros(n_packets, dtype=np.uint8) - abortflag = np.zeros(n_packets, dtype=np.uint8) - startdelay = np.zeros(n_packets, dtype=np.uint16) - p00 = np.zeros(n_packets, dtype=np.uint8) + sid: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + spin: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + abortflag: np.ndarray = np.zeros(n_packets, dtype=np.uint8) + startdelay: np.ndarray = np.zeros(n_packets, dtype=np.uint16) + p00: np.ndarray = np.zeros(n_packets, dtype=np.uint8) # Extract the data array outside of the loop binary_data = packets["packetdata"].data @@ -463,7 +463,7 @@ def process_ultra_cmd_echo(ds: xr.Dataset) -> xr.Dataset: fill = 0xFF max_len = 10 - arg_array = np.full((len(ds["epoch"]), max_len), fill, dtype=np.uint8) + arg_array: np.ndarray = np.full((len(ds["epoch"]), max_len), fill, dtype=np.uint8) for i, arg in enumerate(ds["args"].values): # Converts to the numeric representations of each byte. @@ -508,7 +508,7 @@ def process_ultra_macros_checksum(ds: xr.Dataset) -> xr.Dataset: Dataset with unpacked and decoded checksum values. """ # big endian uint16 - packed_dtype = np.dtype(">u2") + packed_dtype: np.dtype = np.dtype(">u2") fill = np.iinfo(packed_dtype).max n_epochs = ds.sizes["epoch"] max_len = 256 diff --git a/imap_processing/ultra/l1b/de.py b/imap_processing/ultra/l1b/de.py index 69fd75d6f0..0122f756b8 100644 --- a/imap_processing/ultra/l1b/de.py +++ b/imap_processing/ultra/l1b/de.py @@ -116,36 +116,74 @@ def calculate_de( ph_indices = np.nonzero(valid_mask & ph_mask)[0] ssd_indices = np.nonzero(valid_mask & ssd_mask)[0] # Instantiate arrays - xf = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - yf = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - xb = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - yb = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - xc = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - d = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) - r = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - phi = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - theta = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - tof = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - etof = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - ctof = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - tof_energy = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - magnitude_v = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - energy = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - e_bin = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) - e_bin_l1a = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) - species_bin = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) - t2 = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) - event_times = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) - spin_starts = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) + xf: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + yf: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + xb: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + yb: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + xc: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + d: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64) + r: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32) + phi: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + theta: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + tof: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + etof: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + ctof: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + tof_energy: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + magnitude_v: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + energy: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + e_bin: np.ndarray = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) + e_bin_l1a: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + ) + species_bin: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + ) + t2: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float32 + ) + event_times: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64 + ) + spin_starts: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_FLOAT32, dtype=np.float64 + ) shape = (len(de_dataset["epoch"]), 3) - sc_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - sc_dps_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - helio_velocity = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - velocities = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - v_hat = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - r_hat = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) - - start_type = np.full(len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8) + sc_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + sc_dps_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + helio_velocity: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + velocities: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + v_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + r_hat: np.ndarray = np.full(shape, FILLVAL_FLOAT32, dtype=np.float32) + + start_type: np.ndarray = np.full( + len(de_dataset["epoch"]), FILLVAL_UINT8, dtype=np.uint8 + ) quality_flags = np.full( de_dataset["epoch"].shape, ImapDEOutliersUltraFlags.NONE.value, dtype=np.uint16 ) @@ -335,7 +373,7 @@ def calculate_de( repoint_id, et_to_met(event_times[valid_events]) ) # Initialize an array of all events as False - events_to_flag = np.zeros(len(quality_flags), dtype=bool) + events_to_flag: np.ndarray = np.zeros(len(quality_flags), dtype=bool) # Identify valid events that are outside the pointing events_to_flag[valid_events] = ~in_pointing # Update quality flags for valid events that are not in the pointing @@ -375,7 +413,11 @@ def calculate_de( ancillary_files, ) de_dict["event_efficiency"] = get_efficiency( - de_dict["tof_energy"], de_dict["phi"], de_dict["theta"], ancillary_files + de_dict["tof_energy"], + de_dict["phi"], + de_dict["theta"], + ancillary_files, + f"ultra{sensor}", ) de_dict["geometric_factor_blades"] = get_geometric_factor( de_dict["phi"], diff --git a/imap_processing/ultra/l1b/extendedspin.py b/imap_processing/ultra/l1b/extendedspin.py index 8382da622a..147b542afc 100644 --- a/imap_processing/ultra/l1b/extendedspin.py +++ b/imap_processing/ultra/l1b/extendedspin.py @@ -14,7 +14,9 @@ flag_imap_instruments, flag_low_voltage, flag_rates, + flag_spectral_events, flag_statistical_outliers, + flag_upstream_ion, get_binned_energy_ranges, get_binned_spins_edges, get_energy_histogram, @@ -30,6 +32,7 @@ def calculate_extendedspin( dict_datasets: dict[str, xr.Dataset], + de_dataset: xr.Dataset, name: str, instrument_id: int, ) -> xr.Dataset: @@ -40,6 +43,8 @@ def calculate_extendedspin( ---------- dict_datasets : dict Dictionary containing all the datasets. + de_dataset : xarray.Dataset + Dataset containing the direct event data. name : str Name of the dataset. instrument_id : int @@ -52,7 +57,6 @@ def calculate_extendedspin( """ aux_dataset = dict_datasets[f"imap_ultra_l1a_{instrument_id}sensor-aux"] rates_dataset = dict_datasets[f"imap_ultra_l1a_{instrument_id}sensor-rates"] - de_dataset = dict_datasets[f"imap_ultra_l1b_{instrument_id}sensor-de"] status_dataset = dict_datasets[f"imap_ultra_l1b_{instrument_id}sensor-status"] extendedspin_dict = {} @@ -74,6 +78,14 @@ def calculate_extendedspin( spin_tbin_edges = get_binned_spins_edges( spin, spin_period, spin_starttime, spin_bin_size ) + # Calculate goodtime quality flags. + # The culling algorithms should be called in the following order + # 1. Low voltage + # 2. High energy (energy dependent) + # 3. Upstream ion (with first set of energy channels) + # 4. Upstream ion (with second set of energy channels) + # 5. Spectral cull + # 6. Statistical outliers (energy dependent) voltage_qf = flag_low_voltage(spin_tbin_edges, status_dataset) # Get energy bins used at l1c intervals, _, _ = build_energy_bins() @@ -94,6 +106,35 @@ def calculate_extendedspin( mask = ( voltage_qf[np.newaxis, :] | high_energy_qf ) # Shape (n_energy_bins, n_spins_bins) + upstream_ion_qf_1 = flag_upstream_ion( + de_dataset, + spin_tbin_edges, + energy_ranges, + mask, + UltraConstants.UPSTREAM_ION_ENERGY_CHANNELS_1, + instrument_id, + ) + # Update mask to include upstream ion flags from the first set of energy channels + # before flagging with the second set of energy channels + mask = mask | upstream_ion_qf_1 + upstream_ion_qf_2 = flag_upstream_ion( + de_dataset, + spin_tbin_edges, + energy_ranges, + mask, + UltraConstants.UPSTREAM_ION_ENERGY_CHANNELS_2, + instrument_id, + ) + spectral_qf = flag_spectral_events( + de_dataset, + spin_tbin_edges, + energy_ranges, + UltraConstants.SPECTRAL_ENERGY_CHANNELS, + instrument_id, + ) + # Update mask to include upstream ion flags #2 and spectral flags before flagging + # statistical outliers + mask = mask | upstream_ion_qf_2 | spectral_qf stat_outliers_qf, _, _, _ = flag_statistical_outliers( de_dataset, spin_tbin_edges, @@ -128,9 +169,9 @@ def calculate_extendedspin( # Validate that the spin values match valid = (idx < pulses.unique_spins.size) & (pulses.unique_spins[idx] == spin) - start_per_spin = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) - stop_per_spin = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) - coin_per_spin = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) + start_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) + stop_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) + coin_per_spin: np.ndarray = np.full(len(spin), FILLVAL_FLOAT32, dtype=np.float32) # Fill only the valid ones start_per_spin[valid] = pulses.start_per_spin[idx[valid]] @@ -146,13 +187,25 @@ def calculate_extendedspin( stat_outliers_qf = np.bitwise_or.reduce( stat_outliers_qf * energy_bin_flags[:, np.newaxis], axis=0 ) - # Low voltage flag is shape (n_spin_bins,) but we want to convert from a boolean - # to a bitwise flag to be consistent with the other flags, where each spin that - # is flagged will have the bitflag of all the energy flags combined. - voltage_qf = voltage_qf * np.bitwise_or.reduce(energy_bin_flags) + # Low voltage and upstream ion flags are shape (n_spin_bins,) but we want to + # convert from a boolean to a bitwise flag to be consistent with the other flags, + # where each spin that is flagged will have the bitflag of all the energy flags + # combined. + combined_flags = np.bitwise_or.reduce(energy_bin_flags) + voltage_qf = voltage_qf * combined_flags + upstream_ion_qf_1 = upstream_ion_qf_1 * combined_flags + upstream_ion_qf_2 = upstream_ion_qf_2 * combined_flags + spectral_qf = spectral_qf * combined_flags # Expand binned quality flags to individual spins. high_energy_qf = expand_bin_flags_to_spins(len(spin), high_energy_qf, spin_bin_size) voltage_qf = expand_bin_flags_to_spins(len(spin), voltage_qf, spin_bin_size) + spectral_qf = expand_bin_flags_to_spins(len(spin), spectral_qf, spin_bin_size) + upstream_ion_qf_1 = expand_bin_flags_to_spins( + len(spin), upstream_ion_qf_1, spin_bin_size + ) + upstream_ion_qf_2 = expand_bin_flags_to_spins( + len(spin), upstream_ion_qf_2, spin_bin_size + ) stat_outliers_qf = expand_bin_flags_to_spins( len(spin), stat_outliers_qf, spin_bin_size ) @@ -166,14 +219,33 @@ def calculate_extendedspin( extendedspin_dict["quality_hk"] = hk_qf extendedspin_dict["quality_instruments"] = inst_qf extendedspin_dict["quality_low_voltage"] = voltage_qf # shape (nspin,) - # TODO calculate flags for high energy (SEPS) and statistics culling - # Initialize these flags to NONE for now. + extendedspin_dict["quality_upstream_ion_1"] = upstream_ion_qf_1 # shape (nspin,) + extendedspin_dict["quality_upstream_ion_2"] = upstream_ion_qf_2 # shape (nspin,) + extendedspin_dict["quality_spectral"] = spectral_qf # shape (nspin,) extendedspin_dict["quality_statistics"] = stat_outliers_qf # shape (nspin,) extendedspin_dict["quality_high_energy"] = high_energy_qf # shape (nspin,) - # Add an array of flags for each energy bin. Shape: (n_energy_bins) - extendedspin_dict["energy_range_flags"] = energy_bin_flags - # Add energy ranges Shape: (n_energy_bins + 1) - extendedspin_dict["energy_range_edges"] = np.array(energy_ranges) + # ISTP requires stable dimension sizes, so this field must always remain size 16. + # If fewer bins are used, pad the remaining entries with 0. + energy_flags: np.ndarray = np.full( + UltraConstants.MAX_ENERGY_RANGES, 0, dtype=np.uint16 + ) + energy_flags[: len(energy_bin_flags)] = energy_bin_flags + extendedspin_dict["energy_range_flags"] = energy_flags + extendedspin_dict["energy_range_flags_dim"] = np.arange( + UltraConstants.MAX_ENERGY_RANGES + ) + + # Initialize array of energy range edges with fill value, then fill in the valid + # energy ranges. Set the length to be the max number of energy bins we expect to + # use for culling. The number of edges is one more than the number of bins (17). + ranges: np.ndarray = np.full( + (UltraConstants.MAX_ENERGY_RANGE_EDGES,), FILLVAL_FLOAT32, dtype=np.float32 + ) + ranges[: len(energy_ranges)] = energy_ranges + extendedspin_dict["energy_range_edges"] = ranges + extendedspin_dict["energy_range_edges_dim"] = np.arange( + UltraConstants.MAX_ENERGY_RANGE_EDGES + ) extendedspin_dataset = create_dataset(extendedspin_dict, name, "l1b") return extendedspin_dataset diff --git a/imap_processing/ultra/l1b/goodtimes.py b/imap_processing/ultra/l1b/goodtimes.py index 3ed3bb2858..5a57191e5a 100644 --- a/imap_processing/ultra/l1b/goodtimes.py +++ b/imap_processing/ultra/l1b/goodtimes.py @@ -54,9 +54,7 @@ def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Datas ) data_dict = extract_data_dict(filtered_dataset) - goodtimes_dataset = create_dataset(data_dict, name, "l1b") - if goodtimes_dataset["spin_number"].size == 0: goodtimes_dataset = goodtimes_dataset.drop_dims("spin_number") goodtimes_dataset = goodtimes_dataset.expand_dims(spin_number=[FILLVAL_UINT32]) @@ -94,6 +92,15 @@ def calculate_goodtimes(extendedspin_dataset: xr.Dataset, name: str) -> xr.Datas goodtimes_dataset["quality_high_energy"] = xr.DataArray( np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] ) + goodtimes_dataset["quality_upstream_ion_1"] = xr.DataArray( + np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + ) + goodtimes_dataset["quality_upstream_ion_2"] = xr.DataArray( + np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + ) + goodtimes_dataset["quality_spectral"] = xr.DataArray( + np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] + ) goodtimes_dataset["quality_statistics"] = xr.DataArray( np.array([FILLVAL_UINT16], dtype="uint16"), dims=["spin_number"] ) diff --git a/imap_processing/ultra/l1b/lookup_utils.py b/imap_processing/ultra/l1b/lookup_utils.py index 5642aa4730..fb9dad25c8 100644 --- a/imap_processing/ultra/l1b/lookup_utils.py +++ b/imap_processing/ultra/l1b/lookup_utils.py @@ -1,5 +1,7 @@ """Contains tools for lookup tables for l1b.""" +import logging + import numpy as np import numpy.typing as npt import pandas as pd @@ -9,6 +11,8 @@ from imap_processing.quality_flags import ImapDEOutliersUltraFlags from imap_processing.ultra.constants import UltraConstants +logger = logging.getLogger(__name__) + def get_y_adjust(dy_lut: np.ndarray, ancillary_files: dict) -> npt.NDArray: """ @@ -210,7 +214,7 @@ def get_angular_profiles( return lookup_table -def get_energy_efficiencies(ancillary_files: dict) -> pd.DataFrame: +def get_energy_efficiencies(ancillary_files: dict, sensor: str) -> pd.DataFrame: """ Lookup table for efficiencies for theta and phi. @@ -221,14 +225,22 @@ def get_energy_efficiencies(ancillary_files: dict) -> pd.DataFrame: ---------- ancillary_files : dict[Path] Ancillary files. + sensor : str + Sensor name: "ultra45" or "ultra90". Returns ------- lookup_table : DataFrame Efficiencies lookup table for a given sensor. """ - # TODO: add sensor to input when new lookup tables are available. - lookup_table = pd.read_csv(ancillary_files["l1b-45sensor-logistic-interpolation"]) + if sensor == "ultra45": + lookup_table = pd.read_csv( + ancillary_files["l1b-45sensor-logistic-interpolation"] + ) + else: + lookup_table = pd.read_csv( + ancillary_files["l1b-90sensor-logistic-interpolation"] + ) return lookup_table @@ -608,3 +620,71 @@ def get_scattering_thresholds(ancillary_files: dict) -> dict: threshold_dict = {(row[0], row[1]): row[2] for row in thresholds} return threshold_dict + + +def get_de_product_name( + repoint: str, sensor: int, data_level: str, ancillary_files: dict +) -> str: + """ + Get the name of the de product to use for processing. + + This will be either the raw de product or a priority 1-4 de product, depending on + the pointing and data level. + + Note: Currently the lookup tables are identical between ultra45 and ultra90, + but this function accounts for the possibility of them being different in the + future. + + Parameters + ---------- + repoint : str + The repointing ID in the format "repointXXXXX" where XXXXX is the repointing + number. + sensor : int + Sensor number, either 45 or 90. + data_level : str + Data level, either "l1b" or "l1c". + ancillary_files : dict + Ancillary files containing the lookup tables to determine which DE product + to use based on the repointing ID. + + Returns + ------- + de_product_name : str + Name of the de product to use for processing. + """ + if data_level not in ["l1b", "l1c"]: + raise ValueError(f"Invalid data level: {data_level}. Must be 'l1b' or 'l1c'.") + # load the lookup table. + # The lookup table will have columns for repointing_id_start, repointing_id_end, + # and de_product. If repointing_id_end is NaN that indicates that the de_product + # should be used for all repoint IDs greater than or equal to repointing_id_start. + file_name = f"{data_level}-{sensor}sensor-de-product-lookup" + de_lookup = pd.read_csv(ancillary_files[file_name]) + repoint_id = int(repoint.replace("repoint", "")) + # Filter the dataset to find where the current repoint ID falls within the + # repointing_id_start and repointing_id_end range. OR if repointing_id_end is NaN, + # then just check if repoint_id is greater than or equal to repointing_id_start + repoint_row = de_lookup[ + (de_lookup["repointing_id_start"] <= repoint_id) + & ( + (de_lookup["repointing_id_end"] > repoint_id) + | (pd.isna(de_lookup["repointing_id_end"])) + ) + ] + if repoint_row.empty: + raise ValueError( + f"No DE product found for repoint ID {repoint_id} in {file_name}" + ) + if len(repoint_row) > 1: + raise ValueError( + f"Multiple DE products found for repoint ID {repoint_id} using " + f"ancillary file {file_name}. Check that the " + f"repointing_id_start and repointing_id_end values are correct" + f" and not overlapping." + ) + product = repoint_row["de_product"].values[0] + logger.info( + f"Using DE product {product} for repoint ID {repoint_id} based on lookup table" + ) + return product diff --git a/imap_processing/ultra/l1b/quality_flag_filters.py b/imap_processing/ultra/l1b/quality_flag_filters.py index 5782798d13..faa3921a58 100644 --- a/imap_processing/ultra/l1b/quality_flag_filters.py +++ b/imap_processing/ultra/l1b/quality_flag_filters.py @@ -21,6 +21,9 @@ # Then all flags in the array will be used for filtering. ENERGY_DEPENDENT_SPIN_QUALITY_FLAG_FILTERS: list = [ "quality_low_voltage", + "quality_upstream_ion_1", + "quality_upstream_ion_2", + "quality_spectral", "quality_high_energy", "quality_statistics", ] diff --git a/imap_processing/ultra/l1b/ultra_l1b.py b/imap_processing/ultra/l1b/ultra_l1b.py index 34abbf1c8a..d698193145 100644 --- a/imap_processing/ultra/l1b/ultra_l1b.py +++ b/imap_processing/ultra/l1b/ultra_l1b.py @@ -1,11 +1,17 @@ """Calculate ULTRA L1b.""" +import logging +import re + import xarray as xr from imap_processing.ultra.l1b.badtimes import calculate_badtimes from imap_processing.ultra.l1b.de import calculate_de from imap_processing.ultra.l1b.extendedspin import calculate_extendedspin from imap_processing.ultra.l1b.goodtimes import calculate_goodtimes +from imap_processing.ultra.l1b.lookup_utils import get_de_product_name + +logger = logging.getLogger(__name__) def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: @@ -32,15 +38,29 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: 3. l1b extended, goodtimes, badtimes created here """ output_datasets = [] - # Account for possibility of having 45 and 90 in dictionary. for instrument_id in [45, 90]: + # Find any de product if it is in the data_dict + l1a_de_products = [ + name + for name in data_dict.keys() + if re.search(rf"^imap_ultra_l1a_{instrument_id}sensor.*-de$", name) + ] # L1b de data will be created if L1a de data is available - if f"imap_ultra_l1a_{instrument_id}sensor-de" in data_dict: + # Including priority de products + if l1a_de_products: + l1a_de_product = l1a_de_products[0] + if len(l1a_de_products) > 1: + raise ValueError( + f"Multiple L1a de products found for instrument {instrument_id}. " + f"Expected only one but found {len(l1a_de_products)}: " + f"{l1a_de_products}" + ) + l1b_de_product = l1a_de_product.replace("l1a", "l1b") de_dataset = calculate_de( - data_dict[f"imap_ultra_l1a_{instrument_id}sensor-de"], + data_dict[l1a_de_product], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], - f"imap_ultra_l1b_{instrument_id}sensor-de", + l1b_de_product, ancillary_files, ) output_datasets.append(de_dataset) @@ -53,6 +73,23 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: and f"imap_ultra_l1a_{instrument_id}sensor-params" in data_dict and f"imap_ultra_l1b_{instrument_id}sensor-status" in data_dict ): + # get repoint number + repoint = data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"].attrs.get( + "Repointing", None + ) + if repoint is None: + raise ValueError("Repointing ID attribute is missing from the dataset.") + # Determine which l1b de product to use in calculating the goodtimes + # Will be either the raw de product or a priority 1-4 de product. + de_product_desc = get_de_product_name( + repoint, instrument_id, "l1b", ancillary_files + ) + if de_product_desc not in data_dict: + raise ValueError( + f"Selected L1B DE product '{de_product_desc}' for instrument " + f"{instrument_id} is not present in data_dict. Available L1B DE " + f"products: {data_dict.keys()}" + ) extendedspin_dataset = calculate_extendedspin( { f"imap_ultra_l1a_{instrument_id}sensor-aux": data_dict[ @@ -64,13 +101,11 @@ def ultra_l1b(data_dict: dict, ancillary_files: dict) -> list[xr.Dataset]: f"imap_ultra_l1a_{instrument_id}sensor-rates": data_dict[ f"imap_ultra_l1a_{instrument_id}sensor-rates" ], - f"imap_ultra_l1b_{instrument_id}sensor-de": data_dict[ - f"imap_ultra_l1b_{instrument_id}sensor-de" - ], f"imap_ultra_l1b_{instrument_id}sensor-status": data_dict[ f"imap_ultra_l1b_{instrument_id}sensor-status" ], }, + data_dict[de_product_desc], f"imap_ultra_l1b_{instrument_id}sensor-extendedspin", instrument_id, ) diff --git a/imap_processing/ultra/l1b/ultra_l1b_culling.py b/imap_processing/ultra/l1b/ultra_l1b_culling.py index 0f22af6b8a..2261cc9039 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_culling.py +++ b/imap_processing/ultra/l1b/ultra_l1b_culling.py @@ -81,7 +81,7 @@ def get_energy_histogram( spin_df = get_spin_data() unique_spin_number = np.unique(spin_number) - spin_edges = np.append(unique_spin_number, unique_spin_number.max() + 1) + spin_edges: np.ndarray = np.append(unique_spin_number, unique_spin_number.max() + 1) # Counts per spin at each energy bin. hist, _ = np.histogramdd( @@ -321,7 +321,7 @@ def compare_aux_univ_spin_table( .loc[present_in_both] ) - mismatch_indices = np.zeros(len(spins), dtype=bool) + mismatch_indices: np.ndarray = np.zeros(len(spins), dtype=bool) fields_to_compare = [ ("timespinstart", "spin_start_sec_sclk"), @@ -332,7 +332,7 @@ def compare_aux_univ_spin_table( ] # Compare fields - mismatch = np.zeros(len(df_aux), dtype=bool) + mismatch: np.ndarray = np.zeros(len(df_aux), dtype=bool) for aux_field, spin_field in fields_to_compare: mismatch |= df_aux[aux_field].values != df_univ[spin_field].values @@ -555,39 +555,40 @@ def get_energy_and_spin_dependent_rejection_mask( """ # Get the ebin flags for each energy bin from the goodtimes dataset. energy_range_edges = goodtimes_dataset["energy_range_edges"].values + # Filter out fill values from energy_range_edges (negative or zero) + energy_range_edges = energy_range_edges[energy_range_edges > 0] # Get the quality flag arrays "turned on" for energy dependent culling from the # goodtimes dataset. flag_arrays = [ goodtimes_dataset[flag_name].values for flag_name in ENERGY_DEPENDENT_SPIN_QUALITY_FLAG_FILTERS ] - ebin_flags = goodtimes_dataset["energy_range_flags"].values - # Create a dict of spin_number to index in the goodtimes dataset - spin_to_idx = { - spin: idx for idx, spin in enumerate(goodtimes_dataset["spin_number"].values) - } - # Initialize all events to not rejected - rejected = np.full(energy.shape, False, dtype=bool) - # loop through each energy bin and flag events that fall within an energy - # bin and have the corresponding energy bin flag set in the goodtimes dataset. - for i in range(len(energy_range_edges) - 1): - mask = (energy >= energy_range_edges[i]) & (energy < energy_range_edges[i + 1]) - goodtimes_inds = [spin_to_idx[spin] for spin in spin_number[mask]] - # Get the flag value for the current energy bin - energy_bin_flag = ebin_flags[i] - # If the flag is set for any of the quality arrays, then reject - # the event. - flagged_at_spins = ( - np.bitwise_or.reduce( - [qf[goodtimes_inds] & energy_bin_flag for qf in flag_arrays] - ) - > 0 - ) - - # Mark flagged events as rejected - mask_indices = np.where(mask)[0] - rejected[mask_indices[flagged_at_spins]] = True + rejected = np.zeros_like(energy, dtype=bool) + ebin_flags = goodtimes_dataset["energy_range_flags"].values + # Filter out fill values (0s) from energy_range_flags + ebin_flags = ebin_flags[ebin_flags > 0] + # Get the index of the spin number in the goodtimes dataset for each event + # all spin numbers should be present in the goodtimes dataset since we have already + # filtered any events that are not + spin_idx = np.searchsorted(goodtimes_dataset.spin_number, spin_number) + event_energy_bins: NDArray = (np.digitize(energy, energy_range_edges) - 1).astype( + np.intp + ) + in_valid_bin = (event_energy_bins >= 0) & (event_energy_bins < len(ebin_flags)) + # get the flags for each event + event_flags = np.zeros_like(energy, dtype=np.uint16) + event_flags[in_valid_bin] = ebin_flags[event_energy_bins[in_valid_bin]] + for qf_array in flag_arrays: + # select the quality flag for each event + quality_flags_at_events = qf_array[spin_idx] + # If that flag is "turned on" for the spin of that event, and the event is in + # an energy bin that is flagged for culling, then we reject that event. + rejected |= quality_flags_at_events & event_flags > 0 + + logger.info( + "Rejected %d events based on energy and spin dependent flags.", np.sum(rejected) + ) return rejected @@ -650,7 +651,7 @@ def flag_low_voltage( """ spin_bin_size = len(spin_tbin_edges) - 1 # initialize all spins to have no low voltage flag - quality_flags = np.zeros(spin_bin_size, dtype=bool) + quality_flags: np.ndarray = np.zeros(spin_bin_size, dtype=bool) # Get the min voltage across both deflection plate at each epoch min_voltage = np.minimum( status_dataset["rightdeflection_v"].data, @@ -673,7 +674,11 @@ def flag_low_voltage( # For each low voltage ind, flag the corresponding flag quality_flags[lv_spin_inds] = True - # TODO add log summary. + num_culled: int = np.sum(quality_flags) + logger.info( + f"Low voltage culling removed {num_culled} spin bins across all energy " + f"channels. Voltage threshold: {voltage_threshold} V." + ) return quality_flags @@ -730,7 +735,7 @@ def flag_high_energy( # Initialize all spin bins to have no high energy flag spin_bin_size = len(spin_tbin_edges) - 1 - quality_flags = np.zeros((n_energy_bins, spin_bin_size), dtype=bool) + quality_flags: np.ndarray = np.zeros((n_energy_bins, spin_bin_size), dtype=bool) # Get valid events and counts at each spin bin for the # designated culling channel. de_counts = get_valid_de_count_summary( @@ -751,7 +756,12 @@ def flag_high_energy( quality_flags[:, ~mask] = flagged[:, ~mask] else: quality_flags = flagged - # TODO add log summary. E.g Tim's hi goodtimes code + + num_culled: int = np.sum(quality_flags) + logger.info( + f"High energy culling removed {num_culled} spin bins across {n_energy_bins} " + f"energy channels. Energy thresholds: {energy_thresholds.flatten()}, " + ) return quality_flags @@ -831,27 +841,27 @@ def flag_statistical_outliers( curr_mask = mask.copy() # Initialize quality_stats to keep track of which bins are flagged as outliers for # each energy bin - quality_stats = np.zeros((n_energy_bins, spin_bin_size), dtype=bool) + quality_stats: np.ndarray = np.zeros((n_energy_bins, spin_bin_size), dtype=bool) # Initialize a mask to keep track of spin bins that have been flagged across all # energy bins - all_channel_mask = np.zeros(spin_bin_size, dtype=bool) + all_channel_mask: np.ndarray = np.zeros(spin_bin_size, dtype=bool) # Initialize convergence array to keep track of poisson stats convergence = np.full(n_energy_bins, False) # Keep track of how many iterations we have done of flagging outliers and # recalculating stats per energy bin iterations = np.zeros(n_energy_bins) # keep track of the standard deviation difference from poisson stats per energy bin - std_diff = np.zeros(n_energy_bins, dtype=float) + std_diff: np.ndarray = np.zeros(n_energy_bins, dtype=float) count_summary = get_valid_de_count_summary( de_dataset, energy_ranges, spin_tbin_edges, sensor_id=sensor_id ) # shape (n_energy_bins, n_spin_bins) - for e_idx in np.arange(n_energy_bins): + for e_idx in range(n_energy_bins): good_mask = ~curr_mask[e_idx] # spin bins that are not currently flagged for it in range(n_iterations): counts = count_summary[e_idx, good_mask] - # Step 1. check if any energy bins have less than 3 spin bins with counts. + # Step 1. check if there are less than three valid counts. # If so, flag all spins for that energy bin and skip to the next iteration - if np.sum(counts > 0) < 3: + if len(counts) < 3: quality_stats[e_idx] = True curr_mask[e_idx] = True convergence[e_idx] = True @@ -888,7 +898,7 @@ def flag_statistical_outliers( convergence[e_idx] = True num_culled: int = np.sum(quality_stats) - logger.debug( + logger.info( f"Statistical culling removed {num_culled} spin bins across {n_energy_bins}" f" energy channels. Convergence: {convergence} after " f"{iterations} iterations." @@ -932,6 +942,143 @@ def get_poisson_stats(counts: NDArray) -> tuple[float, NDArray]: return std_ratio, sub_mask +def flag_upstream_ion( + de_dataset: xr.Dataset, + spin_tbin_edges: NDArray, + energy_ranges: NDArray, + mask: NDArray, + channels: list, + sensor_id: int = 90, +) -> NDArray: + """ + Flag upstream ion events. + + Parameters + ---------- + de_dataset : xr.Dataset + Direct event dataset. + spin_tbin_edges : NDArray + Edges of the spin time bins. + energy_ranges : NDArray + Array of energy range edges. + mask : NDArray + Mask indicating which events to consider for upstream ion flagging. This should + be a 2d boolean array of shape (n_energy_bins, n_spin_bins) where True + indicates the spin bins that have been flagged in previous steps + and should be excluded from the upstream ion flagging process. + channels : list + List of energy channel indices to use for upstream ion flagging. + sensor_id : int + Sensor ID (e.g., 45 or 90). + + Returns + ------- + flagged : NDArray + Boolean array of shape (n_spin_bins,) where True indicates spin bins flagged for + upstream ions. These flags are energy independent and should be applied across + all energy channels. + """ + # validate that the channels provided are within the bounds of the energy ranges + if not np.all([ch in range(len(energy_ranges) - 1) for ch in channels]): + raise ValueError( + f"Channels provided for upstream ion flagging must be within the bounds" + f" of the energy ranges. Provided channels: {channels}, number of energy" + f" ranges: {len(energy_ranges) - 1}." + ) + counts_sum = get_valid_de_count_summary( + de_dataset, energy_ranges, spin_tbin_edges, sensor_id=sensor_id + )[channels, :] # shape (num_channels, n_spin_bins) + flagged = np.zeros(counts_sum.shape[1], dtype=bool) + channel_mask = ~mask[channels, :] + weights = channel_mask.sum(axis=0) + # Sum counts where the mask is True (valid) + sum_scaled_counts = np.where(channel_mask, counts_sum, 0).sum(axis=0) + # Get 1D array of valid spin bins + valid_bins = np.flatnonzero(weights > 0) + total_scaled = sum_scaled_counts[valid_bins] + if valid_bins.size == 0 or total_scaled.size == 0: + logger.info( + "Upstream Ion culling found no valid spin bins for evaluation; " + "returning all-False upstream ion flags." + ) + return flagged + + total_mean = np.mean(total_scaled) + # Set a threshold based on poisson stats for the total counts across the channels + thresh = total_mean + UltraConstants.UPSTREAM_SIG_THRESHOLD * np.sqrt(total_mean) + # Flag bins where the total counts across the channels exceed the threshold + flagged[valid_bins[total_scaled > thresh]] = True + + num_culled: int = np.sum(flagged) + logger.info( + f"Upstream Ion culling removed {num_culled} spin bins. These are energy" + f" independent flags and will be applied across all {mask.shape[0]} energy" + f" channels." + ) + return flagged + + +def flag_spectral_events( + de_dataset: xr.Dataset, + spin_tbin_edges: NDArray, + energy_ranges: NDArray, + channels: list, + sensor_id: int = 90, +) -> NDArray: + """ + Flag spectral events. + + Parameters + ---------- + de_dataset : xr.Dataset + Direct event dataset. + spin_tbin_edges : NDArray + Edges of the spin time bins. + energy_ranges : NDArray + Array of energy range edges. + channels : list + List of energy channel indices to use for spectral flagging. + sensor_id : int + Sensor ID (e.g., 45 or 90). + + Returns + ------- + flagged : NDArray + Boolean array of shape (n_spin_bins,) where True indicates spin bins flagged for + spectral anomalies. These flags are energy independent and should be applied + across all energy channels. + """ + # validate that the channels provided are within the bounds of the energy ranges + if not np.all([ch in range(len(energy_ranges) - 1) for ch in channels]): + raise ValueError( + f"Channels provided for spectral flagging must be within the bounds" + f" of the energy ranges. Provided channels: {channels}, number of energy" + f" ranges: {len(energy_ranges) - 1}." + ) + counts_sum = get_valid_de_count_summary( + de_dataset, energy_ranges, spin_tbin_edges, sensor_id=sensor_id + )[channels, :] # shape (num_channels, n_spin_bins) + # Flag spin bins where the signed count difference between adjacent selected + # energy channels exceeds a Poisson-based threshold. For each pair of + # adjacent channels, compute np.diff(counts_sum, axis=0) and compare that + # signed difference to a threshold scaled by the combined Poisson + # uncertainty (sqrt(N1 + N2)) of those two channels for each spin bin. + # If any adjacent channel pair exceeds the threshold for a spin bin, that + # spin bin is flagged across all energy ranges. + diff = np.diff(counts_sum, axis=0) - UltraConstants.SPECTRAL_SIG_THRESHOLD * ( + np.sqrt(counts_sum[:-1] + counts_sum[1:]) + ) # shape (num_channels - 1, n_spin_bins) + flagged = np.any(diff > 0, axis=0) # shape (n_spin_bins,) + num_culled: int = np.sum(flagged) + logger.info( + f"Spectral culling removed {num_culled} spin bins using channels" + f" {channels} and threshold {UltraConstants.SPECTRAL_SIG_THRESHOLD}." + f" These are energy independent flags and will be applied across all" + f" energy channels." + ) + return flagged + + def get_valid_de_count_summary( de_dataset: xr.Dataset, energy_ranges: NDArray, @@ -964,7 +1111,9 @@ def get_valid_de_count_summary( valid_events = get_valid_events_per_energy_range( de_dataset, energy_ranges, UltraConstants.EARTH_ANGLE_45_THRESHOLD, sensor_id ) - counts = np.zeros((len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=float) + counts: np.ndarray = np.zeros( + (len(energy_ranges) - 1, len(spin_tbin_edges) - 1), dtype=float + ) for i in range(len(energy_ranges) - 1): counts[i, :], _ = np.histogram( @@ -1008,7 +1157,9 @@ def get_valid_events_per_energy_range( A boolean array of shape (n_energy_ranges, n_events). """ event_energies = de_dataset["energy_spacecraft"].values - valid_events = np.zeros((len(energy_ranges) - 1, len(event_energies)), dtype=bool) + valid_events: np.ndarray = np.zeros( + (len(energy_ranges) - 1, len(event_energies)), dtype=bool + ) valid_outliers = de_dataset["quality_outliers"].values == 0 valid_scattering = de_dataset["quality_scattering"].values == 0 # TODO what about species non-proton? For those psets dont cull based on @@ -1023,7 +1174,7 @@ def get_valid_events_per_energy_range( continue # subset the dataset to events within the energy range de_dataset_subset = de_dataset.isel(epoch=energy_mask) - valid_earth_angle = np.full(np.sum(energy_mask), True, dtype=bool) + valid_earth_angle: np.ndarray = np.full(np.sum(energy_mask), True, dtype=bool) # For ultra45, also apply an Earth angle cut to remove times when # the Earth is in the field of view. ULTRA 90 does not require this since Earth # is always outside the field of view. @@ -1127,7 +1278,7 @@ def get_energy_range_flags(energy_ranges_edges: NDArray) -> NDArray: def get_binned_energy_ranges( energy_bin_edges: list[tuple[float, float]], - max_energy: int | None = UltraConstants.MAX_ENERGY_THRESHOLD, + max_energy: float | None = UltraConstants.MAX_ENERGY_THRESHOLD, ) -> NDArray: """ Create L1C energy ranges by grouping energy bins. @@ -1136,7 +1287,7 @@ def get_binned_energy_ranges( ---------- energy_bin_edges : list[tuple[float, float]] List of (start, stop) tuples for each energy bin. - max_energy : int | None + max_energy : float | None Maximum energy to include in the energy ranges. If None, don't set a max. Returns @@ -1153,13 +1304,16 @@ def get_binned_energy_ranges( ) energy_starts = [energy_bin_edges[i][0] for i in group_start_inds] # Append the stop energy of the last bin to cover the full range - last_group_end_ind = min( - group_start_inds[-1] + UltraConstants.N_CULL_EBINS, len(energy_bin_edges) + last_group_end_ind = int( + min( + int(group_start_inds[-1]) + UltraConstants.N_CULL_EBINS, + len(energy_bin_edges), + ) ) - energy_ranges = np.append( - energy_starts, energy_bin_edges[last_group_end_ind - 1][1] + energy_ranges: np.ndarray = np.append( + energy_starts, + energy_bin_edges[last_group_end_ind - 1][1], ) - if max_energy is not None: # get the first index where the energy range exceeds the max energy # exclude the last edge since it is the stop energy of the last range @@ -1175,8 +1329,11 @@ def get_binned_energy_ranges( energy_ranges_lim = energy_ranges[ : max_reached_idx + 2 ].copy() # include the first edge above max energy and the last edge - # Set the last edge to be the max energy to make the last bin a "catch-all" for - # all energies above the max energy. + # update the last bin to start at the first original edge above the max energy + # and end at the last edge + energy_ranges_lim[-2] = next( + e[0] for e in energy_bin_edges if e[0] > max_energy + ) energy_ranges_lim[-1] = energy_ranges[-1] energy_ranges = energy_ranges_lim @@ -1226,7 +1383,6 @@ def get_binned_spins_edges( spin_tbin_edges, spin_start_times[last_spin_idx] + spin_periods[last_spin_idx] ) return spin_tbin_edges - return spin_tbin_edges def expand_bin_flags_to_spins( @@ -1249,9 +1405,11 @@ def expand_bin_flags_to_spins( quality_flags : NDArray Quality flags mapped to each individual spin. """ - quality_flags = np.full(n_spins, ImapRatesUltraFlags.NONE.value, dtype=np.uint16) + quality_flags: np.ndarray = np.full( + n_spins, ImapRatesUltraFlags.NONE.value, dtype=np.uint16 + ) # Repeat each binned flag for the number of spins in each bin - repeated_flags = np.repeat(binned_quality_flags, spin_bin_size) + repeated_flags: np.ndarray = np.repeat(binned_quality_flags, spin_bin_size) if len(repeated_flags) > n_spins: logger.warning( f"Found incomplete spin bin at the end with" diff --git a/imap_processing/ultra/l1b/ultra_l1b_extended.py b/imap_processing/ultra/l1b/ultra_l1b_extended.py index 1ff755d5eb..e52f025758 100644 --- a/imap_processing/ultra/l1b/ultra_l1b_extended.py +++ b/imap_processing/ultra/l1b/ultra_l1b_extended.py @@ -356,9 +356,9 @@ def get_ssd_back_position_and_tof_offset( indices = np.nonzero(np.isin(de_dataset["stop_type"], StopType.SSD.value))[0] de_filtered = de_dataset.isel(epoch=indices) - yb = np.zeros(len(indices), dtype=np.float64) - ssd_number = np.zeros(len(indices), dtype=int) - tof_offset = np.zeros(len(indices), dtype=np.float64) + yb: np.ndarray = np.zeros(len(indices), dtype=np.float64) + ssd_number: np.ndarray = np.zeros(len(indices), dtype=int) + tof_offset: np.ndarray = np.zeros(len(indices), dtype=np.float64) for i in range(8): ssd_flag_mask = de_filtered[f"ssd_flag_{i}"].data == 1 @@ -480,8 +480,8 @@ def get_coincidence_positions( ] de_bottom = de_dataset.isel(epoch=index_bottom) - etof = np.zeros(len(de_dataset["coin_type"]), dtype=np.float64) - xc_array = np.zeros(len(de_dataset["coin_type"]), dtype=np.float64) + etof: np.ndarray = np.zeros(len(de_dataset["coin_type"]), dtype=np.float64) + xc_array: np.ndarray = np.zeros(len(de_dataset["coin_type"]), dtype=np.float64) # Normalized TDCs # For the stop anode, there are mismatches between the coincidence TDCs, @@ -535,7 +535,7 @@ def get_de_velocity( logger.info("Negative tof values found.") # distances in .1 mm - delta_v = np.empty((len(d), 3), dtype=np.float32) + delta_v: np.ndarray = np.empty((len(d), 3), dtype=np.float32) delta_v[:, 0] = (front_position[0] - back_position[0]) * 0.1 delta_v[:, 1] = (front_position[1] - back_position[1]) * 0.1 delta_v[:, 2] = d * 0.1 @@ -709,12 +709,12 @@ def get_energy_pulse_height( indices_top = np.where(stop_type == 1)[0] indices_bottom = np.where(stop_type == 2)[0] - xlut = np.zeros(len(stop_type), dtype=np.float64) - ylut = np.zeros(len(stop_type), dtype=np.float64) - energy_ph = np.zeros(len(stop_type), dtype=np.float64) + xlut: np.ndarray = np.zeros(len(stop_type), dtype=np.float64) + ylut: np.ndarray = np.zeros(len(stop_type), dtype=np.float64) + energy_ph: np.ndarray = np.zeros(len(stop_type), dtype=np.float64) # Full-length correction arrays - ph_correction = np.zeros(len(stop_type), dtype=np.float64) + ph_correction: np.ndarray = np.zeros(len(stop_type), dtype=np.float64) # Stop type 1 xlut[indices_top] = (xb[indices_top] / 100 - 24.5 / 2) * 20 / 50 # mm @@ -794,7 +794,7 @@ def get_energy_ssd( ssd_indices = np.nonzero(np.isin(de_dataset["stop_type"], StopType.SSD.value))[0] energy = de_dataset["energy_ph"].data[ssd_indices] - composite_energy = np.empty(len(energy), dtype=np.float64) + composite_energy: np.ndarray = np.empty(len(energy), dtype=np.float64) composite_energy[energy >= UltraConstants.COMPOSITE_ENERGY_THRESHOLD] = ( UltraConstants.COMPOSITE_ENERGY_THRESHOLD @@ -843,7 +843,7 @@ def get_ctof( # Multiply times 100 to convert to hundredths of a millimeter. ctof = tof * dmin_ctof * 100 / path_length - magnitude_v = np.full(len(ctof), -1.0e31, dtype=np.float32) + magnitude_v: np.ndarray = np.full(len(ctof), -1.0e31, dtype=np.float32) # Convert from mm/0.1ns to km/s for valid ctof values valid_mask = ctof >= 0 @@ -1170,6 +1170,7 @@ def get_fwhm( def get_efficiency_interpolator( ancillary_files: dict, + sensor: str, ) -> tuple[RegularGridInterpolator, tuple, tuple]: """ Return a callable function that interpolates efficiency values for each event. @@ -1178,6 +1179,8 @@ def get_efficiency_interpolator( ---------- ancillary_files : dict Ancillary files. + sensor : str + Sensor name: "ultra45" or "ultra90". Returns ------- @@ -1188,7 +1191,7 @@ def get_efficiency_interpolator( phi_min_max : tuple Minimum and maximum phi values in the lookup table. """ - lookup_table = get_energy_efficiencies(ancillary_files) + lookup_table = get_energy_efficiencies(ancillary_files, sensor) theta_vals = np.sort(lookup_table["theta (deg)"].unique()) phi_vals = np.sort(lookup_table["phi (deg)"].unique()) @@ -1218,6 +1221,7 @@ def get_efficiency( phi_inst: NDArray, theta_inst: NDArray, ancillary_files: dict, + sensor: str, interpolator: RegularGridInterpolator = None, ) -> np.ndarray: """ @@ -1233,6 +1237,8 @@ def get_efficiency( Instrument-frame elevation angle for each event. ancillary_files : dict Ancillary files. + sensor : str + Sensor name: "ultra45" or "ultra90". interpolator : RegularGridInterpolator, optional Precomputed interpolator to use for efficiency lookup. If None, a new interpolator will be created from the ancillary files. @@ -1243,7 +1249,7 @@ def get_efficiency( Interpolated efficiency values. """ if not interpolator: - interpolator, _, _ = get_efficiency_interpolator(ancillary_files) + interpolator, _, _ = get_efficiency_interpolator(ancillary_files, sensor) return interpolator((theta_inst, phi_inst, energy)) @@ -1424,7 +1430,7 @@ def is_back_tof_valid( top_mask = de_dataset["stop_type"] == StopType.Top.value bottom_mask = de_dataset["stop_type"] == StopType.Bottom.value - valid = np.zeros(len(top_mask), dtype=bool) + valid: np.ndarray = np.zeros(len(top_mask), dtype=bool) diff_tp_min = get_image_params("TOFDiffTpMin", sensor, ancillary_files) diff_tp_max = get_image_params("TOFDiffTpMax", sensor, ancillary_files) diff --git a/imap_processing/ultra/l1c/helio_pset.py b/imap_processing/ultra/l1c/helio_pset.py index 7375d71fc8..9d562aa5f0 100644 --- a/imap_processing/ultra/l1c/helio_pset.py +++ b/imap_processing/ultra/l1c/helio_pset.py @@ -2,6 +2,7 @@ import logging +import astropy_healpix.healpy as hp import numpy as np import xarray as xr @@ -20,7 +21,8 @@ ) from imap_processing.ultra.l1c.l1c_lookup_utils import ( build_energy_bins, - calculate_fwhm_spun_scattering, + calculate_accepted_pixels, + in_restricted_fov, ) from imap_processing.ultra.l1c.make_helio_index_maps import ( make_helio_index_maps_with_nominal_kernels, @@ -75,10 +77,6 @@ def calculate_helio_pset( dataset : xarray.Dataset Dataset containing the data. """ - # Do not cull events based on scattering thresholds - reject_scattering = False - # Do not apply boundary scale factor corrections - apply_bsf = False nside = 32 num_spin_steps = 720 sensor_id = int(parse_filename_like(name)["sensor"][0:2]) @@ -94,7 +92,7 @@ def calculate_helio_pset( rejected = get_de_rejection_mask( species_dataset["quality_scattering"].values, species_dataset["quality_outliers"].values, - reject_scattering, + UltraConstants.APPLY_SCATTERING_REJECTION_L1C, ) species_dataset = species_dataset.isel(epoch=~rejected) # Check if spin_number is in the goodtimes dataset, if not then we can @@ -113,13 +111,6 @@ def calculate_helio_pset( species_dataset["spin"].values, ) species_dataset = species_dataset.isel(epoch=~energy_dependent_rejected) - v_mag_helio_spacecraft = np.linalg.norm( - species_dataset["velocity_dps_helio"].values, axis=1 - ) - vhat_dps_helio = ( - species_dataset["velocity_dps_helio"].values - / v_mag_helio_spacecraft[:, np.newaxis] - ) # Get the start and stop times of the pointing period repoint_id = species_dataset.attrs.get("Repointing", None) if repoint_id is None: @@ -137,7 +128,7 @@ def calculate_helio_pset( spin_duration=15.0, num_steps=num_spin_steps, instrument_frame=instrument_frame, - compute_bsf=apply_bsf, + compute_bsf=UltraConstants.APPLY_BOUNDARY_SCALE_FACTORS_L1C, ) boundary_scale_factors = helio_pointing_ds.bsf theta_vals = helio_pointing_ds.theta @@ -146,38 +137,66 @@ def calculate_helio_pset( logger.info("calculating spun FWHM scattering values.") pixels_below_scattering, scattering_theta, scattering_phi, scattering_thresholds = ( - calculate_fwhm_spun_scattering( + calculate_accepted_pixels( fov_index, theta_vals, phi_vals, ancillary_files, instrument_id, - reject_scattering, + reject_scattering=UltraConstants.APPLY_SCATTERING_REJECTION_L1C, + apply_fov_restriction=UltraConstants.APPLY_FOV_RESTRICTIONS_L1C, ) ) - - counts, latitude, longitude, n_pix = get_spacecraft_histogram( + # Apply restricted FOV filter to counts. + # If a valid event's instrument frame theta/phi falls outside the restricted + # acceptance window it is excluded from the L1C fine energy-bin counts map. + if UltraConstants.APPLY_FOV_RESTRICTIONS_L1C: + fov_accepted = in_restricted_fov( + species_dataset["theta"].values, + species_dataset["phi"].values, + instrument_id, + ) + logger.info( + f"Restricted FOV counts filter: keeping {fov_accepted.sum()} / " + f"{len(fov_accepted)} events." + ) + species_dataset = species_dataset.isel(epoch=fov_accepted) + v_mag_helio_spacecraft = np.linalg.norm( + species_dataset["velocity_dps_helio"].values, axis=1 + ) + vhat_dps_helio = ( + species_dataset["velocity_dps_helio"].values + / v_mag_helio_spacecraft[:, np.newaxis] + ) + counts, counts_n_pix = get_spacecraft_histogram( vhat_dps_helio, species_dataset["energy_heliosphere"].values, intervals, - nside=nside, + nside=UltraConstants.L1C_COUNTS_NSIDE, ) + n_pix = hp.nside2npix(nside) helio_pset_quality_flags = np.full( n_pix, ImapPSETUltraFlags.NONE.value, dtype=np.uint16 ) + counts_healpix = np.arange(counts_n_pix) + # Determine nside for non "counts" variables from the lookup table healpix = np.arange(n_pix) + # Calculate the corresponding longitude (az) latitude (el) + # center coordinates + longitude, latitude = hp.pix2ang(nside, healpix, lonlat=True) + logger.info("Calculating spacecraft exposure times with deadtime correction.") exposure_time, deadtime_ratios = get_spacecraft_exposure_times( rates_dataset, pixels_below_scattering, boundary_scale_factors, aux_dataset, - pointing_range_met, - n_energy_bins=len(energy_bin_geometric_means), + energy_bins=energy_bin_geometric_means, sensor_id=sensor_id, ancillary_files=ancillary_files, - apply_bsf=apply_bsf, + apply_bsf=UltraConstants.APPLY_BOUNDARY_SCALE_FACTORS_L1C, + goodtimes_dataset=goodtimes_dataset, ) logger.info("Calculating spun efficiencies and geometric function.") # calculate efficiency and geometric function as a function of energy @@ -187,8 +206,9 @@ def calculate_helio_pset( theta_vals.values, phi_vals.values, n_pix, + sensor_id, ancillary_files, - apply_bsf, + apply_bsf=UltraConstants.APPLY_BOUNDARY_SCALE_FACTORS_L1C, ) logger.info("Calculating background rates.") @@ -236,6 +256,7 @@ def calculate_helio_pset( pset_dict["background_rates"] = background_rates[np.newaxis, ...] pset_dict["exposure_factor"] = exposure_time[np.newaxis, ...] pset_dict["pixel_index"] = healpix + pset_dict["counts_pixel_index"] = counts_healpix pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[ np.newaxis, ... ] diff --git a/imap_processing/ultra/l1c/l1c_lookup_utils.py b/imap_processing/ultra/l1c/l1c_lookup_utils.py index 1d9d87fa59..15d5f541a3 100644 --- a/imap_processing/ultra/l1c/l1c_lookup_utils.py +++ b/imap_processing/ultra/l1c/l1c_lookup_utils.py @@ -17,6 +17,46 @@ logger = logging.getLogger(__name__) +def in_restricted_fov( + theta: np.ndarray, + phi: np.ndarray, + instrument_id: int, +) -> np.ndarray: + """ + Determine whether the theta/phi pairs are inside the restricted FOV. + + Parameters + ---------- + theta : np.ndarray + Array of theta values in degrees. + phi : np.ndarray + Array of phi values in degrees. + instrument_id : int + Instrument ID, either 45 or 90. + + Returns + ------- + np.ndarray + Boolean array indicating whether each theta/phi pair is within the restricted + FOV. + """ + # Theta limits are dependent on the instrument id + if instrument_id == 90: + low_theta_lim = UltraConstants.RESTRICTED_FOV_THETA_LOW_DEG_90 + high_theta_lim = UltraConstants.RESTRICTED_FOV_THETA_HIGH_DEG_90 + elif instrument_id == 45: + low_theta_lim = UltraConstants.RESTRICTED_FOV_THETA_LOW_DEG_45 + high_theta_lim = UltraConstants.RESTRICTED_FOV_THETA_HIGH_DEG_45 + else: + raise ValueError(f"Invalid instrument ID: {instrument_id}. Must be 45 or 90.") + + return ( + (theta >= low_theta_lim) + & (theta <= high_theta_lim) + & (np.abs(phi) < UltraConstants.FOV_PHI_LIMIT_DEG) + ) + + def mask_below_fwhm_scattering_threshold( theta_coeffs: np.ndarray, phi_coeffs: np.ndarray, @@ -67,18 +107,20 @@ def mask_below_fwhm_scattering_threshold( return scattering_mask, fwhm_theta, fwhm_phi -def calculate_fwhm_spun_scattering( +def calculate_accepted_pixels( # noqa: PLR0912 for_indices_by_spin_phase: xr.DataArray, theta_vals: np.ndarray, phi_vals: np.ndarray, ancillary_files: dict, instrument_id: int, reject_scattering: bool = False, + apply_fov_restriction: bool = False, ) -> tuple[xr.DataArray, NDArray, NDArray, NDArray]: """ - Calculate FWHM scattering values for each pixel, energy bin, and spin phase step. + Calculate the accepted pixels based scattering and FOV restrictions. - This function also calculates a mask for pixels that are below the FWHM threshold. + This function also calculates a mask for pixels that are below the FWHM threshold + and FWHM scattering values for each pixel, energy bin, and spin phase step. Parameters ---------- @@ -99,15 +141,23 @@ def calculate_fwhm_spun_scattering( Instrument ID, either 45 or 90. reject_scattering : bool Whether to reject pixels based on scattering thresholds. + apply_fov_restriction : bool + Whether to apply the restricted FOV theta/phi acceptance test. When + ``True``, any spin-step/pixel combination whose instrument-frame theta + and phi do not satisfy :func:`in_restricted_fov` is skipped entirely: + it is not added to the averaging numerator *or* denominator, and it + does not contribute exposure time. This gates GF, efficiency, and + exposure for the fine L1C energy-bin maps. Returns ------- valid_spun_pixels : xarray.DataArray - Boolean array indicating, for each spin phase step, energy_bin, pixel, - the pixel is inside the Field Of Regard (FOR) and whether the pixel is inside the - FOR at that spin phase and its computed FWHM at that energy is below the - scattering threshold. If reject_scattering is False, this will just reflect - the FOR mask (for_indices_by_spin_phase). + Boolean array of shape ``(spin_phase_step, energy_bin, pixel)`` indicating + which pixel samples are accepted for L1C accumulation at each spin step. + Acceptance always requires the sample to be inside the Field of Regard (FOR), + can optionally require passing the restricted theta/phi FOV criteria when + ``apply_fov_restriction=True``, and can optionally require passing the FWHM + scattering threshold when ``reject_scattering=True``. scattering_fwhm_theta : NDArray Calculated FWHM scatting values for theta at each energy bin and averaged over spin phase. @@ -143,9 +193,10 @@ def calculate_fwhm_spun_scattering( steps = for_indices_by_spin_phase.sizes["spin_phase_step"] energies = energy_bin_geometric_means[np.newaxis, :] # Initialize DataArray to hold boolean of valid pixels at each spin phase step - # If reject_scattering if false, this will just be the FOR mask. + # If reject_scattering and apply_fov_restriction are both False, this will just + # be the FOR mask. spun_dims = ("spin_phase_step", "energy", "pixel") - if reject_scattering: + if reject_scattering or apply_fov_restriction: valid_pixels = xr.DataArray( np.zeros((steps, len(energy_bin_geometric_means), n_pix), dtype=bool), dims=spun_dims, @@ -156,6 +207,8 @@ def calculate_fwhm_spun_scattering( ).transpose(*spun_dims) else: valid_pixels = for_indices_by_spin_phase + # TODO refactor loop below and combine the energy dependent and independent cases + # if possible to avoid code duplication. # The "for_indices_by_spin_phase" lookup table contains the boolean values of each # pixel at each spin phase step, indicating whether the pixel is inside the FOR. # It starts at Spin-phase = 0, and increments in fine steps (1 ms), spinning the @@ -167,14 +220,27 @@ def calculate_fwhm_spun_scattering( if for_inds.ndim > 1: # Energy dependent FOR indices for e_ind in range(len(energy_bin_geometric_means)): - for_inds_energy = for_inds[e_ind, :] - # Skip if no pixels in FOR - if not np.any(for_inds_energy): + if not np.any(for_inds[e_ind, :]): + continue + accepted_pix = np.flatnonzero(for_inds[e_ind, :]) + theta = theta_vals[i, e_ind, accepted_pix] + phi = phi_vals[i, e_ind, accepted_pix] + + # Check if we need to restrict the fov further using theta/phi + # acceptance limits + if apply_fov_restriction: + fov_mask = in_restricted_fov(theta, phi, instrument_id) + # update accepted pixel indices to reflect the FOV restriction + accepted_pix = accepted_pix[fov_mask] + theta = theta[fov_mask] + phi = phi[fov_mask] + # update valid pixels to reflect the FOV restriction + valid_pixels[i, e_ind, accepted_pix] = True + + if len(accepted_pix) == 0: continue - theta = theta_vals[i, e_ind, for_inds_energy] - phi = phi_vals[i, e_ind, for_inds_energy] theta_coeffs, phi_coeffs = get_scattering_coefficients( theta.data, phi.data, lookup_tables=scattering_luts ) @@ -192,19 +258,37 @@ def calculate_fwhm_spun_scattering( ) # If rejecting scattering, store the mask if reject_scattering: - valid_pixels[i, e_ind, for_inds_energy] = scattering_mask.flatten() + valid_pixels[i, e_ind, accepted_pix] = scattering_mask.flatten() # Accumulate FWHM values - fwhm_theta_sum[e_ind, for_inds_energy] += fwhm_theta.flatten() - fwhm_phi_sum[e_ind, for_inds_energy] += fwhm_phi.flatten() - sample_count[e_ind, for_inds_energy] += 1 + fwhm_theta_sum[e_ind, accepted_pix] += fwhm_theta.flatten() + fwhm_phi_sum[e_ind, accepted_pix] += fwhm_phi.flatten() + sample_count[e_ind, accepted_pix] += 1 else: # Energy independent FOR indices if not np.any(for_inds): continue - theta = theta_vals[i, for_inds] - phi = phi_vals[i, for_inds] + accepted_pix = np.flatnonzero(for_inds) # Get indices of pixels in FOR for + # this spin phase step + + theta = theta_vals[i, accepted_pix] + phi = phi_vals[i, accepted_pix] + + # Check if we need to apply the restricted FOV mask before calculating + # scattering coefficients + if apply_fov_restriction: + fov_mask = in_restricted_fov(theta, phi, instrument_id) + # update accepted pixel indices to reflect the FOV restriction + accepted_pix = accepted_pix[fov_mask] + theta = theta[fov_mask] + phi = phi[fov_mask] + # update valid pixels to reflect the FOV restriction + valid_pixels[i, :, accepted_pix] = True + + if len(accepted_pix) == 0: + continue + theta_coeffs, phi_coeffs = get_scattering_coefficients( theta, phi, lookup_tables=scattering_luts ) @@ -216,14 +300,13 @@ def calculate_fwhm_spun_scattering( scattering_thresholds=scattering_thresholds_for_energy_mean, ) ) - if reject_scattering: - valid_pixels[i, :, for_inds] = scattering_mask.T + valid_pixels[i, :, accepted_pix] = scattering_mask.T # Accumulate FWHM values - fwhm_theta_sum[:, for_inds] += fwhm_theta.T - fwhm_phi_sum[:, for_inds] += fwhm_phi.T - sample_count[:, for_inds] += 1 + fwhm_theta_sum[:, accepted_pix] += fwhm_theta.T + fwhm_phi_sum[:, accepted_pix] += fwhm_phi.T + sample_count[:, accepted_pix] += 1 fwhm_phi_avg = np.zeros_like(fwhm_phi_sum) fwhm_theta_avg = np.zeros_like(fwhm_theta_sum) diff --git a/imap_processing/ultra/l1c/spacecraft_pset.py b/imap_processing/ultra/l1c/spacecraft_pset.py index 3506c0e26d..313b39df14 100644 --- a/imap_processing/ultra/l1c/spacecraft_pset.py +++ b/imap_processing/ultra/l1c/spacecraft_pset.py @@ -1,4 +1,4 @@ -"""Calculate Pointing Set Grids.""" +"""Calculate Spacecraft Pointing Set Grids.""" import logging @@ -20,8 +20,9 @@ ) from imap_processing.ultra.l1c.l1c_lookup_utils import ( build_energy_bins, - calculate_fwhm_spun_scattering, + calculate_accepted_pixels, get_spacecraft_pointing_lookup_tables, + in_restricted_fov, ) from imap_processing.ultra.l1c.ultra_l1c_culling import compute_culling_mask from imap_processing.ultra.l1c.ultra_l1c_pset_bins import ( @@ -73,10 +74,6 @@ def calculate_spacecraft_pset( dataset : xarray.Dataset Dataset containing the data. """ - # Do not cull events based on scattering thresholds - reject_scattering = False - # Do not apply boundary scale factor corrections - apply_bsf = False pset_dict: dict[str, np.ndarray] = {} sensor_id = int(parse_filename_like(name)["sensor"][0:2]) @@ -93,7 +90,7 @@ def calculate_spacecraft_pset( de_rejected = get_de_rejection_mask( species_dataset["quality_scattering"].values, species_dataset["quality_outliers"].values, - reject_scattering, + UltraConstants.APPLY_SCATTERING_REJECTION_L1C, ) species_dataset = species_dataset.isel(epoch=~de_rejected) # Check if spin_number is in the goodtimes dataset, if not then we can @@ -112,12 +109,6 @@ def calculate_spacecraft_pset( species_dataset["spin"].values, ) species_dataset = species_dataset.isel(epoch=~energy_dependent_rejected) - v_mag_dps_spacecraft = np.linalg.norm( - species_dataset["velocity_dps_sc"].values, axis=1 - ) - vhat_dps_spacecraft = ( - species_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis] - ) # Get lookup table for FOR indices by spin phase step ( @@ -130,24 +121,54 @@ def calculate_spacecraft_pset( logger.info("calculating spun FWHM scattering values.") valid_spun_pixels, scattering_theta, scattering_phi, scattering_thresholds = ( - calculate_fwhm_spun_scattering( + calculate_accepted_pixels( for_indices_by_spin_phase, theta_vals, phi_vals, ancillary_files, instrument_id, - reject_scattering, + reject_scattering=UltraConstants.APPLY_SCATTERING_REJECTION_L1C, + apply_fov_restriction=UltraConstants.APPLY_FOV_RESTRICTIONS_L1C, ) ) - # Determine nside from the lookup table - nside = hp.npix2nside(for_indices_by_spin_phase.sizes["pixel"]) - counts, latitude, longitude, n_pix = get_spacecraft_histogram( + + # Apply restricted FOV filter to counts. + # If a valid event's instrument frame theta/phi falls outside the restricted + # acceptance window it is excluded from the L1C fine energy-bin counts map. + if UltraConstants.APPLY_FOV_RESTRICTIONS_L1C: + fov_accepted = in_restricted_fov( + species_dataset["theta"].values, + species_dataset["phi"].values, + instrument_id, + ) + logger.info( + f"Restricted FOV counts filter: keeping {fov_accepted.sum()} / " + f"{len(fov_accepted)} events." + ) + species_dataset = species_dataset.isel(epoch=fov_accepted) + + v_mag_dps_spacecraft = np.linalg.norm( + species_dataset["velocity_dps_sc"].values, axis=1 + ) + vhat_dps_spacecraft = ( + species_dataset["velocity_dps_sc"].values / v_mag_dps_spacecraft[:, np.newaxis] + ) + counts, counts_n_pix = get_spacecraft_histogram( vhat_dps_spacecraft, species_dataset["energy_spacecraft"].values, intervals, - nside=nside, + nside=UltraConstants.L1C_COUNTS_NSIDE, ) + counts_healpix = np.arange(counts_n_pix) + # Determine nside for non "counts" variables from the lookup table + n_pix = for_indices_by_spin_phase.sizes["pixel"] + nside = hp.npix2nside(n_pix) healpix = np.arange(n_pix) + + # Calculate the corresponding longitude (az) latitude (el) + # center coordinates + longitude, latitude = hp.pix2ang(nside, healpix, lonlat=True) + # Get the start and stop times of the pointing period repoint_id = species_dataset.attrs.get("Repointing", None) if repoint_id is None: @@ -162,11 +183,11 @@ def calculate_spacecraft_pset( valid_spun_pixels, boundary_scale_factors, aux_dataset, - pointing_range_met, - n_energy_bins=len(energy_bin_geometric_means), + energy_bins=energy_bin_geometric_means, sensor_id=sensor_id, ancillary_files=ancillary_files, - apply_bsf=apply_bsf, + apply_bsf=UltraConstants.APPLY_BOUNDARY_SCALE_FACTORS_L1C, + goodtimes_dataset=goodtimes_dataset, ) logger.info("Calculating spun efficiencies and geometric function.") # calculate efficiency and geometric function as a function of energy @@ -176,8 +197,9 @@ def calculate_spacecraft_pset( theta_vals, phi_vals, n_pix, + sensor_id, ancillary_files, - apply_bsf, + apply_bsf=UltraConstants.APPLY_BOUNDARY_SCALE_FACTORS_L1C, ) sensitivity = efficiencies * geometric_function @@ -225,6 +247,7 @@ def calculate_spacecraft_pset( pset_dict["background_rates"] = background_rates[np.newaxis, ...] pset_dict["exposure_factor"] = exposure_pointing[np.newaxis, ...] pset_dict["pixel_index"] = healpix + pset_dict["counts_pixel_index"] = counts_healpix pset_dict["energy_bin_delta"] = np.diff(intervals, axis=1).squeeze()[ np.newaxis, ... ] diff --git a/imap_processing/ultra/l1c/ultra_l1c.py b/imap_processing/ultra/l1c/ultra_l1c.py index daccfaf377..7fe259e66e 100644 --- a/imap_processing/ultra/l1c/ultra_l1c.py +++ b/imap_processing/ultra/l1c/ultra_l1c.py @@ -3,6 +3,7 @@ import xarray as xr from imap_processing.ultra.constants import UltraConstants +from imap_processing.ultra.l1b.lookup_utils import get_de_product_name from imap_processing.ultra.l1c.helio_pset import calculate_helio_pset from imap_processing.ultra.l1c.spacecraft_pset import calculate_spacecraft_pset @@ -29,17 +30,45 @@ def ultra_l1c( """ output_datasets = [] create_helio_pset = True if "helio" in descriptor else False + + # TODO + # Determine which l1b priority DE product to use in creating the l1c products. + # This will vary per-pointing by an ancillary file produced by the ULTRA team. + # Account for the possibility of having 45 and 90 in the dictionary. for instrument_id in [45, 90]: + # All l1c products require a l1b de dependency so check that first + # and calculate the correct l1b de product to use based on the repointing ID + # and ancillary files. + if f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict: + # get repoint number + repoint = data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"].attrs.get( + "Repointing", None + ) + if repoint is None: + raise ValueError("Repointing ID attribute is missing from the dataset.") + # Determine which l1b de product to use in calculating the l1c products. + # Will be either the raw de product or a priority 1-4 de product. + de_product_desc = get_de_product_name( + repoint, instrument_id, "l1c", ancillary_files + ) + if de_product_desc not in data_dict: + raise ValueError( + f"Selected L1B DE product '{de_product_desc}' for instrument " + f"{instrument_id} is not present in data_dict. Available L1B DE " + f"products: {data_dict.keys()}" + ) + else: + continue if ( f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict - and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict + and de_product_desc in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict and create_helio_pset ): helio_pset = calculate_helio_pset( - data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], + data_dict[de_product_desc], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], @@ -51,12 +80,12 @@ def ultra_l1c( output_datasets = [helio_pset] elif ( f"imap_ultra_l1b_{instrument_id}sensor-goodtimes" in data_dict - and f"imap_ultra_l1b_{instrument_id}sensor-de" in data_dict + and de_product_desc in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-rates" in data_dict and f"imap_ultra_l1a_{instrument_id}sensor-aux" in data_dict ): spacecraft_pset = calculate_spacecraft_pset( - data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], + data_dict[de_product_desc], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], @@ -67,7 +96,7 @@ def ultra_l1c( ) output_datasets = [spacecraft_pset] spacecraft_pset_non_proton = calculate_spacecraft_pset( - data_dict[f"imap_ultra_l1b_{instrument_id}sensor-de"], + data_dict[de_product_desc], data_dict[f"imap_ultra_l1b_{instrument_id}sensor-goodtimes"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-rates"], data_dict[f"imap_ultra_l1a_{instrument_id}sensor-aux"], diff --git a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py index ece5f0f0ef..29ab305c95 100644 --- a/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py +++ b/imap_processing/ultra/l1c/ultra_l1c_pset_bins.py @@ -11,15 +11,15 @@ from imap_processing.spice.geometry import ( cartesian_to_spherical, ) -from imap_processing.spice.spin import ( - get_spin_data, -) from imap_processing.ultra.constants import UltraConstants from imap_processing.ultra.l1b.lookup_utils import ( get_geometric_factor, get_image_params, load_geometric_factor_tables, ) +from imap_processing.ultra.l1b.quality_flag_filters import ( + ENERGY_DEPENDENT_SPIN_QUALITY_FLAG_FILTERS, +) from imap_processing.ultra.l1b.ultra_l1b_culling import ( get_pulses_per_spin, ) @@ -79,7 +79,7 @@ def get_spacecraft_histogram( energy_bin_edges: list[tuple[float, float]], nside: int = 128, nested: bool = False, -) -> tuple[NDArray, NDArray, NDArray, NDArray]: +) -> tuple[NDArray, NDArray]: """ Compute a 2D histogram of the particle data using HEALPix binning. @@ -101,10 +101,6 @@ def get_spacecraft_histogram( ------- hist : np.ndarray A 2D histogram array with shape (n_pix, n_energy_bins). - latitude : np.ndarray - Array of latitude values. - longitude : np.ndarray - Array of longitude values. n_pix : int Number of healpix pixels. @@ -126,10 +122,6 @@ def get_spacecraft_histogram( # Compute number of HEALPix pixels that cover the sphere n_pix = hp.nside2npix(nside) - # Calculate the corresponding longitude (az) latitude (el) - # center coordinates - longitude, latitude = hp.pix2ang(nside, np.arange(n_pix), lonlat=True) - # Get HEALPix pixel indices for each event # HEALPix expects latitude in [-90, 90] so we don't need to change elevation hpix_idx = hp.ang2pix(nside, az, el, nest=nested, lonlat=True) @@ -143,7 +135,7 @@ def get_spacecraft_histogram( # Only count the events that fall within the energy bin hist[i, :] += np.bincount(hpix_idx[mask], minlength=n_pix).astype(np.float64) - return hist, latitude, longitude, n_pix + return hist, n_pix def get_spacecraft_count_rate_uncertainty(hist: NDArray, exposure: NDArray) -> NDArray: @@ -254,7 +246,7 @@ def get_sectored_rates(rates_ds: xr.Dataset) -> xr.Dataset | None: # Get the start indices of each sector mode spin sector_starts = spin_change[spin_run_inds] - sectored_mode_mask = np.zeros(len(spins), dtype=bool) + sectored_mode_mask: np.ndarray = np.zeros(len(spins), dtype=bool) starts = np.asarray(sector_starts) # Create offsets 0..14 and broadcast idx = starts[:, None] + np.arange(15) @@ -306,7 +298,9 @@ def get_deadtime_ratios_by_spin_phase( ) else: num_spin_sectors = 15 - sector_indices = np.arange(len(sectored_rates["epoch"])) % num_spin_sectors + sector_indices: np.ndarray = ( + np.arange(len(sectored_rates["epoch"])) % num_spin_sectors + ) # Get timestamps at the start of each spin (sector 0) spin_start_indices = np.where(sector_indices == 0)[0] met_time = sectored_rates["shcoarse"].values[spin_start_indices] @@ -385,6 +379,7 @@ def calculate_exposure_time( ------- exposure_pointing: xarray.DataArray Adjusted exposure times accounting for dead time. + Shape: ``(energy, pixel)``. """ # nominal spin phase step. nominal_ms_step = 15 / valid_spun_pixels.shape[0] # time step @@ -407,8 +402,8 @@ def get_spacecraft_exposure_times( valid_spun_pixels: xr.DataArray, boundary_scale_factors: xr.DataArray, aux_dataset: xr.Dataset, - pointing_range_met: tuple[float, float], - n_energy_bins: int, + energy_bins: np.ndarray, + goodtimes_dataset: xr.Dataset, sensor_id: int | None = None, ancillary_files: dict | None = None, apply_bsf: bool = True, @@ -431,10 +426,13 @@ def get_spacecraft_exposure_times( Boundary scale factors for each pixel at each spin phase. aux_dataset : xarray.Dataset Auxiliary dataset containing spin information. - pointing_range_met : tuple - Start and stop time of the pointing period in mission elapsed time. - n_energy_bins : int - Number of energy bins. + energy_bins : np.ndarray + Array of energy bin geometric means. + goodtimes_dataset : xarray.Dataset + Dataset containing the quality-filtered spins with energy dependent quality + flags (quality_low_voltage, quality_high_energy, quality_statistics). + Exposure times are computed using only these good spins + and can be adjusted per energy bin based on quality flags. sensor_id : int, optional Sensor ID, either 45 or 90. ancillary_files : dict, optional @@ -448,6 +446,7 @@ def get_spacecraft_exposure_times( Total exposure times of pixels in a Healpix tessellation of the sky in the pointing (dps) frame. + Shape: (n_energy_bins, n_pix). nominal_deadtime_ratios : np.ndarray Deadtime ratios at each spin phase step (1ms res). """ @@ -464,35 +463,65 @@ def get_spacecraft_exposure_times( exposure_time = calculate_exposure_time( nominal_deadtime_ratios, valid_spun_pixels, boundary_scale_factors, apply_bsf ) - # Use the universal spin table to determine the actual number of spins + if exposure_time.ndim != 2: + raise ValueError( + "Exposure time must be 2D with dimensions ('energy', 'pixel'); " + f"got dims {exposure_time.dims} and shape {exposure_time.shape}." + ) nominal_spin_seconds = 15.0 - spin_data = get_spin_data() - # Filter for spins only in pointing - spin_data = spin_data[ - (spin_data["spin_start_met"] >= pointing_range_met[0]) - & (spin_data["spin_start_met"] <= pointing_range_met[1]) + # Use filtered spins from goodtimes dataset to include only the spins that + # passed the quality flag filtering. + spin_periods = goodtimes_dataset["spin_period"].values + energy_flags = goodtimes_dataset["energy_range_flags"].values + # only get valid flags for the energy bins we are using at l1c + # Filter out fill values (0s) from energy_range_flags + energy_flags = energy_flags[energy_flags > 0] + # Filter out fill values from energy_range_edges + energy_range_edges = goodtimes_dataset["energy_range_edges"].values + # Remove fill values (negative or zero) + energy_range_edges_valid = energy_range_edges[energy_range_edges > 0] + # Get the quality flag arrays "turned on" for energy dependent culling from the + # goodtimes dataset. + flag_arrays = [ + goodtimes_dataset[flag_name].values + for flag_name in ENERGY_DEPENDENT_SPIN_QUALITY_FLAG_FILTERS ] - # Get only valid spin data - valid_mask = (spin_data["spin_phase_valid"].values == 1) & ( - spin_data["spin_period_valid"].values == 1 + bin_to_range = np.digitize(energy_bins, energy_range_edges_valid) + valid_spins = ( + np.bitwise_or.reduce(flag_arrays)[np.newaxis, :] & energy_flags[:, np.newaxis] + ) == 0 + # Pad valid_spins with arrays of all true at either end to account for energy bins + # that fall outside the range of the goodtimes dataset energy edges. Energy bins + # outside these ranges were not included in the quality flag filtering, so they are + # considered valid (all true). + valid_spins_padded = np.pad( + valid_spins, + pad_width=((1, 1), (0, 0)), + # pad only the energy axis + mode="constant", + constant_values=True, + ) + # Select the valid spins for each energy bin using the bin_to_range indices + good_spins_per_ebin = valid_spins_padded[bin_to_range] + # Broadcast spin periods to have the correct shape e.g. (energy_bins, spins) + spin_periods_2d = np.broadcast_to( + spin_periods[np.newaxis, :], good_spins_per_ebin.shape ) - n_spins_in_pointing: float = np.sum( - spin_data[valid_mask].spin_period_sec / nominal_spin_seconds + # Calculate total normalized spin count using spin periods from goodtimes + # Shape (n_energy_bins) + n_spins_in_pointing = ( + np.sum(spin_periods_2d, axis=1, where=good_spins_per_ebin) + / nominal_spin_seconds ) + logger.info( - f"Calculated total spins universal spin table. Found {n_spins_in_pointing} " - f"valid spins." + f"Calculated total spins. Found {n_spins_in_pointing.tolist()} valid spins per " + f"energy range." ) - # Adjust exposure time by the actual number of valid spins in the pointing - exposure_pointing_adjusted = n_spins_in_pointing * exposure_time - - if exposure_pointing_adjusted.shape[0] != n_energy_bins: - exposure_pointing_adjusted = np.repeat( - exposure_pointing_adjusted, - n_energy_bins, - axis=0, - ) - return exposure_pointing_adjusted.values, nominal_deadtime_ratios.values + # Shape (n_energy_bins, n_pix) + exposure_pointing_adjusted = exposure_time.data * n_spins_in_pointing[:, np.newaxis] + + return exposure_pointing_adjusted, nominal_deadtime_ratios.values def get_efficiencies_and_geometric_function( @@ -501,6 +530,7 @@ def get_efficiencies_and_geometric_function( theta_vals: np.ndarray, phi_vals: np.ndarray, npix: int, + sensor_id: int, ancillary_files: dict, apply_bsf: bool = True, ) -> tuple[np.ndarray, np.ndarray]: @@ -529,6 +559,8 @@ def get_efficiencies_and_geometric_function( corresponding phi for each pixel (and energy, if present). npix : int Number of HEALPix pixels. + sensor_id : int + Sensor ID, either 45 or 90. ancillary_files : dict Dictionary containing ancillary files. apply_bsf : bool, optional @@ -543,9 +575,10 @@ def get_efficiencies_and_geometric_function( Averaged efficiencies across all spin phases. Shape = (n_energy_bins, npix). """ + sensor_name = f"ultra{sensor_id}" # Load callable efficiency interpolator function eff_interpolator, theta_min_max, phi_min_max = get_efficiency_interpolator( - ancillary_files + ancillary_files, sensor_name ) # load geometric factor lookup table geometric_lookup_table = load_geometric_factor_tables( @@ -622,12 +655,13 @@ def get_efficiencies_and_geometric_function( phi_at_spin_clipped[pixel_inds], theta_at_spin_clipped[pixel_inds], ancillary_files, + sensor_name, interpolator=eff_interpolator, ) # Accumulate and sum eff and gf values bsfs = ( - boundary_scale_factors[pixel_inds, i] + boundary_scale_factors[i, pixel_inds] if apply_bsf else np.ones(len(pixel_inds)) ) diff --git a/imap_processing/ultra/l2/ultra_l2.py b/imap_processing/ultra/l2/ultra_l2.py index a5829cfb7b..b67aa30181 100644 --- a/imap_processing/ultra/l2/ultra_l2.py +++ b/imap_processing/ultra/l2/ultra_l2.py @@ -620,6 +620,7 @@ def ultra_l2( Wrapped in a list for consistency with other product levels. """ inertial_frame = "unknown" + descriptor_duration: str | None = None if descriptor is not None: logger.info( f"Using the provided descriptor '{descriptor}' to set the map structure." @@ -628,6 +629,7 @@ def ultra_l2( map_descriptor = MapDescriptor.from_string(descriptor) output_map_structure = map_descriptor.to_empty_map() inertial_frame = map_descriptor.frame_descriptor + descriptor_duration = str(map_descriptor.duration) inertial_frame_long_name = INERTIAL_FRAME_LONG_NAMES.get(inertial_frame, "unknown") # Object which holds CDF attributes for the map @@ -667,9 +669,13 @@ def ultra_l2( # TODO: replace 1 day in ns below with the actual end time of the last PSET. # Currently assumes the end time of the last PSET is 1 day after its start. map_duration_ns = (pset_epochs.max() + (86400 * 1e9)) - pset_epochs.min() - map_duration_months_int = ns_to_duration_months(map_duration_ns) - map_duration = f"{map_duration_months_int}mo" - + # Use the duration from the descriptor if it is provided, otherwise use the + # calculated duration from the PSET epochs. + if descriptor_duration is None: + map_duration_months_int = ns_to_duration_months(map_duration_ns) + map_duration = f"{map_duration_months_int}mo" + else: + map_duration = descriptor_duration # Always add the common (non-tiling specific) attributes to the attr handler. # These can be updated/overwritten by the tiling specific attributes. cdf_attrs.add_instrument_variable_attrs(instrument="enamaps", level="l2-common") diff --git a/imap_processing/ultra/utils/ultra_l1_utils.py b/imap_processing/ultra/utils/ultra_l1_utils.py index 7efc4d8a59..a22ca75819 100644 --- a/imap_processing/ultra/utils/ultra_l1_utils.py +++ b/imap_processing/ultra/utils/ultra_l1_utils.py @@ -46,6 +46,7 @@ def create_dataset( # noqa: PLR0912 coords = { "epoch": data_dict["epoch"], "pixel_index": data_dict["pixel_index"], + "counts_pixel_index": data_dict["counts_pixel_index"], "energy_bin_geometric_mean": data_dict["energy_bin_geometric_mean"], "spin_phase_step": data_dict["spin_phase_step"], } @@ -97,7 +98,6 @@ def create_dataset( # noqa: PLR0912 # epoch coordinate already created with correct attrs continue elif key == "epoch_delta": - # Create epoch_delta variable dataset[key] = xr.DataArray( data, dims=["epoch"], @@ -107,12 +107,19 @@ def create_dataset( # noqa: PLR0912 "spin_number", "energy_bin_geometric_mean", "pixel_index", + "counts_pixel_index", "spin_phase_step", ]: - # update attrs + # update attrs on existing coords dataset[key].attrs = cdf_manager.get_variable_attributes( key, check_schema=False ) + elif key in ["energy_range_edges_dim", "energy_range_flags_dim"]: + dataset[key] = xr.DataArray( + data, + dims=[key], + attrs=cdf_manager.get_variable_attributes(key, check_schema=False), + ) elif key in velocity_keys: dataset[key] = xr.DataArray( data, @@ -155,7 +162,6 @@ def create_dataset( # noqa: PLR0912 attrs=cdf_manager.get_variable_attributes(key, check_schema=False), ) elif key in { - "counts", "background_rates", "exposure_factor", "helio_exposure_factor", @@ -165,6 +171,12 @@ def create_dataset( # noqa: PLR0912 dims=["epoch", "energy_bin_geometric_mean", "pixel_index"], attrs=cdf_manager.get_variable_attributes(key, check_schema=False), ) + elif key in {"counts"}: + dataset[key] = xr.DataArray( + data, + dims=["epoch", "energy_bin_geometric_mean", "counts_pixel_index"], + attrs=cdf_manager.get_variable_attributes(key, check_schema=False), + ) elif key in { "geometric_function", "scatter_theta", @@ -177,24 +189,22 @@ def create_dataset( # noqa: PLR0912 dims=["energy_bin_geometric_mean", "pixel_index"], attrs=cdf_manager.get_variable_attributes(key, check_schema=False), ) - elif key in { - "dead_time_ratio", - }: + elif key in {"dead_time_ratio"}: dataset[key] = xr.DataArray( data, dims=["spin_phase_step"], attrs=cdf_manager.get_variable_attributes(key, check_schema=False), ) - elif key in {"energy_range_edges"}: + elif key == "energy_range_edges": dataset[key] = xr.DataArray( data, - dims=["energy_range_edges"], + dims=["energy_range_edges_dim"], attrs=cdf_manager.get_variable_attributes(key, check_schema=False), ) - elif key in {"energy_range_flags"}: + elif key == "energy_range_flags": dataset[key] = xr.DataArray( data, - dims=["energy_ranges"], + dims=["energy_range_flags_dim"], attrs=cdf_manager.get_variable_attributes(key, check_schema=False), ) else: @@ -225,7 +235,15 @@ def extract_data_dict(dataset: xr.Dataset) -> dict: data_dict.update( { coord: dataset.coords[coord].values - for coord in ("spin_number", "energy_bin_geometric_mean", "epoch") + for coord in ( + "spin_number", + "energy_bin_geometric_mean", + "epoch", + "energy_range_flags_dim", + "energy_range_edges_dim", + "energy_range_flags", + "energy_range_edges", + ) if coord in dataset.coords } ) diff --git a/poetry.lock b/poetry.lock index 512152d651..a72f1673d2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -807,8 +807,11 @@ files = [ {file = "lxml-5.4.0-cp36-cp36m-win_amd64.whl", hash = "sha256:7ce1a171ec325192c6a636b64c94418e71a1964f56d002cc28122fceff0b6121"}, {file = "lxml-5.4.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:795f61bcaf8770e1b37eec24edf9771b307df3af74d1d6f27d812e15a9ff3872"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29f451a4b614a7b5b6c2e043d7b64a15bd8304d7e767055e8ab68387a8cacf4e"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:891f7f991a68d20c75cb13c5c9142b2a3f9eb161f1f12a9489c82172d1f133c0"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4aa412a82e460571fad592d0f93ce9935a20090029ba08eca05c614f99b0cc92"}, + {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_aarch64.whl", hash = "sha256:ac7ba71f9561cd7d7b55e1ea5511543c0282e2b6450f122672a2694621d63b7e"}, {file = "lxml-5.4.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:c5d32f5284012deaccd37da1e2cd42f081feaa76981f0eaa474351b68df813c5"}, + {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:ce31158630a6ac85bddd6b830cffd46085ff90498b397bd0a259f59d27a12188"}, {file = "lxml-5.4.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:31e63621e073e04697c1b2d23fcb89991790eef370ec37ce4d5d469f40924ed6"}, {file = "lxml-5.4.0-cp37-cp37m-win32.whl", hash = "sha256:be2ba4c3c5b7900246a8f866580700ef0d538f2ca32535e991027bdaba944063"}, {file = "lxml-5.4.0-cp37-cp37m-win_amd64.whl", hash = "sha256:09846782b1ef650b321484ad429217f5154da4d6e786636c38e434fa32e94e49"}, diff --git a/pyproject.toml b/pyproject.toml index 663aa571e2..bfed08d060 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "poetry_dynamic_versioning.backend" [tool.poetry] name = "imap-processing" # Gets updated dynamically by the poetry-dynamic-versioning plugin -version = "0.0.0" +version = "1.0.25.post6.dev0+882be304" description = "IMAP Science Operations Center Processing" authors = ["IMAP SDC Developers "] readme = "README.md" @@ -126,7 +126,7 @@ imap_cli = 'imap_processing.cli:main' imap_xtce = 'imap_processing.ccsds.excel_to_xtce:main' [tool.codespell] -ignore-words-list = "livetime" +ignore-words-list = "livetime,nd" [tool.poetry-dynamic-versioning] enable = true @@ -164,4 +164,3 @@ explicit_package_bases = true follow_imports = 'skip' #may want to remove exclude = ["tests"] packages = ["imap_processing" , "tools"] -plugins = 'numpy.typing.mypy_plugin'