From f540a3ffea07d066fe5ed2373f9fccaa2e96338b Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 00:00:51 -0500 Subject: [PATCH 01/15] new method in MesaData.read_log_data to improve speed --- mesa_reader/__init__.py | 63 ++++++++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index 4833cc4..32868f2 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -1,6 +1,7 @@ import os from os.path import join import re +from ast import literal_eval import numpy as np @@ -148,6 +149,32 @@ def __str__(self): return "MESA model # {:6}, t = {:20.10g} yr".format(model_number, age) except Exception: return "{}".format(self.file_name) + + + def _get_dtype(self, names, data) -> np.ndarray: + """Heuristic datatype determination using the first line of the log file.""" + if not hasattr(data, '__iter__'): + data = np.asarray([data]) + + types = [] + + for i, record in enumerate(data): + try: + record = literal_eval(record) + if type(record) == float: + types.append((names[i], 'float64')) + elif type(record) == int: + types.append((names[i],'int64')) + + except ValueError: + if record == "NaN": + types.append((names[i], 'float64')) + elif type(record) == str: + types.append((names[i], 'U128')) + + dtype = np.dtype(types) + + return dtype def read_data(self): """Decide if data file is log output or a model, then load the data @@ -196,23 +223,25 @@ def read_log_data(self): ------- None """ - self.bulk_data = np.genfromtxt( - self.file_name, - skip_header=MesaData.bulk_names_line - 1, - names=True, - ndmin=1, # Make sure a single entry is still a 1D array - dtype=None, - ) - self.bulk_names = self.bulk_data.dtype.names - header_data = [] - with open(self.file_name) as f: - for i, line in enumerate(f): - if i == MesaData.header_names_line - 1: - self.header_names = line.split() - elif i == MesaData.header_names_line: - header_data = [eval(datum) for datum in line.split()] - elif i > MesaData.header_names_line: - break + # attempting to speed up this process with some dirty tricks + with open(self.file_name, "r") as file: + + for _ in range(MesaData.header_names_line - 1): + file.readline() # skip 1st line + + self.header_names = file.readline().split(None, -1) + header_data = file.readline().split(None, -1) + + for _ in range(2): + file.readline() + + self.bulk_names = file.readline().split(None, -1) + data_elements = file.readline().split(None, -1) + + data_types = self.get_dtype(self.bulk_names, data_elements) + + self.bulk_data = np.loadtxt(file, dtype=data_types, skiprows=MesaData.bulk_names_line) + self.header_data = dict(zip(self.header_names, header_data)) self.remove_backups() From aeb127167b40c0d80ef896f20a3dc23649402301 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 00:04:50 -0500 Subject: [PATCH 02/15] fixed a missing underscore :( --- mesa_reader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index 32868f2..f5e5a84 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -238,7 +238,7 @@ def read_log_data(self): self.bulk_names = file.readline().split(None, -1) data_elements = file.readline().split(None, -1) - data_types = self.get_dtype(self.bulk_names, data_elements) + data_types = self._get_dtype(self.bulk_names, data_elements) self.bulk_data = np.loadtxt(file, dtype=data_types, skiprows=MesaData.bulk_names_line) From dbff63a278384fbfd366689b60e1a9d7234698bb Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 00:15:42 -0500 Subject: [PATCH 03/15] added dtype handling for logicals --- mesa_reader/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index f5e5a84..8aa60b3 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -169,6 +169,8 @@ def _get_dtype(self, names, data) -> np.ndarray: except ValueError: if record == "NaN": types.append((names[i], 'float64')) + if record.lower() in ["true", "false"]: + types.append((names[i], '?')) elif type(record) == str: types.append((names[i], 'U128')) From 5cf022b5c9ca13ed48586ac7e0439c60defc7795 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 01:50:14 -0500 Subject: [PATCH 04/15] fixed rewind file to read bulk_data --- mesa_reader/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index 8aa60b3..fcc3424 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -238,10 +238,12 @@ def read_log_data(self): file.readline() self.bulk_names = file.readline().split(None, -1) - data_elements = file.readline().split(None, -1) + data_elements = file.readline().split(None, -1) data_types = self._get_dtype(self.bulk_names, data_elements) + # rewind & read data + file.seek(0) self.bulk_data = np.loadtxt(file, dtype=data_types, skiprows=MesaData.bulk_names_line) self.header_data = dict(zip(self.header_names, header_data)) From c0bbacd3e41ce764c7c605b2d4e65a3eef5a1e71 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 03:54:38 -0500 Subject: [PATCH 05/15] switched to using numpy.fromfile method --- mesa_reader/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index fcc3424..a07c0f9 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -239,12 +239,17 @@ def read_log_data(self): self.bulk_names = file.readline().split(None, -1) + pos_0 = file.tell() data_elements = file.readline().split(None, -1) + pos_1 = file.tell() + pos_diff = pos_1 - pos_0 # length of data line 1 + data_types = self._get_dtype(self.bulk_names, data_elements) # rewind & read data - file.seek(0) - self.bulk_data = np.loadtxt(file, dtype=data_types, skiprows=MesaData.bulk_names_line) + file.seek(-pos_diff) + self.bulk_data = np.fromfile(file, dtype=data_types, sep=" ") + #self.bulk_data = np.loadtxt(file, dtype=data_types, skiprows=MesaData.bulk_names_line) self.header_data = dict(zip(self.header_names, header_data)) self.remove_backups() From a203eeb062cc8163b0564c1082c127519808d704 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 10:42:44 -0500 Subject: [PATCH 06/15] switched to use pandas.read_csv for extremely fast performance --- mesa_reader/__init__.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index a07c0f9..f25e73e 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -2,6 +2,7 @@ from os.path import join import re from ast import literal_eval +from pandas import read_csv import numpy as np @@ -225,7 +226,9 @@ def read_log_data(self): ------- None """ - # attempting to speed up this process with some dirty tricks + # I'm attempting to speed up this process with some dirty tricks + # Using pandas's read_csv function gives us c-like performance as + # opposed to genfromtxt's native (slow, icky) python with open(self.file_name, "r") as file: for _ in range(MesaData.header_names_line - 1): @@ -238,18 +241,18 @@ def read_log_data(self): file.readline() self.bulk_names = file.readline().split(None, -1) - - pos_0 = file.tell() data_elements = file.readline().split(None, -1) - pos_1 = file.tell() - pos_diff = pos_1 - pos_0 # length of data line 1 - data_types = self._get_dtype(self.bulk_names, data_elements) - # rewind & read data - file.seek(-pos_diff) - self.bulk_data = np.fromfile(file, dtype=data_types, sep=" ") - #self.bulk_data = np.loadtxt(file, dtype=data_types, skiprows=MesaData.bulk_names_line) + # rewind & read + with open(self.file_name, "r") as file: + for _ in range(MesaData.bulk_names_line - 1): + file.readline() + + _dataframe = read_csv(file, sep="\s+", dtype=None) + _records = _dataframe.to_records(index=False) + + self.bulk_data = np.array(_records, dtype=_records.dtype.descr) self.header_data = dict(zip(self.header_names, header_data)) self.remove_backups() From 46da20fcb90c7fc009cd78c3782fee9448594233 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 10:42:56 -0500 Subject: [PATCH 07/15] added pandas --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 58c6229..ed113e6 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ author_email='wolfwm@uwec.edu', license='MIT', packages=['mesa_reader'], - install_requires=['numpy'], + install_requires=['numpy', 'pandas'], classifiers=[ 'Programming Language :: Python :: 3', 'License :: OSI Approved :: MIT License', From d50e39e656ad6005f592282c0c113dd14682ba82 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 11:43:22 -0500 Subject: [PATCH 08/15] changed remove_backups to use vectorized ops --- mesa_reader/__init__.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index f25e73e..2d35e9f 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -2,7 +2,7 @@ from os.path import join import re from ast import literal_eval -from pandas import read_csv +from pandas import DataFrame, read_csv import numpy as np @@ -251,7 +251,7 @@ def read_log_data(self): _dataframe = read_csv(file, sep="\s+", dtype=None) _records = _dataframe.to_records(index=False) - + self.bulk_data = np.array(_records, dtype=_records.dtype.descr) self.header_data = dict(zip(self.header_names, header_data)) @@ -731,18 +731,19 @@ def remove_backups(self, dbg=False): return None if dbg: print("Scrubbing history...") - to_remove = [] - for i in range(len(self.data("model_number")) - 1): - smallest_future = np.min(self.data("model_number")[i + 1 :]) - if self.data("model_number")[i] >= smallest_future: - to_remove.append(i) - if len(to_remove) == 0: + + model_numbers = DataFrame(self.data["model_number"]) + kept_indices = model_numbers.drop_duplicates(keep="last").index + + if len(model_numbers) - len(kept_indices) == 0: if dbg: print("Already clean!") - return None + return if dbg: - print("Removing {} lines.".format(len(to_remove))) - self.bulk_data = np.delete(self.bulk_data, to_remove) + print(f"Found {len(model_numbers) - len(kept_indices)} lines to remove.") + + self.bulk_data = self.bulk_data[kept_indices] + return def __getattr__(self, method_name): if self._any_version(method_name): From ac91c921b56278feef2e19b2fbb7d75c160dff62 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Fri, 3 Jan 2025 12:28:56 -0500 Subject: [PATCH 09/15] fixed a method call --- mesa_reader/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index 2d35e9f..dd75992 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -732,7 +732,7 @@ def remove_backups(self, dbg=False): if dbg: print("Scrubbing history...") - model_numbers = DataFrame(self.data["model_number"]) + model_numbers = DataFrame(self.data("model_number")) kept_indices = model_numbers.drop_duplicates(keep="last").index if len(model_numbers) - len(kept_indices) == 0: From 00f6aa502eed108e4f9011c8baa767b9c2385632 Mon Sep 17 00:00:00 2001 From: d-maclean Date: Sat, 4 Jan 2025 11:28:59 -0500 Subject: [PATCH 10/15] cleaned up read_log_data and removed superfluous heuristic algorihm --- mesa_reader/__init__.py | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index dd75992..17ff400 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -151,33 +151,6 @@ def __str__(self): except Exception: return "{}".format(self.file_name) - - def _get_dtype(self, names, data) -> np.ndarray: - """Heuristic datatype determination using the first line of the log file.""" - if not hasattr(data, '__iter__'): - data = np.asarray([data]) - - types = [] - - for i, record in enumerate(data): - try: - record = literal_eval(record) - if type(record) == float: - types.append((names[i], 'float64')) - elif type(record) == int: - types.append((names[i],'int64')) - - except ValueError: - if record == "NaN": - types.append((names[i], 'float64')) - if record.lower() in ["true", "false"]: - types.append((names[i], '?')) - elif type(record) == str: - types.append((names[i], 'U128')) - - dtype = np.dtype(types) - - return dtype def read_data(self): """Decide if data file is log output or a model, then load the data @@ -240,18 +213,10 @@ def read_log_data(self): for _ in range(2): file.readline() - self.bulk_names = file.readline().split(None, -1) - data_elements = file.readline().split(None, -1) - data_types = self._get_dtype(self.bulk_names, data_elements) - - # rewind & read - with open(self.file_name, "r") as file: - for _ in range(MesaData.bulk_names_line - 1): - file.readline() - _dataframe = read_csv(file, sep="\s+", dtype=None) _records = _dataframe.to_records(index=False) + self.bulk_names = _dataframe.columns.values self.bulk_data = np.array(_records, dtype=_records.dtype.descr) self.header_data = dict(zip(self.header_names, header_data)) From bb8ae7202839e8e7503f4d3ca7f14b7dd420a9b0 Mon Sep 17 00:00:00 2001 From: Bill Wolf Date: Thu, 25 Jun 2026 16:06:23 -0500 Subject: [PATCH 11/15] Fix CI to trigger on master, add CLAUDE.md The test workflow was triggering on the `main` branch, but the repository's default branch is `master`, so CI never ran. Point both the push and pull_request triggers at `master`. Also add a CLAUDE.md with build/lint/test commands and an architecture overview for future Claude Code sessions. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/test-package.yml | 4 +- CLAUDE.md | 73 ++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 CLAUDE.md diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index a4ccf4b..adc42a4 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -5,9 +5,9 @@ name: tests on: push: - branches: [ "main" ] + branches: [ "master" ] pull_request: - branches: [ "main" ] + branches: [ "master" ] jobs: build: diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..dcdba69 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,73 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +`mesa_reader` is a small, single-module Python library for reading and manipulating +output files from [MESA](https://mesastar.org/) (Modules for Experiments in Stellar +Astrophysics). It is published to PyPI as `mesa-reader`. The only runtime dependency +is `numpy`. + +## Commands + +```console +pip install . # install the package locally +pip install -r requirements-dev.txt # install dev tools (ruff, pytest) + +ruff format --check # lint: formatting check (run before committing) +ruff check # lint: static checks +pytest # run the test suite +pytest path/to/test.py::name # run a single test + +make html # build Sphinx docs into docs_build/ (also: bash make_docs.sh) +``` + +CI (`.github/workflows/test-package.yml`) runs `ruff format --check`, `ruff check`, +and `pytest` on Python 3.12 and 3.13. There are currently no committed test files, +so `pytest` collects nothing — add tests under a `tests/` directory or `test_*.py` +files. + +The published HTML docs live in `docs/` (served by GitHub Pages); the reStructuredText +sources are in `docs_source/`. `build/`, `dist/`, and `*.egg-info/` are build artifacts. + +## Architecture + +All code lives in [`mesa_reader/__init__.py`](mesa_reader/__init__.py) — three classes +that build on each other: + +- **`MesaData`** — the core file reader. Reads a single MESA log file (history or + profile, `.data`/`.log`) or a saved model (`.mod`). File type is auto-detected from + the extension. Log files are parsed with `np.genfromtxt` into a record array + (`bulk_data`) plus a header dict; `.mod` files use a hand-rolled line walker + (`read_model_data`) that converts Fortran `D`-exponent notation to Python floats. + History files are scrubbed of backups/restarts so `model_number` is monotonic + (`remove_backups`). + +- **`MesaProfileIndex`** — parses `profiles.index`, providing the + profile-number ↔ model-number mapping. + +- **`MesaLogDir`** — ties a whole LOGS directory together: it owns one `MesaData` + history object plus a `MesaProfileIndex`, and lazily constructs (and optionally + memoizes) per-profile `MesaData` objects via `profile_data`. + +### Key conventions to preserve + +- **Attribute access falls through to data lookup.** Both `MesaData` and + `MesaProfileIndex` override `__getattr__`, so `m.star_age` is equivalent to + `m.data('star_age')` (and falls back to `header(...)`). When adding methods or + attributes, be aware that any unknown attribute is routed to the data accessor. + +- **Logarithm/linear key inference.** `MesaData.data(key)` does more than a dict + lookup: if `key` is absent it searches for a `log_`/`ln_` variant and exponentiates + it (or vice-versa) via the `_log_version`/`_ln_version`/`_exp10_version`/ + `_exp_version` helpers. Preserve this fallback chain when touching `data`. + +- **Parsing layout is configurable via class methods.** `header_names_line`, + `bulk_names_line` (on `MesaData`) and `index_start_line`/`index_names` (on + `MesaProfileIndex`) are class-level and set through classmethods like + `set_data_rows`. Changing them affects all subsequent reads. + +- **Header/model values are parsed with `eval`.** `read_log_data` and + `read_model_data` call `eval` on tokens from the file. This is intentional for + reading numeric/string MESA output; keep input assumed to be trusted MESA files. From 570d7e6483d01e55da124f0fc00cba77299fb4d4 Mon Sep 17 00:00:00 2001 From: Bill Wolf Date: Thu, 25 Jun 2026 16:15:45 -0500 Subject: [PATCH 12/15] Fix ordering and header regressions in pandas backend Two issues in the merged pandas backend (PR #22): 1. remove_backups used DataFrame.drop_duplicates(keep="last"), which only removes exact model_number duplicates. When a run restarts from an earlier model with a different history interval, the stale rows between the restart point and the original run's end are left behind, so model_number is no longer monotonic. Replace it with a vectorized suffix-minimum scan (O(n) form of the original per-row np.min logic): a row survives only if its model number is smaller than every model number that follows it. 2. read_log_data parsed header values as raw strings, dropping the eval() the numpy backend used. Restore eval() so header data are native Python ints/floats/strings again. Also derive the number of skipped lines before the bulk-name row from the configurable line attributes (instead of a hard-coded 2), keep bulk_names a tuple to match the original contract, and use a raw-string separator to avoid an invalid-escape warning. Co-Authored-By: Claude Opus 4.8 --- mesa_reader/__init__.py | 64 +++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index 6e0dd02..ec7a801 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -4,7 +4,7 @@ from pathlib import Path import numpy as np -from pandas import DataFrame, read_csv +from pandas import read_csv class ProfileError(Exception): @@ -150,7 +150,6 @@ def __str__(self): return "MESA model # {:6}, t = {:20.10g} yr".format(model_number, age) except Exception: return "{}".format(self.file_name) - def read_data(self): """Decide if data file is log output or a model, then load the data @@ -170,9 +169,9 @@ def read_data(self): # attempt auto-detection of file_type (if not supplied) if self.file_type is None: - if Path (self.file_name).suffix in [".data", ".log"]: + if Path(self.file_name).suffix in [".data", ".log"]: self.file_type = "log" - elif Path (self.file_name).suffix==".mod": + elif Path(self.file_name).suffix == ".mod": self.file_type = "model" else: raise UnknownFileTypeError( @@ -199,25 +198,25 @@ def read_log_data(self): ------- None """ - # I'm attempting to speed up this process with some dirty tricks - # Using pandas's read_csv function gives us c-like performance as - # opposed to genfromtxt's native (slow, icky) python + # pandas.read_csv parses the bulk table in C, which is dramatically + # faster than numpy.genfromtxt's line-by-line Python parsing for the + # large history/profile files this package targets. with open(self.file_name, "r") as file: - + # Advance to and read the header name/value rows. for _ in range(MesaData.header_names_line - 1): - file.readline() # skip 1st line - - self.header_names = file.readline().split(None, -1) - header_data = file.readline().split(None, -1) - - for _ in range(2): file.readline() + self.header_names = file.readline().split() + header_data = [eval(datum) for datum in file.readline().split()] - _dataframe = read_csv(file, sep="\s+", dtype=None) - _records = _dataframe.to_records(index=False) - - self.bulk_names = _dataframe.columns.values - self.bulk_data = np.array(_records, dtype=_records.dtype.descr) + # Advance to the bulk-name row, which read_csv consumes as its + # column header. The number of intervening lines is derived from + # the (configurable) line positions rather than hard-coded. + for _ in range(MesaData.bulk_names_line - MesaData.header_names_line - 2): + file.readline() + dataframe = read_csv(file, sep=r"\s+", dtype=None) + records = dataframe.to_records(index=False) + self.bulk_names = tuple(dataframe.columns) + self.bulk_data = np.array(records, dtype=records.dtype.descr) self.header_data = dict(zip(self.header_names, header_data)) self.remove_backups() @@ -697,18 +696,27 @@ def remove_backups(self, dbg=False): if dbg: print("Scrubbing history...") - model_numbers = DataFrame(self.data("model_number")) - kept_indices = model_numbers.drop_duplicates(keep="last").index - - if len(model_numbers) - len(kept_indices) == 0: + # A row is genuine only if its model number is smaller than every model + # number that comes after it; otherwise it is cruft superseded by a + # later restart. `suffix_min[i]` is the minimum model number over rows + # i..end, so `suffix_min[i + 1]` is the smallest future model number. + # This is the vectorized (O(n)) form of the original per-row np.min + # scan. Note that drop_duplicates(keep="last") is NOT equivalent: it + # leaves the now-orphaned rows between a restart and its original run, + # breaking the monotonicity of model_number. + model_number = self.data("model_number") + suffix_min = np.minimum.accumulate(model_number[::-1])[::-1] + keep = np.ones(len(model_number), dtype=bool) + keep[:-1] = model_number[:-1] < suffix_min[1:] + + n_removed = np.count_nonzero(~keep) + if n_removed == 0: if dbg: print("Already clean!") - return + return None if dbg: - print(f"Found {len(model_numbers) - len(kept_indices)} lines to remove.") - - self.bulk_data = self.bulk_data[kept_indices] - return + print("Removing {} lines.".format(n_removed)) + self.bulk_data = self.bulk_data[keep] def __getattr__(self, method_name): if self._any_version(method_name): From b69404651a4beebe6c33fbd50d3ebec70db20835 Mon Sep 17 00:00:00 2001 From: Bill Wolf Date: Thu, 25 Jun 2026 16:15:45 -0500 Subject: [PATCH 13/15] Add test suite for MesaData and MesaLogDir Covers bulk/header parsing, attribute access, log/linear key inference, profile<->model mapping, memoization, and select_models. Includes regression tests for the two pandas-backend bugs: history restart ordering (restart_history.data) and typed header values. Co-Authored-By: Claude Opus 4.8 --- tests/data/LOGS/history.data | 10 ++++ tests/data/LOGS/profile1.data | 9 ++++ tests/data/LOGS/profile2.data | 9 ++++ tests/data/LOGS/profiles.index | 3 ++ tests/data/restart_history.data | 31 ++++++++++++ tests/test_mesa_data.py | 88 +++++++++++++++++++++++++++++++++ tests/test_mesa_logdir.py | 58 ++++++++++++++++++++++ 7 files changed, 208 insertions(+) create mode 100644 tests/data/LOGS/history.data create mode 100644 tests/data/LOGS/profile1.data create mode 100644 tests/data/LOGS/profile2.data create mode 100644 tests/data/LOGS/profiles.index create mode 100644 tests/data/restart_history.data create mode 100644 tests/test_mesa_data.py create mode 100644 tests/test_mesa_logdir.py diff --git a/tests/data/LOGS/history.data b/tests/data/LOGS/history.data new file mode 100644 index 0000000..5db5821 --- /dev/null +++ b/tests/data/LOGS/history.data @@ -0,0 +1,10 @@ + 1 2 3 + version_number initial_mass initial_z + '24.08.1' 1.0 0.02 + + 1 2 3 4 + model_number num_zones star_age log_L + 5 100 500.0 0.1 + 10 100 1000.0 0.2 + 15 100 1500.0 0.3 + 20 100 2000.0 0.4 diff --git a/tests/data/LOGS/profile1.data b/tests/data/LOGS/profile1.data new file mode 100644 index 0000000..00dc282 --- /dev/null +++ b/tests/data/LOGS/profile1.data @@ -0,0 +1,9 @@ + 1 2 3 + model_number num_zones star_age + 10 3 1000.0 + + 1 2 3 4 + zone mass logRho L + 1 1.0 2.0 100.0 + 2 0.5 1.0 200.0 + 3 0.1 0.0 300.0 diff --git a/tests/data/LOGS/profile2.data b/tests/data/LOGS/profile2.data new file mode 100644 index 0000000..3e4d648 --- /dev/null +++ b/tests/data/LOGS/profile2.data @@ -0,0 +1,9 @@ + 1 2 3 + model_number num_zones star_age + 20 3 2000.0 + + 1 2 3 4 + zone mass logRho L + 1 2.0 3.0 150.0 + 2 1.0 2.0 250.0 + 3 0.2 1.0 350.0 diff --git a/tests/data/LOGS/profiles.index b/tests/data/LOGS/profiles.index new file mode 100644 index 0000000..cf3c96a --- /dev/null +++ b/tests/data/LOGS/profiles.index @@ -0,0 +1,3 @@ +# comment line giving the number of profiles + 10 2 1 + 20 2 2 diff --git a/tests/data/restart_history.data b/tests/data/restart_history.data new file mode 100644 index 0000000..70a4bae --- /dev/null +++ b/tests/data/restart_history.data @@ -0,0 +1,31 @@ + 1 2 3 + version_number initial_mass initial_z + '24.08.1' 1.0 0.02 + + 1 2 3 + model_number num_zones star_age + 1 100 100.0 + 2 100 200.0 + 3 100 300.0 + 4 100 400.0 + 5 100 500.0 + 6 100 600.0 + 7 100 700.0 + 8 100 800.0 + 9 100 900.0 + 10 100 1000.0 + 11 100 1100.0 + 12 100 1200.0 + 13 100 1300.0 + 14 100 1400.0 + 15 100 1500.0 + 16 100 1600.0 + 17 100 1700.0 + 18 100 1800.0 + 19 100 1900.0 + 20 100 2000.0 + 21 100 2100.0 + 10 100 1000.0 + 15 100 1500.0 + 20 100 2000.0 + 25 100 2500.0 diff --git a/tests/test_mesa_data.py b/tests/test_mesa_data.py new file mode 100644 index 0000000..9528051 --- /dev/null +++ b/tests/test_mesa_data.py @@ -0,0 +1,88 @@ +import os + +import numpy as np +import pytest + +import mesa_reader as mr + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +HISTORY = os.path.join(DATA_DIR, "LOGS", "history.data") +RESTART_HISTORY = os.path.join(DATA_DIR, "restart_history.data") +PROFILE = os.path.join(DATA_DIR, "LOGS", "profile1.data") + + +def test_reads_bulk_data(): + m = mr.MesaData(HISTORY) + assert m.is_history() + assert "model_number" in m.bulk_names + assert isinstance(m.bulk_names, tuple) + np.testing.assert_array_equal(m.model_number, [5, 10, 15, 20]) + np.testing.assert_allclose(m.star_age, [500.0, 1000.0, 1500.0, 2000.0]) + + +def test_header_values_are_typed(): + # The header row must be parsed into native Python values, not left as raw + # strings (regression guard for the pandas backend dropping eval()). + m = mr.MesaData(HISTORY) + assert m.initial_mass == 1.0 + assert isinstance(m.initial_mass, float) + assert m.initial_z == 0.02 + assert m.version_number == "24.08.1" + + +def test_attribute_access_matches_data_method(): + m = mr.MesaData(HISTORY) + np.testing.assert_array_equal(m.star_age, m.data("star_age")) + assert m.initial_mass == m.header("initial_mass") + + +def test_log_linear_inference(): + m = mr.MesaData(PROFILE) + # logRho is present; requesting Rho should exponentiate it. + np.testing.assert_allclose(m.data("Rho"), 10 ** m.data("logRho")) + # L is present; requesting log_L should take its log10. + np.testing.assert_allclose(m.data("log_L"), np.log10(m.data("L"))) + + +def test_invalid_key_raises(): + m = mr.MesaData(HISTORY) + with pytest.raises(KeyError): + m.data("not_a_real_column") + + +def test_remove_backups_restores_monotonic_model_numbers(): + # The file contains a run to model 21 followed by a restart from model 10 + # (1..21, then 10, 15, 20, 25). After scrubbing, model_number must be the + # surviving monotonic sequence with all superseded rows obliterated. + m = mr.MesaData(RESTART_HISTORY) + expected = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25] + np.testing.assert_array_equal(m.model_number, expected) + assert np.all(np.diff(m.model_number) > 0) + # Bulk columns must stay row-aligned with the kept model numbers. + np.testing.assert_allclose(m.star_age, np.array(expected) * 100.0) + + +def test_remove_backups_noop_on_clean_history(): + m = mr.MesaData(HISTORY) + before = m.model_number.copy() + m.remove_backups() + np.testing.assert_array_equal(m.model_number, before) + + +def test_data_at_model_number(): + m = mr.MesaData(HISTORY) + assert m.data_at_model_number("star_age", 15) == 1500.0 + assert m.index_of_model_number(15) == 2 + + +def test_profile_is_not_history(): + p = mr.MesaData(PROFILE) + assert not p.is_history() + assert p.model_number == 10 # from the header, not bulk data + + +def test_unknown_file_type_raises(tmp_path): + bogus = tmp_path / "data.unknown" + bogus.write_text("nothing useful\n") + with pytest.raises(mr.UnknownFileTypeError): + mr.MesaData(str(bogus)) diff --git a/tests/test_mesa_logdir.py b/tests/test_mesa_logdir.py new file mode 100644 index 0000000..1b09ea9 --- /dev/null +++ b/tests/test_mesa_logdir.py @@ -0,0 +1,58 @@ +import os + +import numpy as np +import pytest + +import mesa_reader as mr + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +LOGS = os.path.join(DATA_DIR, "LOGS") + + +def test_reads_history_and_index(): + log = mr.MesaLogDir(LOGS) + assert log.history.is_history() + np.testing.assert_array_equal(log.profile_numbers, [1, 2]) + np.testing.assert_array_equal(log.model_numbers, [10, 20]) + + +def test_profile_model_number_mapping(): + log = mr.MesaLogDir(LOGS) + assert log.profile_with_model_number(20) == 2 + assert log.model_with_profile_number(1) == 10 + assert log.have_profile_with_model_number(10) + assert not log.have_profile_with_model_number(999) + + +def test_profile_data_defaults_to_last_profile(): + log = mr.MesaLogDir(LOGS) + p = log.profile_data() + assert p.model_number == 20 + + +def test_profile_data_by_model_number(): + log = mr.MesaLogDir(LOGS) + p = log.profile_data(model_number=10) + assert p.model_number == 10 + np.testing.assert_allclose(p.data("logRho"), [2.0, 1.0, 0.0]) + + +def test_memoization_returns_same_object(): + log = mr.MesaLogDir(LOGS, memoize_profiles=True) + assert log.profile_data(profile_number=1) is log.profile_data(profile_number=1) + + log_no_memo = mr.MesaLogDir(LOGS, memoize_profiles=False) + first = log_no_memo.profile_data(profile_number=1) + second = log_no_memo.profile_data(profile_number=1) + assert first is not second + + +def test_select_models(): + log = mr.MesaLogDir(LOGS) + selected = log.select_models(lambda age: age > 1200.0, "star_age") + np.testing.assert_array_equal(selected, [20]) + + +def test_bad_path_raises(): + with pytest.raises(mr.BadPathError): + mr.MesaLogDir(os.path.join(DATA_DIR, "does_not_exist")) From 9a5cb35cb8b1ad3195b72410c9a336bc4336a517 Mon Sep 17 00:00:00 2001 From: Bill Wolf Date: Thu, 25 Jun 2026 16:25:45 -0500 Subject: [PATCH 14/15] Move MesaProfileIndex parsing to pandas.read_csv MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make file ingestion uniformly pandas-based: MesaProfileIndex.read_index now uses pandas.read_csv instead of numpy.genfromtxt, matching the history/profile reader. Behavior is preserved exactly (file-order, integer columns, same dict-of-numpy-arrays shape and mappings), verified against the old genfromtxt path on both in-order and out-of-order index files. This also drops a dead np.argsort line: the previous code computed a sorted array but then immediately overwrote index_data with the unsorted dict, so the index was never actually sorted. The new code is honest about preserving file order; the docstring is updated to match. Add direct MesaProfileIndex tests (columns, dtype, mappings, missing keys, file-order) — previously the index was only covered indirectly through MesaLogDir. Update CLAUDE.md to document the pandas-parses / numpy-computes split. Co-Authored-By: Claude Opus 4.8 --- CLAUDE.md | 21 ++++++++---- mesa_reader/__init__.py | 22 ++++++------ tests/test_mesa_profile_index.py | 58 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 tests/test_mesa_profile_index.py diff --git a/CLAUDE.md b/CLAUDE.md index dcdba69..079d7bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -38,14 +38,14 @@ that build on each other: - **`MesaData`** — the core file reader. Reads a single MESA log file (history or profile, `.data`/`.log`) or a saved model (`.mod`). File type is auto-detected from - the extension. Log files are parsed with `np.genfromtxt` into a record array - (`bulk_data`) plus a header dict; `.mod` files use a hand-rolled line walker - (`read_model_data`) that converts Fortran `D`-exponent notation to Python floats. - History files are scrubbed of backups/restarts so `model_number` is monotonic - (`remove_backups`). + the extension. Log files are parsed with `pandas.read_csv` (fast C parser) and + then converted to a numpy structured array (`bulk_data`) plus a header dict; + `.mod` files use a hand-rolled line walker (`read_model_data`) that converts + Fortran `D`-exponent notation to Python floats. History files are scrubbed of + backups/restarts so `model_number` is monotonic (`remove_backups`). -- **`MesaProfileIndex`** — parses `profiles.index`, providing the - profile-number ↔ model-number mapping. +- **`MesaProfileIndex`** — parses `profiles.index` (also via `pandas.read_csv`), + providing the profile-number ↔ model-number mapping. - **`MesaLogDir`** — ties a whole LOGS directory together: it owns one `MesaData` history object plus a `MesaProfileIndex`, and lazily constructs (and optionally @@ -71,3 +71,10 @@ that build on each other: - **Header/model values are parsed with `eval`.** `read_log_data` and `read_model_data` call `eval` on tokens from the file. This is intentional for reading numeric/string MESA output; keep input assumed to be trusted MESA files. + +- **pandas parses, numpy holds and computes.** File ingestion uses + `pandas.read_csv` for speed, but the in-memory data (`bulk_data`, the arrays + returned by `data()` and `MesaProfileIndex.data()`) are numpy, and array math + (`np.exp`/`np.log10`, `np.where`, `np.minimum.accumulate`) stays in numpy. This + split is deliberate — numpy is pandas' own dependency, and the numpy-array + return type is the public API contract. Don't convert these to DataFrames/Series. diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index ec7a801..db774d6 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -786,25 +786,27 @@ def __init__(self, file_name=join(".", "LOGS", "profiles.index")): def read_index(self): """Read (or re-read) data from `self.file_name`. - Read the file into an numpy array, sorting the table in order of - increasing model numbers and establishes the `profile_numbers` and - `model_numbers` attributes. Converts data and names into a dictionary. - Called automatically at instantiation, but may be called again to - refresh data. + Reads the index file (in file order) into per-column numpy arrays, + keyed by `MesaProfileIndex.index_names`, and establishes the + `profile_numbers` and `model_numbers` attributes. Called automatically + at instantiation, but may be called again to refresh data. Returns ------- None """ - temp_index_data = np.genfromtxt( + index_frame = read_csv( self.file_name, - skip_header=MesaProfileIndex.index_start_line - 1, - dtype=None, + skiprows=MesaProfileIndex.index_start_line - 1, + sep=r"\s+", + header=None, ) self.model_number_string = MesaProfileIndex.index_names[0] self.profile_number_string = MesaProfileIndex.index_names[-1] - self.index_data = temp_index_data[np.argsort(temp_index_data[:, 0])] - self.index_data = dict(zip(MesaProfileIndex.index_names, temp_index_data.T)) + self.index_data = { + name: index_frame[column].to_numpy() + for column, name in enumerate(MesaProfileIndex.index_names) + } self.profile_numbers = self.data(self.profile_number_string) self.model_numbers = self.data(self.model_number_string) diff --git a/tests/test_mesa_profile_index.py b/tests/test_mesa_profile_index.py new file mode 100644 index 0000000..c065b29 --- /dev/null +++ b/tests/test_mesa_profile_index.py @@ -0,0 +1,58 @@ +import os + +import numpy as np +import pytest + +import mesa_reader as mr + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +INDEX = os.path.join(DATA_DIR, "LOGS", "profiles.index") + + +def test_reads_index_columns(): + idx = mr.MesaProfileIndex(INDEX) + np.testing.assert_array_equal(idx.model_numbers, [10, 20]) + np.testing.assert_array_equal(idx.profile_numbers, [1, 2]) + np.testing.assert_array_equal(idx.data("priorities"), [2, 2]) + # Integer columns must stay integers, matching the old genfromtxt path. + assert np.issubdtype(idx.data("model_numbers").dtype, np.integer) + + +def test_model_profile_mapping(): + idx = mr.MesaProfileIndex(INDEX) + assert idx.profile_with_model_number(20) == 2 + assert idx.model_with_profile_number(1) == 10 + assert idx.have_profile_with_model_number(10) + assert not idx.have_profile_with_model_number(999) + assert idx.have_profile_with_profile_number(2) + assert not idx.have_profile_with_profile_number(999) + + +def test_missing_profile_raises(): + idx = mr.MesaProfileIndex(INDEX) + with pytest.raises(mr.ProfileError): + idx.profile_with_model_number(999) + with pytest.raises(mr.ProfileError): + idx.model_with_profile_number(999) + + +def test_invalid_column_raises(): + idx = mr.MesaProfileIndex(INDEX) + with pytest.raises(KeyError): + idx.data("not_a_column") + + +def test_preserves_file_order(tmp_path): + # Rows are kept in file order (the index is not re-sorted by model number); + # the mapping must still resolve correctly regardless of row order. + index_file = tmp_path / "profiles.index" + index_file.write_text( + "# comment line\n" + " 20 2 2\n" + " 10 2 1\n" + ) + idx = mr.MesaProfileIndex(str(index_file)) + np.testing.assert_array_equal(idx.model_numbers, [20, 10]) + np.testing.assert_array_equal(idx.profile_numbers, [2, 1]) + assert idx.profile_with_model_number(10) == 1 + assert idx.model_with_profile_number(2) == 20 From b3b2acf0d0f780383fc8c03bd602d333501bf551 Mon Sep 17 00:00:00 2001 From: Bill Wolf Date: Thu, 25 Jun 2026 16:31:11 -0500 Subject: [PATCH 15/15] Sort profile index by model number Make MesaProfileIndex actually sort by increasing model number, matching its long-standing docstring. The pre-pandas code intended to sort (via np.argsort) but discarded the result, so the index was really returned in file order. profiles.index is normally written in model-number order, so this rarely surfaced, but a hand-edited or concatenated index could contradict the documentation. Sort the full frame (stable) so model/profile/priority columns stay row-aligned. Update the file-order test to assert sorted output. Co-Authored-By: Claude Opus 4.8 --- mesa_reader/__init__.py | 16 ++++++++++------ tests/test_mesa_profile_index.py | 11 ++++++----- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index db774d6..862c95f 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -786,23 +786,27 @@ def __init__(self, file_name=join(".", "LOGS", "profiles.index")): def read_index(self): """Read (or re-read) data from `self.file_name`. - Reads the index file (in file order) into per-column numpy arrays, - keyed by `MesaProfileIndex.index_names`, and establishes the - `profile_numbers` and `model_numbers` attributes. Called automatically - at instantiation, but may be called again to refresh data. + Reads the index file into per-column numpy arrays, keyed by + `MesaProfileIndex.index_names`, sorted in order of increasing model + number, and establishes the `profile_numbers` and `model_numbers` + attributes. Called automatically at instantiation, but may be called + again to refresh data. Returns ------- None """ + self.model_number_string = MesaProfileIndex.index_names[0] + self.profile_number_string = MesaProfileIndex.index_names[-1] index_frame = read_csv( self.file_name, skiprows=MesaProfileIndex.index_start_line - 1, sep=r"\s+", header=None, ) - self.model_number_string = MesaProfileIndex.index_names[0] - self.profile_number_string = MesaProfileIndex.index_names[-1] + # Column 0 is the model number; sort rows by it so the index is in + # time order regardless of how the file was written or edited. + index_frame = index_frame.sort_values(by=0, kind="stable") self.index_data = { name: index_frame[column].to_numpy() for column, name in enumerate(MesaProfileIndex.index_names) diff --git a/tests/test_mesa_profile_index.py b/tests/test_mesa_profile_index.py index c065b29..ff950a4 100644 --- a/tests/test_mesa_profile_index.py +++ b/tests/test_mesa_profile_index.py @@ -42,9 +42,9 @@ def test_invalid_column_raises(): idx.data("not_a_column") -def test_preserves_file_order(tmp_path): - # Rows are kept in file order (the index is not re-sorted by model number); - # the mapping must still resolve correctly regardless of row order. +def test_sorts_by_model_number(tmp_path): + # An index whose rows are out of model-number order must be sorted into + # increasing model-number (time) order, matching the documented behavior. index_file = tmp_path / "profiles.index" index_file.write_text( "# comment line\n" @@ -52,7 +52,8 @@ def test_preserves_file_order(tmp_path): " 10 2 1\n" ) idx = mr.MesaProfileIndex(str(index_file)) - np.testing.assert_array_equal(idx.model_numbers, [20, 10]) - np.testing.assert_array_equal(idx.profile_numbers, [2, 1]) + np.testing.assert_array_equal(idx.model_numbers, [10, 20]) + np.testing.assert_array_equal(idx.profile_numbers, [1, 2]) + assert np.all(np.diff(idx.model_numbers) > 0) assert idx.profile_with_model_number(10) == 1 assert idx.model_with_profile_number(2) == 20