diff --git a/.github/workflows/test-package.yml b/.github/workflows/test-package.yml index a4ccf4b..adc42a4 100644 --- a/.github/workflows/test-package.yml +++ b/.github/workflows/test-package.yml @@ -5,9 +5,9 @@ name: tests on: push: - branches: [ "main" ] + branches: [ "master" ] pull_request: - branches: [ "main" ] + branches: [ "master" ] jobs: build: diff --git a/.gitignore b/.gitignore index 0c6347a..6169307 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,7 @@ make.bat *.pyc *.swp *.egg-info +.venv +.pytest_cache .buildinfo .vscode diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..079d7bb --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,80 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Overview + +`mesa_reader` is a small, single-module Python library for reading and manipulating +output files from [MESA](https://mesastar.org/) (Modules for Experiments in Stellar +Astrophysics). It is published to PyPI as `mesa-reader`. The only runtime dependency +is `numpy`. + +## Commands + +```console +pip install . # install the package locally +pip install -r requirements-dev.txt # install dev tools (ruff, pytest) + +ruff format --check # lint: formatting check (run before committing) +ruff check # lint: static checks +pytest # run the test suite +pytest path/to/test.py::name # run a single test + +make html # build Sphinx docs into docs_build/ (also: bash make_docs.sh) +``` + +CI (`.github/workflows/test-package.yml`) runs `ruff format --check`, `ruff check`, +and `pytest` on Python 3.12 and 3.13. There are currently no committed test files, +so `pytest` collects nothing — add tests under a `tests/` directory or `test_*.py` +files. + +The published HTML docs live in `docs/` (served by GitHub Pages); the reStructuredText +sources are in `docs_source/`. `build/`, `dist/`, and `*.egg-info/` are build artifacts. + +## Architecture + +All code lives in [`mesa_reader/__init__.py`](mesa_reader/__init__.py) — three classes +that build on each other: + +- **`MesaData`** — the core file reader. Reads a single MESA log file (history or + profile, `.data`/`.log`) or a saved model (`.mod`). File type is auto-detected from + the extension. Log files are parsed with `pandas.read_csv` (fast C parser) and + then converted to a numpy structured array (`bulk_data`) plus a header dict; + `.mod` files use a hand-rolled line walker (`read_model_data`) that converts + Fortran `D`-exponent notation to Python floats. History files are scrubbed of + backups/restarts so `model_number` is monotonic (`remove_backups`). + +- **`MesaProfileIndex`** — parses `profiles.index` (also via `pandas.read_csv`), + providing the profile-number ↔ model-number mapping. + +- **`MesaLogDir`** — ties a whole LOGS directory together: it owns one `MesaData` + history object plus a `MesaProfileIndex`, and lazily constructs (and optionally + memoizes) per-profile `MesaData` objects via `profile_data`. + +### Key conventions to preserve + +- **Attribute access falls through to data lookup.** Both `MesaData` and + `MesaProfileIndex` override `__getattr__`, so `m.star_age` is equivalent to + `m.data('star_age')` (and falls back to `header(...)`). When adding methods or + attributes, be aware that any unknown attribute is routed to the data accessor. + +- **Logarithm/linear key inference.** `MesaData.data(key)` does more than a dict + lookup: if `key` is absent it searches for a `log_`/`ln_` variant and exponentiates + it (or vice-versa) via the `_log_version`/`_ln_version`/`_exp10_version`/ + `_exp_version` helpers. Preserve this fallback chain when touching `data`. + +- **Parsing layout is configurable via class methods.** `header_names_line`, + `bulk_names_line` (on `MesaData`) and `index_start_line`/`index_names` (on + `MesaProfileIndex`) are class-level and set through classmethods like + `set_data_rows`. Changing them affects all subsequent reads. + +- **Header/model values are parsed with `eval`.** `read_log_data` and + `read_model_data` call `eval` on tokens from the file. This is intentional for + reading numeric/string MESA output; keep input assumed to be trusted MESA files. + +- **pandas parses, numpy holds and computes.** File ingestion uses + `pandas.read_csv` for speed, but the in-memory data (`bulk_data`, the arrays + returned by `data()` and `MesaProfileIndex.data()`) are numpy, and array math + (`np.exp`/`np.log10`, `np.where`, `np.minimum.accumulate`) stays in numpy. This + split is deliberate — numpy is pandas' own dependency, and the numpy-array + return type is the public API contract. Don't convert these to DataFrames/Series. diff --git a/mesa_reader/__init__.py b/mesa_reader/__init__.py index ff07141..862c95f 100644 --- a/mesa_reader/__init__.py +++ b/mesa_reader/__init__.py @@ -1,7 +1,10 @@ -import os, re +import os +import re from os.path import join from pathlib import Path + import numpy as np +from pandas import read_csv class ProfileError(Exception): @@ -166,9 +169,9 @@ def read_data(self): # attempt auto-detection of file_type (if not supplied) if self.file_type is None: - if Path (self.file_name).suffix in [".data", ".log"]: + if Path(self.file_name).suffix in [".data", ".log"]: self.file_type = "log" - elif Path (self.file_name).suffix==".mod": + elif Path(self.file_name).suffix == ".mod": self.file_type = "model" else: raise UnknownFileTypeError( @@ -195,23 +198,26 @@ def read_log_data(self): ------- None """ - self.bulk_data = np.genfromtxt( - self.file_name, - skip_header=MesaData.bulk_names_line - 1, - names=True, - ndmin=1, # Make sure a single entry is still a 1D array - dtype=None, - ) - self.bulk_names = self.bulk_data.dtype.names - header_data = [] - with open(self.file_name) as f: - for i, line in enumerate(f): - if i == MesaData.header_names_line - 1: - self.header_names = line.split() - elif i == MesaData.header_names_line: - header_data = [eval(datum) for datum in line.split()] - elif i > MesaData.header_names_line: - break + # pandas.read_csv parses the bulk table in C, which is dramatically + # faster than numpy.genfromtxt's line-by-line Python parsing for the + # large history/profile files this package targets. + with open(self.file_name, "r") as file: + # Advance to and read the header name/value rows. + for _ in range(MesaData.header_names_line - 1): + file.readline() + self.header_names = file.readline().split() + header_data = [eval(datum) for datum in file.readline().split()] + + # Advance to the bulk-name row, which read_csv consumes as its + # column header. The number of intervening lines is derived from + # the (configurable) line positions rather than hard-coded. + for _ in range(MesaData.bulk_names_line - MesaData.header_names_line - 2): + file.readline() + dataframe = read_csv(file, sep=r"\s+", dtype=None) + records = dataframe.to_records(index=False) + self.bulk_names = tuple(dataframe.columns) + self.bulk_data = np.array(records, dtype=records.dtype.descr) + self.header_data = dict(zip(self.header_names, header_data)) self.remove_backups() @@ -689,18 +695,28 @@ def remove_backups(self, dbg=False): return None if dbg: print("Scrubbing history...") - to_remove = [] - for i in range(len(self.data("model_number")) - 1): - smallest_future = np.min(self.data("model_number")[i + 1 :]) - if self.data("model_number")[i] >= smallest_future: - to_remove.append(i) - if len(to_remove) == 0: + + # A row is genuine only if its model number is smaller than every model + # number that comes after it; otherwise it is cruft superseded by a + # later restart. `suffix_min[i]` is the minimum model number over rows + # i..end, so `suffix_min[i + 1]` is the smallest future model number. + # This is the vectorized (O(n)) form of the original per-row np.min + # scan. Note that drop_duplicates(keep="last") is NOT equivalent: it + # leaves the now-orphaned rows between a restart and its original run, + # breaking the monotonicity of model_number. + model_number = self.data("model_number") + suffix_min = np.minimum.accumulate(model_number[::-1])[::-1] + keep = np.ones(len(model_number), dtype=bool) + keep[:-1] = model_number[:-1] < suffix_min[1:] + + n_removed = np.count_nonzero(~keep) + if n_removed == 0: if dbg: print("Already clean!") return None if dbg: - print("Removing {} lines.".format(len(to_remove))) - self.bulk_data = np.delete(self.bulk_data, to_remove) + print("Removing {} lines.".format(n_removed)) + self.bulk_data = self.bulk_data[keep] def __getattr__(self, method_name): if self._any_version(method_name): @@ -770,25 +786,31 @@ def __init__(self, file_name=join(".", "LOGS", "profiles.index")): def read_index(self): """Read (or re-read) data from `self.file_name`. - Read the file into an numpy array, sorting the table in order of - increasing model numbers and establishes the `profile_numbers` and - `model_numbers` attributes. Converts data and names into a dictionary. - Called automatically at instantiation, but may be called again to - refresh data. + Reads the index file into per-column numpy arrays, keyed by + `MesaProfileIndex.index_names`, sorted in order of increasing model + number, and establishes the `profile_numbers` and `model_numbers` + attributes. Called automatically at instantiation, but may be called + again to refresh data. Returns ------- None """ - temp_index_data = np.genfromtxt( - self.file_name, - skip_header=MesaProfileIndex.index_start_line - 1, - dtype=None, - ) self.model_number_string = MesaProfileIndex.index_names[0] self.profile_number_string = MesaProfileIndex.index_names[-1] - self.index_data = temp_index_data[np.argsort(temp_index_data[:, 0])] - self.index_data = dict(zip(MesaProfileIndex.index_names, temp_index_data.T)) + index_frame = read_csv( + self.file_name, + skiprows=MesaProfileIndex.index_start_line - 1, + sep=r"\s+", + header=None, + ) + # Column 0 is the model number; sort rows by it so the index is in + # time order regardless of how the file was written or edited. + index_frame = index_frame.sort_values(by=0, kind="stable") + self.index_data = { + name: index_frame[column].to_numpy() + for column, name in enumerate(MesaProfileIndex.index_names) + } self.profile_numbers = self.data(self.profile_number_string) self.model_numbers = self.data(self.model_number_string) diff --git a/setup.py b/setup.py index feb2cad..94d485f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ author_email="wolfwm@uwec.edu", license="MIT", packages=["mesa_reader"], - install_requires=["numpy"], + install_requires=["numpy", "pandas"], classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", diff --git a/tests/data/LOGS/history.data b/tests/data/LOGS/history.data new file mode 100644 index 0000000..5db5821 --- /dev/null +++ b/tests/data/LOGS/history.data @@ -0,0 +1,10 @@ + 1 2 3 + version_number initial_mass initial_z + '24.08.1' 1.0 0.02 + + 1 2 3 4 + model_number num_zones star_age log_L + 5 100 500.0 0.1 + 10 100 1000.0 0.2 + 15 100 1500.0 0.3 + 20 100 2000.0 0.4 diff --git a/tests/data/LOGS/profile1.data b/tests/data/LOGS/profile1.data new file mode 100644 index 0000000..00dc282 --- /dev/null +++ b/tests/data/LOGS/profile1.data @@ -0,0 +1,9 @@ + 1 2 3 + model_number num_zones star_age + 10 3 1000.0 + + 1 2 3 4 + zone mass logRho L + 1 1.0 2.0 100.0 + 2 0.5 1.0 200.0 + 3 0.1 0.0 300.0 diff --git a/tests/data/LOGS/profile2.data b/tests/data/LOGS/profile2.data new file mode 100644 index 0000000..3e4d648 --- /dev/null +++ b/tests/data/LOGS/profile2.data @@ -0,0 +1,9 @@ + 1 2 3 + model_number num_zones star_age + 20 3 2000.0 + + 1 2 3 4 + zone mass logRho L + 1 2.0 3.0 150.0 + 2 1.0 2.0 250.0 + 3 0.2 1.0 350.0 diff --git a/tests/data/LOGS/profiles.index b/tests/data/LOGS/profiles.index new file mode 100644 index 0000000..cf3c96a --- /dev/null +++ b/tests/data/LOGS/profiles.index @@ -0,0 +1,3 @@ +# comment line giving the number of profiles + 10 2 1 + 20 2 2 diff --git a/tests/data/restart_history.data b/tests/data/restart_history.data new file mode 100644 index 0000000..70a4bae --- /dev/null +++ b/tests/data/restart_history.data @@ -0,0 +1,31 @@ + 1 2 3 + version_number initial_mass initial_z + '24.08.1' 1.0 0.02 + + 1 2 3 + model_number num_zones star_age + 1 100 100.0 + 2 100 200.0 + 3 100 300.0 + 4 100 400.0 + 5 100 500.0 + 6 100 600.0 + 7 100 700.0 + 8 100 800.0 + 9 100 900.0 + 10 100 1000.0 + 11 100 1100.0 + 12 100 1200.0 + 13 100 1300.0 + 14 100 1400.0 + 15 100 1500.0 + 16 100 1600.0 + 17 100 1700.0 + 18 100 1800.0 + 19 100 1900.0 + 20 100 2000.0 + 21 100 2100.0 + 10 100 1000.0 + 15 100 1500.0 + 20 100 2000.0 + 25 100 2500.0 diff --git a/tests/test_mesa_data.py b/tests/test_mesa_data.py new file mode 100644 index 0000000..9528051 --- /dev/null +++ b/tests/test_mesa_data.py @@ -0,0 +1,88 @@ +import os + +import numpy as np +import pytest + +import mesa_reader as mr + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +HISTORY = os.path.join(DATA_DIR, "LOGS", "history.data") +RESTART_HISTORY = os.path.join(DATA_DIR, "restart_history.data") +PROFILE = os.path.join(DATA_DIR, "LOGS", "profile1.data") + + +def test_reads_bulk_data(): + m = mr.MesaData(HISTORY) + assert m.is_history() + assert "model_number" in m.bulk_names + assert isinstance(m.bulk_names, tuple) + np.testing.assert_array_equal(m.model_number, [5, 10, 15, 20]) + np.testing.assert_allclose(m.star_age, [500.0, 1000.0, 1500.0, 2000.0]) + + +def test_header_values_are_typed(): + # The header row must be parsed into native Python values, not left as raw + # strings (regression guard for the pandas backend dropping eval()). + m = mr.MesaData(HISTORY) + assert m.initial_mass == 1.0 + assert isinstance(m.initial_mass, float) + assert m.initial_z == 0.02 + assert m.version_number == "24.08.1" + + +def test_attribute_access_matches_data_method(): + m = mr.MesaData(HISTORY) + np.testing.assert_array_equal(m.star_age, m.data("star_age")) + assert m.initial_mass == m.header("initial_mass") + + +def test_log_linear_inference(): + m = mr.MesaData(PROFILE) + # logRho is present; requesting Rho should exponentiate it. + np.testing.assert_allclose(m.data("Rho"), 10 ** m.data("logRho")) + # L is present; requesting log_L should take its log10. + np.testing.assert_allclose(m.data("log_L"), np.log10(m.data("L"))) + + +def test_invalid_key_raises(): + m = mr.MesaData(HISTORY) + with pytest.raises(KeyError): + m.data("not_a_real_column") + + +def test_remove_backups_restores_monotonic_model_numbers(): + # The file contains a run to model 21 followed by a restart from model 10 + # (1..21, then 10, 15, 20, 25). After scrubbing, model_number must be the + # surviving monotonic sequence with all superseded rows obliterated. + m = mr.MesaData(RESTART_HISTORY) + expected = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25] + np.testing.assert_array_equal(m.model_number, expected) + assert np.all(np.diff(m.model_number) > 0) + # Bulk columns must stay row-aligned with the kept model numbers. + np.testing.assert_allclose(m.star_age, np.array(expected) * 100.0) + + +def test_remove_backups_noop_on_clean_history(): + m = mr.MesaData(HISTORY) + before = m.model_number.copy() + m.remove_backups() + np.testing.assert_array_equal(m.model_number, before) + + +def test_data_at_model_number(): + m = mr.MesaData(HISTORY) + assert m.data_at_model_number("star_age", 15) == 1500.0 + assert m.index_of_model_number(15) == 2 + + +def test_profile_is_not_history(): + p = mr.MesaData(PROFILE) + assert not p.is_history() + assert p.model_number == 10 # from the header, not bulk data + + +def test_unknown_file_type_raises(tmp_path): + bogus = tmp_path / "data.unknown" + bogus.write_text("nothing useful\n") + with pytest.raises(mr.UnknownFileTypeError): + mr.MesaData(str(bogus)) diff --git a/tests/test_mesa_logdir.py b/tests/test_mesa_logdir.py new file mode 100644 index 0000000..1b09ea9 --- /dev/null +++ b/tests/test_mesa_logdir.py @@ -0,0 +1,58 @@ +import os + +import numpy as np +import pytest + +import mesa_reader as mr + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +LOGS = os.path.join(DATA_DIR, "LOGS") + + +def test_reads_history_and_index(): + log = mr.MesaLogDir(LOGS) + assert log.history.is_history() + np.testing.assert_array_equal(log.profile_numbers, [1, 2]) + np.testing.assert_array_equal(log.model_numbers, [10, 20]) + + +def test_profile_model_number_mapping(): + log = mr.MesaLogDir(LOGS) + assert log.profile_with_model_number(20) == 2 + assert log.model_with_profile_number(1) == 10 + assert log.have_profile_with_model_number(10) + assert not log.have_profile_with_model_number(999) + + +def test_profile_data_defaults_to_last_profile(): + log = mr.MesaLogDir(LOGS) + p = log.profile_data() + assert p.model_number == 20 + + +def test_profile_data_by_model_number(): + log = mr.MesaLogDir(LOGS) + p = log.profile_data(model_number=10) + assert p.model_number == 10 + np.testing.assert_allclose(p.data("logRho"), [2.0, 1.0, 0.0]) + + +def test_memoization_returns_same_object(): + log = mr.MesaLogDir(LOGS, memoize_profiles=True) + assert log.profile_data(profile_number=1) is log.profile_data(profile_number=1) + + log_no_memo = mr.MesaLogDir(LOGS, memoize_profiles=False) + first = log_no_memo.profile_data(profile_number=1) + second = log_no_memo.profile_data(profile_number=1) + assert first is not second + + +def test_select_models(): + log = mr.MesaLogDir(LOGS) + selected = log.select_models(lambda age: age > 1200.0, "star_age") + np.testing.assert_array_equal(selected, [20]) + + +def test_bad_path_raises(): + with pytest.raises(mr.BadPathError): + mr.MesaLogDir(os.path.join(DATA_DIR, "does_not_exist")) diff --git a/tests/test_mesa_profile_index.py b/tests/test_mesa_profile_index.py new file mode 100644 index 0000000..ff950a4 --- /dev/null +++ b/tests/test_mesa_profile_index.py @@ -0,0 +1,59 @@ +import os + +import numpy as np +import pytest + +import mesa_reader as mr + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +INDEX = os.path.join(DATA_DIR, "LOGS", "profiles.index") + + +def test_reads_index_columns(): + idx = mr.MesaProfileIndex(INDEX) + np.testing.assert_array_equal(idx.model_numbers, [10, 20]) + np.testing.assert_array_equal(idx.profile_numbers, [1, 2]) + np.testing.assert_array_equal(idx.data("priorities"), [2, 2]) + # Integer columns must stay integers, matching the old genfromtxt path. + assert np.issubdtype(idx.data("model_numbers").dtype, np.integer) + + +def test_model_profile_mapping(): + idx = mr.MesaProfileIndex(INDEX) + assert idx.profile_with_model_number(20) == 2 + assert idx.model_with_profile_number(1) == 10 + assert idx.have_profile_with_model_number(10) + assert not idx.have_profile_with_model_number(999) + assert idx.have_profile_with_profile_number(2) + assert not idx.have_profile_with_profile_number(999) + + +def test_missing_profile_raises(): + idx = mr.MesaProfileIndex(INDEX) + with pytest.raises(mr.ProfileError): + idx.profile_with_model_number(999) + with pytest.raises(mr.ProfileError): + idx.model_with_profile_number(999) + + +def test_invalid_column_raises(): + idx = mr.MesaProfileIndex(INDEX) + with pytest.raises(KeyError): + idx.data("not_a_column") + + +def test_sorts_by_model_number(tmp_path): + # An index whose rows are out of model-number order must be sorted into + # increasing model-number (time) order, matching the documented behavior. + index_file = tmp_path / "profiles.index" + index_file.write_text( + "# comment line\n" + " 20 2 2\n" + " 10 2 1\n" + ) + idx = mr.MesaProfileIndex(str(index_file)) + np.testing.assert_array_equal(idx.model_numbers, [10, 20]) + np.testing.assert_array_equal(idx.profile_numbers, [1, 2]) + assert np.all(np.diff(idx.model_numbers) > 0) + assert idx.profile_with_model_number(10) == 1 + assert idx.model_with_profile_number(2) == 20