From 0018a017ad762f71a2fca65b02b868cc2a113d6a Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 07:04:38 +0100 Subject: [PATCH 1/9] Make FileSystem::write return GrimpResult This is consistent with FileSystem::read, and means writes can be used from Rust code that doesn't hold the GIL. I/O errors are surfaced as OSError. Assistant-model: Claude Code --- rust/src/errors.rs | 6 +++++- rust/src/filesystem.rs | 15 +++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/rust/src/errors.rs b/rust/src/errors.rs index b2ba6a25..a39a2f83 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -1,6 +1,6 @@ use crate::exceptions; use pyo3::PyErr; -use pyo3::exceptions::{PyFileNotFoundError, PyUnicodeError, PyValueError}; +use pyo3::exceptions::{PyFileNotFoundError, PyOSError, PyUnicodeError, PyValueError}; use ruff_python_parser::ParseError as RuffParseError; use thiserror::Error; @@ -45,6 +45,9 @@ pub enum GrimpError { #[error("{0}")] UndecodableFile(String), + + #[error("{0}")] + Io(#[from] std::io::Error), } pub type GrimpResult = Result; @@ -70,6 +73,7 @@ impl From for PyErr { GrimpError::FileNotFound(_) => PyFileNotFoundError::new_err(value.to_string()), // Not UnicodeDecodeError, as that can't be created from just a message. GrimpError::UndecodableFile(_) => PyUnicodeError::new_err(value.to_string()), + GrimpError::Io(_) => PyOSError::new_err(value.to_string()), } } } diff --git a/rust/src/filesystem.rs b/rust/src/filesystem.rs index 883aaaf4..3199d32e 100644 --- a/rust/src/filesystem.rs +++ b/rust/src/filesystem.rs @@ -68,7 +68,7 @@ pub trait FileSystem: Send + Sync { fn read(&self, file_name: &str) -> GrimpResult; - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()>; + fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()>; } #[derive(Clone)] @@ -183,14 +183,13 @@ impl FileSystem for RealBasicFileSystem { } } - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { + fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()> { let file_path: PathBuf = file_name.into(); if let Some(patent_dir) = file_path.parent() { fs::create_dir_all(patent_dir)?; } - File::create(file_path)? - .write_all(contents.as_bytes()) - .map_err(Into::into) + File::create(file_path)?.write_all(contents.as_bytes())?; + Ok(()) } } @@ -226,7 +225,7 @@ impl PyRealBasicFileSystem { } fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - self.inner.write(file_name, contents) + Ok(self.inner.write(file_name, contents)?) } } @@ -313,7 +312,7 @@ impl FileSystem for FakeBasicFileSystem { } #[allow(unused_variables)] - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { + fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()> { let mut contents_mut = self.contents.lock().unwrap(); contents_mut.insert(file_name.to_string(), contents.to_string()); Ok(()) @@ -354,7 +353,7 @@ impl PyFakeBasicFileSystem { } fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - self.inner.write(file_name, contents) + Ok(self.inner.write(file_name, contents)?) } // Temporary workaround method for Python tests. From d79daf18ad1af4a47850ba75387fb6f73eb38eb9 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 07:04:39 +0100 Subject: [PATCH 2/9] Extract into_py_err helper Converts a GrimpError into a Python exception, including translating syntax errors into SourceSyntaxError. This will be reused when building graphs in Rust. Assistant-model: Claude Code --- rust/src/import_scanning.rs | 46 ++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/rust/src/import_scanning.rs b/rust/src/import_scanning.rs index ca5ae99d..05ddf354 100644 --- a/rust/src/import_scanning.rs +++ b/rust/src/import_scanning.rs @@ -420,39 +420,39 @@ pub fn scan_for_imports<'py>( }) .collect(); - let imports_by_module_result = py.detach(|| { - scan_for_imports_no_py( - &file_system_boxed, - &found_packages_rust, - include_external_packages, - &modules_rust, - exclude_type_checking_imports, - ) - }); - - match imports_by_module_result { - Err(GrimpError::ParseError { + let imports_by_module = py + .detach(|| { + scan_for_imports_no_py( + &file_system_boxed, + &found_packages_rust, + include_external_packages, + &modules_rust, + exclude_type_checking_imports, + ) + }) + .map_err(|e| into_py_err(py, e))?; + + Ok(imports_by_module_to_py(py, imports_by_module)) +} + +/// Convert a `GrimpError` to a Python exception, translating syntax errors in the scanned +/// source code to `grimp.exceptions.SourceSyntaxError`. +pub fn into_py_err(py: Python, error: GrimpError) -> PyErr { + match error { + GrimpError::ParseError { module_filename, line_number, text, .. - }) => { + } => { // TODO: define SourceSyntaxError using pyo3. let exceptions_pymodule = PyModule::import(py, "grimp.exceptions").unwrap(); let py_exception_class = exceptions_pymodule.getattr("SourceSyntaxError").unwrap(); let exception = py_exception_class .call1((module_filename, line_number, text)) .unwrap(); - return Err(PyErr::from_value(exception)); - } - Err(e) => { - return Err(e.into()); + PyErr::from_value(exception) } - _ => (), + e => e.into(), } - let imports_by_module = imports_by_module_result.unwrap(); - - let imports_by_module_py = imports_by_module_to_py(py, imports_by_module); - - Ok(imports_by_module_py) } From 90daa924f3360e3fdb598d19701168f32e4aaeba Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 07:04:51 +0100 Subject: [PATCH 3/9] Add build_graph to the Rust extension Builds an import graph entirely in Rust: finding modules, reading and writing the cache, scanning imports and assembling the graph. This avoids the cost of converting everything to and from Python objects. - find_package is a port of the Python ModuleFinder, including namespace packages. It also returns file mtimes, in the same form as os.path.getmtime. - Cache reads and writes the same files as the Python cache, so the two are interchangeable. serde_json's float_roundtrip feature is needed for mtimes to compare equal after being read back. - Import scanning reuses the existing scan_for_imports_no_py. Assistant-model: Claude Code --- rust/Cargo.lock | 63 ++++++++++++++++ rust/Cargo.toml | 4 +- rust/src/caching.rs | 135 +++++++++++++++++++++++++++++++++- rust/src/filesystem.rs | 4 ++ rust/src/graph/mod.rs | 4 ++ rust/src/graph_building.rs | 128 +++++++++++++++++++++++++++++++++ rust/src/import_scanning.rs | 2 +- rust/src/lib.rs | 4 ++ rust/src/module_finding.rs | 139 +++++++++++++++++++++++++++++++++++- 9 files changed, 478 insertions(+), 5 deletions(-) create mode 100644 rust/src/graph_building.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8e192b6e..9e0054b2 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -7,6 +7,7 @@ name = "_rustgrimp" version = "0.1.0" dependencies = [ "bimap", + "blake2", "const_format", "derive-new", "encoding_rs", @@ -29,6 +30,7 @@ dependencies = [ "string-interner", "tap", "thiserror", + "unicode-ident", "unindent", ] @@ -95,6 +97,24 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.12.3" @@ -187,6 +207,16 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "derive-new" version = "0.7.0" @@ -209,6 +239,17 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "either" version = "1.16.0" @@ -236,6 +277,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "get-size-derive2" version = "0.10.1" @@ -893,6 +944,12 @@ dependencies = [ "serde", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.118" @@ -957,6 +1014,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a82574df..d1e7ef0f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -25,10 +25,12 @@ ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.1 ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.137" +serde_json = { version = "1.0.137", features = ["float_roundtrip"] } serde_yaml = "0.9" unindent = "0.2.4" encoding_rs = "0.8.35" +blake2 = "0.10.6" +unicode-ident = "1.0.24" pyo3 = "0.29.0" [features] diff --git a/rust/src/caching.rs b/rust/src/caching.rs index cd33562a..7c535b83 100644 --- a/rust/src/caching.rs +++ b/rust/src/caching.rs @@ -1,7 +1,9 @@ use crate::errors::{GrimpError, GrimpResult}; -use crate::filesystem::get_file_system_boxed; +use crate::filesystem::{FileSystem, get_file_system_boxed}; use crate::import_scanning::{DirectImport, imports_by_module_to_py}; -use crate::module_finding::Module; +use crate::module_finding::{FoundPackage, Module}; +use blake2::Blake2bVar; +use blake2::digest::{Update, VariableOutput}; use pyo3::types::PyAnyMethods; use pyo3::types::{PyDict, PySet}; use pyo3::types::{PyDictMethods, PySetMethods}; @@ -155,3 +157,132 @@ pub fn parse_json_to_map( Ok(parsed_map) } + +/// A cache of the imports of each module, for building a graph. +/// +/// The files written are the same as those written by the Python `grimp.adaptors.caching.Cache`, +/// so the two are interchangeable. +pub struct Cache<'a> { + directory: &'a str, + found_packages: &'a HashSet, + data_file_name: String, +} + +impl<'a> Cache<'a> { + pub fn new( + directory: &'a str, + found_packages: &'a HashSet, + include_external_packages: bool, + exclude_type_checking_imports: bool, + ) -> Self { + // The data file is specific to the packages and options used. + let mut package_names: Vec<&str> = found_packages.iter().map(|p| p.name.as_str()).collect(); + package_names.sort(); + let mut identifier = package_names.join(","); + if include_external_packages { + identifier += ":external"; + } + if exclude_type_checking_imports { + identifier += ":no_type_checking"; + } + // Hash the identifier to avoid filenames that are too long for the filesystem. + let mut hasher = Blake2bVar::new(20).unwrap(); + hasher.update(identifier.as_bytes()); + let mut hash = [0; 20]; + hasher.finalize_variable(&mut hash).unwrap(); + let hex_hash: String = hash.iter().map(|byte| format!("{byte:02x}")).collect(); + + Self { + directory, + found_packages, + data_file_name: format!("{hex_hash}.data.json"), + } + } + + /// Reads the imports of the modules that haven't been modified since they were cached. + /// + /// Anything that can't be read (e.g. the cache doesn't exist yet) is treated as a cache miss. + /// + /// Args: + /// - mtimes: the current modification time of each module, by module name. + pub fn read( + &self, + file_system: &dyn FileSystem, + mtimes: &HashMap, + ) -> HashMap> { + let mut cached_mtimes: HashMap = HashMap::new(); + for found_package in self.found_packages { + let meta_file = self.file_path(file_system, &meta_file_name(found_package)); + let package_mtimes = file_system + .read(&meta_file) + .ok() + .and_then(|contents| serde_json::from_str::>(&contents).ok()); + cached_mtimes.extend(package_mtimes.unwrap_or_default()); + } + + let data_file = self.file_path(file_system, &self.data_file_name); + let imports_by_module = file_system + .read(&data_file) + .and_then(|contents| parse_json_to_map(&contents, &data_file)) + .unwrap_or_default(); + + imports_by_module + .into_iter() + .filter(|(module, _)| { + cached_mtimes + .get(&module.name) + .is_some_and(|cached| mtimes.get(&module.name) == Some(cached)) + }) + .collect() + } + + pub fn write( + &self, + file_system: &mut dyn FileSystem, + mtimes: &HashMap, + imports_by_module: &HashMap>, + ) -> GrimpResult<()> { + for (file_name, contents) in MARKER_FILES { + let path = self.file_path(file_system, file_name); + if !file_system.exists(&path) { + file_system.write(&path, contents)?; + } + } + + let data_file = self.file_path(file_system, &self.data_file_name); + file_system.write(&data_file, &serialize_imports_by_module(imports_by_module))?; + + for found_package in self.found_packages { + let package_mtimes: HashMap<&str, f64> = found_package + .module_files + .iter() + .map(|module_file| { + let name = module_file.module.name.as_str(); + (name, mtimes[name]) + }) + .collect(); + let meta_file = self.file_path(file_system, &meta_file_name(found_package)); + let contents = serde_json::to_string(&package_mtimes).expect("Failed to serialize"); + file_system.write(&meta_file, &contents)?; + } + Ok(()) + } + + fn file_path(&self, file_system: &dyn FileSystem, file_name: &str) -> String { + file_system.join(vec![self.directory.to_string(), file_name.to_string()]) + } +} + +fn meta_file_name(found_package: &FoundPackage) -> String { + format!("{}.meta.json", found_package.name) +} + +const MARKER_FILES: [(&str, &str); 2] = [ + (".gitignore", "# Automatically created by Grimp.\n*"), + ( + "CACHEDIR.TAG", + "Signature: 8a477f597d28d172789f06886806bc55\n\ + # This file is a cache directory tag automatically created by Grimp.\n\ + # For information about cache directory tags see https://bford.info/cachedir/", + ), +]; diff --git a/rust/src/filesystem.rs b/rust/src/filesystem.rs index 3199d32e..baaf66c1 100644 --- a/rust/src/filesystem.rs +++ b/rust/src/filesystem.rs @@ -452,6 +452,10 @@ fn parse_indented_file_system_string(file_system_string: &str) -> HashMap Box { + Box::new(RealBasicFileSystem {}) +} + #[allow(clippy::borrowed_box)] pub fn get_file_system_boxed<'py>( file_system: &Bound<'py, PyAny>, diff --git a/rust/src/graph/mod.rs b/rust/src/graph/mod.rs index 40ff48cb..f5077913 100644 --- a/rust/src/graph/mod.rs +++ b/rust/src/graph/mod.rs @@ -81,6 +81,10 @@ pub struct GraphWrapper { } impl GraphWrapper { + pub fn from_graph(graph: Graph) -> Self { + GraphWrapper { _graph: graph } + } + fn get_visible_module_by_name(&self, name: &str) -> Result<&Module, ModuleNotPresent> { self._graph .get_module_by_name(name) diff --git a/rust/src/graph_building.rs b/rust/src/graph_building.rs new file mode 100644 index 00000000..255a2a7e --- /dev/null +++ b/rust/src/graph_building.rs @@ -0,0 +1,128 @@ +use crate::caching::Cache; +use crate::errors::GrimpResult; +use crate::filesystem::real_file_system; +use crate::graph::{Graph, GraphWrapper}; +use crate::import_scanning::{DirectImport, into_py_err, scan_for_imports_no_py}; +use crate::module_finding::{FoundPackage, Module, find_package}; +use pyo3::prelude::*; +use std::collections::{HashMap, HashSet}; + +/// Builds an import graph of the supplied packages. +/// +/// This is the Rust equivalent of `grimp.application.usecases.build_graph`. +/// +/// Args: +/// - packages: (name, directory) of each package. A package may have more than one directory. +/// - include_external_packages: whether to include imports of external modules. +/// - exclude_type_checking_imports: whether to exclude imports behind TYPE_CHECKING guards. +/// - cache_dir: the directory to use for caching, or None for no caching. +#[pyfunction] +pub fn build_graph( + py: Python, + packages: Vec<(String, String)>, + include_external_packages: bool, + exclude_type_checking_imports: bool, + cache_dir: Option, +) -> PyResult { + py.detach(|| { + build_graph_no_py( + &packages, + include_external_packages, + exclude_type_checking_imports, + cache_dir.as_deref(), + ) + }) + .map(GraphWrapper::from_graph) + .map_err(|e| into_py_err(py, e)) +} + +fn build_graph_no_py( + packages: &[(String, String)], + include_external_packages: bool, + exclude_type_checking_imports: bool, + cache_dir: Option<&str>, +) -> GrimpResult { + let mut file_system = real_file_system(); + + // Find the modules. + let mut found_packages = HashSet::new(); + let mut mtimes = HashMap::new(); + for (name, directory) in packages { + let (found_package, package_mtimes) = find_package(name, directory)?; + found_packages.insert(found_package); + mtimes.extend(package_mtimes); + } + + // Read the imports of each module, from the cache if possible, otherwise from the source. + let cache = cache_dir.map(|directory| { + Cache::new( + directory, + &found_packages, + include_external_packages, + exclude_type_checking_imports, + ) + }); + let mut imports_by_module = match &cache { + Some(cache) => cache.read(&*file_system, &mtimes), + None => HashMap::new(), + }; + let modules_to_scan: HashSet = found_packages + .iter() + .flat_map(|found_package| &found_package.module_files) + .map(|module_file| module_file.module.clone()) + .filter(|module| !imports_by_module.contains_key(module)) + .collect(); + if !modules_to_scan.is_empty() { + imports_by_module.extend(scan_for_imports_no_py( + &file_system, + &found_packages, + include_external_packages, + &modules_to_scan, + exclude_type_checking_imports, + )?); + } + if let Some(cache) = &cache { + cache.write(&mut *file_system, &mtimes, &imports_by_module)?; + } + + Ok(assemble_graph(&found_packages, &imports_by_module)) +} + +fn assemble_graph( + found_packages: &HashSet, + imports_by_module: &HashMap>, +) -> Graph { + let mut graph = Graph::default(); + + for found_package in found_packages { + for namespace_package in &found_package.namespace_packages { + graph.get_or_add_module(&namespace_package.name); + } + } + + let is_external = |module: &str| { + !found_packages.iter().any(|found_package| { + module == found_package.name || module.starts_with(&format!("{}.", found_package.name)) + }) + }; + for (module, direct_imports) in imports_by_module { + let importer = graph.get_or_add_module(&module.name).token(); + for direct_import in direct_imports { + // External modules are squashed. + let imported = if is_external(&direct_import.imported) { + graph.get_or_add_squashed_module(&direct_import.imported) + } else { + graph.get_or_add_module(&direct_import.imported) + } + .token(); + graph.add_detailed_import( + importer, + imported, + direct_import.line_number as u32, + &direct_import.line_contents, + direct_import.is_lazy, + ); + } + } + graph +} diff --git a/rust/src/import_scanning.rs b/rust/src/import_scanning.rs index 05ddf354..3ee50512 100644 --- a/rust/src/import_scanning.rs +++ b/rust/src/import_scanning.rs @@ -81,7 +81,7 @@ fn module_is_descendant(module_name: &str, potential_ancestor: &str) -> bool { /// Statically analyses the given module and returns a set of Modules that /// it imports. #[allow(clippy::borrowed_box)] -fn scan_for_imports_no_py( +pub fn scan_for_imports_no_py( file_system: &Box, found_packages: &HashSet, include_external_packages: bool, diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 000e0df7..f98cffdd 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -3,6 +3,7 @@ pub mod errors; pub mod exceptions; mod filesystem; pub mod graph; +mod graph_building; pub mod import_parsing; mod import_scanning; pub mod module_expressions; @@ -24,6 +25,9 @@ mod _rustgrimp { #[pymodule_export] use crate::graph::GraphWrapper; + #[pymodule_export] + use crate::graph_building::build_graph; + #[pymodule_export] use crate::filesystem::{PyFakeBasicFileSystem, PyRealBasicFileSystem}; diff --git a/rust/src/module_finding.rs b/rust/src/module_finding.rs index 868e3f31..c909ef41 100644 --- a/rust/src/module_finding.rs +++ b/rust/src/module_finding.rs @@ -1,6 +1,10 @@ +use crate::errors::GrimpResult; use pyo3::{prelude::*, types::PyFrozenSet}; -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, FromPyObject)] pub struct ModuleFile { @@ -76,3 +80,136 @@ impl<'a, 'py> FromPyObject<'a, 'py> for FoundPackage { }) } } + +/// Finds the modules and namespace packages inside a package directory. +/// +/// This is a port of `grimp.adaptors.modulefinder.ModuleFinder`. +/// +/// Also returns the mtime of each module's file (keyed by module name), in the same form as +/// Python's `os.path.getmtime`, so that it's interchangeable with the Python cache. +pub fn find_package( + name: &str, + directory: &str, +) -> GrimpResult<(FoundPackage, HashMap)> { + let mut python_files = vec![]; + let mut namespace_dirs = vec![]; + walk( + Path::new(directory), + false, + &mut python_files, + &mut namespace_dirs, + )?; + + let module_name = |path: &Path| -> String { + let relative_path = path.strip_prefix(directory).unwrap(); + let mut components = vec![name.to_string()]; + components.extend( + relative_path + .iter() + .map(|c| c.to_string_lossy().into_owned()), + ); + if components.last().unwrap() == "__init__" { + components.pop(); + } + components.join(".") + }; + + let mut module_files = BTreeSet::new(); + let mut mtimes = HashMap::new(); + for path in &python_files { + let module = Module { + name: module_name(&path.with_extension("")), + }; + mtimes.insert(module.name.clone(), mtime(path)?); + module_files.insert(ModuleFile { module }); + } + let namespace_packages = namespace_dirs + .iter() + .map(|dir| Module { + name: module_name(dir), + }) + .collect(); + + let found_package = FoundPackage { + name: name.to_string(), + directory: directory.to_string(), + module_files, + namespace_packages, + }; + Ok((found_package, mtimes)) +} + +/// Recursively collects the Python files and namespace directories within a directory. +/// +/// A directory is a namespace directory if it isn't (within) a regular package, but does +/// contain Python files somewhere below it. +fn walk( + directory: &Path, + in_regular_package: bool, + python_files: &mut Vec, + namespace_dirs: &mut Vec, +) -> GrimpResult<()> { + let mut file_names = vec![]; + let mut subdirectories = vec![]; + for entry in fs::read_dir(directory)? { + let entry = entry?; + let Ok(name) = entry.file_name().into_string() else { + continue; + }; + // Follow symlinks to directories, like os.walk(followlinks=True) did. + let file_type = entry.file_type()?; + if file_type.is_dir() || (file_type.is_symlink() && entry.path().is_dir()) { + subdirectories.push((name, entry.path())); + } else { + file_names.push(name); + } + } + + let has_init = file_names.iter().any(|name| name == "__init__.py"); + if in_regular_package && !has_init { + // Don't drill down into directories that aren't packages. + return Ok(()); + } + let in_regular_package = in_regular_package || has_init; + + let python_files_before = python_files.len(); + python_files.extend( + file_names + .iter() + .filter(|name| is_python_file(name)) + .map(|name| directory.join(name)), + ); + for (name, path) in &subdirectories { + if is_identifier(name) { + walk(path, in_regular_package, python_files, namespace_dirs)?; + } + } + + if !in_regular_package && python_files.len() > python_files_before { + namespace_dirs.push(directory.to_path_buf()); + } + Ok(()) +} + +/// Files like `some.module.py` and hidden files aren't treated as Python files. +fn is_python_file(name: &str) -> bool { + name.ends_with(".py") && !name.starts_with('.') && name.matches('.').count() == 1 +} + +/// Equivalent to Python's `str.isidentifier`. +fn is_identifier(name: &str) -> bool { + let mut chars = name.chars(); + chars + .next() + .is_some_and(|c| c == '_' || unicode_ident::is_xid_start(c)) + && chars.all(unicode_ident::is_xid_continue) +} + +/// Equivalent to Python's `os.path.getmtime`, including the floating point arithmetic. +fn mtime(path: &Path) -> GrimpResult { + let since_epoch = fs::metadata(path)? + .modified()? + .duration_since(UNIX_EPOCH) + .map_err(std::io::Error::other)?; + Ok(since_epoch.as_secs() as f64 + 1e-9 * since_epoch.subsec_nanos() as f64) +} From 9de4165d5d5df8a38d26d6d25644c1aee5e81693 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 07:05:00 +0100 Subject: [PATCH 4/9] Add build_graph_rust A faster alternative to build_graph, which builds the graph in Rust. It takes the same arguments, and shares its cache with build_graph. The functional tests now run against both implementations, via a build_graph fixture. The test that checks the cache logs a message only asserts that for the Python implementation, as the Rust one doesn't log. Assistant-model: Claude Code --- .importlinter | 1 + src/grimp/__init__.py | 3 +- src/grimp/application/graph.py | 6 +++ src/grimp/application/usecases.py | 36 ++++++++++++++++ src/grimp/main.py | 4 +- tests/functional/conftest.py | 11 +++++ tests/functional/test_build_and_use_graph.py | 42 +++++++++---------- ...build_and_use_graph_with_multiple_roots.py | 10 ++--- .../test_build_graph_on_real_packages.py | 5 +++ tests/functional/test_caching.py | 39 ++++++++++++++--- tests/functional/test_encoding_handling.py | 22 +++++----- tests/functional/test_error_handling.py | 4 +- tests/functional/test_lazy_imports.py | 7 +--- tests/functional/test_namespace_packages.py | 24 +++++------ 14 files changed, 146 insertions(+), 68 deletions(-) create mode 100644 tests/functional/conftest.py diff --git a/.importlinter b/.importlinter index c58f0d92..097c591d 100644 --- a/.importlinter +++ b/.importlinter @@ -21,3 +21,4 @@ ignore_imports = grimp.application.graph -> grimp grimp.adaptors.filesystem -> grimp grimp.application.scanning -> grimp + grimp.application.usecases -> grimp diff --git a/src/grimp/__init__.py b/src/grimp/__init__.py index 1e43bb16..c7685e91 100644 --- a/src/grimp/__init__.py +++ b/src/grimp/__init__.py @@ -3,7 +3,7 @@ from .application.graph import DetailedImport, Import, ImportGraph from .domain.analysis import PackageDependency, Route from .domain.valueobjects import DirectImport, Layer, Module -from .main import build_graph +from .main import build_graph, build_graph_rust __all__ = [ "DetailedImport", @@ -15,4 +15,5 @@ "PackageDependency", "Route", "build_graph", + "build_graph_rust", ] diff --git a/src/grimp/application/graph.py b/src/grimp/application/graph.py index 3c679621..04050b04 100644 --- a/src/grimp/application/graph.py +++ b/src/grimp/application/graph.py @@ -40,6 +40,12 @@ def __init__(self) -> None: self._cached_modules: set[str] | None = None self._rustgraph = rust.Graph() + @classmethod + def from_rustgraph(cls, rustgraph: rust.Graph) -> ImportGraph: + graph = cls() + graph._rustgraph = rustgraph + return graph + # Mechanics # --------- diff --git a/src/grimp/application/usecases.py b/src/grimp/application/usecases.py index 34c4e934..2fafa3b6 100644 --- a/src/grimp/application/usecases.py +++ b/src/grimp/application/usecases.py @@ -8,6 +8,8 @@ from collections.abc import Iterable, Sequence from typing import cast +from grimp import _rustgrimp as rust # type: ignore[attr-defined] + from ..application.graph import ImportGraph from ..application.ports import caching from ..application.ports.filesystem import AbstractFileSystem, BasicFileSystem @@ -71,6 +73,40 @@ def build_graph( return graph +def build_graph_rust( + package_name: str, + *additional_package_names: str, + include_external_packages: bool = False, + exclude_type_checking_imports: bool = False, + cache_dir: str | type[NotSupplied] | None = NotSupplied, +) -> ImportGraph: + """ + Build and return an import graph for the supplied package name(s), using Rust throughout. + + This takes the same arguments as, and is intended as a faster drop-in replacement for, + ``build_graph``. It shares its cache with ``build_graph``. + """ + package_names = _validate_package_names_are_strings([package_name, *additional_package_names]) + packages = [ + (name, directory) + for name in package_names + for directory in settings.PACKAGE_FINDER.determine_package_directories( + package_name=name, file_system=settings.FILE_SYSTEM + ) + ] + if cache_dir is NotSupplied: + cache_dir = settings.CACHE_CLASS.cache_dir_or_default(None) + + rust_graph = rust.build_graph( + packages, + # Tolerate a non-boolean (Import Linter has been known to pass None). + include_external_packages=bool(include_external_packages), + exclude_type_checking_imports=exclude_type_checking_imports, + cache_dir=cache_dir, + ) + return ImportGraph.from_rustgraph(rust_graph) + + def _find_packages( file_system: AbstractFileSystem, package_names: Sequence[object] ) -> set[FoundPackage]: diff --git a/src/grimp/main.py b/src/grimp/main.py index 57d02f47..88c5fc7f 100644 --- a/src/grimp/main.py +++ b/src/grimp/main.py @@ -1,4 +1,4 @@ -__all__ = ["build_graph"] +__all__ = ["build_graph", "build_graph_rust"] from .adaptors.caching import Cache from .adaptors.filesystem import FileSystem @@ -7,7 +7,7 @@ from .adaptors.timing import SystemClockTimer from .application.config import settings from .application.graph import ImportGraph -from .application.usecases import build_graph +from .application.usecases import build_graph, build_graph_rust settings.configure( MODULE_FINDER=ModuleFinder(), diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py new file mode 100644 index 00000000..18807d3f --- /dev/null +++ b/tests/functional/conftest.py @@ -0,0 +1,11 @@ +import pytest + +import grimp + + +@pytest.fixture(params=["python", "rust"]) +def build_graph(request): + """ + Provides each implementation of build_graph, so that the functional tests cover both. + """ + return grimp.build_graph if request.param == "python" else grimp.build_graph_rust diff --git a/tests/functional/test_build_and_use_graph.py b/tests/functional/test_build_and_use_graph.py index bf365b11..3d22fa8e 100644 --- a/tests/functional/test_build_and_use_graph.py +++ b/tests/functional/test_build_and_use_graph.py @@ -1,7 +1,5 @@ import pytest -from grimp import build_graph - """ For ease of reference, these are the imports of all the files: @@ -30,7 +28,7 @@ # --------- -def test_modules(): +def test_modules(build_graph): graph = build_graph("testpackage", cache_dir=None) assert graph.modules == { @@ -53,7 +51,7 @@ def test_modules(): } -def test_add_module(): +def test_add_module(build_graph): graph = build_graph("testpackage", cache_dir=None) number_of_modules = len(graph.modules) @@ -62,7 +60,7 @@ def test_add_module(): assert number_of_modules + 1 == len(graph.modules) -def test_remove_module(): +def test_remove_module(build_graph): graph = build_graph("testpackage", cache_dir=None) number_of_modules = len(graph.modules) @@ -71,7 +69,7 @@ def test_remove_module(): assert number_of_modules - 1 == len(graph.modules) -def test_add_and_remove_import(): +def test_add_and_remove_import(build_graph): graph = build_graph("testpackage", cache_dir=None) a = "testpackage.one.delta.blue" b = "testpackage.two.alpha" @@ -91,7 +89,7 @@ def test_add_and_remove_import(): # ----------- -def test_find_children(): +def test_find_children(build_graph): graph = build_graph("testpackage", cache_dir=None) assert graph.find_children("testpackage.one") == { @@ -102,7 +100,7 @@ def test_find_children(): } -def test_find_descendants(): +def test_find_descendants(build_graph): graph = build_graph("testpackage", cache_dir=None) assert graph.find_descendants("testpackage.one") == { @@ -118,7 +116,7 @@ def test_find_descendants(): # -------------- -def test_find_modules_directly_imported_by(): +def test_find_modules_directly_imported_by(build_graph): graph = build_graph("testpackage", cache_dir=None) result = graph.find_modules_directly_imported_by("testpackage.utils") @@ -126,7 +124,7 @@ def test_find_modules_directly_imported_by(): assert {"testpackage.one", "testpackage.two.alpha"} == result -def test_find_modules_that_directly_import(): +def test_find_modules_that_directly_import(build_graph): graph = build_graph("testpackage", cache_dir=None) result = graph.find_modules_that_directly_import("testpackage.one.alpha") @@ -141,7 +139,7 @@ def test_find_modules_that_directly_import(): } == result -def test_direct_import_exists(): +def test_direct_import_exists(build_graph): graph = build_graph("testpackage", cache_dir=None) assert False is graph.direct_import_exists( @@ -155,7 +153,7 @@ def test_direct_import_exists(): ) -def test_get_import_details(): +def test_get_import_details(build_graph): graph = build_graph("testpackage", cache_dir=None) assert [ @@ -174,7 +172,7 @@ def test_get_import_details(): class TestPathExists: - def test_as_packages_false(self): + def test_as_packages_false(self, build_graph): graph = build_graph("testpackage", cache_dir=None) assert not graph.chain_exists( @@ -183,7 +181,7 @@ def test_as_packages_false(self): assert graph.chain_exists(imported="testpackage.one.alpha", importer="testpackage.utils") - def test_as_packages_true(self): + def test_as_packages_true(self, build_graph): graph = build_graph("testpackage", cache_dir=None) assert graph.chain_exists( @@ -195,7 +193,7 @@ def test_as_packages_true(self): ) -def test_find_shortest_chain(): +def test_find_shortest_chain(build_graph): graph = build_graph("testpackage", cache_dir=None) assert ( @@ -247,7 +245,7 @@ def test_find_shortest_chain(): ), ], ) -def test_find_shortest_chains(as_packages: bool | None, expected_result: set[tuple]): +def test_find_shortest_chains(build_graph, as_packages: bool | None, expected_result: set[tuple]): importer = "testpackage.three" imported = "testpackage.one.alpha" @@ -264,7 +262,7 @@ def test_find_shortest_chains(as_packages: bool | None, expected_result: set[tup class TestFindDownstreamModules: - def test_as_package_false(self): + def test_as_package_false(self, build_graph): graph = build_graph("testpackage", cache_dir=None) result = graph.find_downstream_modules("testpackage.one.alpha") @@ -282,7 +280,7 @@ def test_as_package_false(self): "testpackage.three.gamma", } == result - def test_as_package_true(self): + def test_as_package_true(self, build_graph): graph = build_graph("testpackage", cache_dir=None) result = graph.find_downstream_modules("testpackage.one", as_package=True) @@ -300,7 +298,7 @@ def test_as_package_true(self): class TestFindUpstreamModules: - def test_as_package_false(self): + def test_as_package_false(self, build_graph): graph = build_graph("testpackage", cache_dir=None) assert graph.find_upstream_modules("testpackage.one.alpha") == set() @@ -311,7 +309,7 @@ def test_as_package_false(self): "testpackage.one.alpha", } - def test_as_package_true(self): + def test_as_package_true(self, build_graph): graph = build_graph("testpackage", cache_dir=None) assert graph.find_upstream_modules("testpackage.two", as_package=True) == { @@ -322,13 +320,13 @@ def test_as_package_true(self): class TestExcludeTypeCheckingImports: - def test_exclude_false(self): + def test_exclude_false(self, build_graph): graph = build_graph("testpackage", cache_dir=None, exclude_type_checking_imports=False) assert True is graph.direct_import_exists( importer="testpackage.one.beta", imported="testpackage.two.alpha" ) - def test_exclude_true(self): + def test_exclude_true(self, build_graph): graph = build_graph("testpackage", cache_dir=None, exclude_type_checking_imports=True) assert False is graph.direct_import_exists( importer="testpackage.one.beta", imported="testpackage.two.alpha" diff --git a/tests/functional/test_build_and_use_graph_with_multiple_roots.py b/tests/functional/test_build_and_use_graph_with_multiple_roots.py index 13a23f7f..7740ef35 100644 --- a/tests/functional/test_build_and_use_graph_with_multiple_roots.py +++ b/tests/functional/test_build_and_use_graph_with_multiple_roots.py @@ -1,7 +1,5 @@ import pytest -from grimp import build_graph - """ For ease of reference, these are the imports of all the files: @@ -25,7 +23,9 @@ @pytest.mark.parametrize("root_packages", PACKAGES_IN_DIFFERENT_ORDERS) class TestBuildGraph: - def test_graph_has_correct_modules_regardless_of_package_order(self, root_packages): + def test_graph_has_correct_modules_regardless_of_package_order( + self, build_graph, root_packages + ): graph = build_graph(*root_packages, cache_dir=None) assert graph.modules == { @@ -39,7 +39,7 @@ def test_graph_has_correct_modules_regardless_of_package_order(self, root_packag "rootpackagegreen.two", } - def test_stores_import_within_package(self, root_packages): + def test_stores_import_within_package(self, build_graph, root_packages): graph = build_graph(*root_packages, cache_dir=None) assert [ @@ -54,7 +54,7 @@ def test_stores_import_within_package(self, root_packages): importer="rootpackageblue.two", imported="rootpackageblue.one.alpha" ) - def test_stores_import_between_root_packages(self, root_packages): + def test_stores_import_between_root_packages(self, build_graph, root_packages): graph = build_graph(*root_packages, cache_dir=None) assert [ diff --git a/tests/functional/test_build_graph_on_real_packages.py b/tests/functional/test_build_graph_on_real_packages.py index 63e5a4bf..66d6d6f5 100644 --- a/tests/functional/test_build_graph_on_real_packages.py +++ b/tests/functional/test_build_graph_on_real_packages.py @@ -12,6 +12,11 @@ def test_build_graph_on_real_package(package_name, snapshot): imports = graph.find_matching_direct_imports(import_expression="** -> **") assert imports == snapshot + graph_from_rust = grimp.build_graph_rust(package_name, cache_dir=None) + imports_from_rust = graph_from_rust.find_matching_direct_imports(import_expression="** -> **") + sort_key = lambda i: (i["importer"], i["imported"]) # noqa: E731 + assert sorted(imports_from_rust, key=sort_key) == sorted(imports, key=sort_key) + @pytest.mark.parametrize( "package_name", diff --git a/tests/functional/test_caching.py b/tests/functional/test_caching.py index 4f966d22..089556ff 100644 --- a/tests/functional/test_caching.py +++ b/tests/functional/test_caching.py @@ -6,7 +6,7 @@ import pytest -from grimp import build_graph +import grimp """ For ease of reference, these are the imports of all the files: @@ -43,7 +43,7 @@ def copied_cachingpackage(): shutil.rmtree(str(PACKAGE_COPY_DESTINATION)) -def test_build_graph_uses_cache(copied_cachingpackage): +def test_build_graph_uses_cache(build_graph, copied_cachingpackage): with tempfile.TemporaryDirectory() as cache_dir: graph = build_graph("cachingpackage", cache_dir=cache_dir) @@ -110,7 +110,7 @@ def test_build_graph_uses_cache(copied_cachingpackage): ) -def test_cache_preserves_is_lazy(): +def test_cache_preserves_is_lazy(build_graph): with tempfile.TemporaryDirectory() as cache_dir: # First build populates the cache from source. build_graph("lazyimports", cache_dir=cache_dir) @@ -131,7 +131,7 @@ def test_cache_preserves_is_lazy(): ] -def test_data_file_records_version_and_is_lazy(copied_cachingpackage): +def test_data_file_records_version_and_is_lazy(build_graph, copied_cachingpackage): with tempfile.TemporaryDirectory() as cache_dir: build_graph("cachingpackage", cache_dir=cache_dir) @@ -146,7 +146,7 @@ def test_data_file_records_version_and_is_lazy(copied_cachingpackage): ] -def test_cache_with_different_version_is_rebuilt(copied_cachingpackage, caplog): +def test_cache_with_different_version_is_rebuilt(build_graph, copied_cachingpackage, caplog): with tempfile.TemporaryDirectory() as cache_dir: build_graph("cachingpackage", cache_dir=cache_dir) @@ -163,7 +163,8 @@ def test_cache_with_different_version_is_rebuilt(copied_cachingpackage, caplog): graph = build_graph("cachingpackage", cache_dir=cache_dir) # The mismatched cache is ignored (not treated as corrupt) and rebuilt from source. - assert any("different version of Grimp" in record.message for record in caplog.records) + if build_graph is grimp.build_graph: # Only the Python implementation logs. + assert any("different version of Grimp" in record.message for record in caplog.records) assert graph.get_import_details( importer="cachingpackage.two.alpha", imported="cachingpackage.one.alpha", @@ -178,6 +179,32 @@ def test_cache_with_different_version_is_rebuilt(copied_cachingpackage, caplog): ] +@pytest.mark.parametrize( + "build_graph_writing_cache, build_graph_reading_cache", + ( + (grimp.build_graph, grimp.build_graph_rust), + (grimp.build_graph_rust, grimp.build_graph), + ), +) +def test_cache_is_shared_between_implementations( + copied_cachingpackage, build_graph_writing_cache, build_graph_reading_cache +): + with tempfile.TemporaryDirectory() as cache_dir: + build_graph_writing_cache("cachingpackage", cache_dir=cache_dir) + + data_file = Path(cache_dir) / "27aa562ad2a4745eb20ecf156430dbfeb0e90610.data.json" + snippet = "from ..one import alpha" + replacement = snippet + " # Inserted by test" + _manipulate_data_file(data_file, snippet, replacement) + + graph = build_graph_reading_cache("cachingpackage", cache_dir=cache_dir) + + [import_details] = graph.get_import_details( + importer="cachingpackage.two.alpha", imported="cachingpackage.one.alpha" + ) + assert import_details["line_contents"] == replacement + + def _manipulate_data_file(data_file: Path, snippet: str, replacement: str) -> None: with open(data_file) as file: filedata = file.read() diff --git a/tests/functional/test_encoding_handling.py b/tests/functional/test_encoding_handling.py index 446cb107..f7a60f4b 100644 --- a/tests/functional/test_encoding_handling.py +++ b/tests/functional/test_encoding_handling.py @@ -2,14 +2,12 @@ import pytest -import grimp - -def test_build_graph_of_non_ascii_source(): +def test_build_graph_of_non_ascii_source(build_graph): """ Tests we can cope with non ascii Python source files. """ - graph = grimp.build_graph("encodingpackage", cache_dir=None) + graph = build_graph("encodingpackage", cache_dir=None) result = graph.get_import_details( importer="encodingpackage.importer", imported="encodingpackage.imported" @@ -26,11 +24,11 @@ def test_build_graph_of_non_ascii_source(): ] == result -def test_build_graph_of_non_utf8_source(): +def test_build_graph_of_non_utf8_source(build_graph): """ Tests we can cope with non UTF-8 Python source files. """ - graph = grimp.build_graph("encodingpackage", cache_dir=None) + graph = build_graph("encodingpackage", cache_dir=None) result = graph.get_import_details( importer="encodingpackage.shift_jis_importer", imported="encodingpackage.imported" @@ -62,7 +60,7 @@ def test_build_graph_of_non_utf8_source(): ), ) def test_build_graph_of_source_declaring_encoding_with_python_specific_name( - tmp_path, monkeypatch, declared_encoding, codec, imported_name + build_graph, tmp_path, monkeypatch, declared_encoding, codec, imported_name ): """ Tests we can cope with source files that declare their encoding using a name that Python @@ -79,7 +77,7 @@ def test_build_graph_of_source_declaring_encoding_with_python_specific_name( ) monkeypatch.syspath_prepend(str(tmp_path)) - graph = grimp.build_graph("declaredencodingpackage", cache_dir=None) + graph = build_graph("declaredencodingpackage", cache_dir=None) result = graph.get_import_details( importer="declaredencodingpackage.importer", imported="declaredencodingpackage.imported" @@ -100,7 +98,7 @@ def test_build_graph_of_source_declaring_encoding_with_python_specific_name( "declared_encoding", ("latin-1", "latin_1", "iso-8859-1", "iso_8859_1"), ) -def test_build_graph_of_true_latin_1_source(tmp_path, monkeypatch, declared_encoding): +def test_build_graph_of_true_latin_1_source(build_graph, tmp_path, monkeypatch, declared_encoding): """ Tests that Latin-1 is decoded as true byte-to-code-point Latin-1, rather than Windows-1252. @@ -115,7 +113,7 @@ def test_build_graph_of_true_latin_1_source(tmp_path, monkeypatch, declared_enco (package_directory / "importer.py").write_bytes(source.encode("latin-1")) monkeypatch.syspath_prepend(str(tmp_path)) - graph = grimp.build_graph("latin1package", cache_dir=None) + graph = build_graph("latin1package", cache_dir=None) result = graph.get_import_details( importer="latin1package.importer", imported="latin1package.imported" @@ -149,7 +147,7 @@ def test_build_graph_of_true_latin_1_source(tmp_path, monkeypatch, declared_enco ), ) def test_undecodable_source_raises_unicode_error_including_filename( - tmp_path, monkeypatch, contents, expected_problem + build_graph, tmp_path, monkeypatch, contents, expected_problem ): package_directory = tmp_path / "undecodablepackage" package_directory.mkdir() @@ -162,4 +160,4 @@ def test_undecodable_source_raises_unicode_error_including_filename( UnicodeError, match=re.escape(f"Failed to decode file {module_filename} {expected_problem}"), ): - grimp.build_graph("undecodablepackage", cache_dir=None) + build_graph("undecodablepackage", cache_dir=None) diff --git a/tests/functional/test_error_handling.py b/tests/functional/test_error_handling.py index 955cc14f..8dbdd674 100644 --- a/tests/functional/test_error_handling.py +++ b/tests/functional/test_error_handling.py @@ -2,10 +2,10 @@ import pytest -from grimp import build_graph, exceptions +from grimp import exceptions -def test_syntax_error_includes_module(): +def test_syntax_error_includes_module(build_graph): filename = str( ( Path(__file__).parent.parent / "assets" / "syntaxerrorpackage" / "foo" / "one.py" diff --git a/tests/functional/test_lazy_imports.py b/tests/functional/test_lazy_imports.py index d12ac714..ba6c7948 100644 --- a/tests/functional/test_lazy_imports.py +++ b/tests/functional/test_lazy_imports.py @@ -1,14 +1,11 @@ -import grimp - - -def test_build_graph_with_lazy_imports(): +def test_build_graph_with_lazy_imports(build_graph): """ Tests we can cope with lazy imports (Python 3.15+). Under the hood we use ruff's parser, which understands lazy imports even when this is running under a different Python version. """ - graph = grimp.build_graph("lazyimports", cache_dir=None) + graph = build_graph("lazyimports", cache_dir=None) result = graph.find_modules_directly_imported_by("lazyimports.one") diff --git a/tests/functional/test_namespace_packages.py b/tests/functional/test_namespace_packages.py index 53c80424..07fbd00b 100644 --- a/tests/functional/test_namespace_packages.py +++ b/tests/functional/test_namespace_packages.py @@ -1,7 +1,5 @@ import pytest -from grimp import build_graph - """ For ease of reference, these are the imports of all the files: @@ -22,7 +20,7 @@ BLUE_MODULES = {"mynamespace.blue", "mynamespace.blue.alpha", "mynamespace.blue.beta"} -def test_build_graph_for_namespace(): +def test_build_graph_for_namespace(build_graph): graph = build_graph("mynamespace", cache_dir=None) assert graph.modules == {"mynamespace"} | YELLOW_MODULES | GREEN_MODULES | BLUE_MODULES @@ -42,13 +40,13 @@ def test_build_graph_for_namespace(): ), ), ) -def test_modules_for_namespace_child(package, expected_modules): +def test_modules_for_namespace_child(build_graph, package, expected_modules): graph = build_graph(package, cache_dir=None) assert graph.modules == expected_modules -def test_modules_for_multiple_namespace_children(): +def test_modules_for_multiple_namespace_children(build_graph): graph = build_graph("mynamespace.green", "mynamespace.blue", cache_dir=None) assert graph.modules == GREEN_MODULES | BLUE_MODULES @@ -75,7 +73,7 @@ def test_modules_for_multiple_namespace_children(): ), ) def test_external_packages_handling( - packages, expected_internal_modules, expected_external_modules + build_graph, packages, expected_internal_modules, expected_external_modules ): graph = build_graph(*packages, include_external_packages=True, cache_dir=None) @@ -83,7 +81,7 @@ def test_external_packages_handling( assert all(graph.is_module_squashed(m) for m in expected_external_modules) -def test_import_within_namespace_child(): +def test_import_within_namespace_child(build_graph): graph = build_graph("mynamespace.blue", cache_dir=None) assert graph.direct_import_exists( @@ -91,7 +89,7 @@ def test_import_within_namespace_child(): ) -def test_import_between_namespace_children(): +def test_import_between_namespace_children(build_graph): graph = build_graph("mynamespace.blue", "mynamespace.green", cache_dir=None) assert graph.direct_import_exists( @@ -147,7 +145,7 @@ def test_import_between_namespace_children(): ), ], ) -def test_build_graph_for_nested_namespace(package_name, expected): +def test_build_graph_for_nested_namespace(build_graph, package_name, expected): graph = build_graph(package_name, cache_dir=None) assert graph.modules == expected @@ -179,13 +177,13 @@ def test_build_graph_for_nested_namespace(package_name, expected): ), ), ) -def test_modules_for_nested_namespace_child(package, expected_modules): +def test_modules_for_nested_namespace_child(build_graph, package, expected_modules): graph = build_graph(package, cache_dir=None) assert graph.modules == expected_modules -def test_import_within_nested_namespace_child(): +def test_import_within_nested_namespace_child(build_graph): graph = build_graph( "nestednamespace.foo.alpha.blue", cache_dir=None, @@ -210,7 +208,7 @@ def test_import_within_nested_namespace_child(): ), ), ) -def test_import_between_nested_namespace_children(root_packages): +def test_import_between_nested_namespace_children(build_graph, root_packages): graph = build_graph(*root_packages, cache_dir=None) assert graph.direct_import_exists( @@ -257,7 +255,7 @@ def test_import_between_nested_namespace_children(root_packages): ), ) def test_external_packages_handling_for_nested_namespaces( - packages, expected_internal_modules, expected_external_modules + build_graph, packages, expected_internal_modules, expected_external_modules ): graph = build_graph(*packages, include_external_packages=True, cache_dir=None) From a8a1d11e493dae5ab169dc18a620b08e1c962228 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 07:05:04 +0100 Subject: [PATCH 5/9] Document build_graph_rust, and add benchmarks Assistant-model: Claude Code --- CHANGELOG.rst | 5 +++++ docs/usage.rst | 8 ++++++++ tests/benchmarking/test_benchmarking.py | 17 +++++++++++++++++ 3 files changed, 30 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2182c72b..cd2c0de2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,11 @@ Changelog ========= +latest +------ + +* Add ``build_graph_rust``, a faster alternative to ``build_graph`` that does everything in Rust. + 3.20 (2026-11-13) ----------------- diff --git a/docs/usage.rst b/docs/usage.rst index 447dd96f..fccd15f5 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -86,6 +86,14 @@ Building the graph :return: An import graph that you can use to analyse the package. :rtype: ``ImportGraph`` +.. py:function:: grimp.build_graph_rust(package_name, *additional_package_names, include_external_packages=False, exclude_type_checking_imports=False, cache_dir=...) + + A faster alternative to ``grimp.build_graph``, which does everything (finding modules, reading the cache and + assembling the graph) in Rust. It takes the same parameters and returns an equivalent graph. It uses the same cache + files as ``grimp.build_graph``, so the two can be used interchangeably. + + Unlike ``grimp.build_graph``, it doesn't log anything about cache usage or skipped modules. + .. _typing module documentation: https://docs.python.org/3/library/typing.html#typing.TYPE_CHECKING Methods for analysing the module tree diff --git a/tests/benchmarking/test_benchmarking.py b/tests/benchmarking/test_benchmarking.py index abcf0d16..a1cf9b9a 100644 --- a/tests/benchmarking/test_benchmarking.py +++ b/tests/benchmarking/test_benchmarking.py @@ -294,6 +294,13 @@ def test_build_django_uncached(benchmark): benchmark(grimp.build_graph, "django", cache_dir=None) +def test_build_django_uncached_rust(benchmark): + """ + Benchmarks building a graph of real package using build_graph_rust, with the cache turned off. + """ + benchmark(grimp.build_graph_rust, "django", cache_dir=None) + + def test_build_django_from_cache_no_misses(benchmark): """ Benchmarks building a graph of real package - in this case Django. @@ -306,6 +313,16 @@ def test_build_django_from_cache_no_misses(benchmark): benchmark(grimp.build_graph, "django") +def test_build_django_from_cache_no_misses_rust(benchmark): + """ + Benchmarks building a graph of real package using build_graph_rust, fully utilizing the cache. + """ + # Populate the cache first, before beginning the benchmark. + grimp.build_graph_rust("django") + + benchmark(grimp.build_graph_rust, "django") + + @pytest.mark.parametrize( "number_of_misses", ( From b5257497d1a2145d1de57eaad1521a58e2ea124c Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 07:59:02 +0100 Subject: [PATCH 6/9] Port unit tests of the Python pipeline to functional tests The unit tests of module finding, scanning, caching and build_graph's arguments use an in-memory file system, so they only cover the Python implementation. The Rust implementation that is about to replace it reads the file system directly, so there is nothing to substitute. Keeping the in-memory file system for the tests alone would mean keeping production code that only they use: the Python-facing scanning and cache functions, and the fake file system itself. Port them instead to functional tests that write real files to a temporary directory. These test the code path that users actually run. They run against both implementations (via the build_graph fixture), ahead of replacing the Python implementation, which shows that the Rust one behaves the same. The tests are still fast: the ported tests take under a second, and the whole suite still takes around 2 seconds. Behaviours that are only observable through log messages are not ported, as the Rust implementation doesn't log. A write_files fixture is added, which writes files to a temporary directory on sys.path. Assistant-model: Claude Code --- tests/conftest.py | 35 + .../functional/test_build_graph_arguments.py | 147 +++ tests/functional/test_caching.py | 335 +++++++ tests/functional/test_finding_modules.py | 183 ++++ tests/functional/test_scanning_imports.py | 949 ++++++++++++++++++ 5 files changed, 1649 insertions(+) create mode 100644 tests/functional/test_build_graph_arguments.py create mode 100644 tests/functional/test_finding_modules.py create mode 100644 tests/functional/test_scanning_imports.py diff --git a/tests/conftest.py b/tests/conftest.py index 974fa891..45ae0a8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,10 @@ +import importlib import sys +import textwrap from pathlib import Path +import pytest + ASSETS_PATH = Path(__file__).parent / "assets" TEST_ASSETS = ( @@ -13,3 +17,34 @@ # Add test assets to the Python path. sys.path.extend([str((ASSETS_PATH / path).resolve()) for path in TEST_ASSETS]) + + +@pytest.fixture +def write_files(tmp_path, monkeypatch): + """ + Returns a function that writes files to a temporary directory that is on sys.path, so that the + packages they contain can be found by build_graph. + + The function takes a dict mapping file paths (relative to the temporary directory) to their + contents. It returns the directory that the files were written to. A different directory + (relative to the temporary directory) can be passed as root, which is useful for creating + namespace packages that span several directories. + """ + + modules_before = set(sys.modules) + + def write(files: dict[str, str], root: str = ".") -> Path: + root_directory = tmp_path / root + for relative_path, contents in files.items(): + path = root_directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(contents), encoding="utf-8") + monkeypatch.syspath_prepend(str(root_directory)) + importlib.invalidate_caches() + return root_directory + + yield write + + # Finding a package can import its parents, so don't let them leak between tests. + for module_name in set(sys.modules) - modules_before: + del sys.modules[module_name] diff --git a/tests/functional/test_build_graph_arguments.py b/tests/functional/test_build_graph_arguments.py new file mode 100644 index 00000000..0c759566 --- /dev/null +++ b/tests/functional/test_build_graph_arguments.py @@ -0,0 +1,147 @@ +import re + +import pytest + + +@pytest.mark.parametrize("include_external_packages", (True, False)) +def test_happy_path(build_graph, write_files, include_external_packages): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "import mypackage.foo.two.green\nfrom .. import Something", + "mypackage/foo/two/__init__.py": "", + "mypackage/foo/two/green.py": ( + "import mypackage.foo.two.blue\nfrom external.subpackage import foobar\nimport decimal" + ), + "mypackage/foo/two/blue.py": "", + } + ) + + graph = build_graph( + "mypackage", include_external_packages=include_external_packages, cache_dir=None + ) + + expected_import_map = { + "mypackage": set(), + "mypackage.foo": set(), + "mypackage.foo.one": {"mypackage.foo.two.green", "mypackage"}, + "mypackage.foo.two": set(), + "mypackage.foo.two.green": {"mypackage.foo.two.blue"}, + "mypackage.foo.two.blue": set(), + } + if include_external_packages: + expected_import_map["decimal"] = set() + expected_import_map["external"] = set() + expected_import_map["mypackage.foo.two.green"] |= {"external", "decimal"} + + assert set(expected_import_map.keys()) == graph.modules + for importer, imported_modules in expected_import_map.items(): + assert graph.find_modules_directly_imported_by(importer) == imported_modules + + # Check that the external packages are squashed modules. + if include_external_packages: + for module in ("external", "decimal"): + with pytest.raises( + ValueError, match=re.escape("Cannot find children of a squashed module.") + ): + graph.find_children(module) + + +def test_boolean_additional_package_raises_type_error(build_graph): + """ + Tests that a useful error message if build_graph is called + with a boolean as the second argument. + + This is because earlier versions of build_graph took include_external_packages + as the second argument, and it's possible it might have been called + as a positional argument. + """ + with pytest.raises(TypeError, match=re.escape("Package names must be strings, got bool.")): + build_graph("mypackage", True) + + +class TestCacheDir: + @pytest.fixture(autouse=True) + def package(self, write_files, tmp_path, monkeypatch): + write_files({"mypackage/__init__.py": ""}) + # Default caching writes to the current directory. + monkeypatch.chdir(tmp_path) + + def test_supplied(self, build_graph, tmp_path): + build_graph("mypackage", cache_dir=str(tmp_path / "supplied")) + + assert (tmp_path / "supplied" / "mypackage.meta.json").exists() + assert not (tmp_path / ".grimp_cache").exists() + + def test_not_supplied_uses_default(self, build_graph, tmp_path): + build_graph("mypackage") + + assert (tmp_path / ".grimp_cache" / "mypackage.meta.json").exists() + + def test_none_disables_caching(self, build_graph, tmp_path): + build_graph("mypackage", cache_dir=None) + + assert not (tmp_path / ".grimp_cache").exists() + + +def test_forgives_wrong_type_being_passed_to_include_external_packages(build_graph, write_files): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "import mypackage.foo.two.green\nfrom .. import Something", + } + ) + + graph = build_graph( + "mypackage", + # Note: this should be a bool, but we want to tolerate it, + # as Import Linter currently has a bug where it will pass it as None. + include_external_packages=None, + cache_dir=None, + ) + + assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.foo.one"} + + +def test_namespace_package_passed_as_root(build_graph, write_files): + # A namespace package can span several directories. + write_files( + { + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "import mypackage\nfrom . import two", + "mypackage/foo/two/__init__.py": "", + "mypackage/foo/two/green.py": "", + "mypackage/foo/two/blue.py": "", + "mypackage/bar/__init__.py": "", + "mypackage/bar/three.py": "", + }, + root="one", + ) + write_files( + { + "mypackage/foobar/__init__.py": "", + "mypackage/foobar/four.py": "", + }, + root="two", + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == { + "mypackage", + "mypackage.foo", + "mypackage.foo.one", + "mypackage.foo.two", + "mypackage.foo.two.green", + "mypackage.foo.two.blue", + "mypackage.bar", + "mypackage.bar.three", + "mypackage.foobar", + "mypackage.foobar.four", + } + assert graph.find_modules_directly_imported_by("mypackage.foo.one") == { + "mypackage", + "mypackage.foo.two", + } diff --git a/tests/functional/test_caching.py b/tests/functional/test_caching.py index 089556ff..717a4f84 100644 --- a/tests/functional/test_caching.py +++ b/tests/functional/test_caching.py @@ -1,5 +1,6 @@ import json import logging +import os import shutil import tempfile from pathlib import Path @@ -213,3 +214,337 @@ def _manipulate_data_file(data_file: Path, snippet: str, replacement: str) -> No with open(data_file, "w") as file: file.write(filedata) + + +# Tests of the cache's behaviour, using real files. +# --------------------------------------------------- + +MYPACKAGE_FILES = { + "mypackage/__init__.py": "", + "mypackage/one.py": "from . import two", + "mypackage/two.py": "from . import three", + "mypackage/three.py": "", +} +ANOTHERPACKAGE_FILES = { + "anotherpackage/__init__.py": "", + "anotherpackage/one.py": "import externalpackage\nfrom . import two", + "anotherpackage/two.py": "", +} +MARKER = " # Inserted by test" + + +@pytest.fixture +def cache_dir(tmp_path): + return tmp_path / "cache" + + +def _data_file(cache_dir: Path) -> Path: + [data_file] = cache_dir.glob("*.data.json") + return data_file + + +def _insert_marker_in_cache(cache_dir: Path) -> None: + """ + Alter the cached line contents of "from . import two", so we can tell if the cache was used. + """ + _manipulate_data_file(_data_file(cache_dir), "from . import two", "from . import two" + MARKER) + + +def _line_contents(graph, importer: str, imported: str) -> str: + [import_details] = graph.get_import_details(importer=importer, imported=imported) + return import_details["line_contents"] + + +class TestCacheFiles: + @pytest.mark.parametrize( + "include_external_packages, exclude_type_checking_imports, expected_data_file_name", + ( + # Blake2B 20-character hash of "bluepackage,greenpackage". + (False, False, "3d0968b3c17583a69a90610a871276fe2110c6f4.data.json"), + # Blake2B 20-character hash of "bluepackage,greenpackage:external". + (True, False, "ff83c6ee50a447a433a0c7a345d09de4b0d82a69.data.json"), + # Blake2B 20-character hash of "bluepackage,greenpackage:no_type_checking". + (False, True, "5e8b57eaed0aabd7826b5b0cc6cb2b7db8c82fb8.data.json"), + # Blake2B 20-character hash of "bluepackage,greenpackage:external:no_type_checking". + (True, True, "91f51d0e59ed9e09cfc3e8a94c9249c4c3ab7184.data.json"), + ), + ) + @pytest.mark.parametrize( + "package_names", (("bluepackage", "greenpackage"), ("greenpackage", "bluepackage")) + ) + def test_file_names( + self, + build_graph, + write_files, + cache_dir, + package_names, + include_external_packages, + exclude_type_checking_imports, + expected_data_file_name, + ): + write_files( + { + "bluepackage/__init__.py": "", + "greenpackage/__init__.py": "", + } + ) + + build_graph( + *package_names, + include_external_packages=include_external_packages, + exclude_type_checking_imports=exclude_type_checking_imports, + cache_dir=str(cache_dir), + ) + + # There's a meta file per package, and a data file that depends on the packages + # (but not the order they were passed in) and options. + assert {path.name for path in cache_dir.iterdir()} == { + ".gitignore", + "CACHEDIR.TAG", + "bluepackage.meta.json", + "greenpackage.meta.json", + expected_data_file_name, + } + + @pytest.mark.parametrize("cache_dir_suffix", ("", "/")) + @pytest.mark.parametrize( + "include_external_packages, expected_data_file_name", + ( + (False, "3d0968b3c17583a69a90610a871276fe2110c6f4.data.json"), + (True, "ff83c6ee50a447a433a0c7a345d09de4b0d82a69.data.json"), + ), + ) + def test_contents( + self, + build_graph, + write_files, + cache_dir, + cache_dir_suffix, + include_external_packages, + expected_data_file_name, + ): + root = write_files( + { + "bluepackage/__init__.py": "", + "bluepackage/one.py": "from . import two\nimport externalpackage", + "bluepackage/two.py": "", + "greenpackage/__init__.py": "", + "greenpackage/one.py": "", + "greenpackage/two.py": "from . import one", + } + ) + + build_graph( + "bluepackage", + "greenpackage", + include_external_packages=include_external_packages, + cache_dir=f"{cache_dir}{cache_dir_suffix}", + ) + + # The meta files contain the mtime of each module. + for package, module_files in ( + ( + "bluepackage", + {"bluepackage": "__init__", "bluepackage.one": "one", "bluepackage.two": "two"}, + ), + ( + "greenpackage", + {"greenpackage": "__init__", "greenpackage.one": "one", "greenpackage.two": "two"}, + ), + ): + assert json.loads((cache_dir / f"{package}.meta.json").read_text()) == { + module: os.path.getmtime(root / package / f"{file_stem}.py") + for module, file_stem in module_files.items() + } + + # The data file contains the imports of each module. + expected_blue_one_imports = {("bluepackage.two", False, 1, "from . import two")} + if include_external_packages: + expected_blue_one_imports.add(("externalpackage", False, 2, "import externalpackage")) + data = json.loads((cache_dir / expected_data_file_name).read_text()) + assert data["version"] == 2 + assert { + module: {tuple(imported) for imported in imports} + for module, imports in data["imports_by_module"].items() + } == { + "bluepackage": set(), + "bluepackage.one": expected_blue_one_imports, + "bluepackage.two": set(), + "greenpackage": set(), + "greenpackage.one": set(), + "greenpackage.two": {("greenpackage.one", False, 1, "from . import one")}, + } + + def test_default_cache_dir(self, build_graph, write_files, tmp_path, monkeypatch): + write_files({"bluepackage/__init__.py": ""}) + monkeypatch.chdir(tmp_path) + + build_graph("bluepackage") + + assert (tmp_path / ".grimp_cache" / "bluepackage.meta.json").exists() + + def test_adds_marker_files(self, build_graph, write_files, cache_dir): + write_files({"bluepackage/__init__.py": ""}) + + build_graph("bluepackage", cache_dir=str(cache_dir)) + + assert (cache_dir / ".gitignore").read_text() == "# Automatically created by Grimp.\n*" + assert (cache_dir / "CACHEDIR.TAG").read_text() == ( + "Signature: 8a477f597d28d172789f06886806bc55\n" + "# This file is a cache directory tag automatically created by Grimp.\n" + "# For information about cache directory tags see https://bford.info/cachedir/" + ) + + def test_doesnt_overwrite_marker_files(self, build_graph, write_files, cache_dir): + write_files({"bluepackage/__init__.py": ""}) + cache_dir.mkdir() + (cache_dir / ".gitignore").write_text("Something else") + + build_graph("bluepackage", cache_dir=str(cache_dir)) + + assert (cache_dir / ".gitignore").read_text() == "Something else" + + +class TestCacheUse: + def test_uses_cache_for_unmodified_module(self, build_graph, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == ( + "from . import two" + MARKER + ) + + def test_ignores_cache_for_module_with_different_mtime( + self, build_graph, write_files, cache_dir + ): + root = write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + path = root / "mypackage" / "one.py" + os.utime(path, (path.stat().st_atime, path.stat().st_mtime + 100)) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + # The modified module isn't read from the cache, but the others are. + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + assert _line_contents(graph, "mypackage.two", "mypackage.three") == "from . import three" + + def test_ignores_cache_for_new_module(self, build_graph, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + write_files({"mypackage/four.py": "from . import one"}) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert graph.direct_import_exists(importer="mypackage.four", imported="mypackage.one") + + def test_ignores_cache_for_module_with_no_cached_mtime( + self, build_graph, write_files, cache_dir + ): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + meta_file = cache_dir / "mypackage.meta.json" + mtimes = json.loads(meta_file.read_text()) + del mtimes["mypackage.one"] + meta_file.write_text(json.dumps(mtimes)) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + + def test_ignores_cache_for_module_missing_from_data(self, build_graph, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + data_file = _data_file(cache_dir) + data = json.loads(data_file.read_text()) + del data["imports_by_module"]["mypackage.one"] + data_file.write_text(json.dumps(data)) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert graph.direct_import_exists(importer="mypackage.one", imported="mypackage.two") + + @pytest.mark.parametrize( + "serialized_meta", + ( + "INVALID_JSON", + '{"mypackage.one": ["wrong", "type"]}', + ), + ) + def test_ignores_corrupt_meta_file(self, build_graph, write_files, cache_dir, serialized_meta): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + (cache_dir / "mypackage.meta.json").write_text(serialized_meta) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + + @pytest.mark.parametrize( + "serialized_data", + ( + "INVALID_JSON", + '{"version": 2, "imports_by_module": {"mypackage.one": [["wrong", "type"]]}}', + ), + ) + def test_ignores_corrupt_data_file(self, build_graph, write_files, cache_dir, serialized_data): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _data_file(cache_dir).write_text(serialized_data) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + # The cache is repaired. + data = json.loads(_data_file(cache_dir).read_text()) + assert data["imports_by_module"]["mypackage.one"] == [ + ["mypackage.two", False, 1, "from . import two"] + ] + + @pytest.mark.parametrize( + "package_names, options", + ( + (("mypackage", "anotherpackage"), {}), + (("mypackage",), {"include_external_packages": True}), + (("mypackage",), {"exclude_type_checking_imports": True}), + ), + ) + def test_doesnt_use_cache_for_different_packages_or_options( + self, build_graph, write_files, cache_dir, package_names, options + ): + write_files(MYPACKAGE_FILES | ANOTHERPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + + graph = build_graph(*package_names, cache_dir=str(cache_dir), **options) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + + @pytest.mark.parametrize("include_external_packages", (False, True)) + def test_uses_cache_for_multiple_packages( + self, build_graph, write_files, cache_dir, include_external_packages + ): + write_files(MYPACKAGE_FILES | ANOTHERPACKAGE_FILES) + kwargs = dict( + include_external_packages=include_external_packages, cache_dir=str(cache_dir) + ) + build_graph("mypackage", "anotherpackage", **kwargs) + _insert_marker_in_cache(cache_dir) + + graph = build_graph("mypackage", "anotherpackage", **kwargs) + + expected_line_contents = "from . import two" + MARKER + assert _line_contents(graph, "mypackage.one", "mypackage.two") == expected_line_contents + assert ( + _line_contents(graph, "anotherpackage.one", "anotherpackage.two") + == expected_line_contents + ) + expected_imports = {"anotherpackage.two"} + if include_external_packages: + expected_imports.add("externalpackage") + assert graph.find_modules_directly_imported_by("anotherpackage.one") == expected_imports diff --git a/tests/functional/test_finding_modules.py b/tests/functional/test_finding_modules.py new file mode 100644 index 00000000..f529fe03 --- /dev/null +++ b/tests/functional/test_finding_modules.py @@ -0,0 +1,183 @@ +import pytest + +# Files are empty unless stated otherwise. +HAPPY_PATH_FILES = { + "mypackage/__init__.py": "", + "mypackage/not-a-python-file.txt": "", + "mypackage/.hidden": "", + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "", + "mypackage/foo/two/__init__.py": "", + "mypackage/foo/two/green.py": "", + "mypackage/foo/two/blue.py": "", +} + + +def test_happy_path(build_graph, write_files): + write_files(HAPPY_PATH_FILES) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == { + "mypackage", + "mypackage.foo", + "mypackage.foo.one", + "mypackage.foo.two", + "mypackage.foo.two.green", + "mypackage.foo.two.blue", + } + + +NAMESPACE_FILES = { + "somenamespace/foo/blue/__init__.py": "", + "somenamespace/foo/blue/one.py": "", + "somenamespace/foo/blue/two/__init__.py": "", + "somenamespace/foo/blue/two/alpha.py": "", + "somenamespace/foo/blue/noinitpackage/three.py": "", + "somenamespace/foo/blue/noinitpackage/orphan/__init__.py": "", + "somenamespace/foo/blue/noinitpackage/orphan/four.py": "", + "somenamespace/foo/green/five/__init__.py": "", + "somenamespace/foo/green/five/beta.py": "", + "somenamespace/foo/non_python_directory/six/README.txt": "", +} +MODULES_FOO_BLUE = { + "somenamespace.foo.blue", + "somenamespace.foo.blue.one", + "somenamespace.foo.blue.two", + "somenamespace.foo.blue.two.alpha", +} +MODULES_FOO_GREEN_FIVE = { + "somenamespace.foo.green.five", + "somenamespace.foo.green.five.beta", +} + + +@pytest.mark.parametrize( + "package_name, expected_modules", + [ + ( + "somenamespace", + MODULES_FOO_BLUE + | MODULES_FOO_GREEN_FIVE + | {"somenamespace", "somenamespace.foo", "somenamespace.foo.green"}, + ), + ( + "somenamespace.foo", + MODULES_FOO_BLUE + | MODULES_FOO_GREEN_FIVE + | {"somenamespace.foo", "somenamespace.foo.green"}, + ), + ("somenamespace.foo.blue", MODULES_FOO_BLUE), + ], +) +def test_namespaced_packages(build_graph, write_files, package_name, expected_modules): + write_files(NAMESPACE_FILES) + + graph = build_graph(package_name, cache_dir=None) + + # Note: namespace directories are only included if they (indirectly) contain Python files. + assert graph.modules == expected_modules + + +def test_ignores_orphaned_python_files(build_graph, write_files): + # Python files in directories that don't contain an __init__.py should not be discovered. + write_files( + { + "mypackage/__init__.py": "", + "mypackage/two/__init__.py": "", + "mypackage/two/green.py": "", + "mypackage/noinitpackage/green.py": "", + "mypackage/noinitpackage/orphan/__init__.py": "", + "mypackage/noinitpackage/orphan/red.py": "", + } + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.two", "mypackage.two.green"} + + +def test_ignores_invalid_identifier_directories(build_graph, write_files): + write_files( + { + f"namespacepackage/{path}": "" + for path in ( + "foo/valid_underscore_name/__init__.py", + "foo/valid_underscore_name/mod.py", + "foo/invalid-hyphenated-name/__init__.py", + "foo/invalid-hyphenated-name/mod.py", + "foo/1nvalid-number-prefixed/__init__.py", + "foo/1nvalid-number-prefixed/mod.py", + "foo/valid_non_àscii/__init__.py", + "foo/valid_non_àscii/mod.py", + "foo/invalid_non_😃scii/__init__.py", + "foo/invalid_non_😃scii/mod.py", + "bar/__init__.py", + "bar/mod.py", + ) + } + ) + + graph = build_graph("namespacepackage", cache_dir=None) + + assert graph.modules == { + "namespacepackage", + "namespacepackage.foo", + "namespacepackage.foo.valid_underscore_name", + "namespacepackage.foo.valid_underscore_name.mod", + "namespacepackage.foo.valid_non_àscii", + "namespacepackage.foo.valid_non_àscii.mod", + "namespacepackage.bar", + "namespacepackage.bar.mod", + } + + +@pytest.mark.parametrize("extension", ("py", "txt")) +def test_ignores_dotted_python_files(build_graph, write_files, extension): + # Python files containing dots (other than the one before .py) should not be discovered. + write_files( + { + "mypackage/__init__.py": "", + "mypackage/foo.py": "", + "mypackage/bar/__init__.py": "", + f"mypackage/bar/baz.dotted.{extension}": "", + } + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.bar"} + + +def test_ignores_hidden_directories(build_graph, write_files): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/two/__init__.py": "", + "mypackage/two/green.py": "", + "mypackage/.hidden/green.py": "", + "mypackage/.hidden/orphan/__init__.py": "", + "mypackage/.hidden/orphan/red.py": "", + } + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.two", "mypackage.two.green"} + + +def test_follows_symlinked_directories(build_graph, write_files, tmp_path): + write_files( + { + "mypackage/__init__.py": "", + "elsewhere/__init__.py": "", + "elsewhere/green.py": "", + } + ) + (tmp_path / "mypackage" / "linked").symlink_to( + tmp_path / "elsewhere", target_is_directory=True + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.linked", "mypackage.linked.green"} diff --git a/tests/functional/test_scanning_imports.py b/tests/functional/test_scanning_imports.py new file mode 100644 index 00000000..30cfbd43 --- /dev/null +++ b/tests/functional/test_scanning_imports.py @@ -0,0 +1,949 @@ +""" +Tests of the imports found when building a graph. + +(Ported from unit tests of the scanning of imports, which used an in-memory file system.) +""" + +import textwrap + +import pytest + +from grimp.domain.valueobjects import DirectImport, Module + + +def _source(code: str) -> str: + """Dedent code, and drop the leading newline so that line numbers start from 1.""" + return textwrap.dedent(code).lstrip("\n") + + +def _direct_imports(graph, importer: str) -> set[DirectImport]: + return { + DirectImport( + importer=Module(details["importer"]), + imported=Module(details["imported"]), + line_number=details["line_number"], + line_contents=details["line_contents"], + is_lazy=details["is_lazy"], + ) + for imported in graph.find_modules_directly_imported_by(importer) + for details in graph.get_import_details(importer=importer, imported=imported) + } + + +FOO_PACKAGE_PATHS = ( + "foo/__init__.py", + "foo/one/__init__.py", + "foo/one/blue.py", + "foo/one/green.py", + "foo/two/__init__.py", + "foo/two/brown.py", + "foo/two/yellow.py", + "foo/three.py", +) + + +def _foo_package_files(contents_by_path: dict[str, str]) -> dict[str, str]: + """ + Files in the foo package, which are empty unless otherwise specified. + """ + files = dict.fromkeys(FOO_PACKAGE_PATHS, "") + files.update({path: _source(contents) for path, contents in contents_by_path.items()}) + return files + + +@pytest.mark.parametrize( + "include_external_packages, expected_result", + ( + ( + False, + { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ) + }, + ), + ( + True, + { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("externalone"), + line_number=2, + line_contents="import externalone", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("externaltwo"), + line_number=3, + line_contents="import externaltwo.subpackage # with comment afterwards.", + ), + }, + ), + ), +) +def test_absolute_imports(build_graph, write_files, include_external_packages, expected_result): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(""" + import foo.two + import externalone + import externaltwo.subpackage # with comment afterwards. + arbitrary_expression = 1 + """), + "foo/two.py": "", + } + ) + + graph = build_graph( + "foo", + include_external_packages=include_external_packages, + cache_dir=None, + ) + + assert _direct_imports(graph, "foo.one") == expected_result + + +def test_lazy_imports(build_graph, write_files): + write_files( + _foo_package_files( + { + "foo/one/blue.py": """ + # Absolute no-from + lazy import foo.two + lazy import externalone + lazy import externaltwo.subpackage # with comment afterwards. + arbitrary_expression = 1 + + # Absolute from + lazy from foo.one import green + lazy from foo.two import yellow + if t.TYPE_CHECKING: + lazy from foo import three + lazy from external import one + lazy from external.two import blue # with comment afterwards. + + # Relative from + lazy from . import green + lazy from ..two import yellow + lazy from .. import three + """ + } + ) + ) + + graph = build_graph("foo", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, "foo.one.blue") == { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two"), + line_number=2, + line_contents="lazy import foo.two", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("externalone"), + line_number=3, + line_contents="lazy import externalone", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("externaltwo"), + line_number=4, + line_contents="lazy import externaltwo.subpackage # with comment afterwards.", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=8, + line_contents="lazy from foo.one import green", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=9, + line_contents="lazy from foo.two import yellow", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=11, + line_contents="lazy from foo import three", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=12, + line_contents="lazy from external import one", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=13, + line_contents="lazy from external.two import blue # with comment afterwards.", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=16, + line_contents="lazy from . import green", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=17, + line_contents="lazy from ..two import yellow", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=18, + line_contents="lazy from .. import three", + is_lazy=True, + ), + } + + +def test_non_ascii(build_graph, write_files): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/blue.py": _source(""" + from ñon_ascii_变 import * + from . import ñon_ascii_变 + import mypackage.ñon_ascii_变.ラーメン + """), + "mypackage/ñon_ascii_变/__init__.py": "", + "mypackage/ñon_ascii_变/ラーメン.py": "", + } + ) + + graph = build_graph("mypackage", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, "mypackage.blue") == { + DirectImport( + importer=Module("mypackage.blue"), + imported=Module("ñon_ascii_变"), + line_number=1, + line_contents="from ñon_ascii_变 import *", + ), + DirectImport( + importer=Module("mypackage.blue"), + imported=Module("mypackage.ñon_ascii_变"), + line_number=2, + line_contents="from . import ñon_ascii_变", + ), + DirectImport( + importer=Module("mypackage.blue"), + imported=Module("mypackage.ñon_ascii_变.ラーメン"), + line_number=3, + line_contents="import mypackage.ñon_ascii_变.ラーメン", + ), + } + + +def test_single_namespace_package_portion(build_graph, write_files): + MODULE_ONE = Module("namespace.foo.one") + MODULE_TWO = Module("namespace.foo.two") + MODULE_THREE = Module("namespace.foo.three") + MODULE_FOUR = Module("namespace.foo.four") + MODULE_BLUE = Module("namespace.foo.two.blue") + MODULE_ALPHA = Module("namespace.foo.two.green.alpha") + write_files( + { + "namespace/foo/__init__.py": "", + "namespace/foo/one.py": _source(""" + import namespace.foo.two + from namespace.foo import three + from . import four + """), + "namespace/foo/two/__init__.py": "", + "namespace/foo/three.py": "", + "namespace/foo/four.py": "", + "namespace/foo/two/green/__init__.py": "", + "namespace/foo/two/blue.py": "", + "namespace/foo/two/green/alpha.py": _source(""" + from .. import blue + from ... import three + """), + } + ) + + graph = build_graph("namespace.foo", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, MODULE_ONE.name) == { + DirectImport( + importer=MODULE_ONE, + imported=MODULE_TWO, + line_number=1, + line_contents="import namespace.foo.two", + ), + DirectImport( + importer=MODULE_ONE, + imported=MODULE_THREE, + line_number=2, + line_contents="from namespace.foo import three", + ), + DirectImport( + importer=MODULE_ONE, + imported=MODULE_FOUR, + line_number=3, + line_contents="from . import four", + ), + } + assert _direct_imports(graph, MODULE_ALPHA.name) == { + DirectImport( + importer=MODULE_ALPHA, + imported=MODULE_BLUE, + line_number=1, + line_contents="from .. import blue", + ), + DirectImport( + importer=MODULE_ALPHA, + imported=MODULE_THREE, + line_number=2, + line_contents="from ... import three", + ), + } + + +@pytest.mark.parametrize("include_external_packages", (True, False)) +def test_import_of_portion_not_in_graph(build_graph, write_files, include_external_packages): + # Include two namespaces, one deeper than the other. + # + # Note: unlike the (in-memory) layout this was originally written against, foo.one can't + # have both a module file and a submodule (blue), so the blue module is namespace.foo.two.blue. + # It's at the same depth, so its relative imports resolve in the same way. + MODULE_FOO_ONE = Module("namespace.foo.one") + MODULE_FOO_BLUE = Module("namespace.foo.two.blue") + write_files( + { + "namespace/foo/__init__.py": "", + "namespace/foo/one.py": _source(""" + import namespace.bar.one.orange + from namespace.yellow import one + from .. import mauve + if t.TYPE_CHECKING: + from ..cyan import one + from ..magenta.one import alpha + """), + "namespace/foo/two/__init__.py": "", + "namespace/foo/two/blue.py": _source(""" + from ... import teal + from ...pink import one + from ...scarlet.one import alpha + """), + "namespace/bar/one/green/__init__.py": "", + "namespace/bar/one/green/alpha.py": "", + } + ) + + graph = build_graph( + "namespace.foo", + "namespace.bar.one.green", + include_external_packages=include_external_packages, + cache_dir=None, + ) + + if include_external_packages: + assert _direct_imports(graph, MODULE_FOO_ONE.name) == { + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.bar.one.orange"), + line_number=1, + line_contents="import namespace.bar.one.orange", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.yellow"), + line_number=2, + line_contents="from namespace.yellow import one", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.mauve"), + line_number=3, + line_contents="from .. import mauve", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.cyan"), + line_number=5, + line_contents="from ..cyan import one", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.magenta"), + line_number=6, + line_contents="from ..magenta.one import alpha", + ), + } + assert _direct_imports(graph, MODULE_FOO_BLUE.name) == { + DirectImport( + importer=MODULE_FOO_BLUE, + imported=Module("namespace.teal"), + line_number=1, + line_contents="from ... import teal", + ), + DirectImport( + importer=MODULE_FOO_BLUE, + imported=Module("namespace.pink"), + line_number=2, + line_contents="from ...pink import one", + ), + DirectImport( + importer=MODULE_FOO_BLUE, + imported=Module("namespace.scarlet"), + line_number=3, + line_contents="from ...scarlet.one import alpha", + ), + } + else: + assert _direct_imports(graph, MODULE_FOO_ONE.name) == set() + assert _direct_imports(graph, MODULE_FOO_BLUE.name) == set() + + +@pytest.mark.parametrize( + "include_external_packages, expected_result", + ( + ( + False, + { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=1, + line_contents="from foo.one import green", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=2, + line_contents="from foo.two import yellow", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=4, + line_contents="from foo import three", + ), + }, + ), + ( + True, + { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=1, + line_contents="from foo.one import green", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=2, + line_contents="from foo.two import yellow", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=4, + line_contents="from foo import three", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=5, + line_contents="from external import one", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=6, + line_contents="from external.two import blue # with comment afterwards.", + ), + }, + ), + ), +) +def test_absolute_from_imports( + build_graph, write_files, include_external_packages, expected_result +): + write_files( + _foo_package_files( + { + "foo/one/blue.py": """ + from foo.one import green + from foo.two import yellow + if t.TYPE_CHECKING: + from foo import three + from external import one + from external.two import blue # with comment afterwards. + arbitrary_expression = 1 + """ + } + ) + ) + + graph = build_graph( + "foo", + include_external_packages=include_external_packages, + cache_dir=None, + ) + + assert _direct_imports(graph, "foo.one.blue") == expected_result + + +@pytest.mark.parametrize("module_to_scan_is_package", (True, False)) +def test_relative_from_imports(build_graph, write_files, module_to_scan_is_package): + if module_to_scan_is_package: + module_to_scan = Module("foo.one") + module_path = "foo/one/__init__.py" + else: + module_to_scan = Module("foo.one.blue") + module_path = "foo/one/blue.py" + write_files( + _foo_package_files( + { + module_path: """ + from . import green + from ..two import yellow + from .. import three + arbitrary_expression = 1 + """ + } + ) + ) + + graph = build_graph( + "foo", + include_external_packages=False, + exclude_type_checking_imports=True, + cache_dir=None, + ) + + assert _direct_imports(graph, module_to_scan.name) == { + DirectImport( + importer=module_to_scan, + imported=Module("foo.one.green"), + line_number=1, + line_contents="from . import green", + ), + DirectImport( + importer=module_to_scan, + imported=Module("foo.two.yellow"), + line_number=2, + line_contents="from ..two import yellow", + ), + DirectImport( + importer=module_to_scan, + imported=Module("foo.three"), + line_number=3, + line_contents="from .. import three", + ), + } + + +@pytest.mark.parametrize( + "import_source", + ("from .two.yellow import my_function", "from foo.two.yellow import my_function"), +) +def test_trims_to_known_modules(build_graph, write_files, import_source): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": import_source, + "foo/two/__init__.py": "", + "foo/two/yellow.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two.yellow"), + line_number=1, + line_contents=import_source, + ) + } + + +def test_trims_to_known_modules_within_init_file(build_graph, write_files): + write_files( + { + "foo/__init__.py": "", + "foo/one/__init__.py": "from .yellow import my_function", + "foo/one/yellow.py": "", + "foo/one/blue/__init__.py": "from .alpha import my_function", + "foo/one/blue/alpha.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.one.yellow"), + line_number=1, + line_contents="from .yellow import my_function", + ) + } + assert _direct_imports(graph, "foo.one.blue") == { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.blue.alpha"), + line_number=1, + line_contents="from .alpha import my_function", + ) + } + + +def test_trims_whitespace_from_start_of_line_contents(build_graph, write_files): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(""" + def my_function(): + from . import two + """), + "foo/two.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=2, + line_contents="from . import two", + ), + } + + +@pytest.mark.parametrize( + "statement, expected_module_name", + ( + # External packages that share a namespace with an internal module resolve + # to the shallowest component that does not clash with an internal module namespace. + ("import namespace", None), + ("import namespace.foo", None), + ("from namespace import foo", None), + ("import namespace.bar", "namespace.bar"), + ("from namespace import bar", "namespace.bar"), + ("from ... import bar", "namespace.bar"), + ("import namespace.bar.orange", "namespace.bar"), + ("from namespace.bar import orange", "namespace.bar"), + ("from ...bar import orange", "namespace.bar"), + ("import namespace.foo.green", "namespace.foo.green"), + ("from namespace.foo import green", "namespace.foo.green"), + ("from .. import green", "namespace.foo.green"), + ("import namespace.foo.green.alpha", "namespace.foo.green"), + ("from namespace.foo.green import alpha", "namespace.foo.green"), + ("from ..green import alpha", "namespace.foo.green"), + ("import namespace.foo.green.alpha.one", "namespace.foo.green"), + ("from namespace.foo.green.alpha import one", "namespace.foo.green"), + ("from ..green.alpha import one", "namespace.foo.green"), + ), +) +def test_external_package_imports_for_namespace_packages( + build_graph, write_files, statement, expected_module_name +): + write_files( + { + "namespace/foo/blue/__init__.py": "", + "namespace/foo/blue/alpha.py": statement, + "namespace/foo/blue/beta.py": "", + } + ) + + graph = build_graph("namespace.foo.blue", include_external_packages=True, cache_dir=None) + + module_to_scan = Module("namespace.foo.blue.alpha") + if expected_module_name: + assert _direct_imports(graph, module_to_scan.name) == { + DirectImport( + importer=module_to_scan, + imported=Module(expected_module_name), + line_number=1, + line_contents=statement, + ), + } + else: + assert _direct_imports(graph, module_to_scan.name) == set() + + +@pytest.mark.parametrize("statement", ("import bar.blue", "from bar import blue")) +def test_scans_multiple_packages(build_graph, write_files, statement): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(f""" + import foo.two + {statement} + import externalone + + arbitrary_expression = 1 + """), + "foo/two.py": "", + "bar/__init__.py": "", + "bar/green.py": "", + "bar/blue.py": "", + } + ) + + graph = build_graph("foo", "bar", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("bar.blue"), + line_number=2, + line_contents=statement, + ), + } + + +@pytest.mark.parametrize("exclude_type_checking_imports", (True, False)) +@pytest.mark.parametrize( + "statement, is_statement_valid", + ( + ("if t.TYPE_CHECKING:", True), + ("if TYPE_CHECKING:", True), + ("if typing.TYPE_CHECKING:", True), + ("if WEIRD_ALIAS.TYPE_CHECKING:", True), + ("if type_checking:", False), + ("while TYPE_CHECKING:", False), + ), +) +def test_exclude_type_checking_imports( + build_graph, write_files, exclude_type_checking_imports, statement, is_statement_valid +): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(f""" + import foo.two + {statement} + import foo.three + import foo.four + import foo.five + arbitrary_expression = 1 + """), + "foo/two.py": "", + "foo/three.py": "", + "foo/four.py": "", + "foo/five.py": "", + } + ) + + graph = build_graph( + "foo", + include_external_packages=False, + exclude_type_checking_imports=exclude_type_checking_imports, + cache_dir=None, + ) + + if exclude_type_checking_imports and is_statement_valid: + expected_result = { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.five"), + line_number=5, + line_contents="import foo.five", + ), + } + else: + expected_result = { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.three"), + line_number=3, + line_contents="import foo.three", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.four"), + line_number=4, + line_contents="import foo.four", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.five"), + line_number=5, + line_contents="import foo.five", + ), + } + assert _direct_imports(graph, "foo.one") == expected_result + + +@pytest.mark.parametrize( + "code, expected_imported_modules", + ( + pytest.param( + """ + if TYPE_CHECKING: + import foo.two + else: + import foo.three + """, + {"foo.three"}, + id="else", + ), + pytest.param( + """ + if typing.TYPE_CHECKING: + import foo.two + elif sys.version_info >= (3, 12): + import foo.three + else: + import foo.four + """, + {"foo.three", "foo.four"}, + id="elif-and-else", + ), + pytest.param( + """ + if sys.version_info >= (3, 12): + import foo.two + elif TYPE_CHECKING: + import foo.three + else: + import foo.four + """, + {"foo.two", "foo.four"}, + id="elif-type-checking", + ), + pytest.param( + """ + if TYPE_CHECKING: + if sys.version_info >= (3, 12): + import foo.two + else: + import foo.three + if TYPE_CHECKING: + import foo.four + import foo.five + """, + set(), + id="nested", + ), + ), +) +def test_exclude_type_checking_imports_only_excludes_type_checking_branches( + build_graph, write_files, code, expected_imported_modules +): + files = {"foo/__init__.py": ""} + files.update({f"foo/{name}.py": "" for name in ("two", "three", "four", "five")}) + files["foo/one.py"] = _source(code) + write_files(files) + + graph = build_graph( + "foo", + include_external_packages=False, + exclude_type_checking_imports=True, + cache_dir=None, + ) + + imported_modules = { + direct_import.imported.name for direct_import in _direct_imports(graph, "foo.one") + } + assert imported_modules == expected_imported_modules + + +def test_t_string_syntax(build_graph, write_files): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(""" + import foo.two + # t-strings are only valid in Python 3.14+ + hello = t"Hello" + """), + "foo/two.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + } + + +@pytest.mark.parametrize( + "import_statement", + ( + "from namespace.portion.nonexistent import something", + "from .nonexistent import something", + "from ..nonexistent import something", + ), +) +def test_missing_module_in_dotted_root_package_is_ignored( + build_graph, write_files, import_statement: str +): + """ + When a dotted root package (namespace package) contains an import of a missing module + within that same package, and include_external_packages=True, the Rust code should not + panic with an index-out-of-bounds error. + + Regression test for https://github.com/python-grimp/grimp/issues/308. + """ + write_files( + { + "namespace/portion/__init__.py": "", + "namespace/portion/subpackage/__init__.py": "", + "namespace/portion/subpackage/importer.py": import_statement, + } + ) + + graph = build_graph("namespace.portion", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, "namespace.portion.subpackage.importer") == set() From fe2ad0de16779e94b3aef97b40063193537aeeca Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 08:08:50 +0100 Subject: [PATCH 7/9] Replace build_graph with the Rust implementation build_graph now builds the graph in Rust, and build_graph_rust is removed. Remove everything that only the Python implementation used: - Python: the module finder, cache and file system adaptors and ports, the scanning module, and the Settings entries and copy method they needed. The package finder now uses os.path rather than a file system port. - Rust: the Python-facing scan_for_imports and cache functions, the conversions to and from Python objects, the in-memory file system and the Python wrappers around the real one, and the cache exceptions, which were never raised to Python by the new implementation. Rust only parses the cache to decide whether it can be used, so parse_json_to_map now returns an Option. - Tests: the unit tests of the removed code (ported to functional tests in the previous commit), their fakes, and the build_graph fixture which ran the functional tests against both implementations. Assistant-model: Claude Code --- .importlinter | 3 - AGENTS.md | 14 +- CHANGELOG.rst | 4 +- docs/usage.rst | 8 - rust/Cargo.lock | 7 - rust/Cargo.toml | 1 - rust/src/caching.rs | 108 +- rust/src/errors.rs | 10 - rust/src/exceptions.rs | 2 - rust/src/filesystem.rs | 307 +--- rust/src/graph_building.rs | 2 +- rust/src/import_scanning.rs | 143 +- rust/src/lib.rs | 15 +- rust/src/module_finding.rs | 57 +- src/grimp/__init__.py | 3 +- src/grimp/adaptors/caching.py | 235 --- src/grimp/adaptors/filesystem.py | 52 - src/grimp/adaptors/modulefinder.py | 181 --- src/grimp/adaptors/packagefinder.py | 15 +- src/grimp/application/config.py | 5 - src/grimp/application/ports/caching.py | 61 - src/grimp/application/ports/filesystem.py | 115 -- src/grimp/application/ports/modulefinder.py | 42 - src/grimp/application/ports/packagefinder.py | 6 +- src/grimp/application/scanning.py | 30 - src/grimp/application/usecases.py | 179 +-- src/grimp/main.py | 12 +- tests/adaptors/filesystem.py | 267 ---- tests/adaptors/modulefinder.py | 17 - tests/adaptors/packagefinder.py | 15 - tests/benchmarking/test_benchmarking.py | 17 - tests/config.py | 14 - tests/functional/conftest.py | 11 - tests/functional/test_build_and_use_graph.py | 42 +- ...build_and_use_graph_with_multiple_roots.py | 10 +- .../functional/test_build_graph_arguments.py | 16 +- .../test_build_graph_on_real_packages.py | 5 - tests/functional/test_caching.py | 74 +- tests/functional/test_encoding_handling.py | 22 +- tests/functional/test_error_handling.py | 4 +- tests/functional/test_finding_modules.py | 16 +- tests/functional/test_lazy_imports.py | 7 +- tests/functional/test_namespace_packages.py | 24 +- tests/functional/test_scanning_imports.py | 37 +- tests/unit/adaptors/test_caching.py | 738 ---------- tests/unit/adaptors/test_filesystem.py | 278 ---- tests/unit/adaptors/test_modulefinder.py | 317 ----- tests/unit/adaptors/test_packagefinder.py | 8 +- tests/unit/application/test_scanning.py | 1254 ----------------- tests/unit/application/test_usecases.py | 223 --- tests/unit/conftest.py | 15 - 51 files changed, 168 insertions(+), 4880 deletions(-) delete mode 100644 src/grimp/adaptors/caching.py delete mode 100644 src/grimp/adaptors/filesystem.py delete mode 100644 src/grimp/adaptors/modulefinder.py delete mode 100644 src/grimp/application/ports/caching.py delete mode 100644 src/grimp/application/ports/filesystem.py delete mode 100644 src/grimp/application/ports/modulefinder.py delete mode 100644 src/grimp/application/scanning.py delete mode 100644 tests/adaptors/filesystem.py delete mode 100644 tests/adaptors/modulefinder.py delete mode 100644 tests/adaptors/packagefinder.py delete mode 100644 tests/config.py delete mode 100644 tests/functional/conftest.py delete mode 100644 tests/unit/adaptors/test_caching.py delete mode 100644 tests/unit/adaptors/test_filesystem.py delete mode 100644 tests/unit/adaptors/test_modulefinder.py delete mode 100644 tests/unit/application/test_scanning.py delete mode 100644 tests/unit/application/test_usecases.py delete mode 100644 tests/unit/conftest.py diff --git a/.importlinter b/.importlinter index 097c591d..4c5187c4 100644 --- a/.importlinter +++ b/.importlinter @@ -17,8 +17,5 @@ ignore_imports = ; Grimp doesn't understand extension modules: it thinks grimp._rustgrimp ; is an object within grimp, rather than a module in its own right, ; so we ignore these imports here. - grimp.adaptors.caching -> grimp grimp.application.graph -> grimp - grimp.adaptors.filesystem -> grimp - grimp.application.scanning -> grimp grimp.application.usecases -> grimp diff --git a/AGENTS.md b/AGENTS.md index 718d6005..f6a1636d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Grimp builds a queryable graph of the imports within one or more Python packages. It's a hybrid Python/Rust project: the public API and orchestration are in Python -(`src/grimp/`), and the import scanning and graph algorithms are implemented in Rust +(`src/grimp/`), and the graph building and graph algorithms are implemented in Rust (`rust/`) as a PyO3 extension module (`grimp._rustgrimp`), built with maturin. ## Architecture @@ -11,18 +11,18 @@ hybrid Python/Rust project: the public API and orchestration are in Python (see `.importlinter`): - `domain/` — core types (`valueobjects.py`, `analysis.py`), no dependencies on other layers. -- `application/` — use cases and ports (`usecases.py`, `graph.py`, `scanning.py`, `config.py`). -- `adaptors/` — implementations of application ports (filesystem, module/package finding, - caching, timing). +- `application/` — use cases and ports (`usecases.py`, `graph.py`, `config.py`). +- `adaptors/` — implementations of application ports (package finding, timing). - `main.py` — top-level entry point (`grimp.build_graph`), wires adaptors to use cases. - `exceptions.py` — shared exceptions, importable from any layer. Respect this layering: `domain` must not import from `application` or `adaptors`, etc. Run `just lint-python` (which includes `lint-imports`) to check. -The Rust side (`rust/src/`) does the heavy lifting: import parsing (via `ruff`'s parser), -module/import scanning, and graph algorithms (`rust/src/graph/`: pathfinding, cycle -detection, hierarchy queries, etc.). +The Rust side (`rust/src/`) does the heavy lifting. `grimp.build_graph` calls into +`rust/src/graph_building.rs`, which finds the modules, reads and writes the cache, scans the +imports (parsed via `ruff`'s parser) and assembles the graph. It also implements the graph +algorithms (`rust/src/graph/`: pathfinding, cycle detection, hierarchy queries, etc.). ## Setup diff --git a/CHANGELOG.rst b/CHANGELOG.rst index cd2c0de2..090260a8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -5,7 +5,9 @@ Changelog latest ------ -* Add ``build_graph_rust``, a faster alternative to ``build_graph`` that does everything in Rust. +* Speed up ``build_graph`` by building the graph in Rust. This includes finding the modules, reading + and writing the cache, and assembling the graph. The cache files are unchanged. +* ``build_graph`` no longer logs messages about the cache, or about skipping modules with dots in their name. 3.20 (2026-11-13) ----------------- diff --git a/docs/usage.rst b/docs/usage.rst index fccd15f5..447dd96f 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -86,14 +86,6 @@ Building the graph :return: An import graph that you can use to analyse the package. :rtype: ``ImportGraph`` -.. py:function:: grimp.build_graph_rust(package_name, *additional_package_names, include_external_packages=False, exclude_type_checking_imports=False, cache_dir=...) - - A faster alternative to ``grimp.build_graph``, which does everything (finding modules, reading the cache and - assembling the graph) in Rust. It takes the same parameters and returns an equivalent graph. It uses the same cache - files as ``grimp.build_graph``, so the two can be used interchangeably. - - Unlike ``grimp.build_graph``, it doesn't log anything about cache usage or skipped modules. - .. _typing module documentation: https://docs.python.org/3/library/typing.html#typing.TYPE_CHECKING Methods for analysing the module tree diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 9e0054b2..3b988f13 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -31,7 +31,6 @@ dependencies = [ "tap", "thiserror", "unicode-ident", - "unindent", ] [[package]] @@ -1069,12 +1068,6 @@ dependencies = [ "rand", ] -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index d1e7ef0f..40ad1e3e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -27,7 +27,6 @@ ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15. serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.137", features = ["float_roundtrip"] } serde_yaml = "0.9" -unindent = "0.2.4" encoding_rs = "0.8.35" blake2 = "0.10.6" unicode-ident = "1.0.24" diff --git a/rust/src/caching.rs b/rust/src/caching.rs index 7c535b83..2e55046d 100644 --- a/rust/src/caching.rs +++ b/rust/src/caching.rs @@ -1,82 +1,11 @@ -use crate::errors::{GrimpError, GrimpResult}; -use crate::filesystem::{FileSystem, get_file_system_boxed}; -use crate::import_scanning::{DirectImport, imports_by_module_to_py}; +use crate::errors::GrimpResult; +use crate::filesystem::FileSystem; +use crate::import_scanning::DirectImport; use crate::module_finding::{FoundPackage, Module}; use blake2::Blake2bVar; use blake2::digest::{Update, VariableOutput}; -use pyo3::types::PyAnyMethods; -use pyo3::types::{PyDict, PySet}; -use pyo3::types::{PyDictMethods, PySetMethods}; -use pyo3::{Borrowed, Bound, FromPyObject, PyAny, PyErr, PyResult, Python, pyfunction}; use std::collections::{HashMap, HashSet}; -/// Writes the cache file containing all the imports for a given package. -/// Args: -/// - filename: str -/// - imports_by_module: dict[Module, Set[DirectImport]] -/// - file_system: The file system interface to use. (A BasicFileSystem.) -#[pyfunction] -pub fn write_cache_data_map_file<'py>( - filename: &str, - imports_by_module: Bound<'py, PyDict>, - file_system: Bound<'py, PyAny>, -) -> PyResult<()> { - let mut file_system_boxed = get_file_system_boxed(&file_system)?; - - let ImportsByModule(imports_by_module_rust) = imports_by_module.extract()?; - - let file_contents = serialize_imports_by_module(&imports_by_module_rust); - - file_system_boxed.write(filename, &file_contents)?; - - Ok(()) -} - -/// Reads the cache file containing all the imports for a given package. -/// Args: -/// - filename: str -/// - file_system: The file system interface to use. (A BasicFileSystem.) -/// Returns Dict[Module, Set[DirectImport]] -#[pyfunction] -pub fn read_cache_data_map_file<'py>( - py: Python<'py>, - filename: &str, - file_system: Bound<'py, PyAny>, -) -> PyResult> { - let file_system_boxed = get_file_system_boxed(&file_system)?; - - let file_contents = file_system_boxed.read(filename)?; - - let imports_by_module = parse_json_to_map(&file_contents, filename)?; - - Ok(imports_by_module_to_py(py, imports_by_module)) -} - -/// A newtype wrapper for HashMap> that implements FromPyObject. -pub struct ImportsByModule(pub HashMap>); - -impl<'a, 'py> FromPyObject<'a, 'py> for ImportsByModule { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult { - let py_dict = ob.cast::()?; - let mut imports_by_module_rust = HashMap::new(); - - for (py_key, py_value) in py_dict.iter() { - let module: Module = py_key.extract()?; - let py_set = py_value.cast::()?; - let mut hashset: HashSet = HashSet::new(); - for element in py_set.iter() { - let direct_import: DirectImport = element.extract()?; - hashset.insert(direct_import); - } - imports_by_module_rust.insert(module, hashset); - } - - Ok(ImportsByModule(imports_by_module_rust)) - } -} - /// The version of the data cache file format. /// /// Bump this whenever the serialized structure changes. On read, a file with a @@ -115,24 +44,18 @@ fn serialize_imports_by_module( serde_json::to_string(&envelope).expect("Failed to serialize to JSON") } -pub fn parse_json_to_map( - json_str: &str, - filename: &str, -) -> GrimpResult>> { - // Parse into a generic value first, so we can distinguish genuinely corrupt - // JSON from a cache file written by a different format version (e.g. by an - // older Grimp). The latter should be silently rebuilt, not warned about. - let value: serde_json::Value = serde_json::from_str(json_str) - .map_err(|_| GrimpError::CorruptCache(filename.to_string()))?; +/// Parses the contents of a data file. +/// +/// Returns None if it's corrupt, or was written by a different version of Grimp. +fn parse_json_to_map(json_str: &str) -> Option>> { + let value: serde_json::Value = serde_json::from_str(json_str).ok()?; - let version = value.get("version").and_then(|v| v.as_u64()); - if version != Some(CACHE_SCHEMA_VERSION as u64) { - return Err(GrimpError::CacheVersionMismatch(filename.to_string())); + if value.get("version").and_then(|v| v.as_u64()) != Some(CACHE_SCHEMA_VERSION as u64) { + return None; } let raw_map: HashMap> = - serde_json::from_value(value.get("imports_by_module").cloned().unwrap_or_default()) - .map_err(|_| GrimpError::CorruptCache(filename.to_string()))?; + serde_json::from_value(value.get("imports_by_module").cloned().unwrap_or_default()).ok()?; let mut parsed_map: HashMap> = HashMap::new(); @@ -155,13 +78,13 @@ pub fn parse_json_to_map( parsed_map.insert(module, import_set); } - Ok(parsed_map) + Some(parsed_map) } /// A cache of the imports of each module, for building a graph. /// -/// The files written are the same as those written by the Python `grimp.adaptors.caching.Cache`, -/// so the two are interchangeable. +/// The file names and formats must not change without good reason, so that caches written by +/// earlier versions of Grimp can still be used. pub struct Cache<'a> { directory: &'a str, found_packages: &'a HashSet, @@ -223,7 +146,8 @@ impl<'a> Cache<'a> { let data_file = self.file_path(file_system, &self.data_file_name); let imports_by_module = file_system .read(&data_file) - .and_then(|contents| parse_json_to_map(&contents, &data_file)) + .ok() + .and_then(|contents| parse_json_to_map(&contents)) .unwrap_or_default(); imports_by_module diff --git a/rust/src/errors.rs b/rust/src/errors.rs index a39a2f83..081a54ce 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -34,12 +34,6 @@ pub enum GrimpError { parse_error: RuffParseError, }, - #[error("Could not use corrupt cache file {0}.")] - CorruptCache(String), - - #[error("Cache file {0} was written by a different version of Grimp.")] - CacheVersionMismatch(String), - #[error("{0}")] FileNotFound(String), @@ -66,10 +60,6 @@ impl From for PyErr { GrimpError::ParseError { line_number, text, .. } => PyErr::new::((line_number, text)), - GrimpError::CorruptCache(_) => exceptions::CorruptCache::new_err(value.to_string()), - GrimpError::CacheVersionMismatch(_) => { - exceptions::CacheVersionMismatch::new_err(value.to_string()) - } GrimpError::FileNotFound(_) => PyFileNotFoundError::new_err(value.to_string()), // Not UnicodeDecodeError, as that can't be created from just a message. GrimpError::UndecodableFile(_) => PyUnicodeError::new_err(value.to_string()), diff --git a/rust/src/exceptions.rs b/rust/src/exceptions.rs index f4824fdb..f25a78cd 100644 --- a/rust/src/exceptions.rs +++ b/rust/src/exceptions.rs @@ -5,8 +5,6 @@ use pyo3::prelude::*; create_exception!(_rustgrimp, ModuleNotPresent, PyException); create_exception!(_rustgrimp, NoSuchContainer, PyException); create_exception!(_rustgrimp, InvalidModuleExpression, PyException); -create_exception!(_rustgrimp, CorruptCache, PyException); -create_exception!(_rustgrimp, CacheVersionMismatch, PyException); // We need to use here `pyclass(extends=PyException)` instead of `create_exception!` // since the exception contains custom data. See: diff --git a/rust/src/filesystem.rs b/rust/src/filesystem.rs index baaf66c1..bc6b222f 100644 --- a/rust/src/filesystem.rs +++ b/rust/src/filesystem.rs @@ -1,16 +1,10 @@ use crate::errors::{GrimpError, GrimpResult}; -use itertools::Itertools; -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::*; use regex::Regex; -use std::collections::HashMap; -use std::ffi::OsStr; use std::fs; use std::fs::File; use std::io::prelude::*; use std::path::{Path, PathBuf}; -use std::sync::{Arc, LazyLock, Mutex}; -use unindent::unindent; +use std::sync::LazyLock; static ENCODING_RE: LazyLock = LazyLock::new(|| Regex::new(r"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)").unwrap()); @@ -58,8 +52,6 @@ fn lookup_encoding(name: &str) -> Option { } pub trait FileSystem: Send + Sync { - fn sep(&self) -> &str; - fn join(&self, components: Vec) -> String; fn split(&self, file_name: &str) -> (String, String); @@ -71,22 +63,10 @@ pub trait FileSystem: Send + Sync { fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()>; } -#[derive(Clone)] -#[pyclass(from_py_object)] -struct RealBasicFileSystem {} - -// Implements a BasicFileSystem (defined in grimp.application.ports.filesystem.BasicFileSystem) -// that actually reads files. -#[pyclass(name = "RealBasicFileSystem")] -pub struct PyRealBasicFileSystem { - inner: RealBasicFileSystem, -} - -impl FileSystem for RealBasicFileSystem { - fn sep(&self) -> &str { - std::path::MAIN_SEPARATOR_STR - } +/// A file system that actually reads and writes files. +struct RealFileSystem {} +impl FileSystem for RealFileSystem { fn join(&self, components: Vec) -> String { let mut path = PathBuf::new(); for component in components { @@ -193,283 +173,6 @@ impl FileSystem for RealBasicFileSystem { } } -#[pymethods] -impl PyRealBasicFileSystem { - #[new] - fn new() -> Self { - PyRealBasicFileSystem { - inner: RealBasicFileSystem {}, - } - } - - #[getter] - fn sep(&self) -> &str { - self.inner.sep() - } - - #[pyo3(signature = (*components))] - fn join(&self, components: Vec) -> String { - self.inner.join(components) - } - - fn split(&self, file_name: &str) -> (String, String) { - self.inner.split(file_name) - } - - fn exists(&self, file_name: &str) -> bool { - self.inner.exists(file_name) - } - - fn read(&self, file_name: &str) -> PyResult { - Ok(self.inner.read(file_name)?) - } - - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - Ok(self.inner.write(file_name, contents)?) - } -} - -type FileSystemContents = HashMap; - -#[derive(Clone)] -struct FakeBasicFileSystem { - contents: Arc>, -} - -// Implements BasicFileSystem (defined in grimp.application.ports.filesystem.BasicFileSystem). -#[pyclass(name = "FakeBasicFileSystem")] -pub struct PyFakeBasicFileSystem { - inner: FakeBasicFileSystem, -} - -impl FakeBasicFileSystem { - fn new(contents: Option<&str>, content_map: Option>) -> PyResult { - let mut parsed_contents = match contents { - Some(contents) => parse_indented_file_system_string(contents), - None => HashMap::new(), - }; - if let Some(content_map) = content_map { - let unindented_map: HashMap = content_map - .into_iter() - .map(|(key, val)| (key, unindent(&val).trim().to_string())) - .collect(); - parsed_contents.extend(unindented_map); - }; - Ok(FakeBasicFileSystem { - contents: Arc::new(Mutex::new(parsed_contents)), - }) - } -} - -impl FileSystem for FakeBasicFileSystem { - fn sep(&self) -> &str { - "/" - } - - fn join(&self, components: Vec) -> String { - let sep = self.sep(); - components - .into_iter() - .map(|c| c.trim_end_matches(sep).to_string()) - .join(sep) - } - - fn split(&self, file_name: &str) -> (String, String) { - let path; - let head; - let tail; - if let Some(file_name_without_trailing_slash) = file_name.strip_suffix("/") { - head = Path::new(file_name_without_trailing_slash); - tail = OsStr::new(""); - } else { - path = Path::new(file_name); - head = path.parent().unwrap_or(Path::new("")); - tail = path.file_name().unwrap_or(OsStr::new("")); - } - ( - head.to_str() - .expect("Path components should be valid unicode") - .to_string(), - tail.to_str() - .expect("Path components should be valid unicode") - .to_string(), - ) - } - - /// Checks if a file or directory exists within the file system. - fn exists(&self, file_name: &str) -> bool { - self.contents.lock().unwrap().contains_key(file_name) - } - - fn read(&self, file_name: &str) -> GrimpResult { - let contents = self.contents.lock().unwrap(); - match contents.get(file_name) { - Some(file_contents) => Ok(file_contents.clone()), - None => Err(GrimpError::FileNotFound(format!( - "No such file: {file_name}" - ))), - } - } - - #[allow(unused_variables)] - fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()> { - let mut contents_mut = self.contents.lock().unwrap(); - contents_mut.insert(file_name.to_string(), contents.to_string()); - Ok(()) - } -} - -#[pymethods] -impl PyFakeBasicFileSystem { - #[pyo3(signature = (contents=None, content_map=None))] - #[new] - fn new(contents: Option<&str>, content_map: Option>) -> PyResult { - Ok(PyFakeBasicFileSystem { - inner: FakeBasicFileSystem::new(contents, content_map)?, - }) - } - - #[getter] - fn sep(&self) -> &str { - self.inner.sep() - } - - #[pyo3(signature = (*components))] - fn join(&self, components: Vec) -> String { - self.inner.join(components) - } - - fn split(&self, file_name: &str) -> (String, String) { - self.inner.split(file_name) - } - - /// Checks if a file or directory exists within the file system. - fn exists(&self, file_name: &str) -> bool { - self.inner.exists(file_name) - } - - fn read(&self, file_name: &str) -> PyResult { - Ok(self.inner.read(file_name)?) - } - - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - Ok(self.inner.write(file_name, contents)?) - } - - // Temporary workaround method for Python tests. - fn convert_to_basic(&self) -> PyResult { - Ok(PyFakeBasicFileSystem { - inner: self.inner.clone(), - }) - } -} - -/// Parses an indented string representing a file system structure -/// into a HashMap where keys are full file paths. -/// See tests.adaptors.filesystem.FakeFileSystem for the API. -fn parse_indented_file_system_string(file_system_string: &str) -> HashMap { - let mut file_paths_map: HashMap = HashMap::new(); - let mut path_stack: Vec = Vec::new(); // Stores current directory path components - let mut first_line = true; // Flag to handle the very first path component - let mut first_line_indent: usize = 0; - // Normalize newlines and split into lines - let buffer = file_system_string.replace("\r\n", "\n"); - let lines: Vec<&str> = buffer.split('\n').collect(); - - let first_line_starts_with_slash = lines[0].trim().starts_with('/'); - for line_raw in lines { - let line = line_raw.trim_end(); // Remove trailing whitespace - if line.is_empty() { - continue; // Skip empty lines - } - let current_indent = - line.chars().take_while(|&c| c.is_whitespace()).count() - first_line_indent; - let trimmed_line = line.trim_start(); - - // Assuming 4 spaces per indentation level for calculating depth - let current_depth = current_indent / 4; - if first_line { - // The first non-empty line sets the base path. - // It might be absolute (/a/b/) or relative (a/b/). - let root_component = trimmed_line.trim_end_matches('/').to_string(); - path_stack.push(root_component); - first_line = false; - first_line_indent = current_indent; - } else { - // Adjust the path_stack based on indentation level - // Pop elements from the stack until we reach the correct parent directory depth - while path_stack.len() > current_depth { - path_stack.pop(); - } - // If the current line is a file, append it to the path for inserting into map, - // then pop it off so that subsequent siblings are correctly handled. - // If it's a directory, append it and it stays on the stack for its children. - let component_to_add = trimmed_line.trim_end_matches('/').to_string(); - if !component_to_add.is_empty() { - // Avoid pushing empty strings due to lines like just "/" - path_stack.push(component_to_add); - } - } - - // Construct the full path - // Join components on the stack. If the first component started with '/', - // ensure the final path also starts with '/'. - let full_path = if !path_stack.is_empty() { - let mut joined = path_stack.join("/"); - // If the original root started with a slash, ensure the final path does too. - // But be careful not to double-slash if a component is e.g. "/root" - if first_line_starts_with_slash && !joined.starts_with('/') { - joined = format!("/{joined}"); - } - joined - } else { - "".to_string() - }; - - // If it's a file (doesn't end with '/'), add it to the map - // A file is not a directory, so its name should be removed from the stack after processing - // so that sibling items are at the correct level. - if !trimmed_line.ends_with('/') { - file_paths_map.insert(full_path, String::new()); // Value can be empty or actual content - if !path_stack.is_empty() { - path_stack.pop(); // Pop the file name off the stack - } - } - } - - // Edge case: If the very first line was a file and it ended up on the stack, it needs to be processed. - // This handles single-file inputs like "myfile.txt" - if !path_stack.is_empty() - && !path_stack - .last() - .expect("path_stack should be non-empty") - .ends_with('/') - && !file_paths_map.contains_key(&path_stack.join("/")) - { - file_paths_map.insert(path_stack.join("/"), String::new()); - } - - file_paths_map -} - pub fn real_file_system() -> Box { - Box::new(RealBasicFileSystem {}) -} - -#[allow(clippy::borrowed_box)] -pub fn get_file_system_boxed<'py>( - file_system: &Bound<'py, PyAny>, -) -> PyResult> { - let file_system_boxed: Box; - if let Ok(py_real) = file_system.extract::>() { - file_system_boxed = Box::new(py_real.inner.clone()); - } else if let Ok(py_fake) = file_system.extract::>() { - file_system_boxed = Box::new(py_fake.inner.clone()); - } else { - return Err(PyTypeError::new_err( - "file_system must be an instance of RealBasicFileSystem or FakeBasicFileSystem", - )); - } - - Ok(file_system_boxed) + Box::new(RealFileSystem {}) } diff --git a/rust/src/graph_building.rs b/rust/src/graph_building.rs index 255a2a7e..7748c06e 100644 --- a/rust/src/graph_building.rs +++ b/rust/src/graph_building.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; /// Builds an import graph of the supplied packages. /// -/// This is the Rust equivalent of `grimp.application.usecases.build_graph`. +/// This is called by `grimp.application.usecases.build_graph`. /// /// Args: /// - packages: (name, directory) of each package. A package may have more than one directory. diff --git a/rust/src/import_scanning.rs b/rust/src/import_scanning.rs index 3ee50512..6ceb0b39 100644 --- a/rust/src/import_scanning.rs +++ b/rust/src/import_scanning.rs @@ -1,10 +1,9 @@ use crate::errors::{GrimpError, GrimpResult}; -use crate::filesystem::{FileSystem, get_file_system_boxed}; +use crate::filesystem::FileSystem; +use crate::import_parsing; use crate::module_finding::{FoundPackage, Module}; -use crate::{import_parsing, module_finding}; use itertools::Itertools; use pyo3::prelude::*; -use pyo3::types::{PyDict, PySet}; /// Statically analyses some Python modules for import statements within their shared package. use rayon::prelude::*; use std::collections::{HashMap, HashSet}; @@ -19,44 +18,6 @@ pub struct DirectImport { pub is_lazy: bool, } -impl<'a, 'py> FromPyObject<'a, 'py> for DirectImport { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult { - let importer: String = ob.getattr("importer")?.getattr("name")?.extract()?; - let imported: String = ob.getattr("imported")?.getattr("name")?.extract()?; - let line_number: usize = ob.getattr("line_number")?.extract()?; - let line_contents: String = ob.getattr("line_contents")?.extract()?; - let is_lazy: bool = ob.getattr("is_lazy")?.extract()?; - - Ok(DirectImport { - importer, - imported, - line_number, - line_contents, - is_lazy, - }) - } -} - -fn py_found_packages_to_rust(py_found_packages: &Bound<'_, PyAny>) -> HashSet { - let py_set = py_found_packages - .cast::() - .expect("Expected py_found_packages to be a Python set."); - - let mut rust_found_packages = HashSet::new(); - // Iterate over the elements in the Python set. - for py_found_package_any in py_set.iter() { - // Extract each Python 'FoundPackage' object into a Rust 'FoundPackage' struct. - // Panics if extraction fails, as specified. - let rust_found_package: FoundPackage = py_found_package_any - .extract() - .expect("Failed to extract Python FoundPackage to Rust FoundPackage."); - rust_found_packages.insert(rust_found_package); - } - rust_found_packages -} - fn get_modules_from_found_packages(found_packages: &HashSet) -> HashSet { let mut modules = HashSet::new(); for package in found_packages { @@ -179,36 +140,6 @@ fn scan_for_imports_no_py_single_module( Ok(imports) } -fn to_py_direct_imports<'a>( - py: Python<'a>, - rust_imports: &HashSet, -) -> Bound<'a, PySet> { - let valueobjects_pymodule = PyModule::import(py, "grimp.domain.valueobjects").unwrap(); - let py_module_class = valueobjects_pymodule.getattr("Module").unwrap(); - let py_direct_import_class = valueobjects_pymodule.getattr("DirectImport").unwrap(); - - let pyset = PySet::empty(py).unwrap(); - - for rust_import in rust_imports { - let importer = py_module_class.call1((&rust_import.importer,)).unwrap(); - let imported = py_module_class.call1((&rust_import.imported,)).unwrap(); - let kwargs = PyDict::new(py); - kwargs.set_item("importer", &importer).unwrap(); - kwargs.set_item("imported", &imported).unwrap(); - kwargs - .set_item("line_number", rust_import.line_number) - .unwrap(); - kwargs - .set_item("line_contents", &rust_import.line_contents) - .unwrap(); - kwargs.set_item("is_lazy", rust_import.is_lazy).unwrap(); - let py_direct_import = py_direct_import_class.call((), Some(&kwargs)).unwrap(); - pyset.add(&py_direct_import).unwrap(); - } - - pyset -} - #[allow(clippy::borrowed_box)] fn _determine_module_filename( module: &Module, @@ -365,76 +296,6 @@ fn _distill_external_module( } } -/// Convert the rust data structure into a Python dict[Module, set[DirectImport]]. -pub fn imports_by_module_to_py( - py: Python, - imports_by_module: HashMap>, -) -> Bound { - let valueobjects_pymodule = PyModule::import(py, "grimp.domain.valueobjects").unwrap(); - let py_module_class = valueobjects_pymodule.getattr("Module").unwrap(); - - let imports_by_module_py = PyDict::new(py); - for (module, imports) in imports_by_module.iter() { - let py_module_instance = py_module_class.call1((module.name.clone(),)).unwrap(); - let py_imports = to_py_direct_imports(py, imports); - imports_by_module_py - .set_item(py_module_instance, py_imports) - .unwrap(); - } - imports_by_module_py -} - -/// Statically analyses the given module and returns a set of Modules that -/// it imports. -/// Python args: -/// -/// - module_files The modules to scan. -/// - found_packages: Set of FoundPackages containing all the modules -/// for analysis. -/// - include_external_packages: Whether to include imports of external modules (i.e. -/// modules not contained in modules_by_package_directory) -/// in the results. -/// - exclude_type_checking_imports: If True, don't include imports behind TYPE_CHECKING guards. -/// - file_system: The file system interface to use. (A BasicFileSystem.) -/// -/// Returns dict[Module, set[DirectImport]]. -#[pyfunction] -pub fn scan_for_imports<'py>( - py: Python<'py>, - module_files: Vec>, - found_packages: Bound<'py, PyAny>, - include_external_packages: bool, - exclude_type_checking_imports: bool, - file_system: Bound<'py, PyAny>, -) -> PyResult> { - let file_system_boxed = get_file_system_boxed(&file_system)?; - let found_packages_rust = py_found_packages_to_rust(&found_packages); - let modules_rust: HashSet = module_files - .iter() - .map(|module_file| { - module_file - .getattr("module") - .unwrap() - .extract::() - .unwrap() - }) - .collect(); - - let imports_by_module = py - .detach(|| { - scan_for_imports_no_py( - &file_system_boxed, - &found_packages_rust, - include_external_packages, - &modules_rust, - exclude_type_checking_imports, - ) - }) - .map_err(|e| into_py_err(py, e))?; - - Ok(imports_by_module_to_py(py, imports_by_module)) -} - /// Convert a `GrimpError` to a Python exception, translating syntax errors in the scanned /// source code to `grimp.exceptions.SourceSyntaxError`. pub fn into_py_err(py: Python, error: GrimpError) -> PyErr { diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f98cffdd..0f5142e3 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -13,27 +13,14 @@ use pyo3::prelude::*; #[pymodule] mod _rustgrimp { - #[pymodule_export] - use crate::import_scanning::scan_for_imports; - - #[pymodule_export] - use crate::caching::read_cache_data_map_file; - - #[pymodule_export] - use crate::caching::write_cache_data_map_file; - #[pymodule_export] use crate::graph::GraphWrapper; #[pymodule_export] use crate::graph_building::build_graph; - #[pymodule_export] - use crate::filesystem::{PyFakeBasicFileSystem, PyRealBasicFileSystem}; - #[pymodule_export] use crate::exceptions::{ - CacheVersionMismatch, CorruptCache, InvalidModuleExpression, ModuleNotPresent, - NoSuchContainer, ParseError, + InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError, }; } diff --git a/rust/src/module_finding.rs b/rust/src/module_finding.rs index c909ef41..fc54bd7f 100644 --- a/rust/src/module_finding.rs +++ b/rust/src/module_finding.rs @@ -1,19 +1,18 @@ use crate::errors::GrimpResult; -use pyo3::{prelude::*, types::PyFrozenSet}; use std::collections::{BTreeSet, HashMap}; use std::fmt; use std::fs; use std::path::{Path, PathBuf}; use std::time::UNIX_EPOCH; -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, FromPyObject)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct ModuleFile { pub module: Module, } /// A Python module. /// The string is the importable name, e.g. "mypackage.foo". -#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, FromPyObject)] +#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash)] pub struct Module { pub name: String, } @@ -34,59 +33,11 @@ pub struct FoundPackage { pub namespace_packages: BTreeSet, } -/// Implements conversion from a Python 'FoundPackage' object to the Rust 'FoundPackage' struct. -/// It extracts 'name', 'directory', and converts the 'module_files' frozenset into a Rust BTreeSet. -impl<'a, 'py> FromPyObject<'a, 'py> for FoundPackage { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult { - let name: String = ob.getattr("name")?.extract()?; - // Extract the 'directory' attribute. - let directory: String = ob.getattr("directory")?.extract()?; - - // Access the 'module_files' attribute. - let module_files_py = ob.getattr("module_files")?; - // Downcast the PyAny object to a PyFrozenSet, as Python 'FrozenSet' maps to 'PyFrozenSet'. - let module_files_frozenset = module_files_py.cast::()?; - - let mut module_files = BTreeSet::new(); - // Iterate over the Python frozenset. - for py_module_file_any in module_files_frozenset.iter() { - // Extract each element (PyAny) into a Rust 'ModuleFile'. - let module_file: ModuleFile = py_module_file_any.extract()?; - module_files.insert(module_file); - } - - // Access the 'namespace_packages' attribute. - let namespace_packages_py = ob.getattr("namespace_packages")?; - // Downcast the PyAny object to a PyFrozenSet, as Python 'FrozenSet' maps to 'PyFrozenSet'. - let namespace_packages_frozenset = namespace_packages_py.cast::()?; - - let mut namespace_packages = BTreeSet::new(); - // Iterate over the Python frozenset. - for py_namespace_any in namespace_packages_frozenset.iter() { - // Extract each element (PyAny) into a Rust 'ModuleFile'. - let namespace_package: String = py_namespace_any.extract()?; - namespace_packages.insert(Module { - name: namespace_package, - }); - } - - Ok(FoundPackage { - name, - directory, - module_files, - namespace_packages, - }) - } -} - /// Finds the modules and namespace packages inside a package directory. /// -/// This is a port of `grimp.adaptors.modulefinder.ModuleFinder`. -/// /// Also returns the mtime of each module's file (keyed by module name), in the same form as -/// Python's `os.path.getmtime`, so that it's interchangeable with the Python cache. +/// Python's `os.path.getmtime`, so that caches written by earlier versions of Grimp (which used +/// that) can still be used. pub fn find_package( name: &str, directory: &str, diff --git a/src/grimp/__init__.py b/src/grimp/__init__.py index c7685e91..1e43bb16 100644 --- a/src/grimp/__init__.py +++ b/src/grimp/__init__.py @@ -3,7 +3,7 @@ from .application.graph import DetailedImport, Import, ImportGraph from .domain.analysis import PackageDependency, Route from .domain.valueobjects import DirectImport, Layer, Module -from .main import build_graph, build_graph_rust +from .main import build_graph __all__ = [ "DetailedImport", @@ -15,5 +15,4 @@ "PackageDependency", "Route", "build_graph", - "build_graph_rust", ] diff --git a/src/grimp/adaptors/caching.py b/src/grimp/adaptors/caching.py deleted file mode 100644 index 84d96057..00000000 --- a/src/grimp/adaptors/caching.py +++ /dev/null @@ -1,235 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import logging -from typing import Any - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import BasicFileSystem -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - -from ..application.ports.caching import Cache as AbstractCache -from ..application.ports.caching import CacheMiss - -logger = logging.getLogger(__name__) -PrimitiveFormat = dict[str, list[tuple[str, int | None, str]]] - - -class CacheFileNamer: - @classmethod - def make_meta_file_name(cls, found_package: FoundPackage) -> str: - return f"{found_package.name}.meta.json" - - @classmethod - def make_data_file_name( - cls, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, - ) -> str: - identifier = cls.make_data_file_unique_string( - found_packages, include_external_packages, exclude_type_checking_imports - ) - - bytes_identifier = identifier.encode() - # Use a hash algorithm with a limited size to avoid cache filenames that are too long - # the filesystem, which can happen if there are more than a few root packages - # being analyzed. - safe_unicode_identifier = hashlib.blake2b( - bytes_identifier, digest_size=20, usedforsecurity=False - ).hexdigest() - return f"{safe_unicode_identifier}.data.json" - - @classmethod - def make_data_file_unique_string( - cls, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, - ) -> str: - """ - Construct a unique string that identifies the analysis parameters. - - Doesn't need to be safe to use for a filename. - """ - package_names = (p.name for p in found_packages) - csv_packages = ",".join(sorted(package_names)) - include_external_packages_option = ":external" if include_external_packages else "" - exclude_type_checking_imports_option = ( - ":no_type_checking" if exclude_type_checking_imports else "" - ) - return ( - csv_packages + include_external_packages_option + exclude_type_checking_imports_option - ) - - -class Cache(AbstractCache): - DEFAULT_CACHE_DIR = ".grimp_cache" - - def __init__(self, *args: Any, namer: type[CacheFileNamer], **kwargs: Any) -> None: - """ - Don't instantiate Cache directly; use Cache.setup(). - """ - super().__init__(*args, **kwargs) - self._mtime_map: dict[str, float] = {} - self._data_map: dict[Module, set[DirectImport]] = {} - self._namer = namer - - @classmethod - def setup( - cls, - file_system: BasicFileSystem, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool = False, - cache_dir: str | None = None, - namer: type[CacheFileNamer] = CacheFileNamer, - ) -> Cache: - cache = cls( - file_system=file_system, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cls.cache_dir_or_default(cache_dir), - namer=namer, - ) - cache._build_mtime_map() - cache._build_data_map() - assert cache.cache_dir - return cache - - @classmethod - def cache_dir_or_default(cls, cache_dir: str | None) -> str: - return cache_dir or cls.DEFAULT_CACHE_DIR - - def read_imports(self, module_file: ModuleFile) -> set[DirectImport]: - try: - cached_mtime = self._mtime_map[module_file.module.name] - except KeyError as exc: - raise CacheMiss from exc - if cached_mtime != module_file.mtime: - raise CacheMiss - - try: - return self._data_map[module_file.module] - except KeyError as exc: - # While we would expect the module to be in here, - # there's no point in crashing if, for some reason, it's not. - raise CacheMiss from exc - - def write( - self, - imports_by_module: dict[Module, set[DirectImport]], - ) -> None: - self._write_marker_files_if_not_already_there() - # Write data file. - data_cache_filename = self.file_system.join( - self.cache_dir, - self._namer.make_data_file_name( - found_packages=self.found_packages, - include_external_packages=self.include_external_packages, - exclude_type_checking_imports=self.exclude_type_checking_imports, - ), - ) - rust.write_cache_data_map_file( - filename=data_cache_filename, - imports_by_module=imports_by_module, - file_system=self.file_system, - ) - - logger.info(f"Wrote data cache file {data_cache_filename}.") - - # Write meta files. - for found_package in self.found_packages: - meta_filename = self.file_system.join( - self.cache_dir, self._namer.make_meta_file_name(found_package) - ) - mtime_map = { - module_file.module.name: module_file.mtime - for module_file in found_package.module_files - } - serialized_meta = json.dumps(mtime_map) - self.file_system.write(meta_filename, serialized_meta) - logger.info(f"Wrote meta cache file {meta_filename}.") - - def _build_mtime_map(self) -> None: - self._mtime_map = self._read_mtime_map_files() - - def _read_mtime_map_files(self) -> dict[str, float]: - all_mtimes: dict[str, float] = {} - for found_package in self.found_packages: - all_mtimes.update(self._read_mtime_map_file(found_package)) - return all_mtimes - - def _read_mtime_map_file(self, found_package: FoundPackage) -> dict[str, float]: - meta_cache_filename = self.file_system.join( - self.cache_dir, self._namer.make_meta_file_name(found_package) - ) - try: - serialized = self.file_system.read(meta_cache_filename) - except FileNotFoundError: - logger.info(f"No cache file: {meta_cache_filename}.") - return {} - try: - deserialized = json.loads(serialized) - except json.JSONDecodeError: - logger.warning(f"Could not use corrupt cache file {meta_cache_filename}.") - return {} - - logger.info(f"Used cache meta file {meta_cache_filename}.") - return deserialized - - def _build_data_map(self) -> None: - self._data_map = self._read_data_map_file() - - def _read_data_map_file(self) -> dict[Module, set[DirectImport]]: - data_cache_filename = self.file_system.join( - self.cache_dir, - self._namer.make_data_file_name( - found_packages=self.found_packages, - include_external_packages=self.include_external_packages, - exclude_type_checking_imports=self.exclude_type_checking_imports, - ), - ) - try: - imports_by_module = rust.read_cache_data_map_file( - data_cache_filename, self.file_system - ) - except FileNotFoundError: - logger.info(f"No cache file: {data_cache_filename}.") - return {} - except rust.CorruptCache: - logger.warning(f"Could not use corrupt cache file {data_cache_filename}.") - return {} - except rust.CacheVersionMismatch: - logger.info( - f"Cache file {data_cache_filename} was written by a different version of Grimp; " - "rebuilding." - ) - return {} - - logger.info(f"Used cache data file {data_cache_filename}.") - return imports_by_module - - def _build_data_cache_filename(self, found_package: FoundPackage) -> str: - return self.file_system.join(self.cache_dir, f"{found_package.name}.data.json") - - def _write_marker_files_if_not_already_there(self) -> None: - marker_files_info = ( - (".gitignore", "# Automatically created by Grimp.\n*"), - ( - "CACHEDIR.TAG", - ( - "Signature: 8a477f597d28d172789f06886806bc55\n" - "# This file is a cache directory tag automatically created by Grimp.\n" - "# For information about cache directory tags see https://bford.info/cachedir/" - ), - ), - ) - - for filename, contents in marker_files_info: - full_filename = self.file_system.join(self.cache_dir, filename) - if not self.file_system.exists(full_filename): - self.file_system.write(full_filename, contents) diff --git a/src/grimp/adaptors/filesystem.py b/src/grimp/adaptors/filesystem.py deleted file mode 100644 index 0a396cee..00000000 --- a/src/grimp/adaptors/filesystem.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import os -import tokenize -from collections.abc import Iterator - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import AbstractFileSystem, BasicFileSystem - - -class FileSystem(AbstractFileSystem): - """ - Abstraction around file system calls. - """ - - @property - def sep(self) -> str: - return os.sep - - def dirname(self, filename: str) -> str: - return os.path.dirname(filename) - - def walk(self, directory_name: str) -> Iterator[tuple[str, list[str], list[str]]]: - yield from os.walk(directory_name, followlinks=True) - - def join(self, *components: str) -> str: - return os.path.join(*components) - - def split(self, file_name: str) -> tuple[str, str]: - return os.path.split(file_name) - - def read(self, file_name: str) -> str: - # Use tokenize.open to give us a better chance of successfully decoding - # source code in a non-ascii compatible encoding. - with tokenize.open(file_name) as file: - return file.read() - - def exists(self, file_name: str) -> bool: - return os.path.isfile(file_name) - - def get_mtime(self, file_name: str) -> float: - return os.path.getmtime(file_name) - - def write(self, file_name: str, contents: str) -> None: - dirname = os.path.dirname(file_name) - if not os.path.exists(dirname): - os.makedirs(dirname) - with open(file_name, "w") as file: - print(contents, file=file) - - def convert_to_basic(self) -> BasicFileSystem: - return rust.RealBasicFileSystem() diff --git a/src/grimp/adaptors/modulefinder.py b/src/grimp/adaptors/modulefinder.py deleted file mode 100644 index 00f77374..00000000 --- a/src/grimp/adaptors/modulefinder.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Iterable, Set - -from grimp.application.ports import modulefinder -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.domain.valueobjects import Module - -logger = logging.getLogger(__name__) - - -class ModuleFinder(modulefinder.AbstractModuleFinder): - def find_package( - self, package_name: str, package_directory: str, file_system: AbstractFileSystem - ) -> modulefinder.FoundPackage: - self.file_system = file_system - - module_files: list[modulefinder.ModuleFile] = [] - - python_files, namespace_dirs = self._get_python_files_and_namespace_dirs_inside_package( - package_directory - ) - for module_filename in python_files: - module_name = self._module_name_from_filename( - package_name, module_filename, package_directory - ) - module_mtime = self.file_system.get_mtime(module_filename) - module_files.append( - modulefinder.ModuleFile(module=Module(module_name), mtime=module_mtime) - ) - - namespace_packages = frozenset( - { - self._namespace_from_dir(package_name, namespace_dir, package_directory) - for namespace_dir in namespace_dirs - } - ) - - return modulefinder.FoundPackage( - name=package_name, - directory=package_directory, - module_files=frozenset(module_files), - namespace_packages=namespace_packages, - ) - - def _get_python_files_and_namespace_dirs_inside_package( - self, directory: str - ) -> tuple[Iterable[str], Set[str]]: - """ - Search the supplied package directory for Python files and namespaces. - - Return tuple consisting of: - 1. Iterable of Python file names. - 2. Set of namespace directories encountered. - """ - python_files: list[str] = [] - candidate_namespace_dirs: list[str] = [] - portion_dirs: set[str] = set() - - for dirpath, dirs, files in self.file_system.walk(directory): - if self._is_in_portion(dirpath, portion_dirs): - # Are we somewhere inside a non-namespace package? - if "__init__.py" not in files: - # Don't drill down further in this directory. - # (This means we won't include 'orphans' - Python packages deeply nested - # in a package that has already included __init__.py files. - for d in list(dirs): - dirs.remove(d) - continue - elif "__init__.py" in files: - # This directory is a portion (i.e. it has a top-level __init__.py). - portion_dirs.add(dirpath) - else: - # We don't yet know whether this is a namespace dir. It'll only be one if we find - # a Python file somewhere within it. - candidate_namespace_dirs.append(dirpath) - - # Don't include directories that aren't valid identifiers. - dirs_to_remove = [d for d in dirs if self._should_ignore_dir(d)] - for d in dirs_to_remove: - dirs.remove(d) - - python_files.extend( - self.file_system.join(dirpath, filename) - for filename in files - if self._is_python_file(filename, dirpath) - ) - - namespace_dirs = self._determine_namespace_dirs(candidate_namespace_dirs, python_files) - return python_files, namespace_dirs - - def _is_in_portion(self, directory: str, portions: Set[str]) -> bool: - return any(directory.startswith(portion) for portion in portions) - - def _should_ignore_dir(self, directory: str) -> bool: - # TODO: make this configurable. - return not directory.isidentifier() - - def _determine_namespace_dirs( - self, candidates: Iterable[str], python_files: Iterable[str] - ) -> set[str]: - namespace_dirs: set[str] = set() - for candidate in candidates: - candidate_with_trailing_sep = candidate + self.file_system.sep - for python_file in python_files: - if python_file.startswith(candidate_with_trailing_sep): - namespace_dirs.add(candidate) - break - return namespace_dirs - - def _is_python_file(self, filename: str, dirpath: str) -> bool: - """ - Given a filename, return whether it's a Python file. - - Files with extra dots in the name won't be treated as Python files. - - Args: - filename: the filename, excluding the path. - dirpath: the directory path containing the file. - - Returns: - Whether it's a Python file. - """ - # Ignore hidden files. - if filename.startswith("."): - return False - - if not filename.endswith(".py"): - return False - - # Ignore files like some.module.py. - if filename.count(".") > 1: - logger.warning( - "Warning: skipping module with too many dots in the name: " - f"{dirpath}{self.file_system.sep}{filename}" - ) - return False - - return True - - def _module_name_from_filename( - self, package_name: str, filename_and_path: str, package_directory: str - ) -> str: - """ - Args: - package_name: the importable name of the top level package. Could - be namespaced. - filename_and_path: the full name of the Python file. - package_directory: the full path of the top level Python package directory. - - Returns: - Absolute module name for importing. - """ - internal_filename_and_path = filename_and_path[len(package_directory) :] - internal_filename_and_path_without_extension = internal_filename_and_path[1:-3] - components = [ - package_name, - *internal_filename_and_path_without_extension.split(self.file_system.sep), - ] - if components[-1] == "__init__": - components.pop() - return ".".join(components) - - def _namespace_from_dir( - self, package_name: str, namespace_dir: str, package_directory: str - ) -> str: - """ - Args: - package_name: the importable name of the top level package. Could - be namespaced. - namespace_dir: the full name of the namespace directory. - package_directory: the full path of the top level Python package directory. - - Returns: - Absolute module name for importing (string). - """ - parent_of_package_directory = package_directory[: -len(package_name)] - directory_relative_to_parent = namespace_dir[len(parent_of_package_directory) :] - components = directory_relative_to_parent.split(self.file_system.sep) - return ".".join(components) diff --git a/src/grimp/adaptors/packagefinder.py b/src/grimp/adaptors/packagefinder.py index 7da528dd..8f900078 100644 --- a/src/grimp/adaptors/packagefinder.py +++ b/src/grimp/adaptors/packagefinder.py @@ -2,11 +2,11 @@ import importlib.util import logging +import os import sys from importlib.machinery import ModuleSpec from grimp import exceptions -from grimp.application.ports.filesystem import AbstractFileSystem from grimp.application.ports.packagefinder import AbstractPackageFinder from grimp.domain.valueobjects import Module @@ -14,9 +14,7 @@ class ImportLibPackageFinder(AbstractPackageFinder): - def determine_package_directories( - self, package_name: str, file_system: AbstractFileSystem - ) -> set[str]: + def determine_package_directories(self, package_name: str) -> set[str]: # Attempt to locate the package file. spec = importlib.util.find_spec(package_name) if not spec: @@ -27,19 +25,16 @@ def determine_package_directories( if ( spec.has_location and spec.origin - and ( - not self._is_a_package(spec, file_system) or self._has_a_non_namespace_parent(spec) - ) + and (not self._is_a_package(spec) or self._has_a_non_namespace_parent(spec)) ): raise exceptions.NotATopLevelModule assert spec.submodule_search_locations # This should be the case if spec.has_location. return set(spec.submodule_search_locations) - def _is_a_package(self, spec: ModuleSpec, file_system: AbstractFileSystem) -> bool: + def _is_a_package(self, spec: ModuleSpec) -> bool: assert spec.origin - filename = file_system.split(spec.origin)[1] - return filename == "__init__.py" + return os.path.basename(spec.origin) == "__init__.py" def _has_a_non_namespace_parent(self, spec: ModuleSpec) -> bool: module = Module(spec.name) diff --git a/src/grimp/application/config.py b/src/grimp/application/config.py index 6bec409e..61eacedd 100644 --- a/src/grimp/application/config.py +++ b/src/grimp/application/config.py @@ -15,10 +15,5 @@ def __getattr__(self, name: str) -> Any: return self._config[name] return super().__getattribute__(name) - def copy(self) -> Settings: - new_instance = self.__class__() - new_instance.configure(**self._config) - return new_instance - settings = Settings() diff --git a/src/grimp/application/ports/caching.py b/src/grimp/application/ports/caching.py deleted file mode 100644 index 46f10997..00000000 --- a/src/grimp/application/ports/caching.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - -from .filesystem import BasicFileSystem - - -class CacheMiss(Exception): - pass - - -class Cache: - def __init__( - self, - file_system: BasicFileSystem, - include_external_packages: bool, - exclude_type_checking_imports: bool, - found_packages: set[FoundPackage], - cache_dir: str, - ) -> None: - """ - Don't instantiate Cache directly; use Cache.setup(). - """ - self.file_system = file_system - self.found_packages = found_packages - self.include_external_packages = include_external_packages - self.exclude_type_checking_imports = exclude_type_checking_imports - self.cache_dir = cache_dir - - @classmethod - def setup( - cls, - file_system: BasicFileSystem, - found_packages: set[FoundPackage], - *, - include_external_packages: bool, - exclude_type_checking_imports: bool = False, - cache_dir: str | None = None, - ) -> Cache: - cache = cls( - file_system=file_system, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cls.cache_dir_or_default(cache_dir), - ) - return cache - - def read_imports(self, module_file: ModuleFile) -> set[DirectImport]: - raise NotImplementedError - - def write( - self, - imports_by_module: dict[Module, set[DirectImport]], - ) -> None: - raise NotImplementedError - - @classmethod - def cache_dir_or_default(cls, cache_dir: str | None) -> str: - raise NotImplementedError diff --git a/src/grimp/application/ports/filesystem.py b/src/grimp/application/ports/filesystem.py deleted file mode 100644 index 92e5504c..00000000 --- a/src/grimp/application/ports/filesystem.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import abc -from collections.abc import Iterator -from typing import Protocol - - -class AbstractFileSystem(abc.ABC): - """ - Abstraction around file system calls. - """ - - @property - @abc.abstractmethod - def sep(self) -> str: - r""" - Return the file separator for the FileSystem. - - E.G. '/' for UNIX systems and '\\' for Windows systems - """ - - @abc.abstractmethod - def dirname(self, filename: str) -> str: - """ - Return the full path to the directory name of the supplied filename. - - E.g. '/path/to/filename.py' will return '/path/to'. - """ - raise NotImplementedError - - @abc.abstractmethod - def walk(self, directory_name: str) -> Iterator[tuple[str, list[str], list[str]]]: - """ - Given a directory, walk the file system recursively. - - For each directory in the tree rooted at directory top (including top itself), - it yields a 3-tuple (dirpath, dirnames, filenames). - """ - raise NotImplementedError - - @abc.abstractmethod - def join(self, *components: str) -> str: - raise NotImplementedError - - @abc.abstractmethod - def split(self, file_name: str) -> tuple[str, str]: - """ - Split the pathname path into a pair, (head, tail) where tail is the last pathname component - and head is everything leading up to that. The tail part will never contain a slash; - if path ends in a slash, tail will be empty. If there is no slash in path, head will be - empty. If path is empty, both head and tail are empty. Trailing slashes are stripped from - head unless it is the root (one or more slashes only). In all cases, join(head, tail) - returns a path to the same location as path (but the strings may differ). - """ - raise NotImplementedError - - @abc.abstractmethod - def read(self, file_name: str) -> str: - """ - Given a file name, return the contents of the file. - """ - raise NotImplementedError - - @abc.abstractmethod - def exists(self, file_name: str) -> bool: - """ - Return whether a file exists. - """ - raise NotImplementedError - - @abc.abstractmethod - def get_mtime(self, file_name: str) -> float: - """ - Return the mtime of a file. - - Raises FileNotFoundError if the file does not exist. - """ - raise NotImplementedError - - @abc.abstractmethod - def write(self, file_name: str, contents: str) -> None: - """ - Write the contents to a file. - """ - raise NotImplementedError - - @abc.abstractmethod - def convert_to_basic(self) -> BasicFileSystem: - """ - Convert this file system to a BasicFileSystem. - """ - raise NotImplementedError - - -class BasicFileSystem(Protocol): - """ - A more limited file system. - - Having two different file system APIs is an interim approach, allowing us to - implement BasicFileSystem in Rust without needing to implement the full range - of methods used by other parts of Grimp. - - Eventually we'll move the full implementation to Rust, and have a single - FileSystem interface. - """ - - def join(self, *components: str) -> str: ... - - def split(self, file_name: str) -> tuple[str, str]: ... - - def read(self, file_name: str) -> str: ... - - def write(self, file_name: str, contents: str) -> None: ... - - def exists(self, file_name: str) -> bool: ... diff --git a/src/grimp/application/ports/modulefinder.py b/src/grimp/application/ports/modulefinder.py deleted file mode 100644 index 1d59f936..00000000 --- a/src/grimp/application/ports/modulefinder.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import abc -from collections.abc import Set -from dataclasses import dataclass - -from grimp.domain.valueobjects import Module - -from .filesystem import AbstractFileSystem - - -@dataclass(frozen=True, order=True) -class ModuleFile: - module: Module - mtime: float - - -@dataclass(frozen=True, order=True) -class FoundPackage: - """ - Set of modules found under a single package, together with metadata. - """ - - name: str - directory: str - module_files: Set[ModuleFile] - namespace_packages: Set[str] = frozenset() - - -class AbstractModuleFinder(abc.ABC): - """ - Finds Python modules inside a package. - """ - - @abc.abstractmethod - def find_package( - self, package_name: str, package_directory: str, file_system: AbstractFileSystem - ) -> FoundPackage: - """ - Searches the package for all importable Python modules. - """ - raise NotImplementedError diff --git a/src/grimp/application/ports/packagefinder.py b/src/grimp/application/ports/packagefinder.py index 317f5318..09478f47 100644 --- a/src/grimp/application/ports/packagefinder.py +++ b/src/grimp/application/ports/packagefinder.py @@ -2,12 +2,8 @@ import abc -from .filesystem import AbstractFileSystem - class AbstractPackageFinder(abc.ABC): @abc.abstractmethod - def determine_package_directories( - self, package_name: str, file_system: AbstractFileSystem - ) -> set[str]: + def determine_package_directories(self, package_name: str) -> set[str]: raise NotImplementedError diff --git a/src/grimp/application/scanning.py b/src/grimp/application/scanning.py deleted file mode 100644 index 47b85506..00000000 --- a/src/grimp/application/scanning.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from collections.abc import Collection - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.config import settings -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - - -def scan_imports( - module_files: Collection[ModuleFile], - *, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, -) -> dict[ModuleFile, set[DirectImport]]: - file_system: AbstractFileSystem = settings.FILE_SYSTEM - basic_file_system = file_system.convert_to_basic() - imports_by_module: dict[Module, set[DirectImport]] = rust.scan_for_imports( - module_files=tuple(module_files), - found_packages=found_packages, - # Ensure that the passed exclude_type_checking_imports is definitely a boolean, - # otherwise the Rust class will error. - include_external_packages=bool(include_external_packages), - exclude_type_checking_imports=exclude_type_checking_imports, - file_system=basic_file_system, - ) - return {module_file: imports_by_module[module_file.module] for module_file in module_files} diff --git a/src/grimp/application/usecases.py b/src/grimp/application/usecases.py index 2fafa3b6..aef04930 100644 --- a/src/grimp/application/usecases.py +++ b/src/grimp/application/usecases.py @@ -4,20 +4,15 @@ from __future__ import annotations -import itertools -from collections.abc import Iterable, Sequence +from collections.abc import Sequence from typing import cast from grimp import _rustgrimp as rust # type: ignore[attr-defined] from ..application.graph import ImportGraph -from ..application.ports import caching -from ..application.ports.filesystem import AbstractFileSystem, BasicFileSystem -from ..application.ports.modulefinder import AbstractModuleFinder, FoundPackage, ModuleFile -from ..application.ports.packagefinder import AbstractPackageFinder -from ..domain.valueobjects import DirectImport, Module from .config import settings -from .scanning import scan_imports + +DEFAULT_CACHE_DIR = ".grimp_cache" class NotSupplied: @@ -53,49 +48,14 @@ def build_graph( "mypackage", "anotherpackage", "onemore", include_external_packages=True, ) """ - file_system: AbstractFileSystem = settings.FILE_SYSTEM - - found_packages = _find_packages( - file_system=file_system, - package_names=[package_name, *additional_package_names], - ) - - imports_by_module = _scan_packages( - found_packages=found_packages, - file_system=file_system.convert_to_basic(), - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cache_dir, - ) - - graph = _assemble_graph(found_packages, imports_by_module) - - return graph - - -def build_graph_rust( - package_name: str, - *additional_package_names: str, - include_external_packages: bool = False, - exclude_type_checking_imports: bool = False, - cache_dir: str | type[NotSupplied] | None = NotSupplied, -) -> ImportGraph: - """ - Build and return an import graph for the supplied package name(s), using Rust throughout. - - This takes the same arguments as, and is intended as a faster drop-in replacement for, - ``build_graph``. It shares its cache with ``build_graph``. - """ package_names = _validate_package_names_are_strings([package_name, *additional_package_names]) packages = [ (name, directory) for name in package_names - for directory in settings.PACKAGE_FINDER.determine_package_directories( - package_name=name, file_system=settings.FILE_SYSTEM - ) + for directory in settings.PACKAGE_FINDER.determine_package_directories(name) ] if cache_dir is NotSupplied: - cache_dir = settings.CACHE_CLASS.cache_dir_or_default(None) + cache_dir = DEFAULT_CACHE_DIR rust_graph = rust.build_graph( packages, @@ -107,31 +67,6 @@ def build_graph_rust( return ImportGraph.from_rustgraph(rust_graph) -def _find_packages( - file_system: AbstractFileSystem, package_names: Sequence[object] -) -> set[FoundPackage]: - package_names = _validate_package_names_are_strings(package_names) - - module_finder: AbstractModuleFinder = settings.MODULE_FINDER - package_finder: AbstractPackageFinder = settings.PACKAGE_FINDER - - found_packages: set[FoundPackage] = set() - - for package_name in package_names: - package_directories = package_finder.determine_package_directories( - package_name=package_name, file_system=file_system - ) - for package_directory in package_directories: - found_package = module_finder.find_package( - package_name=package_name, - package_directory=package_directory, - file_system=file_system, - ) - found_packages.add(found_package) - - return found_packages - - def _validate_package_names_are_strings( package_names: Sequence[object], ) -> Sequence[str]: @@ -140,107 +75,3 @@ def _validate_package_names_are_strings( msg = f"Package names must be strings, got {name.__class__.__name__}." raise TypeError(msg) return cast("Sequence[str]", package_names) - - -def _scan_packages( - found_packages: set[FoundPackage], - file_system: BasicFileSystem, - *, - include_external_packages: bool, - exclude_type_checking_imports: bool, - cache_dir: str | type[NotSupplied] | None, -) -> dict[Module, set[DirectImport]]: - if cache_dir is not None: - cache_dir_if_supplied = cache_dir if cache_dir != NotSupplied else None - cache: caching.Cache = settings.CACHE_CLASS.setup( - file_system=file_system, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cache_dir_if_supplied, - ) - - module_files_to_scan = { - module_file - for found_package in found_packages - for module_file in found_package.module_files - } - - imports_by_module_file: dict[ModuleFile, set[DirectImport]] = {} - - if cache_dir is not None: - imports_by_module_file.update(_read_imports_from_cache(module_files_to_scan, cache=cache)) - - remaining_module_files_to_scan = module_files_to_scan.difference(imports_by_module_file) - if remaining_module_files_to_scan: - imports_by_module_file.update( - scan_imports( - remaining_module_files_to_scan, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - ) - - imports_by_module: dict[Module, set[DirectImport]] = { - k.module: v for k, v in imports_by_module_file.items() - } - - if cache_dir is not None: - cache.write(imports_by_module) - - return imports_by_module - - -def _assemble_graph( - found_packages: set[FoundPackage], - imports_by_module: dict[Module, set[DirectImport]], -) -> ImportGraph: - graph: ImportGraph = settings.IMPORT_GRAPH_CLASS() - - for namespace_package in itertools.chain.from_iterable( - found_package.namespace_packages for found_package in found_packages - ): - graph.add_module(namespace_package) - - package_modules = {Module(found_package.name) for found_package in found_packages} - - for module, direct_imports in imports_by_module.items(): - graph.add_module(module.name) - for direct_import in direct_imports: - # Before we add the import, check to see if the imported module is in fact an - # external module, and if so, tell the graph that it is a squashed module. - graph.add_module( - direct_import.imported.name, - is_squashed=_is_external(direct_import.imported, package_modules), - ) - - graph.add_import( - importer=direct_import.importer.name, - imported=direct_import.imported.name, - line_number=direct_import.line_number, - line_contents=direct_import.line_contents, - is_lazy=direct_import.is_lazy, - ) - return graph - - -def _is_external(module: Module, package_modules: set[Module]) -> bool: - return not any( - module.is_descendant_of(package_module) or module == package_module - for package_module in package_modules - ) - - -def _read_imports_from_cache( - module_files: Iterable[ModuleFile], *, cache: caching.Cache -) -> dict[ModuleFile, set[DirectImport]]: - imports_by_module_file: dict[ModuleFile, set[DirectImport]] = {} - for module_file in module_files: - try: - direct_imports = cache.read_imports(module_file) - except caching.CacheMiss: - continue - else: - imports_by_module_file[module_file] = direct_imports - return imports_by_module_file diff --git a/src/grimp/main.py b/src/grimp/main.py index 88c5fc7f..9878d3f6 100644 --- a/src/grimp/main.py +++ b/src/grimp/main.py @@ -1,19 +1,11 @@ -__all__ = ["build_graph", "build_graph_rust"] +__all__ = ["build_graph"] -from .adaptors.caching import Cache -from .adaptors.filesystem import FileSystem -from .adaptors.modulefinder import ModuleFinder from .adaptors.packagefinder import ImportLibPackageFinder from .adaptors.timing import SystemClockTimer from .application.config import settings -from .application.graph import ImportGraph -from .application.usecases import build_graph, build_graph_rust +from .application.usecases import build_graph settings.configure( - MODULE_FINDER=ModuleFinder(), - FILE_SYSTEM=FileSystem(), - IMPORT_GRAPH_CLASS=ImportGraph, PACKAGE_FINDER=ImportLibPackageFinder(), - CACHE_CLASS=Cache, TIMER=SystemClockTimer(), ) diff --git a/tests/adaptors/filesystem.py b/tests/adaptors/filesystem.py deleted file mode 100644 index 6b9809d3..00000000 --- a/tests/adaptors/filesystem.py +++ /dev/null @@ -1,267 +0,0 @@ -from __future__ import annotations - -from collections.abc import Generator -from typing import Any - -import yaml - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import AbstractFileSystem, BasicFileSystem - -DEFAULT_MTIME = 10000.0 - - -class FakeFileSystem(AbstractFileSystem): - def __init__( - self, - contents: str | None = None, - content_map: dict[str, str] | None = None, - mtime_map: dict[str, float] | None = None, - ) -> None: - """ - Files can be declared as existing in the file system in two different ways, either - in a contents string (which is a quick way of defining a lot of files), or in content_map - (which specifies the actual contents of a file in the file system). For a file to be - treated as existing, it needs to be declared in at least one of these. If it isn't - declared in content_map, the file will behave as an empty file. - - Args: - contents: a string in the following format: - - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - - content_map: A dictionary keyed with filenames, with values that are the contents. - If present in content_map, .read(filename) will return the string. - { - '/path/to/foo/__init__.py': "from . import one", - } - mtime_map: A dictionary keyed with filenames, with values that are the mtimes - i.e. last modified times. - """ - self.contents = self._parse_contents(contents) - self._raw_contents = contents - self.content_map = content_map or {} - self.mtime_map: dict[str, float] = mtime_map or {} - - @property - def sep(self) -> str: - return "/" - - def dirname(self, filename: str) -> str: - """ - Return the full path to the directory name of the supplied filename. - - E.g. '/path/to/filename.py' will return '/path/to'. - """ - return self.split(filename)[0] - - def walk(self, directory_name): - """ - Given a directory, walk the file system recursively. - - For each directory in the tree rooted at directory top (including top itself), - it yields a 3-tuple (dirpath, dirnames, filenames). - """ - # Navigate through the nested structure to find the directory - directory_components = [c for c in directory_name.split("/") if c] - - contents = self.contents - for component in directory_components: - key_with_prefix = "/" + component - if key_with_prefix in contents: - contents = contents[key_with_prefix] - elif component in contents: - contents = contents[component] - else: - return [] - - # If we found a file (None) instead of a directory, return empty - if contents is None: - return [] - - yield from self._walk_contents(contents, containing_directory=directory_name) - - def _walk_contents( - self, directory_contents: dict[str, Any], containing_directory: str - ) -> Generator[tuple[str, list[str], list[str]]]: - directories = [] - files = [] - for key, value in directory_contents.items(): - if value is None: - files.append(key) - else: - directories.append(key) - - yield (containing_directory, directories, files) - - if directories: - for directory in directories: - yield from self._walk_contents( - directory_contents=directory_contents[directory], - containing_directory=self.join(containing_directory, directory), - ) - - def join(self, *components: str) -> str: - return self.sep.join(c.rstrip(self.sep) for c in components) - - def split(self, file_name: str) -> tuple[str, str]: - components = file_name.split(self.sep) - if len(components) == 2: - # Handle case where file is child of the root, i.e. /some-file.txt. - # In this case, to conform with the interface we ensure the leading slash - # is included in the returned head. - components.insert(0, "") - return (self.sep.join(components[:-1]), components[-1]) - - def _parse_contents(self, raw_contents: str | None) -> dict[str, Any]: - """ - Returns the raw contents parsed in the form: - { - "/path": { - "/to": - "/mypackage": { - "__init__.py": None, - "foo": { - "__init__.py": None, - "one.py": None, - "two": { - "__init__.py": None, - "blue.py": None, - "green.py": None, - } - }, - }, - }, - } - } - """ - if raw_contents is None: - return {} - - raw_lines = [line for line in raw_contents.split("\n") if line.strip()] - dedented_lines = self._dedent(raw_lines) - - # Group lines by their root paths - # A root path is a line that starts with "/" and has no indentation - root_path_groups = [] - current_group: list[str] = [] - - for line in dedented_lines: - if line.startswith("/") and not line.startswith(" "): - # This is a new root path - if current_group: - root_path_groups.append(current_group) - current_group = [line] - else: - current_group.append(line) - - if current_group: - root_path_groups.append(current_group) - - # Process each root path group - result: dict[str, Any] = {} - for group in root_path_groups: - # First line is the root path - root_path = group[0].rstrip().rstrip("/") - path_components = [c for c in root_path.split("/") if c] - - # Remaining lines are the file tree - yamlified_lines = [] - for line in group[1:]: - trimmed_line = line.rstrip().rstrip("/") - yamlified_line = trimmed_line + ":" - yamlified_lines.append(yamlified_line) - - yamlified_string = "\n".join(yamlified_lines) - nested_contents = yaml.safe_load(yamlified_string) if yamlified_lines else {} - - # Build the nested structure from path components - group_result = nested_contents - for component in reversed(path_components): - group_result = {"/" + component: group_result} - - # Merge into result - self._deep_merge(result, group_result) - - return result - - def _deep_merge(self, target: dict, source: dict) -> None: - """Merge source dict into target dict recursively.""" - for key, value in source.items(): - if key in target and isinstance(target[key], dict) and isinstance(value, dict): - self._deep_merge(target[key], value) - else: - target[key] = value - - def _dedent(self, lines: list[str]) -> list[str]: - """ - Dedent all lines by the same amount. - """ - first_line = lines[0] - first_line_indent = len(first_line) - len(first_line.lstrip()) - return [line[first_line_indent:] for line in lines] - - def read(self, file_name: str) -> str: - if not self.exists(file_name): - raise FileNotFoundError - try: - file_contents = self.content_map[file_name] - except KeyError: - return "" - raw_lines = [line for line in file_contents.split("\n") if line.strip()] - dedented_lines = self._dedent(raw_lines) - return "\n".join(dedented_lines) - - def exists(self, file_name: str) -> bool: - # The file should exist if it's either declared in contents or in content_map. - if file_name in self.content_map.keys(): - return True - - # Split the file path into components and navigate through nested structure - file_components = [c for c in file_name.split("/") if c] - - contents = self.contents - for i, component in enumerate(file_components): - # First, try with "/" prefix (for the top-level path structure) - key_with_prefix = "/" + component - if key_with_prefix in contents: - contents = contents[key_with_prefix] - # If not found, try without prefix (for files and directories in the tree) - elif component in contents: - contents = contents[component] - else: - return False - - # If we've reached None, it's a file (leaf node) - if contents is None: - return True - - # If we've navigated through all components and haven't hit None, - # it's a directory, not a file - return False - return False - - def get_mtime(self, file_name: str) -> float: - if not self.exists(file_name): - raise FileNotFoundError(f"{file_name} does not exist.") - return self.mtime_map.get(file_name, DEFAULT_MTIME) - - def write(self, file_name: str, contents: str) -> None: - self.content_map[file_name] = contents - self.mtime_map[file_name] = DEFAULT_MTIME - - def convert_to_basic(self) -> BasicFileSystem: - """ - Convert this file system to a BasicFileSystem. - """ - return rust.FakeBasicFileSystem( - contents=self._raw_contents, - content_map=self.content_map, - ) diff --git a/tests/adaptors/modulefinder.py b/tests/adaptors/modulefinder.py deleted file mode 100644 index 50acdd2c..00000000 --- a/tests/adaptors/modulefinder.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.application.ports.modulefinder import AbstractModuleFinder, FoundPackage, ModuleFile - - -class BaseFakeModuleFinder(AbstractModuleFinder): - module_files_by_package_name: dict[str, frozenset[ModuleFile]] = {} - - def find_package( - self, package_name: str, package_directory: str, file_system: AbstractFileSystem - ) -> FoundPackage: - return FoundPackage( - name=package_name, - directory=package_directory, - module_files=self.module_files_by_package_name[package_name], - ) diff --git a/tests/adaptors/packagefinder.py b/tests/adaptors/packagefinder.py deleted file mode 100644 index 81d1b70b..00000000 --- a/tests/adaptors/packagefinder.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping - -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.application.ports.packagefinder import AbstractPackageFinder - - -class BaseFakePackageFinder(AbstractPackageFinder): - directory_map: Mapping[str, set[str]] = {} - - def determine_package_directories( - self, package_name: str, file_system: AbstractFileSystem - ) -> set[str]: - return self.directory_map[package_name] diff --git a/tests/benchmarking/test_benchmarking.py b/tests/benchmarking/test_benchmarking.py index a1cf9b9a..abcf0d16 100644 --- a/tests/benchmarking/test_benchmarking.py +++ b/tests/benchmarking/test_benchmarking.py @@ -294,13 +294,6 @@ def test_build_django_uncached(benchmark): benchmark(grimp.build_graph, "django", cache_dir=None) -def test_build_django_uncached_rust(benchmark): - """ - Benchmarks building a graph of real package using build_graph_rust, with the cache turned off. - """ - benchmark(grimp.build_graph_rust, "django", cache_dir=None) - - def test_build_django_from_cache_no_misses(benchmark): """ Benchmarks building a graph of real package - in this case Django. @@ -313,16 +306,6 @@ def test_build_django_from_cache_no_misses(benchmark): benchmark(grimp.build_graph, "django") -def test_build_django_from_cache_no_misses_rust(benchmark): - """ - Benchmarks building a graph of real package using build_graph_rust, fully utilizing the cache. - """ - # Populate the cache first, before beginning the benchmark. - grimp.build_graph_rust("django") - - benchmark(grimp.build_graph_rust, "django") - - @pytest.mark.parametrize( "number_of_misses", ( diff --git a/tests/config.py b/tests/config.py deleted file mode 100644 index cfae0556..00000000 --- a/tests/config.py +++ /dev/null @@ -1,14 +0,0 @@ -from grimp.application.config import settings - - -class override_settings: # noqa: N801 - def __init__(self, **settings_to_override): - self.settings_to_override = settings_to_override - - def __enter__(self): - self.original_settings = settings.copy() - settings.configure(**self.settings_to_override) - - def __exit__(self, exc_type, exc_val, exc_tb): - for setting_name in self.settings_to_override: - settings.configure(**{setting_name: getattr(self.original_settings, setting_name)}) diff --git a/tests/functional/conftest.py b/tests/functional/conftest.py deleted file mode 100644 index 18807d3f..00000000 --- a/tests/functional/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - -import grimp - - -@pytest.fixture(params=["python", "rust"]) -def build_graph(request): - """ - Provides each implementation of build_graph, so that the functional tests cover both. - """ - return grimp.build_graph if request.param == "python" else grimp.build_graph_rust diff --git a/tests/functional/test_build_and_use_graph.py b/tests/functional/test_build_and_use_graph.py index 3d22fa8e..bf365b11 100644 --- a/tests/functional/test_build_and_use_graph.py +++ b/tests/functional/test_build_and_use_graph.py @@ -1,5 +1,7 @@ import pytest +from grimp import build_graph + """ For ease of reference, these are the imports of all the files: @@ -28,7 +30,7 @@ # --------- -def test_modules(build_graph): +def test_modules(): graph = build_graph("testpackage", cache_dir=None) assert graph.modules == { @@ -51,7 +53,7 @@ def test_modules(build_graph): } -def test_add_module(build_graph): +def test_add_module(): graph = build_graph("testpackage", cache_dir=None) number_of_modules = len(graph.modules) @@ -60,7 +62,7 @@ def test_add_module(build_graph): assert number_of_modules + 1 == len(graph.modules) -def test_remove_module(build_graph): +def test_remove_module(): graph = build_graph("testpackage", cache_dir=None) number_of_modules = len(graph.modules) @@ -69,7 +71,7 @@ def test_remove_module(build_graph): assert number_of_modules - 1 == len(graph.modules) -def test_add_and_remove_import(build_graph): +def test_add_and_remove_import(): graph = build_graph("testpackage", cache_dir=None) a = "testpackage.one.delta.blue" b = "testpackage.two.alpha" @@ -89,7 +91,7 @@ def test_add_and_remove_import(build_graph): # ----------- -def test_find_children(build_graph): +def test_find_children(): graph = build_graph("testpackage", cache_dir=None) assert graph.find_children("testpackage.one") == { @@ -100,7 +102,7 @@ def test_find_children(build_graph): } -def test_find_descendants(build_graph): +def test_find_descendants(): graph = build_graph("testpackage", cache_dir=None) assert graph.find_descendants("testpackage.one") == { @@ -116,7 +118,7 @@ def test_find_descendants(build_graph): # -------------- -def test_find_modules_directly_imported_by(build_graph): +def test_find_modules_directly_imported_by(): graph = build_graph("testpackage", cache_dir=None) result = graph.find_modules_directly_imported_by("testpackage.utils") @@ -124,7 +126,7 @@ def test_find_modules_directly_imported_by(build_graph): assert {"testpackage.one", "testpackage.two.alpha"} == result -def test_find_modules_that_directly_import(build_graph): +def test_find_modules_that_directly_import(): graph = build_graph("testpackage", cache_dir=None) result = graph.find_modules_that_directly_import("testpackage.one.alpha") @@ -139,7 +141,7 @@ def test_find_modules_that_directly_import(build_graph): } == result -def test_direct_import_exists(build_graph): +def test_direct_import_exists(): graph = build_graph("testpackage", cache_dir=None) assert False is graph.direct_import_exists( @@ -153,7 +155,7 @@ def test_direct_import_exists(build_graph): ) -def test_get_import_details(build_graph): +def test_get_import_details(): graph = build_graph("testpackage", cache_dir=None) assert [ @@ -172,7 +174,7 @@ def test_get_import_details(build_graph): class TestPathExists: - def test_as_packages_false(self, build_graph): + def test_as_packages_false(self): graph = build_graph("testpackage", cache_dir=None) assert not graph.chain_exists( @@ -181,7 +183,7 @@ def test_as_packages_false(self, build_graph): assert graph.chain_exists(imported="testpackage.one.alpha", importer="testpackage.utils") - def test_as_packages_true(self, build_graph): + def test_as_packages_true(self): graph = build_graph("testpackage", cache_dir=None) assert graph.chain_exists( @@ -193,7 +195,7 @@ def test_as_packages_true(self, build_graph): ) -def test_find_shortest_chain(build_graph): +def test_find_shortest_chain(): graph = build_graph("testpackage", cache_dir=None) assert ( @@ -245,7 +247,7 @@ def test_find_shortest_chain(build_graph): ), ], ) -def test_find_shortest_chains(build_graph, as_packages: bool | None, expected_result: set[tuple]): +def test_find_shortest_chains(as_packages: bool | None, expected_result: set[tuple]): importer = "testpackage.three" imported = "testpackage.one.alpha" @@ -262,7 +264,7 @@ def test_find_shortest_chains(build_graph, as_packages: bool | None, expected_re class TestFindDownstreamModules: - def test_as_package_false(self, build_graph): + def test_as_package_false(self): graph = build_graph("testpackage", cache_dir=None) result = graph.find_downstream_modules("testpackage.one.alpha") @@ -280,7 +282,7 @@ def test_as_package_false(self, build_graph): "testpackage.three.gamma", } == result - def test_as_package_true(self, build_graph): + def test_as_package_true(self): graph = build_graph("testpackage", cache_dir=None) result = graph.find_downstream_modules("testpackage.one", as_package=True) @@ -298,7 +300,7 @@ def test_as_package_true(self, build_graph): class TestFindUpstreamModules: - def test_as_package_false(self, build_graph): + def test_as_package_false(self): graph = build_graph("testpackage", cache_dir=None) assert graph.find_upstream_modules("testpackage.one.alpha") == set() @@ -309,7 +311,7 @@ def test_as_package_false(self, build_graph): "testpackage.one.alpha", } - def test_as_package_true(self, build_graph): + def test_as_package_true(self): graph = build_graph("testpackage", cache_dir=None) assert graph.find_upstream_modules("testpackage.two", as_package=True) == { @@ -320,13 +322,13 @@ def test_as_package_true(self, build_graph): class TestExcludeTypeCheckingImports: - def test_exclude_false(self, build_graph): + def test_exclude_false(self): graph = build_graph("testpackage", cache_dir=None, exclude_type_checking_imports=False) assert True is graph.direct_import_exists( importer="testpackage.one.beta", imported="testpackage.two.alpha" ) - def test_exclude_true(self, build_graph): + def test_exclude_true(self): graph = build_graph("testpackage", cache_dir=None, exclude_type_checking_imports=True) assert False is graph.direct_import_exists( importer="testpackage.one.beta", imported="testpackage.two.alpha" diff --git a/tests/functional/test_build_and_use_graph_with_multiple_roots.py b/tests/functional/test_build_and_use_graph_with_multiple_roots.py index 7740ef35..13a23f7f 100644 --- a/tests/functional/test_build_and_use_graph_with_multiple_roots.py +++ b/tests/functional/test_build_and_use_graph_with_multiple_roots.py @@ -1,5 +1,7 @@ import pytest +from grimp import build_graph + """ For ease of reference, these are the imports of all the files: @@ -23,9 +25,7 @@ @pytest.mark.parametrize("root_packages", PACKAGES_IN_DIFFERENT_ORDERS) class TestBuildGraph: - def test_graph_has_correct_modules_regardless_of_package_order( - self, build_graph, root_packages - ): + def test_graph_has_correct_modules_regardless_of_package_order(self, root_packages): graph = build_graph(*root_packages, cache_dir=None) assert graph.modules == { @@ -39,7 +39,7 @@ def test_graph_has_correct_modules_regardless_of_package_order( "rootpackagegreen.two", } - def test_stores_import_within_package(self, build_graph, root_packages): + def test_stores_import_within_package(self, root_packages): graph = build_graph(*root_packages, cache_dir=None) assert [ @@ -54,7 +54,7 @@ def test_stores_import_within_package(self, build_graph, root_packages): importer="rootpackageblue.two", imported="rootpackageblue.one.alpha" ) - def test_stores_import_between_root_packages(self, build_graph, root_packages): + def test_stores_import_between_root_packages(self, root_packages): graph = build_graph(*root_packages, cache_dir=None) assert [ diff --git a/tests/functional/test_build_graph_arguments.py b/tests/functional/test_build_graph_arguments.py index 0c759566..a58bbcb0 100644 --- a/tests/functional/test_build_graph_arguments.py +++ b/tests/functional/test_build_graph_arguments.py @@ -2,9 +2,11 @@ import pytest +from grimp import build_graph + @pytest.mark.parametrize("include_external_packages", (True, False)) -def test_happy_path(build_graph, write_files, include_external_packages): +def test_happy_path(write_files, include_external_packages): write_files( { "mypackage/__init__.py": "", @@ -48,7 +50,7 @@ def test_happy_path(build_graph, write_files, include_external_packages): graph.find_children(module) -def test_boolean_additional_package_raises_type_error(build_graph): +def test_boolean_additional_package_raises_type_error(): """ Tests that a useful error message if build_graph is called with a boolean as the second argument. @@ -68,24 +70,24 @@ def package(self, write_files, tmp_path, monkeypatch): # Default caching writes to the current directory. monkeypatch.chdir(tmp_path) - def test_supplied(self, build_graph, tmp_path): + def test_supplied(self, tmp_path): build_graph("mypackage", cache_dir=str(tmp_path / "supplied")) assert (tmp_path / "supplied" / "mypackage.meta.json").exists() assert not (tmp_path / ".grimp_cache").exists() - def test_not_supplied_uses_default(self, build_graph, tmp_path): + def test_not_supplied_uses_default(self, tmp_path): build_graph("mypackage") assert (tmp_path / ".grimp_cache" / "mypackage.meta.json").exists() - def test_none_disables_caching(self, build_graph, tmp_path): + def test_none_disables_caching(self, tmp_path): build_graph("mypackage", cache_dir=None) assert not (tmp_path / ".grimp_cache").exists() -def test_forgives_wrong_type_being_passed_to_include_external_packages(build_graph, write_files): +def test_forgives_wrong_type_being_passed_to_include_external_packages(write_files): write_files( { "mypackage/__init__.py": "", @@ -105,7 +107,7 @@ def test_forgives_wrong_type_being_passed_to_include_external_packages(build_gra assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.foo.one"} -def test_namespace_package_passed_as_root(build_graph, write_files): +def test_namespace_package_passed_as_root(write_files): # A namespace package can span several directories. write_files( { diff --git a/tests/functional/test_build_graph_on_real_packages.py b/tests/functional/test_build_graph_on_real_packages.py index 66d6d6f5..63e5a4bf 100644 --- a/tests/functional/test_build_graph_on_real_packages.py +++ b/tests/functional/test_build_graph_on_real_packages.py @@ -12,11 +12,6 @@ def test_build_graph_on_real_package(package_name, snapshot): imports = graph.find_matching_direct_imports(import_expression="** -> **") assert imports == snapshot - graph_from_rust = grimp.build_graph_rust(package_name, cache_dir=None) - imports_from_rust = graph_from_rust.find_matching_direct_imports(import_expression="** -> **") - sort_key = lambda i: (i["importer"], i["imported"]) # noqa: E731 - assert sorted(imports_from_rust, key=sort_key) == sorted(imports, key=sort_key) - @pytest.mark.parametrize( "package_name", diff --git a/tests/functional/test_caching.py b/tests/functional/test_caching.py index 717a4f84..bfc6cca1 100644 --- a/tests/functional/test_caching.py +++ b/tests/functional/test_caching.py @@ -1,5 +1,4 @@ import json -import logging import os import shutil import tempfile @@ -7,7 +6,7 @@ import pytest -import grimp +from grimp import build_graph """ For ease of reference, these are the imports of all the files: @@ -44,7 +43,7 @@ def copied_cachingpackage(): shutil.rmtree(str(PACKAGE_COPY_DESTINATION)) -def test_build_graph_uses_cache(build_graph, copied_cachingpackage): +def test_build_graph_uses_cache(copied_cachingpackage): with tempfile.TemporaryDirectory() as cache_dir: graph = build_graph("cachingpackage", cache_dir=cache_dir) @@ -111,7 +110,7 @@ def test_build_graph_uses_cache(build_graph, copied_cachingpackage): ) -def test_cache_preserves_is_lazy(build_graph): +def test_cache_preserves_is_lazy(): with tempfile.TemporaryDirectory() as cache_dir: # First build populates the cache from source. build_graph("lazyimports", cache_dir=cache_dir) @@ -132,7 +131,7 @@ def test_cache_preserves_is_lazy(build_graph): ] -def test_data_file_records_version_and_is_lazy(build_graph, copied_cachingpackage): +def test_data_file_records_version_and_is_lazy(copied_cachingpackage): with tempfile.TemporaryDirectory() as cache_dir: build_graph("cachingpackage", cache_dir=cache_dir) @@ -147,7 +146,7 @@ def test_data_file_records_version_and_is_lazy(build_graph, copied_cachingpackag ] -def test_cache_with_different_version_is_rebuilt(build_graph, copied_cachingpackage, caplog): +def test_cache_with_different_version_is_rebuilt(copied_cachingpackage): with tempfile.TemporaryDirectory() as cache_dir: build_graph("cachingpackage", cache_dir=cache_dir) @@ -160,12 +159,9 @@ def test_cache_with_different_version_is_rebuilt(build_graph, copied_cachingpack } data_file.write_text(json.dumps(legacy)) - with caplog.at_level(logging.INFO): - graph = build_graph("cachingpackage", cache_dir=cache_dir) + graph = build_graph("cachingpackage", cache_dir=cache_dir) - # The mismatched cache is ignored (not treated as corrupt) and rebuilt from source. - if build_graph is grimp.build_graph: # Only the Python implementation logs. - assert any("different version of Grimp" in record.message for record in caplog.records) + # The mismatched cache is ignored and rebuilt from source. assert graph.get_import_details( importer="cachingpackage.two.alpha", imported="cachingpackage.one.alpha", @@ -180,32 +176,6 @@ def test_cache_with_different_version_is_rebuilt(build_graph, copied_cachingpack ] -@pytest.mark.parametrize( - "build_graph_writing_cache, build_graph_reading_cache", - ( - (grimp.build_graph, grimp.build_graph_rust), - (grimp.build_graph_rust, grimp.build_graph), - ), -) -def test_cache_is_shared_between_implementations( - copied_cachingpackage, build_graph_writing_cache, build_graph_reading_cache -): - with tempfile.TemporaryDirectory() as cache_dir: - build_graph_writing_cache("cachingpackage", cache_dir=cache_dir) - - data_file = Path(cache_dir) / "27aa562ad2a4745eb20ecf156430dbfeb0e90610.data.json" - snippet = "from ..one import alpha" - replacement = snippet + " # Inserted by test" - _manipulate_data_file(data_file, snippet, replacement) - - graph = build_graph_reading_cache("cachingpackage", cache_dir=cache_dir) - - [import_details] = graph.get_import_details( - importer="cachingpackage.two.alpha", imported="cachingpackage.one.alpha" - ) - assert import_details["line_contents"] == replacement - - def _manipulate_data_file(data_file: Path, snippet: str, replacement: str) -> None: with open(data_file) as file: filedata = file.read() @@ -274,7 +244,6 @@ class TestCacheFiles: ) def test_file_names( self, - build_graph, write_files, cache_dir, package_names, @@ -316,7 +285,6 @@ def test_file_names( ) def test_contents( self, - build_graph, write_files, cache_dir, cache_dir_suffix, @@ -375,7 +343,7 @@ def test_contents( "greenpackage.two": {("greenpackage.one", False, 1, "from . import one")}, } - def test_default_cache_dir(self, build_graph, write_files, tmp_path, monkeypatch): + def test_default_cache_dir(self, write_files, tmp_path, monkeypatch): write_files({"bluepackage/__init__.py": ""}) monkeypatch.chdir(tmp_path) @@ -383,7 +351,7 @@ def test_default_cache_dir(self, build_graph, write_files, tmp_path, monkeypatch assert (tmp_path / ".grimp_cache" / "bluepackage.meta.json").exists() - def test_adds_marker_files(self, build_graph, write_files, cache_dir): + def test_adds_marker_files(self, write_files, cache_dir): write_files({"bluepackage/__init__.py": ""}) build_graph("bluepackage", cache_dir=str(cache_dir)) @@ -395,7 +363,7 @@ def test_adds_marker_files(self, build_graph, write_files, cache_dir): "# For information about cache directory tags see https://bford.info/cachedir/" ) - def test_doesnt_overwrite_marker_files(self, build_graph, write_files, cache_dir): + def test_doesnt_overwrite_marker_files(self, write_files, cache_dir): write_files({"bluepackage/__init__.py": ""}) cache_dir.mkdir() (cache_dir / ".gitignore").write_text("Something else") @@ -406,7 +374,7 @@ def test_doesnt_overwrite_marker_files(self, build_graph, write_files, cache_dir class TestCacheUse: - def test_uses_cache_for_unmodified_module(self, build_graph, write_files, cache_dir): + def test_uses_cache_for_unmodified_module(self, write_files, cache_dir): write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) _insert_marker_in_cache(cache_dir) @@ -417,9 +385,7 @@ def test_uses_cache_for_unmodified_module(self, build_graph, write_files, cache_ "from . import two" + MARKER ) - def test_ignores_cache_for_module_with_different_mtime( - self, build_graph, write_files, cache_dir - ): + def test_ignores_cache_for_module_with_different_mtime(self, write_files, cache_dir): root = write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) _insert_marker_in_cache(cache_dir) @@ -432,7 +398,7 @@ def test_ignores_cache_for_module_with_different_mtime( assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" assert _line_contents(graph, "mypackage.two", "mypackage.three") == "from . import three" - def test_ignores_cache_for_new_module(self, build_graph, write_files, cache_dir): + def test_ignores_cache_for_new_module(self, write_files, cache_dir): write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) write_files({"mypackage/four.py": "from . import one"}) @@ -441,9 +407,7 @@ def test_ignores_cache_for_new_module(self, build_graph, write_files, cache_dir) assert graph.direct_import_exists(importer="mypackage.four", imported="mypackage.one") - def test_ignores_cache_for_module_with_no_cached_mtime( - self, build_graph, write_files, cache_dir - ): + def test_ignores_cache_for_module_with_no_cached_mtime(self, write_files, cache_dir): write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) _insert_marker_in_cache(cache_dir) @@ -456,7 +420,7 @@ def test_ignores_cache_for_module_with_no_cached_mtime( assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" - def test_ignores_cache_for_module_missing_from_data(self, build_graph, write_files, cache_dir): + def test_ignores_cache_for_module_missing_from_data(self, write_files, cache_dir): write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) data_file = _data_file(cache_dir) @@ -475,7 +439,7 @@ def test_ignores_cache_for_module_missing_from_data(self, build_graph, write_fil '{"mypackage.one": ["wrong", "type"]}', ), ) - def test_ignores_corrupt_meta_file(self, build_graph, write_files, cache_dir, serialized_meta): + def test_ignores_corrupt_meta_file(self, write_files, cache_dir, serialized_meta): write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) _insert_marker_in_cache(cache_dir) @@ -492,7 +456,7 @@ def test_ignores_corrupt_meta_file(self, build_graph, write_files, cache_dir, se '{"version": 2, "imports_by_module": {"mypackage.one": [["wrong", "type"]]}}', ), ) - def test_ignores_corrupt_data_file(self, build_graph, write_files, cache_dir, serialized_data): + def test_ignores_corrupt_data_file(self, write_files, cache_dir, serialized_data): write_files(MYPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) _data_file(cache_dir).write_text(serialized_data) @@ -515,7 +479,7 @@ def test_ignores_corrupt_data_file(self, build_graph, write_files, cache_dir, se ), ) def test_doesnt_use_cache_for_different_packages_or_options( - self, build_graph, write_files, cache_dir, package_names, options + self, write_files, cache_dir, package_names, options ): write_files(MYPACKAGE_FILES | ANOTHERPACKAGE_FILES) build_graph("mypackage", cache_dir=str(cache_dir)) @@ -527,7 +491,7 @@ def test_doesnt_use_cache_for_different_packages_or_options( @pytest.mark.parametrize("include_external_packages", (False, True)) def test_uses_cache_for_multiple_packages( - self, build_graph, write_files, cache_dir, include_external_packages + self, write_files, cache_dir, include_external_packages ): write_files(MYPACKAGE_FILES | ANOTHERPACKAGE_FILES) kwargs = dict( diff --git a/tests/functional/test_encoding_handling.py b/tests/functional/test_encoding_handling.py index f7a60f4b..446cb107 100644 --- a/tests/functional/test_encoding_handling.py +++ b/tests/functional/test_encoding_handling.py @@ -2,12 +2,14 @@ import pytest +import grimp -def test_build_graph_of_non_ascii_source(build_graph): + +def test_build_graph_of_non_ascii_source(): """ Tests we can cope with non ascii Python source files. """ - graph = build_graph("encodingpackage", cache_dir=None) + graph = grimp.build_graph("encodingpackage", cache_dir=None) result = graph.get_import_details( importer="encodingpackage.importer", imported="encodingpackage.imported" @@ -24,11 +26,11 @@ def test_build_graph_of_non_ascii_source(build_graph): ] == result -def test_build_graph_of_non_utf8_source(build_graph): +def test_build_graph_of_non_utf8_source(): """ Tests we can cope with non UTF-8 Python source files. """ - graph = build_graph("encodingpackage", cache_dir=None) + graph = grimp.build_graph("encodingpackage", cache_dir=None) result = graph.get_import_details( importer="encodingpackage.shift_jis_importer", imported="encodingpackage.imported" @@ -60,7 +62,7 @@ def test_build_graph_of_non_utf8_source(build_graph): ), ) def test_build_graph_of_source_declaring_encoding_with_python_specific_name( - build_graph, tmp_path, monkeypatch, declared_encoding, codec, imported_name + tmp_path, monkeypatch, declared_encoding, codec, imported_name ): """ Tests we can cope with source files that declare their encoding using a name that Python @@ -77,7 +79,7 @@ def test_build_graph_of_source_declaring_encoding_with_python_specific_name( ) monkeypatch.syspath_prepend(str(tmp_path)) - graph = build_graph("declaredencodingpackage", cache_dir=None) + graph = grimp.build_graph("declaredencodingpackage", cache_dir=None) result = graph.get_import_details( importer="declaredencodingpackage.importer", imported="declaredencodingpackage.imported" @@ -98,7 +100,7 @@ def test_build_graph_of_source_declaring_encoding_with_python_specific_name( "declared_encoding", ("latin-1", "latin_1", "iso-8859-1", "iso_8859_1"), ) -def test_build_graph_of_true_latin_1_source(build_graph, tmp_path, monkeypatch, declared_encoding): +def test_build_graph_of_true_latin_1_source(tmp_path, monkeypatch, declared_encoding): """ Tests that Latin-1 is decoded as true byte-to-code-point Latin-1, rather than Windows-1252. @@ -113,7 +115,7 @@ def test_build_graph_of_true_latin_1_source(build_graph, tmp_path, monkeypatch, (package_directory / "importer.py").write_bytes(source.encode("latin-1")) monkeypatch.syspath_prepend(str(tmp_path)) - graph = build_graph("latin1package", cache_dir=None) + graph = grimp.build_graph("latin1package", cache_dir=None) result = graph.get_import_details( importer="latin1package.importer", imported="latin1package.imported" @@ -147,7 +149,7 @@ def test_build_graph_of_true_latin_1_source(build_graph, tmp_path, monkeypatch, ), ) def test_undecodable_source_raises_unicode_error_including_filename( - build_graph, tmp_path, monkeypatch, contents, expected_problem + tmp_path, monkeypatch, contents, expected_problem ): package_directory = tmp_path / "undecodablepackage" package_directory.mkdir() @@ -160,4 +162,4 @@ def test_undecodable_source_raises_unicode_error_including_filename( UnicodeError, match=re.escape(f"Failed to decode file {module_filename} {expected_problem}"), ): - build_graph("undecodablepackage", cache_dir=None) + grimp.build_graph("undecodablepackage", cache_dir=None) diff --git a/tests/functional/test_error_handling.py b/tests/functional/test_error_handling.py index 8dbdd674..955cc14f 100644 --- a/tests/functional/test_error_handling.py +++ b/tests/functional/test_error_handling.py @@ -2,10 +2,10 @@ import pytest -from grimp import exceptions +from grimp import build_graph, exceptions -def test_syntax_error_includes_module(build_graph): +def test_syntax_error_includes_module(): filename = str( ( Path(__file__).parent.parent / "assets" / "syntaxerrorpackage" / "foo" / "one.py" diff --git a/tests/functional/test_finding_modules.py b/tests/functional/test_finding_modules.py index f529fe03..91e983af 100644 --- a/tests/functional/test_finding_modules.py +++ b/tests/functional/test_finding_modules.py @@ -1,5 +1,7 @@ import pytest +from grimp import build_graph + # Files are empty unless stated otherwise. HAPPY_PATH_FILES = { "mypackage/__init__.py": "", @@ -13,7 +15,7 @@ } -def test_happy_path(build_graph, write_files): +def test_happy_path(write_files): write_files(HAPPY_PATH_FILES) graph = build_graph("mypackage", cache_dir=None) @@ -70,7 +72,7 @@ def test_happy_path(build_graph, write_files): ("somenamespace.foo.blue", MODULES_FOO_BLUE), ], ) -def test_namespaced_packages(build_graph, write_files, package_name, expected_modules): +def test_namespaced_packages(write_files, package_name, expected_modules): write_files(NAMESPACE_FILES) graph = build_graph(package_name, cache_dir=None) @@ -79,7 +81,7 @@ def test_namespaced_packages(build_graph, write_files, package_name, expected_mo assert graph.modules == expected_modules -def test_ignores_orphaned_python_files(build_graph, write_files): +def test_ignores_orphaned_python_files(write_files): # Python files in directories that don't contain an __init__.py should not be discovered. write_files( { @@ -97,7 +99,7 @@ def test_ignores_orphaned_python_files(build_graph, write_files): assert graph.modules == {"mypackage", "mypackage.two", "mypackage.two.green"} -def test_ignores_invalid_identifier_directories(build_graph, write_files): +def test_ignores_invalid_identifier_directories(write_files): write_files( { f"namespacepackage/{path}": "" @@ -133,7 +135,7 @@ def test_ignores_invalid_identifier_directories(build_graph, write_files): @pytest.mark.parametrize("extension", ("py", "txt")) -def test_ignores_dotted_python_files(build_graph, write_files, extension): +def test_ignores_dotted_python_files(write_files, extension): # Python files containing dots (other than the one before .py) should not be discovered. write_files( { @@ -149,7 +151,7 @@ def test_ignores_dotted_python_files(build_graph, write_files, extension): assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.bar"} -def test_ignores_hidden_directories(build_graph, write_files): +def test_ignores_hidden_directories(write_files): write_files( { "mypackage/__init__.py": "", @@ -166,7 +168,7 @@ def test_ignores_hidden_directories(build_graph, write_files): assert graph.modules == {"mypackage", "mypackage.two", "mypackage.two.green"} -def test_follows_symlinked_directories(build_graph, write_files, tmp_path): +def test_follows_symlinked_directories(write_files, tmp_path): write_files( { "mypackage/__init__.py": "", diff --git a/tests/functional/test_lazy_imports.py b/tests/functional/test_lazy_imports.py index ba6c7948..d12ac714 100644 --- a/tests/functional/test_lazy_imports.py +++ b/tests/functional/test_lazy_imports.py @@ -1,11 +1,14 @@ -def test_build_graph_with_lazy_imports(build_graph): +import grimp + + +def test_build_graph_with_lazy_imports(): """ Tests we can cope with lazy imports (Python 3.15+). Under the hood we use ruff's parser, which understands lazy imports even when this is running under a different Python version. """ - graph = build_graph("lazyimports", cache_dir=None) + graph = grimp.build_graph("lazyimports", cache_dir=None) result = graph.find_modules_directly_imported_by("lazyimports.one") diff --git a/tests/functional/test_namespace_packages.py b/tests/functional/test_namespace_packages.py index 07fbd00b..53c80424 100644 --- a/tests/functional/test_namespace_packages.py +++ b/tests/functional/test_namespace_packages.py @@ -1,5 +1,7 @@ import pytest +from grimp import build_graph + """ For ease of reference, these are the imports of all the files: @@ -20,7 +22,7 @@ BLUE_MODULES = {"mynamespace.blue", "mynamespace.blue.alpha", "mynamespace.blue.beta"} -def test_build_graph_for_namespace(build_graph): +def test_build_graph_for_namespace(): graph = build_graph("mynamespace", cache_dir=None) assert graph.modules == {"mynamespace"} | YELLOW_MODULES | GREEN_MODULES | BLUE_MODULES @@ -40,13 +42,13 @@ def test_build_graph_for_namespace(build_graph): ), ), ) -def test_modules_for_namespace_child(build_graph, package, expected_modules): +def test_modules_for_namespace_child(package, expected_modules): graph = build_graph(package, cache_dir=None) assert graph.modules == expected_modules -def test_modules_for_multiple_namespace_children(build_graph): +def test_modules_for_multiple_namespace_children(): graph = build_graph("mynamespace.green", "mynamespace.blue", cache_dir=None) assert graph.modules == GREEN_MODULES | BLUE_MODULES @@ -73,7 +75,7 @@ def test_modules_for_multiple_namespace_children(build_graph): ), ) def test_external_packages_handling( - build_graph, packages, expected_internal_modules, expected_external_modules + packages, expected_internal_modules, expected_external_modules ): graph = build_graph(*packages, include_external_packages=True, cache_dir=None) @@ -81,7 +83,7 @@ def test_external_packages_handling( assert all(graph.is_module_squashed(m) for m in expected_external_modules) -def test_import_within_namespace_child(build_graph): +def test_import_within_namespace_child(): graph = build_graph("mynamespace.blue", cache_dir=None) assert graph.direct_import_exists( @@ -89,7 +91,7 @@ def test_import_within_namespace_child(build_graph): ) -def test_import_between_namespace_children(build_graph): +def test_import_between_namespace_children(): graph = build_graph("mynamespace.blue", "mynamespace.green", cache_dir=None) assert graph.direct_import_exists( @@ -145,7 +147,7 @@ def test_import_between_namespace_children(build_graph): ), ], ) -def test_build_graph_for_nested_namespace(build_graph, package_name, expected): +def test_build_graph_for_nested_namespace(package_name, expected): graph = build_graph(package_name, cache_dir=None) assert graph.modules == expected @@ -177,13 +179,13 @@ def test_build_graph_for_nested_namespace(build_graph, package_name, expected): ), ), ) -def test_modules_for_nested_namespace_child(build_graph, package, expected_modules): +def test_modules_for_nested_namespace_child(package, expected_modules): graph = build_graph(package, cache_dir=None) assert graph.modules == expected_modules -def test_import_within_nested_namespace_child(build_graph): +def test_import_within_nested_namespace_child(): graph = build_graph( "nestednamespace.foo.alpha.blue", cache_dir=None, @@ -208,7 +210,7 @@ def test_import_within_nested_namespace_child(build_graph): ), ), ) -def test_import_between_nested_namespace_children(build_graph, root_packages): +def test_import_between_nested_namespace_children(root_packages): graph = build_graph(*root_packages, cache_dir=None) assert graph.direct_import_exists( @@ -255,7 +257,7 @@ def test_import_between_nested_namespace_children(build_graph, root_packages): ), ) def test_external_packages_handling_for_nested_namespaces( - build_graph, packages, expected_internal_modules, expected_external_modules + packages, expected_internal_modules, expected_external_modules ): graph = build_graph(*packages, include_external_packages=True, cache_dir=None) diff --git a/tests/functional/test_scanning_imports.py b/tests/functional/test_scanning_imports.py index 30cfbd43..c84e0d54 100644 --- a/tests/functional/test_scanning_imports.py +++ b/tests/functional/test_scanning_imports.py @@ -8,6 +8,7 @@ import pytest +from grimp import build_graph from grimp.domain.valueobjects import DirectImport, Module @@ -90,7 +91,7 @@ def _foo_package_files(contents_by_path: dict[str, str]) -> dict[str, str]: ), ), ) -def test_absolute_imports(build_graph, write_files, include_external_packages, expected_result): +def test_absolute_imports(write_files, include_external_packages, expected_result): write_files( { "foo/__init__.py": "", @@ -113,7 +114,7 @@ def test_absolute_imports(build_graph, write_files, include_external_packages, e assert _direct_imports(graph, "foo.one") == expected_result -def test_lazy_imports(build_graph, write_files): +def test_lazy_imports(write_files): write_files( _foo_package_files( { @@ -224,7 +225,7 @@ def test_lazy_imports(build_graph, write_files): } -def test_non_ascii(build_graph, write_files): +def test_non_ascii(write_files): write_files( { "mypackage/__init__.py": "", @@ -262,7 +263,7 @@ def test_non_ascii(build_graph, write_files): } -def test_single_namespace_package_portion(build_graph, write_files): +def test_single_namespace_package_portion(write_files): MODULE_ONE = Module("namespace.foo.one") MODULE_TWO = Module("namespace.foo.two") MODULE_THREE = Module("namespace.foo.three") @@ -328,7 +329,7 @@ def test_single_namespace_package_portion(build_graph, write_files): @pytest.mark.parametrize("include_external_packages", (True, False)) -def test_import_of_portion_not_in_graph(build_graph, write_files, include_external_packages): +def test_import_of_portion_not_in_graph(write_files, include_external_packages): # Include two namespaces, one deeper than the other. # # Note: unlike the (in-memory) layout this was originally written against, foo.one can't @@ -486,9 +487,7 @@ def test_import_of_portion_not_in_graph(build_graph, write_files, include_extern ), ), ) -def test_absolute_from_imports( - build_graph, write_files, include_external_packages, expected_result -): +def test_absolute_from_imports(write_files, include_external_packages, expected_result): write_files( _foo_package_files( { @@ -515,7 +514,7 @@ def test_absolute_from_imports( @pytest.mark.parametrize("module_to_scan_is_package", (True, False)) -def test_relative_from_imports(build_graph, write_files, module_to_scan_is_package): +def test_relative_from_imports(write_files, module_to_scan_is_package): if module_to_scan_is_package: module_to_scan = Module("foo.one") module_path = "foo/one/__init__.py" @@ -568,7 +567,7 @@ def test_relative_from_imports(build_graph, write_files, module_to_scan_is_packa "import_source", ("from .two.yellow import my_function", "from foo.two.yellow import my_function"), ) -def test_trims_to_known_modules(build_graph, write_files, import_source): +def test_trims_to_known_modules(write_files, import_source): write_files( { "foo/__init__.py": "", @@ -590,7 +589,7 @@ def test_trims_to_known_modules(build_graph, write_files, import_source): } -def test_trims_to_known_modules_within_init_file(build_graph, write_files): +def test_trims_to_known_modules_within_init_file(write_files): write_files( { "foo/__init__.py": "", @@ -621,7 +620,7 @@ def test_trims_to_known_modules_within_init_file(build_graph, write_files): } -def test_trims_whitespace_from_start_of_line_contents(build_graph, write_files): +def test_trims_whitespace_from_start_of_line_contents(write_files): write_files( { "foo/__init__.py": "", @@ -671,7 +670,7 @@ def my_function(): ), ) def test_external_package_imports_for_namespace_packages( - build_graph, write_files, statement, expected_module_name + write_files, statement, expected_module_name ): write_files( { @@ -698,7 +697,7 @@ def test_external_package_imports_for_namespace_packages( @pytest.mark.parametrize("statement", ("import bar.blue", "from bar import blue")) -def test_scans_multiple_packages(build_graph, write_files, statement): +def test_scans_multiple_packages(write_files, statement): write_files( { "foo/__init__.py": "", @@ -747,7 +746,7 @@ def test_scans_multiple_packages(build_graph, write_files, statement): ), ) def test_exclude_type_checking_imports( - build_graph, write_files, exclude_type_checking_imports, statement, is_statement_valid + write_files, exclude_type_checking_imports, statement, is_statement_valid ): write_files( { @@ -873,7 +872,7 @@ def test_exclude_type_checking_imports( ), ) def test_exclude_type_checking_imports_only_excludes_type_checking_branches( - build_graph, write_files, code, expected_imported_modules + write_files, code, expected_imported_modules ): files = {"foo/__init__.py": ""} files.update({f"foo/{name}.py": "" for name in ("two", "three", "four", "five")}) @@ -893,7 +892,7 @@ def test_exclude_type_checking_imports_only_excludes_type_checking_branches( assert imported_modules == expected_imported_modules -def test_t_string_syntax(build_graph, write_files): +def test_t_string_syntax(write_files): write_files( { "foo/__init__.py": "", @@ -926,9 +925,7 @@ def test_t_string_syntax(build_graph, write_files): "from ..nonexistent import something", ), ) -def test_missing_module_in_dotted_root_package_is_ignored( - build_graph, write_files, import_statement: str -): +def test_missing_module_in_dotted_root_package_is_ignored(write_files, import_statement: str): """ When a dotted root package (namespace package) contains an import of a missing module within that same package, and include_external_packages=True, the Rust code should not diff --git a/tests/unit/adaptors/test_caching.py b/tests/unit/adaptors/test_caching.py deleted file mode 100644 index 2b684b89..00000000 --- a/tests/unit/adaptors/test_caching.py +++ /dev/null @@ -1,738 +0,0 @@ -import json -import logging - -import pytest - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.adaptors.caching import Cache, CacheFileNamer -from grimp.application.ports.caching import CacheMiss -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - - -class SimplisticFileNamer(CacheFileNamer): - """ - Simplistic version of the file namer that makes testing easier. - - Does not base64 encode the data filenames. - """ - - @classmethod - def make_data_file_name( - cls, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, - ) -> str: - unsafe_name = cls.make_data_file_unique_string( - found_packages, include_external_packages, exclude_type_checking_imports - ) - return f"{unsafe_name}.data.json" - - -class TestCacheFileNamer: - def test_names_meta_cache_per_package(self): - result = CacheFileNamer.make_meta_file_name( - FoundPackage( - name="some-package", - directory="whatever", - module_files=set(), - ) - ) - - assert result == "some-package.meta.json" - - @pytest.mark.parametrize( - "include_external_packages, exclude_type_checking_imports, expected", - ( - # Blake2B 20-character hash of "hyphenated-package,underscore_package". - (False, False, "a857d066514de048b7f94fa8d385e8bd7b048406.data.json"), - # Blake2B 20-character hash "hyphenated-package,underscore_package:external". - ( - True, - False, - "021977b6de56b09810ae52f5c9d067622c1ea30f.data.json", - ), - # Blake2B 20-character hash - # "hyphenated-package,underscore_package:external:no_type_checking". - ( - True, - True, - "4c2deb1d787161187915e159b5a17ea8b27cd0d4.data.json", - ), - # Blake2B 20-character hash "hyphenated-package,underscore_package:no_type_checking". - ( - False, - True, - "815e7686179e2f3f817c130eec1121d53e62ff1c.data.json", - ), - ), - ) - def test_make_data_file_name( - self, include_external_packages, exclude_type_checking_imports, expected - ): - result = CacheFileNamer.make_data_file_name( - found_packages={ - FoundPackage( - name="hyphenated-package", - directory="whatever", - module_files=frozenset(), - ), - FoundPackage( - name="underscore_package", - directory="whatever", - module_files=frozenset(), - ), - }, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - - assert result == expected - - @pytest.mark.parametrize( - "include_external_packages, exclude_type_checking_imports, expected", - ( - (False, False, "hyphenated-package,underscore_package"), - (True, False, "hyphenated-package,underscore_package:external"), - (True, True, "hyphenated-package,underscore_package:external:no_type_checking"), - (False, True, "hyphenated-package,underscore_package:no_type_checking"), - ), - ) - def test_make_data_file_unique_string( - self, include_external_packages, exclude_type_checking_imports, expected - ): - result = CacheFileNamer.make_data_file_unique_string( - found_packages={ - FoundPackage( - name="hyphenated-package", - directory="whatever", - module_files=frozenset(), - ), - FoundPackage( - name="underscore_package", - directory="whatever", - module_files=frozenset(), - ), - }, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - - assert result == expected - - -class TestCache: - SOME_MTIME = 1676645081.4935088 - FILE_SYSTEM = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - new.py - unmodified.py - modified.py - /path/to/anotherpackage/ - __init__.py - new.py - unmodified.py - modified.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.unmodified": {SOME_MTIME}, - "mypackage.foo.modified": {SOME_MTIME} - }}""", - ".grimp_cache/anotherpackage.meta.json": f"""{{ - "anotherpackage.unmodified": {SOME_MTIME}, - "anotherpackage.modified": {SOME_MTIME} - }}""", - ".grimp_cache/mypackage.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], ["brown", false, 22, "import brown"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - ".grimp_cache/mypackage:external.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], - ["brown", false, 22, "import brown"], - ["external", false, 100, "import external"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - ".grimp_cache/anotherpackage,mypackage.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], ["brown", false, 22, "import brown"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ], - "anotherpackage.unmodified": [ - ["purple", false, 11, "import purple"], ["green", false, 22, "import green"] - ], - "anotherpackage.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - ".grimp_cache/anotherpackage,mypackage:external.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], - ["brown", false, 22, "import brown"], - ["external", false, 100, "import external"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ], - "anotherpackage.unmodified": [ - ["purple", false, 11, "import purple"], - ["green", false, 22, "import green"], - ["anotherexternal", false, 100, "import anotherexternal"] - ], - "anotherpackage.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - }, - ) - MODULE_FILE_UNMODIFIED = ModuleFile( - module=Module("mypackage.foo.unmodified"), mtime=SOME_MTIME - ) - MODULE_FILE_MODIFIED = ModuleFile( - module=Module("mypackage.foo.modified"), mtime=SOME_MTIME + 100.0 - ) - MODULE_FILE_NEW = ModuleFile(module=Module("mypackage.foo.new"), mtime=SOME_MTIME) - - FOUND_PACKAGES = { - FoundPackage( - name="mypackage", - directory="/path/to/mypackage/", - module_files=frozenset( - {MODULE_FILE_MODIFIED, MODULE_FILE_UNMODIFIED, MODULE_FILE_NEW} - ), - ), - } - - @pytest.mark.parametrize( - "include_external_packages, expected_data_file", - ( - (True, ".grimp_cache/mypackage:external.data.json"), - (False, ".grimp_cache/mypackage.data.json"), - ), - ) - def test_logs_successful_cache_file_reading( - self, include_external_packages: bool, expected_data_file: str, caplog - ): - caplog.set_level(logging.INFO, logger=Cache.__module__) - - Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - include_external_packages=include_external_packages, - namer=SimplisticFileNamer, - ) - - assert caplog.messages == [ - "Used cache meta file .grimp_cache/mypackage.meta.json.", - f"Used cache data file {expected_data_file}.", - ] - - def test_logs_missing_cache_files(self, caplog): - caplog.set_level(logging.INFO, logger=Cache.__module__) - - Cache.setup( - file_system=rust.FakeBasicFileSystem(), # No cache files. - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - assert caplog.messages == [ - "No cache file: .grimp_cache/mypackage.meta.json.", - "No cache file: .grimp_cache/mypackage.data.json.", - ] - - @pytest.mark.parametrize("serialized_mtime", ("INVALID_JSON", '["wrong", "type"]')) - def test_logs_corrupt_cache_meta_file_reading(self, serialized_mtime: str, caplog): - caplog.set_level(logging.WARNING, logger=Cache.__module__) - - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {serialized_mtime}, - }}""", - ".grimp_cache/mypackage.data.json": "{}", - }, - ) - Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - assert caplog.messages == [ - "Could not use corrupt cache file .grimp_cache/mypackage.meta.json.", - ] - - def test_logs_corrupt_cache_data_file_reading(self, caplog): - caplog.set_level(logging.WARNING, logger=Cache.__module__) - - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {self.SOME_MTIME - 1} - }}""", - ".grimp_cache/mypackage.data.json": "INVALID JSON", - }, - ) - - Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - assert caplog.messages == [ - "Could not use corrupt cache file .grimp_cache/mypackage.data.json.", - ] - - @pytest.mark.parametrize("include_external_packages", (True, False)) - def test_raises_cache_miss_for_module_with_different_mtime( - self, include_external_packages: bool - ): - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - include_external_packages=include_external_packages, - namer=SimplisticFileNamer, - ) - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_MODIFIED) - - @pytest.mark.parametrize("include_external_packages", (True, False)) - def test_raises_cache_miss_for_module_with_no_cached_mtime( - self, include_external_packages: bool - ): - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=include_external_packages, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_NEW) - - @pytest.mark.parametrize( - "include_external_packages, expected_additional_imports", - ( - (False, set()), - ( - True, - { - DirectImport( - importer=MODULE_FILE_UNMODIFIED.module, - imported=Module("external"), - line_number=100, - line_contents="import external", - ), - }, - ), - ), - ) - def test_uses_cache_for_module_with_same_mtime( - self, include_external_packages, expected_additional_imports - ): - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=include_external_packages, - ) - - assert ( - cache.read_imports(self.MODULE_FILE_UNMODIFIED) - == { - DirectImport( - importer=self.MODULE_FILE_UNMODIFIED.module, - imported=Module("yellow"), - line_number=11, - line_contents="import yellow", - ), - DirectImport( - importer=self.MODULE_FILE_UNMODIFIED.module, - imported=Module("brown"), - line_number=22, - line_contents="import brown", - ), - } - | expected_additional_imports - ) - - def test_raises_cache_miss_for_missing_module_from_data(self): - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - somemodule.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.somemodule": {self.SOME_MTIME} - }}""", - ".grimp_cache/mypackage.data.json": """{}""", - }, - ) - module_file = ModuleFile(module=Module("mypackage.somemodule"), mtime=self.SOME_MTIME) - cache = Cache.setup( - file_system=file_system, - found_packages={ - FoundPackage( - name="mypackage", - directory="/path/to/mypackage/", - module_files=frozenset({module_file}), - ), - }, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(module_file) - - @pytest.mark.parametrize("serialized_mtime", ("INVALID_JSON", '["wrong", "type"]')) - def test_raises_cache_miss_for_corrupt_meta_file(self, serialized_mtime): - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - new.py - unmodified.py - modified.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {serialized_mtime} - }}""", - ".grimp_cache/mypackage.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - }, - ) - cache = Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_MODIFIED) - - @pytest.mark.parametrize("serialized_import", ("INVALID_JSON", '["wrong", "type"]')) - def test_raises_cache_miss_for_corrupt_data_file(self, serialized_import): - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - new.py - unmodified.py - modified.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {self.SOME_MTIME - 1} - }}""", - ".grimp_cache/mypackage.data.json": f"""{{ - "version": 2, - "imports_by_module": {{ - "mypackage.foo.modified": [ - {serialized_import} - ] - }} - }}""", - }, - ) - cache = Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_MODIFIED) - - @pytest.mark.parametrize( - "include_external_packages, expected_additional_imports", - ( - (False, set()), - ( - True, - { - DirectImport( - importer=Module("anotherpackage.unmodified"), - imported=Module("anotherexternal"), - line_number=100, - line_contents="import anotherexternal", - ), - }, - ), - ), - ) - def test_uses_cache_multiple_packages( - self, include_external_packages, expected_additional_imports - ): - module_file = ModuleFile(module=Module("anotherpackage.unmodified"), mtime=self.SOME_MTIME) - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES - | { - FoundPackage( - name="anotherpackage", - directory="/path/to/anotherpackage/", - module_files=frozenset({module_file}), - ), - }, - namer=SimplisticFileNamer, - include_external_packages=include_external_packages, - ) - - result = cache.read_imports(module_file) - - assert ( - result - == { - DirectImport( - importer=module_file.module, - imported=Module("purple"), - line_number=11, - line_contents="import purple", - ), - DirectImport( - importer=module_file.module, - imported=Module("green"), - line_number=22, - line_contents="import green", - ), - } - | expected_additional_imports - ) - - @pytest.mark.parametrize( - "cache_dir", - ( - "/tmp/some-cache-dir", - "/tmp/some-cache-dir/", - None, - ), - ) - @pytest.mark.parametrize( - "include_external_packages, expected_data_file_name", - ( - (False, "blue,green.data.json"), - (True, "blue,green:external.data.json"), - ), - ) - def test_write_to_cache( - self, include_external_packages, expected_data_file_name, cache_dir, caplog - ): - caplog.set_level(logging.INFO, logger=Cache.__module__) - file_system = rust.FakeBasicFileSystem() - blue_one = Module(name="blue.one") - blue_two = Module(name="blue.two") - green_one = Module(name="green.one") - green_two = Module(name="green.two") - mtimes = { - blue_one: 10000.1, - blue_two: 20000.2, - green_one: 30000.3, - green_two: 40000.4, - } - cache = Cache.setup( - file_system=file_system, - cache_dir=cache_dir, - found_packages={ - FoundPackage( - name="blue", - module_files=frozenset( - { - ModuleFile(module=blue_one, mtime=mtimes[blue_one]), - ModuleFile(module=blue_two, mtime=mtimes[blue_two]), - } - ), - directory="-", - ), - FoundPackage( - name="green", - module_files=frozenset( - { - ModuleFile(module=green_one, mtime=mtimes[green_one]), - ModuleFile(module=green_two, mtime=mtimes[green_two]), - } - ), - directory="-", - ), - }, - include_external_packages=include_external_packages, - namer=SimplisticFileNamer, - ) - - cache.write( - imports_by_module={ - blue_one: { - DirectImport( - importer=blue_one, - imported=blue_two, - line_number=11, - line_contents="from . import two", - ), - DirectImport( - importer=blue_one, - imported=Module("externalpackage"), - line_number=22, - line_contents="import externalpackage", - ), - }, - blue_two: set(), - green_one: set(), - green_two: { - DirectImport( - importer=green_two, - imported=green_one, - line_number=33, - line_contents="from . import one", - ), - }, - }, - ) - - # Assert the cache is written afterwards. - expected_cache_dir = cache_dir.rstrip(file_system.sep) if cache_dir else ".grimp_cache" - assert set(caplog.messages) == { - f"No cache file: {expected_cache_dir}/{expected_data_file_name}.", - f"No cache file: {expected_cache_dir}/blue.meta.json.", - f"No cache file: {expected_cache_dir}/green.meta.json.", - f"Wrote data cache file {expected_cache_dir}/{expected_data_file_name}.", - f"Wrote meta cache file {expected_cache_dir}/blue.meta.json.", - f"Wrote meta cache file {expected_cache_dir}/green.meta.json.", - } - expected = { - f"{expected_cache_dir}/blue.meta.json": { - blue_one.name: mtimes[blue_one], - blue_two.name: mtimes[blue_two], - }, - f"{expected_cache_dir}/green.meta.json": { - green_one.name: mtimes[green_one], - green_two.name: mtimes[green_two], - }, - f"{expected_cache_dir}/{expected_data_file_name}": { - "version": 2, - "imports_by_module": { - blue_one.name: { - (blue_two.name, False, 11, "from . import two"), - ("externalpackage", False, 22, "import externalpackage"), - }, - blue_two.name: set(), - green_one.name: set(), - green_two.name: { - (green_one.name, False, 33, "from . import one"), - }, - }, - }, - } - for filename, expected_deserialized in expected.items(): - serialized = file_system.read(filename) - deserialized = _unflake(json.loads(serialized)) - assert deserialized == expected_deserialized - - def test_write_to_cache_adds_marker_files(self): - some_cache_dir = "/tmp/some-cache-dir" - file_system = rust.FakeBasicFileSystem() - cache = Cache.setup( - file_system=file_system, - cache_dir=some_cache_dir, - found_packages=set(), - include_external_packages=False, # Value shouldn't matter. - namer=SimplisticFileNamer, - ) - - cache.write( - imports_by_module={}, - ) - - assert file_system.read(f"{some_cache_dir}/.gitignore") == ( - "# Automatically created by Grimp.\n*" - ) - assert file_system.read(f"{some_cache_dir}/CACHEDIR.TAG") == ( - "Signature: 8a477f597d28d172789f06886806bc55\n" - "# This file is a cache directory tag automatically created by Grimp.\n" - "# For information about cache directory tags see https://bford.info/cachedir/" - ) - - -def _unflake(flakey_data): - # Takes the deserialized data that could come in any - # order and order it consistently, so the test isn't flakey. - non_flakey_data = {} - for key, value in flakey_data.items(): - if isinstance(value, list): - non_flakey_value = {tuple(i) for i in value} - elif isinstance(value, dict): - non_flakey_value = _unflake(value) - else: - non_flakey_value = value - non_flakey_data[key] = non_flakey_value - return non_flakey_data diff --git a/tests/unit/adaptors/test_filesystem.py b/tests/unit/adaptors/test_filesystem.py deleted file mode 100644 index 3526e55c..00000000 --- a/tests/unit/adaptors/test_filesystem.py +++ /dev/null @@ -1,278 +0,0 @@ -from __future__ import annotations - -from copy import copy -from typing import TypeAlias - -import pytest - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import BasicFileSystem -from tests.adaptors.filesystem import FakeFileSystem - - -class _Base: - """ - Tests for methods that AbstractFileSystem and BasicFileSystem share. - """ - - file_system_cls: type[BasicFileSystem] - - @pytest.mark.parametrize("path", ("/path/to", "/path/to/")) - def test_join(self, path): - file_system = self.file_system_cls() - assert "/path/to/mypackage/file.py" == file_system.join(path, "mypackage", "file.py") - - @pytest.mark.parametrize( - "path, expected", - [ - ("", ("", "")), - ("/path", ("/", "path")), - ("some-file", ("", "some-file")), - ("/path/to/mypackage/", ("/path/to/mypackage", "")), - ("/path/to/mypackage/some-file", ("/path/to/mypackage", "some-file")), - ("/path/to/mypackage/some-file.py", ("/path/to/mypackage", "some-file.py")), - ], - ) - def test_split(self, path, expected): - file_system = self.file_system_cls() - assert file_system.split(path) == expected - - @pytest.mark.parametrize( - "file_name, expected", - [ - ("/path/to/mypackage", False), - ("/path/to/mypackage/", False), - ("/path/to/mypackage/readme.txt", True), - ("/path/to/mypackage/foo/one.txt", True), - ("/path/to/mypackage/foo/two/green.txt", True), - ("/path/to/mypackage/bar/blue.txt", True), - ("/path/to/nonexistent.txt", False), - ("/path/to/mypackage/purple.txt", False), - ], - ) - def test_exists_content_only(self, file_name, expected): - file_system = self.file_system_cls( - contents=""" - /path/to/mypackage/ - readme.txt - foo/ - one.txt - two/ - green.txt - bar/ - blue.txt - """ - ) - - assert file_system.exists(file_name) == expected - - @pytest.mark.parametrize( - "file_name, expected", - [ - ("/path/to/", False), - ("/path/to/file.txt", True), - ("/path/to/a/deeper/file.txt", True), - ("/path/to/nonexistent.txt", False), - ], - ) - def test_exists_content_map_only(self, file_name, expected): - file_system = self.file_system_cls( - content_map={ - "/path/to/file.txt": "hello", - "/path/to/a/deeper/file.txt": "hello", - } - ) - - assert file_system.exists(file_name) == expected - - @pytest.mark.parametrize( - "file_name, expected", - [ - ("/path/to/file.txt", True), - ("/path/to/mypackage/foo/two/green.txt", True), - ("/path/to/nonexistent.txt", False), - ], - ) - def test_exists_content_map_and_content(self, file_name, expected): - file_system = self.file_system_cls( - contents=""" - /path/to/mypackage/ - readme.txt - foo/ - one.txt - two/ - green.txt - """, - content_map={ - "/path/to/file.txt": "hello", - "/path/to/a/deeper/file.txt": "hello", - }, - ) - - assert file_system.exists(file_name) == expected - - @pytest.mark.parametrize( - "file_name, expected_contents", - ( - ("/path/to/mypackage/readme.txt", "Lorem"), - ("/path/to/mypackage/foo/one.txt", "Ipsum"), - # Listed in contents, but not in content_map. - ("/path/to/mypackage/foo/two/green.txt", ""), - # Listed in content_map, but not in contents. - ("/path/to/mypackage/foo/two/blue.txt", "Dolor sic"), - ( - "/path/to/mypackage/indented.txt", - "This is indented text.\n We should dedent it.", - ), - ("/path/to/mypackage/nonexistent", FileNotFoundError), - ("/path/to/mypackage/foo/three/nonexistent", FileNotFoundError), - # TODO - should we raise an exception if we attempt to read a directory? - ), - ) - def test_read(self, file_name, expected_contents): - file_system = self.file_system_cls( - contents=""" - /path/to/mypackage/ - readme.txt - foo/ - one.txt - two/ - green.txt - """, - content_map={ - "/path/to/mypackage/readme.txt": "Lorem", - "/path/to/mypackage/foo/one.txt": "Ipsum", - "/path/to/mypackage/foo/two/blue.txt": "Dolor sic", - "/path/to/mypackage/indented.txt": """ - This is indented text. - We should dedent it. - """, - }, - ) - if isinstance(expected_contents, type) and issubclass(expected_contents, Exception): - with pytest.raises(expected_contents): - file_system.read(file_name) - else: - assert file_system.read(file_name) == expected_contents - - def test_write(self): - some_filename, some_contents = "path/to/some-file.txt", "Some contents." - file_system = self.file_system_cls() - - file_system.write(some_filename, some_contents) - - assert file_system.read(some_filename) == some_contents - - -WalkReturn: TypeAlias = tuple[str, list[str], list[str]] - - -class TestFakeFileSystem(_Base): - file_system_cls = FakeFileSystem - - MYPACKAGE: WalkReturn = ("/path/to/mypackage", ["foo"], ["__init__.py"]) - MYPACKAGE_FOO: WalkReturn = ("/path/to/mypackage/foo", ["two"], ["__init__.py", "one.py"]) - MYPACKAGE_FOO_TWO: WalkReturn = ( - "/path/to/mypackage/foo/two", - [], - ["__init__.py", "green.py", "blue.py"], - ) - - @pytest.mark.parametrize( - "directory, expected", - [ - ( - "/path/to/mypackage", - [MYPACKAGE, MYPACKAGE_FOO, MYPACKAGE_FOO_TWO], - ), - ( - "/path/to/mypackage/foo", - [MYPACKAGE_FOO, MYPACKAGE_FOO_TWO], - ), - ( - "/path/to/mypackage/foo/two", - [MYPACKAGE_FOO_TWO], - ), - ( - "/anotherpackage", - [("/anotherpackage", [], ["another.txt"])], - ), - ], - ) - def test_walk(self, directory: str, expected: list[WalkReturn]): - file_system = self.file_system_cls( - """ - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - /anotherpackage/ - another.txt - """ - ) - - result = list(file_system.walk(directory)) - - assert result == expected - - def test_empty_if_directory_does_not_exist(self): - file_system = self.file_system_cls( - """ - /path/to/mypackage/ - __init__.py - """ - ) - assert [] == list(file_system.walk("/path/to/nonexistent/package")) - - def test_dirname(self): - file_system = self.file_system_cls() - assert "/path/to" == file_system.dirname("/path/to/file.txt") - - def test_dirnames_can_be_modified_in_place(self): - """ - From the os.walk docs: - The caller can modify the dirnames list in-place (perhaps using del or slice - assignment), and walk() will only recurse into the subdirectories whose names - remain in dirnames; this can be used to prune the search, impose a specific order - of visiting, or even to inform walk() about directories the caller creates or renames - before it resumes walk() again. - """ - file_system = self.file_system_cls( - """ - /path/to/mypackage/ - foo/ - one.txt - skipme/ - two.txt - dontskip/ - three.txt - bar/ - four.txt - """ - ) - - expected_tuples = [ - ("/path/to/mypackage", ["foo", "bar"], []), - ("/path/to/mypackage/foo", ["skipme", "dontskip"], ["one.txt"]), - ("/path/to/mypackage/foo/dontskip", [], ["three.txt"]), - ("/path/to/mypackage/bar", [], ["four.txt"]), - ] - - actual_tuples = [] - for dirpath, dirs, files in file_system.walk("/path/to/mypackage"): - # Ensure we make a copy of dirs (since we change it). - actual_tuples.append((dirpath, copy(dirs), files)) - if "skipme" in dirs: - dirs.remove("skipme") - continue - - assert expected_tuples == actual_tuples - - -class TestFakeBasicFileSystem(_Base): - file_system_cls = rust.FakeBasicFileSystem diff --git a/tests/unit/adaptors/test_modulefinder.py b/tests/unit/adaptors/test_modulefinder.py deleted file mode 100644 index ad5a84a6..00000000 --- a/tests/unit/adaptors/test_modulefinder.py +++ /dev/null @@ -1,317 +0,0 @@ -import pytest - -from grimp.adaptors.modulefinder import ModuleFinder -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import Module -from tests.adaptors.filesystem import DEFAULT_MTIME, FakeFileSystem - - -def test_happy_path(): - module_finder = ModuleFinder() - SOME_MTIME = 12340000.3 - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - not-a-python-file.txt - .hidden - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - """, - mtime_map={ - "/path/to/mypackage/foo/one.py": SOME_MTIME, - }, - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files=frozenset( - { - ModuleFile(module=Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo.one"), mtime=SOME_MTIME), - ModuleFile(module=Module("mypackage.foo.two"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo.two.green"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo.two.blue"), mtime=DEFAULT_MTIME), - } - ), - ) - - -MODULE_FILES_FOO_BLUE = { - ModuleFile(module=Module("somenamespace.foo.blue"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.blue.one"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.blue.two"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.blue.two.alpha"), mtime=DEFAULT_MTIME), -} -MODULE_FILES_FOO_GREEN_FIVE = { - ModuleFile(module=Module("somenamespace.foo.green.five"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.green.five.beta"), mtime=DEFAULT_MTIME), -} - - -@pytest.mark.parametrize( - "package_name, package_directory, expected", - [ - ( - "somenamespace", - "/path/to/somenamespace", - FoundPackage( - name="somenamespace", - directory="/path/to/somenamespace", - module_files=MODULE_FILES_FOO_BLUE | MODULE_FILES_FOO_GREEN_FIVE, - namespace_packages=frozenset( - { - "somenamespace", - "somenamespace.foo", - "somenamespace.foo.green", - } - ), - ), - ), - ( - "somenamespace.foo", - "/path/to/somenamespace/foo", - FoundPackage( - name="somenamespace.foo", - directory="/path/to/somenamespace/foo", - module_files=MODULE_FILES_FOO_BLUE | MODULE_FILES_FOO_GREEN_FIVE, - namespace_packages=frozenset( - { - "somenamespace.foo", - "somenamespace.foo.green", - } - ), - ), - ), - ( - "somenamespace.foo.blue", - "/path/to/somenamespace/foo/blue", - FoundPackage( - name="somenamespace.foo.blue", - directory="/path/to/somenamespace/foo/blue", - module_files=MODULE_FILES_FOO_BLUE, - ), - ), - ], -) -def test_namespaced_packages(package_name: str, package_directory: str, expected: FoundPackage): - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/somenamespace/ - foo/ - blue/ - __init__.py - one.py - two/ - __init__.py - alpha.py - noinitpackage/ - three.py - orphan/ - __init__.py - four.py - green/ - five/ - __init__.py - beta.py - non_python_directory/ - six/ - README.txt - """ - ) - - result = module_finder.find_package( - package_name=package_name, - package_directory=package_directory, - file_system=file_system, - ) - - assert result == expected - - -def test_ignores_orphaned_python_files(): - # Python files in directories that don't contain an __init__.py should not be discovered. - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - two/ - __init__.py - green.py - noinitpackage/ - green.py - orphan/ - __init__.py - red.py - """ - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files={ - ModuleFile(module=Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.two"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.two.green"), mtime=DEFAULT_MTIME), - }, - ) - - -def test_ignores_invalid_identifier_directories(): - # Python files in directories that don't contain an __init__.py should not be discovered. - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/namespacepackage/ - foo/ - valid_underscore_name/ - __init__.py - mod.py - invalid-hyphenated-name/ - __init__.py - mod.py - 1nvalid-number-prefixed/ - __init__.py - mod.py - valid_non_àscii/ - __init__.py - mod.py - invalid_non_😃scii/ - __init__.py - mod.py - bar/ - __init__.py - mod.py - """ - ) - - result = module_finder.find_package( - package_name="namespacepackage", - package_directory="/path/to/namespacepackage", - file_system=file_system, - ) - - module_files = { - ModuleFile(module=Module(name), mtime=DEFAULT_MTIME) - for name in ( - "namespacepackage.foo.valid_underscore_name", - "namespacepackage.foo.valid_underscore_name.mod", - "namespacepackage.foo.valid_non_àscii", - "namespacepackage.foo.valid_non_àscii.mod", - "namespacepackage.bar", - "namespacepackage.bar.mod", - ) - } - assert result == FoundPackage( - name="namespacepackage", - directory="/path/to/namespacepackage", - module_files=module_files, - namespace_packages=frozenset({"namespacepackage", "namespacepackage.foo"}), - ) - - -@pytest.mark.parametrize( - "extension, should_warn", - ( - ("py", True), - ("txt", False), # Anything other than .py. - ), -) -def test_ignores_dotted_python_files(extension, should_warn, caplog): - # Python files containing dots (other than the one before .py) should not be discovered. - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=f""" - /path/to/mypackage/ - __init__.py - foo.py - bar/ - __init__.py - baz.dotted.{extension} - """ - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files={ - ModuleFile(module=Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.bar"), mtime=DEFAULT_MTIME), - }, - ) - if should_warn: - assert caplog.messages == [ - ( - "Warning: skipping module with too many dots in the name: " - f"/path/to/mypackage/bar/baz.dotted.{extension}" - ) - ] - else: - assert caplog.messages == [] - - -def test_ignores_hidden_directories(): - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - two/ - __init__.py - green.py - .hidden/ - green.py - orphan/ - __init__.py - red.py - """ - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files={ - ModuleFile(Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(Module("mypackage.two"), mtime=DEFAULT_MTIME), - ModuleFile(Module("mypackage.two.green"), mtime=DEFAULT_MTIME), - }, - ) diff --git a/tests/unit/adaptors/test_packagefinder.py b/tests/unit/adaptors/test_packagefinder.py index 2f687a29..ce1b150b 100644 --- a/tests/unit/adaptors/test_packagefinder.py +++ b/tests/unit/adaptors/test_packagefinder.py @@ -3,9 +3,7 @@ import pytest from grimp import exceptions -from grimp.adaptors.filesystem import FileSystem from grimp.adaptors.packagefinder import ImportLibPackageFinder -from tests.adaptors.filesystem import FakeFileSystem assets = (Path(__file__).parent.parent.parent / "assets").resolve() @@ -44,9 +42,7 @@ ), ) def test_determine_package_directories(package, expected): - assert ( - ImportLibPackageFinder().determine_package_directories(package, FileSystem()) == expected - ) + assert ImportLibPackageFinder().determine_package_directories(package) == expected @pytest.mark.parametrize( @@ -63,4 +59,4 @@ def test_determine_package_directories_doesnt_support_non_top_level_modules(pack with pytest.raises( exceptions.NotATopLevelModule, ): - ImportLibPackageFinder().determine_package_directories(package, FakeFileSystem()) + ImportLibPackageFinder().determine_package_directories(package) diff --git a/tests/unit/application/test_scanning.py b/tests/unit/application/test_scanning.py deleted file mode 100644 index 7f209442..00000000 --- a/tests/unit/application/test_scanning.py +++ /dev/null @@ -1,1254 +0,0 @@ -from __future__ import annotations - -from collections.abc import Set - -import pytest - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application import scanning -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module -from tests.config import override_settings - - -@pytest.mark.parametrize( - "include_external_packages, expected_result", - ( - ( - False, - { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ) - }, - ), - ( - True, - { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("externalone"), - line_number=2, - line_contents="import externalone", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("externaltwo"), - line_number=3, - line_contents="import externaltwo.subpackage # with comment afterwards.", - ), - }, - ), - ), -) -def test_absolute_imports(include_external_packages, expected_result): - module_file_to_scan = _module_to_module_file(Module("foo.one")) - all_module_files = frozenset({module_file_to_scan, _module_to_module_file(Module("foo.two"))}) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": """ - import foo.two - import externalone - import externaltwo.subpackage # with comment afterwards. - arbitrary_expression = 1 - """ - } - ) - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages={ - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=all_module_files, - ) - }, - include_external_packages=include_external_packages, - exclude_type_checking_imports=False, - ) - - assert {module_file_to_scan: expected_result} == result - - -def test_lazy_imports(): - module_file_to_scan = _module_to_module_file(Module("foo.one.blue")) - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.blue"), - Module("foo.one.green"), - Module("foo.two"), - Module("foo.two.brown"), - Module("foo.two.yellow"), - Module("foo.three"), - } - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - blue.py - green.py - two/ - __init__.py - brown.py - yellow.py - three.py - """, - content_map={ - "/path/to/foo/one/blue.py": """ - # Absolute no-from - lazy import foo.two - lazy import externalone - lazy import externaltwo.subpackage # with comment afterwards. - arbitrary_expression = 1 - - # Absolute from - lazy from foo.one import green - lazy from foo.two import yellow - if t.TYPE_CHECKING: - lazy from foo import three - lazy from external import one - lazy from external.two import blue # with comment afterwards. - - # Relative from - lazy from . import green - lazy from ..two import yellow - lazy from .. import three - """ - }, - ) - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages={ - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - }, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - expected = { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two"), - line_number=2, - line_contents="lazy import foo.two", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("externalone"), - line_number=3, - line_contents="lazy import externalone", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("externaltwo"), - line_number=4, - line_contents="lazy import externaltwo.subpackage # with comment afterwards.", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=8, - line_contents="lazy from foo.one import green", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=9, - line_contents="lazy from foo.two import yellow", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=11, - line_contents="lazy from foo import three", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=12, - line_contents="lazy from external import one", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=13, - line_contents="lazy from external.two import blue # with comment afterwards.", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=16, - line_contents="lazy from . import green", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=17, - line_contents="lazy from ..two import yellow", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=18, - line_contents="lazy from .. import three", - is_lazy=True, - ), - } - } - assert result == expected - - -def test_non_ascii(): - blue_module = Module("mypackage.blue") - blue_module_file = _module_to_module_file(blue_module) - non_ascii_modules = { - Module("mypackage.ñon_ascii_变"), - Module("mypackage.ñon_ascii_变.ラーメン"), - } - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/mypackage/blue.py": """ - from ñon_ascii_变 import * - from . import ñon_ascii_变 - import mypackage.ñon_ascii_变.ラーメン - """, - "/path/to/mypackage/ñon_ascii_变/__init__.py": "", - "/path/to/mypackage/ñon_ascii_变/ラーメン.py": "", - }, - ) - found_packages = { - FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files=frozenset({blue_module_file}) - | _modules_to_module_files(non_ascii_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {blue_module_file}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - assert result == { - blue_module_file: { - DirectImport( - importer=Module("mypackage.blue"), - imported=Module("ñon_ascii_变"), - line_number=1, - line_contents="from ñon_ascii_变 import *", - ), - DirectImport( - importer=Module("mypackage.blue"), - imported=Module("mypackage.ñon_ascii_变"), - line_number=2, - line_contents="from . import ñon_ascii_变", - ), - DirectImport( - importer=Module("mypackage.blue"), - imported=Module("mypackage.ñon_ascii_变.ラーメン"), - line_number=3, - line_contents="import mypackage.ñon_ascii_变.ラーメン", - ), - } - } - - -def test_single_namespace_package_portion(): - MODULE_FOO = Module("namespace.foo") - MODULE_ONE = Module("namespace.foo.one") - MODULE_TWO = Module("namespace.foo.two") - MODULE_THREE = Module("namespace.foo.three") - MODULE_FOUR = Module("namespace.foo.four") - MODULE_GREEN = Module("namespace.foo.two.green") - MODULE_BLUE = Module("namespace.foo.two.blue") - MODULE_ALPHA = Module("namespace.foo.two.green.alpha") - - all_modules = { - MODULE_FOO, - MODULE_ONE, - MODULE_TWO, - MODULE_THREE, - MODULE_FOUR, - MODULE_GREEN, - MODULE_BLUE, - MODULE_ALPHA, - } - module_one_file = _module_to_module_file(MODULE_ONE) - module_alpha_file = _module_to_module_file(MODULE_ALPHA) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/namespace/foo/one.py": """ - import namespace.foo.two - from namespace.foo import three - from . import four - """, - "/path/to/namespace/foo/two/green/alpha.py": """ - from .. import blue - from ... import three - """, - } - ) - found_packages = { - FoundPackage( - name="namespace.foo", - directory="/path/to/namespace/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_one_file, module_alpha_file}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - assert result == { - module_one_file: { - DirectImport( - importer=MODULE_ONE, - imported=MODULE_TWO, - line_number=1, - line_contents="import namespace.foo.two", - ), - DirectImport( - importer=MODULE_ONE, - imported=MODULE_THREE, - line_number=2, - line_contents="from namespace.foo import three", - ), - DirectImport( - importer=MODULE_ONE, - imported=MODULE_FOUR, - line_number=3, - line_contents="from . import four", - ), - }, - module_alpha_file: { - DirectImport( - importer=MODULE_ALPHA, - imported=MODULE_BLUE, - line_number=1, - line_contents="from .. import blue", - ), - DirectImport( - importer=MODULE_ALPHA, - imported=MODULE_THREE, - line_number=2, - line_contents="from ... import three", - ), - }, - } - - -@pytest.mark.parametrize("include_external_packages", (True, False)) -def test_import_of_portion_not_in_graph(include_external_packages): - # Include two namespaces, one deeper than the other. - MODULE_BAR_ONE_GREEN = Module("namespace.bar.one.green") - MODULE_BAR_ONE_GREEN_ALPHA = Module("namespace.bar.one.green.alpha") - - MODULE_FOO = Module("namespace.foo") - MODULE_FOO_ONE = Module("namespace.foo.one") - MODULE_FOO_BLUE = Module("namespace.foo.one.blue") - module_foo_one_file = _module_to_module_file(MODULE_FOO_ONE) - module_foo_blue_file = _module_to_module_file(MODULE_FOO_BLUE) - all_modules = { - MODULE_FOO, - MODULE_FOO_ONE, - MODULE_FOO_BLUE, - } - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/namespace/foo/one.py": """ - import namespace.bar.one.orange - from namespace.yellow import one - from .. import mauve - if t.TYPE_CHECKING: - from ..cyan import one - from ..magenta.one import alpha - """, - "/path/to/namespace/foo/one/blue.py": """ - from ... import teal - from ...pink import one - from ...scarlet.one import alpha - """, - } - ) - - found_packages = { - FoundPackage( - name="namespace.foo", - directory="/path/to/namespace/foo", - module_files=_modules_to_module_files(all_modules), - ), - FoundPackage( - name="namespace.bar.one.green", - directory="/path/to/namespace/bar/one/green", - module_files=_modules_to_module_files( - { - MODULE_BAR_ONE_GREEN, - MODULE_BAR_ONE_GREEN_ALPHA, - } - ), - ), - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_file, module_foo_blue_file}, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=False, - ) - - if include_external_packages: - assert result == { - module_foo_one_file: { - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.bar.one.orange"), - line_number=1, - line_contents="import namespace.bar.one.orange", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.yellow"), - line_number=2, - line_contents="from namespace.yellow import one", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.mauve"), - line_number=3, - line_contents="from .. import mauve", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.cyan"), - line_number=5, - line_contents="from ..cyan import one", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.magenta"), - line_number=6, - line_contents="from ..magenta.one import alpha", - ), - }, - module_foo_blue_file: { - DirectImport( - importer=MODULE_FOO_BLUE, - imported=Module("namespace.teal"), - line_number=1, - line_contents="from ... import teal", - ), - DirectImport( - importer=MODULE_FOO_BLUE, - imported=Module("namespace.pink"), - line_number=2, - line_contents="from ...pink import one", - ), - DirectImport( - importer=MODULE_FOO_BLUE, - imported=Module("namespace.scarlet"), - line_number=3, - line_contents="from ...scarlet.one import alpha", - ), - }, - } - else: - assert result == { - module_foo_one_file: set(), - module_foo_blue_file: set(), - } - - -@pytest.mark.parametrize( - "include_external_packages, expected_result", - ( - ( - False, - { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=1, - line_contents="from foo.one import green", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=2, - line_contents="from foo.two import yellow", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=4, - line_contents="from foo import three", - ), - }, - ), - ( - True, - { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=1, - line_contents="from foo.one import green", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=2, - line_contents="from foo.two import yellow", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=4, - line_contents="from foo import three", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=5, - line_contents="from external import one", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=6, - line_contents="from external.two import blue # with comment afterwards.", - ), - }, - ), - ), -) -def test_absolute_from_imports(include_external_packages, expected_result): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.blue"), - Module("foo.one.green"), - Module("foo.two"), - Module("foo.two.brown"), - Module("foo.two.yellow"), - Module("foo.three"), - } - module_foo_one_blue_file = _module_to_module_file(Module("foo.one.blue")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - blue.py - green.py - two/ - __init__.py - brown.py - yellow.py - three.py - """, - content_map={ - "/path/to/foo/one/blue.py": """ - from foo.one import green - from foo.two import yellow - if t.TYPE_CHECKING: - from foo import three - from external import one - from external.two import blue # with comment afterwards. - arbitrary_expression = 1 - """ - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_blue_file}, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=False, - ) - - assert {module_foo_one_blue_file: expected_result} == result - - -@pytest.mark.parametrize("module_to_scan_is_package", (True, False)) -def test_relative_from_imports(module_to_scan_is_package): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.blue"), - Module("foo.one.green"), - Module("foo.two.brown"), - Module("foo.two.yellow"), - Module("foo.three"), - } - if module_to_scan_is_package: - module_to_scan = Module("foo.one") - module_filename = "/path/to/foo/one/__init__.py" - else: - module_to_scan = Module("foo.one.blue") - module_filename = "/path/to/foo/one/blue.py" - module_file_to_scan = _module_to_module_file(module_to_scan) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - blue.py - green.py - two/ - __init__.py - brown.py - yellow.py - three.py - """, - content_map={ - module_filename: """ - from . import green - from ..two import yellow - from .. import three - arbitrary_expression = 1 - """ - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=True, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=module_to_scan, - imported=Module("foo.one.green"), - line_number=1, - line_contents="from . import green", - ), - DirectImport( - importer=module_to_scan, - imported=Module("foo.two.yellow"), - line_number=2, - line_contents="from ..two import yellow", - ), - DirectImport( - importer=module_to_scan, - imported=Module("foo.three"), - line_number=3, - line_contents="from .. import three", - ), - } - } - - -@pytest.mark.parametrize( - "import_source", - ("from .two.yellow import my_function", "from foo.two.yellow import my_function"), -) -def test_trims_to_known_modules(import_source): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.two"), - Module("foo.two.yellow"), - } - module_file_to_scan = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one.py - two/ - __init__.py - yellow.py - """, - content_map={"/path/to/foo/one.py": import_source}, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two.yellow"), - line_number=1, - line_contents=import_source, - ) - } - } - - -def test_trims_to_known_modules_within_init_file(): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.yellow"), - Module("foo.one.blue"), - Module("foo.one.blue.alpha"), - } - module_file_foo_one = _module_to_module_file(Module("foo.one")) - module_file_foo_one_blue = _module_to_module_file(Module("foo.one.blue")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - yellow.py - blue/ - __init__.py - alpha.py - """, - content_map={ - "/path/to/foo/one/__init__.py": "from .yellow import my_function", - "/path/to/foo/one/blue/__init__.py": "from .alpha import my_function", - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_foo_one, module_file_foo_one_blue}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_foo_one: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.one.yellow"), - line_number=1, - line_contents="from .yellow import my_function", - ) - }, - module_file_foo_one_blue: { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.blue.alpha"), - line_number=1, - line_contents="from .alpha import my_function", - ) - }, - } - - -def test_trims_whitespace_from_start_of_line_contents(): - all_modules = {Module("foo"), Module("foo.one"), Module("foo.two")} - module_file_to_scan = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one.py - two.py - """, - content_map={ - "/path/to/foo/one.py": """ - def my_function(): - from . import two - """ - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=2, - line_contents="from . import two", - ), - } - } - - -@pytest.mark.parametrize( - "statement, expected_module_name", - ( - # External packages that share a namespace with an internal module resolve - # to the shallowest component that does not clash with an internal module namespace. - ("import namespace", None), - ("import namespace.foo", None), - ("from namespace import foo", None), - ("import namespace.bar", "namespace.bar"), - ("from namespace import bar", "namespace.bar"), - ("from ... import bar", "namespace.bar"), - ("import namespace.bar.orange", "namespace.bar"), - ("from namespace.bar import orange", "namespace.bar"), - ("from ...bar import orange", "namespace.bar"), - ("import namespace.foo.green", "namespace.foo.green"), - ("from namespace.foo import green", "namespace.foo.green"), - ("from .. import green", "namespace.foo.green"), - ("import namespace.foo.green.alpha", "namespace.foo.green"), - ("from namespace.foo.green import alpha", "namespace.foo.green"), - ("from ..green import alpha", "namespace.foo.green"), - ("import namespace.foo.green.alpha.one", "namespace.foo.green"), - ("from namespace.foo.green.alpha import one", "namespace.foo.green"), - ("from ..green.alpha import one", "namespace.foo.green"), - ), -) -def test_external_package_imports_for_namespace_packages(statement, expected_module_name): - module_to_scan = Module("namespace.foo.blue.alpha") - module_file_to_scan = _module_to_module_file(module_to_scan) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/namespace/foo/blue/alpha.py": statement, - } - ) - found_packages = { - FoundPackage( - name="namespace.foo.blue", - directory="/path/to/namespace/foo/blue", - module_files=_modules_to_module_files( - { - Module("namespace.foo.blue"), - module_to_scan, - Module("namespace.foo.blue.beta"), - } - ), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - if expected_module_name: - assert { - module_file_to_scan: { - DirectImport( - importer=module_to_scan, - imported=Module(expected_module_name), - line_number=1, - line_contents=statement, - ), - } - } == result - else: - assert {module_file_to_scan: set()} == result - - -@pytest.mark.parametrize("statement", ("import bar.blue", "from bar import blue")) -def test_scans_multiple_packages(statement): - foo_modules = {Module("foo"), Module("foo.one"), Module("foo.two")} - bar_modules = {Module("bar"), Module("bar.green"), Module("bar.blue")} - foo_one_module_file = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": f""" - import foo.two - {statement} - import externalone - - arbitrary_expression = 1 - """ - } - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(foo_modules), - ), - FoundPackage( - name="bar", - directory="/path/to/bar", - module_files=_modules_to_module_files(bar_modules), - ), - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {foo_one_module_file}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert { - foo_one_module_file: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("bar.blue"), - line_number=2, - line_contents=statement, - ), - } - } == result - - -@pytest.mark.parametrize("exclude_type_checking_imports", (True, False)) -@pytest.mark.parametrize( - "statement, is_statement_valid", - ( - ("if t.TYPE_CHECKING:", True), - ("if TYPE_CHECKING:", True), - ("if typing.TYPE_CHECKING:", True), - ("if WEIRD_ALIAS.TYPE_CHECKING:", True), - ("if type_checking:", False), - ("while TYPE_CHECKING:", False), - ), -) -def test_exclude_type_checking_imports( - exclude_type_checking_imports, statement, is_statement_valid -): - all_modules = { - Module("foo.one"), - Module("foo.two"), - Module("foo.three"), - Module("foo.four"), - Module("foo.five"), - } - module_foo_one_file = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": f""" - import foo.two - {statement} - import foo.three - import foo.four - import foo.five - arbitrary_expression = 1 - """ - } - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_file}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - - if exclude_type_checking_imports and is_statement_valid: - expected_result = { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.five"), - line_number=5, - line_contents="import foo.five", - ), - } - else: - expected_result = { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.three"), - line_number=3, - line_contents="import foo.three", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.four"), - line_number=4, - line_contents="import foo.four", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.five"), - line_number=5, - line_contents="import foo.five", - ), - } - assert {module_foo_one_file: expected_result} == result - - -@pytest.mark.parametrize( - "code, expected_imported_modules", - ( - pytest.param( - """ - if TYPE_CHECKING: - import foo.two - else: - import foo.three - """, - {"foo.three"}, - id="else", - ), - pytest.param( - """ - if typing.TYPE_CHECKING: - import foo.two - elif sys.version_info >= (3, 12): - import foo.three - else: - import foo.four - """, - {"foo.three", "foo.four"}, - id="elif-and-else", - ), - pytest.param( - """ - if sys.version_info >= (3, 12): - import foo.two - elif TYPE_CHECKING: - import foo.three - else: - import foo.four - """, - {"foo.two", "foo.four"}, - id="elif-type-checking", - ), - pytest.param( - """ - if TYPE_CHECKING: - if sys.version_info >= (3, 12): - import foo.two - else: - import foo.three - if TYPE_CHECKING: - import foo.four - import foo.five - """, - set(), - id="nested", - ), - ), -) -def test_exclude_type_checking_imports_only_excludes_type_checking_branches( - code, expected_imported_modules -): - all_modules = {Module(f"foo.{name}") for name in ("one", "two", "three", "four", "five")} - module_foo_one_file = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem(content_map={"/path/to/foo/one.py": code}) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_file}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=True, - ) - - imported_modules = { - direct_import.imported.name for direct_import in result[module_foo_one_file] - } - assert imported_modules == expected_imported_modules - - -def test_t_string_syntax(): - module_file_to_scan = _module_to_module_file(Module("foo.one")) - - all_module_files = frozenset({module_file_to_scan, _module_to_module_file(Module("foo.two"))}) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": """ - import foo.two - # t-strings are only valid in Python 3.14+ - hello = t"Hello" - """ - } - ) - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages={ - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=all_module_files, - ) - }, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - } - } - - -@pytest.mark.parametrize( - "import_statement", - ( - "from namespace.portion.nonexistent import something", - "from .nonexistent import something", - "from ..nonexistent import something", - ), -) -def test_missing_module_in_dotted_root_package_is_ignored(import_statement: str): - """ - When a dotted root package (namespace package) contains an import of a missing module - within that same package, and include_external_packages=True, the Rust code should not - panic with an index-out-of-bounds error. - - Regression test for https://github.com/python-grimp/grimp/issues/308. - """ - module_to_scan = Module("namespace.portion.subpackage.importer") - module_file_to_scan = _module_to_module_file(module_to_scan) - all_modules = { - Module("namespace.portion"), - Module("namespace.portion.subpackage"), - module_to_scan, - } - file_system = rust.FakeBasicFileSystem( - content_map={"/path/to/namespace/portion/subpackage/importer.py": import_statement}, - ) - found_packages = { - FoundPackage( - name="namespace.portion", - directory="/path/to/namespace/portion", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, # Required field, not relevant. - ) - - assert result == {module_file_to_scan: set()} - - -def _module_to_module_file(module: Module) -> ModuleFile: - some_mtime = 100933.4 - return ModuleFile(module=module, mtime=some_mtime) - - -def _modules_to_module_files(modules: Set[Module]) -> frozenset[ModuleFile]: - return frozenset({_module_to_module_file(module) for module in modules}) diff --git a/tests/unit/application/test_usecases.py b/tests/unit/application/test_usecases.py deleted file mode 100644 index 7c3da83e..00000000 --- a/tests/unit/application/test_usecases.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import re -from unittest.mock import sentinel - -import pytest - -from grimp.application import usecases -from grimp.application.ports.caching import Cache -from grimp.application.ports.modulefinder import ModuleFile -from grimp.domain.valueobjects import DirectImport, Module -from tests.adaptors.filesystem import FakeFileSystem -from tests.adaptors.packagefinder import BaseFakePackageFinder -from tests.config import override_settings - - -class TestBuildGraph: - @pytest.mark.parametrize("include_external_packages", (True, False)) - def test_happy_path(self, include_external_packages): - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - """, - content_map={ - "/path/to/mypackage/foo/one.py": ( - "import mypackage.foo.two.green\nfrom .. import Something" - ), - "/path/to/mypackage/foo/two/green.py": "import mypackage.foo.two.blue\n" - "from external.subpackage import foobar\n" - "import decimal", - }, - ) - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = {"mypackage": {"/path/to/mypackage"}} - - with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()): - graph = usecases.build_graph( - "mypackage", include_external_packages=include_external_packages - ) - - expected_import_map = { - "mypackage": set(), - "mypackage.foo": set(), - "mypackage.foo.one": {"mypackage.foo.two.green", "mypackage"}, - "mypackage.foo.two": set(), - "mypackage.foo.two.green": {"mypackage.foo.two.blue"}, - "mypackage.foo.two.blue": set(), - } - if include_external_packages: - expected_import_map["decimal"] = set() - expected_import_map["external"] = set() - expected_import_map["mypackage.foo.two.green"] |= {"external", "decimal"} - - assert set(expected_import_map.keys()) == graph.modules - for importer, imported_modules in expected_import_map.items(): - assert graph.find_modules_directly_imported_by(importer) == imported_modules - - # Check that the external packages are squashed modules. - if include_external_packages: - for module in ("external", "decimal"): - with pytest.raises( - ValueError, match=re.escape("Cannot find children of a squashed module.") - ): - graph.find_children(module) - - def test_boolean_additional_package_raises_type_error(self): - """ - Tests that a useful error message if build_graph is called - with a boolean as the second argument. - - This is because earlier versions of build_graph took include_external_packages - as the second argument, and it's possible it might have been called - as a positional argument. - """ - with pytest.raises(TypeError, match=re.escape("Package names must be strings, got bool.")): - usecases.build_graph("mypackage", True) - - @pytest.mark.parametrize( - "supplied_cache_dir", ("/path/to/somewhere", None, sentinel.not_supplied) - ) - def test_build_graph_respects_cache_dir(self, supplied_cache_dir): - file_system = FakeFileSystem() - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = {"mypackage": {"/path/to/mypackage"}} - - SOME_DEFAULT_CACHE_DIR = ".some_default" - - class AssertingCache(Cache): - @classmethod - def cache_dir_or_default(cls, cache_dir: str | None) -> str: - return cache_dir or SOME_DEFAULT_CACHE_DIR - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - - # Assertions. - - if supplied_cache_dir is None: - raise RuntimeError("Cache should not be instantiated if caching is disabled.") - - expected_cache_dir = ( - SOME_DEFAULT_CACHE_DIR - if supplied_cache_dir is sentinel.not_supplied - else supplied_cache_dir - ) - assert self.cache_dir == expected_cache_dir - - def read_imports(self, module_file: ModuleFile) -> set[DirectImport]: - return set() - - def write( - self, - imports_by_module: dict[Module, set[DirectImport]], - ) -> None: - pass - - with override_settings( - FILE_SYSTEM=file_system, - PACKAGE_FINDER=FakePackageFinder(), - CACHE_CLASS=AssertingCache, - ): - kwargs = dict( - include_external_packages=True, - ) - if supplied_cache_dir is not sentinel.not_supplied: - kwargs["cache_dir"] = supplied_cache_dir - usecases.build_graph("mypackage", **kwargs) - - def test_forgives_wrong_type_being_passed_to_include_external_packages(self): - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - """, - content_map={ - "/path/to/mypackage/foo/one.py": ( - "import mypackage.foo.two.green\nfrom .. import Something" - ), - }, - ) - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = {"mypackage": {"/path/to/mypackage"}} - - with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()): - graph = usecases.build_graph( - "mypackage", - # Note: this should be a bool, but we want to tolerate it, - # as Import Linter currently has a bug where it will pass it as None. - include_external_packages=False, - ) - - expected_modules = { - "mypackage", - "mypackage.foo", - "mypackage.foo.one", - } - assert expected_modules == graph.modules - - def test_namespace_package_passed_as_root(self): - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - bar/ - __init__.py - three.py - /different-path/to/mypackage/ - foobar/ - __init__.py - four.py - """, - content_map={ - "/path/to/mypackage/foo/one.py": "import mypackage\nfrom . import two", - }, - ) - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = { - "mypackage": { - "/path/to/mypackage", - "/different-path/to/mypackage", - }, - } - - with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()): - graph = usecases.build_graph("mypackage") - - assert graph.modules == { - "mypackage", - "mypackage.foo", - "mypackage.foo.one", - "mypackage.foo.two", - "mypackage.foo.two.green", - "mypackage.foo.two.blue", - "mypackage.bar", - "mypackage.bar.three", - "mypackage.foobar", - "mypackage.foobar.four", - } - assert graph.find_modules_directly_imported_by("mypackage.foo.one") == { - "mypackage", - "mypackage.foo.two", - } diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py deleted file mode 100644 index dcd47608..00000000 --- a/tests/unit/conftest.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest - -from grimp.adaptors.modulefinder import ModuleFinder -from grimp.application.graph import ImportGraph -from tests.config import override_settings - - -@pytest.fixture(scope="module", autouse=True) -def configure_unit_tests(): - with override_settings( - IMPORT_GRAPH_CLASS=ImportGraph, - MODULE_FINDER=ModuleFinder(), - FILE_SYSTEM=None, - ): - yield From 39f2e304bc60fcaa06bdb33a45fc79b7e0be655d Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 09:16:15 +0100 Subject: [PATCH 8/9] Log from Python what Rust finds when building the graph Rust can't use Python's logging directly, so rust.build_graph now returns the messages that should be logged, along with the graph, and build_graph logs them. The loggers and messages are the same as before the module finder and the cache were ported to Rust, so that nothing changes for anyone who configures them: - grimp.adaptors.modulefinder: the warning about skipping a module with dots in its name. - grimp.adaptors.caching: whether each cache file was used, missing, corrupt or written by a different version of Grimp, and the files written. This restores the assertions on the log messages that couldn't be ported to the functional tests, and adds tests of the cache messages. An alternative would be the pyo3-log crate, which forwards messages from Rust's log crate to Python's logging as they happen. It isn't used because it would add two dependencies (log and pyo3-log), and mean taking the GIL from code that deliberately runs without it, for what is only a handful of messages per build. It also caches the levels of the loggers, which can go stale if logging is configured after import. Returning the messages keeps Python's logging out of the Rust code, and is simple to test. Assistant-model: Claude Code --- CHANGELOG.rst | 1 - rust/src/caching.rs | 95 +++++++++++++++++++----- rust/src/graph_building.rs | 28 +++++-- rust/src/lib.rs | 1 + rust/src/log_records.rs | 27 +++++++ rust/src/module_finding.rs | 37 ++++++--- src/grimp/application/usecases.py | 7 +- tests/functional/test_caching.py | 65 +++++++++++++++- tests/functional/test_finding_modules.py | 19 ++++- 9 files changed, 235 insertions(+), 45 deletions(-) create mode 100644 rust/src/log_records.rs diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 090260a8..ed71b2eb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -7,7 +7,6 @@ latest * Speed up ``build_graph`` by building the graph in Rust. This includes finding the modules, reading and writing the cache, and assembling the graph. The cache files are unchanged. -* ``build_graph`` no longer logs messages about the cache, or about skipping modules with dots in their name. 3.20 (2026-11-13) ----------------- diff --git a/rust/src/caching.rs b/rust/src/caching.rs index 2e55046d..4baa3170 100644 --- a/rust/src/caching.rs +++ b/rust/src/caching.rs @@ -1,11 +1,14 @@ -use crate::errors::GrimpResult; +use crate::errors::{GrimpError, GrimpResult}; use crate::filesystem::FileSystem; use crate::import_scanning::DirectImport; +use crate::log_records::LogRecord; use crate::module_finding::{FoundPackage, Module}; use blake2::Blake2bVar; use blake2::digest::{Update, VariableOutput}; use std::collections::{HashMap, HashSet}; +const LOGGER: &str = "grimp.adaptors.caching"; + /// The version of the data cache file format. /// /// Bump this whenever the serialized structure changes. On read, a file with a @@ -44,18 +47,24 @@ fn serialize_imports_by_module( serde_json::to_string(&envelope).expect("Failed to serialize to JSON") } +/// Why the contents of a cache file can't be used. +enum Unusable { + Corrupt, + /// The file was written by a different version of Grimp. + DifferentVersion, +} + /// Parses the contents of a data file. -/// -/// Returns None if it's corrupt, or was written by a different version of Grimp. -fn parse_json_to_map(json_str: &str) -> Option>> { - let value: serde_json::Value = serde_json::from_str(json_str).ok()?; +fn parse_json_to_map(json_str: &str) -> Result>, Unusable> { + let value: serde_json::Value = serde_json::from_str(json_str).map_err(|_| Unusable::Corrupt)?; if value.get("version").and_then(|v| v.as_u64()) != Some(CACHE_SCHEMA_VERSION as u64) { - return None; + return Err(Unusable::DifferentVersion); } let raw_map: HashMap> = - serde_json::from_value(value.get("imports_by_module").cloned().unwrap_or_default()).ok()?; + serde_json::from_value(value.get("imports_by_module").cloned().unwrap_or_default()) + .map_err(|_| Unusable::Corrupt)?; let mut parsed_map: HashMap> = HashMap::new(); @@ -78,7 +87,46 @@ fn parse_json_to_map(json_str: &str) -> Option( + file_system: &dyn FileSystem, + file: &str, + kind: &str, + parse: impl FnOnce(&str) -> Result, + log: &mut Vec, +) -> Option { + let unusable = match file_system.read(file) { + Ok(contents) => match parse(&contents) { + Ok(value) => { + log.push(LogRecord::info( + LOGGER, + format!("Used cache {kind} file {file}."), + )); + return Some(value); + } + Err(unusable) => unusable, + }, + Err(GrimpError::FileNotFound(_)) => { + log.push(LogRecord::info(LOGGER, format!("No cache file: {file}."))); + return None; + } + Err(_) => Unusable::Corrupt, + }; + log.push(match unusable { + Unusable::Corrupt => { + LogRecord::warning(LOGGER, format!("Could not use corrupt cache file {file}.")) + } + Unusable::DifferentVersion => LogRecord::info( + LOGGER, + format!("Cache file {file} was written by a different version of Grimp; rebuilding."), + ), + }); + None } /// A cache of the imports of each module, for building a graph. @@ -128,27 +176,29 @@ impl<'a> Cache<'a> { /// /// Args: /// - mtimes: the current modification time of each module, by module name. + /// - log: anything to be logged is added to this. pub fn read( &self, file_system: &dyn FileSystem, mtimes: &HashMap, + log: &mut Vec, ) -> HashMap> { let mut cached_mtimes: HashMap = HashMap::new(); for found_package in self.found_packages { let meta_file = self.file_path(file_system, &meta_file_name(found_package)); - let package_mtimes = file_system - .read(&meta_file) - .ok() - .and_then(|contents| serde_json::from_str::>(&contents).ok()); - cached_mtimes.extend(package_mtimes.unwrap_or_default()); + let parse = |contents: &str| { + serde_json::from_str::>(contents) + .map_err(|_| Unusable::Corrupt) + }; + cached_mtimes.extend( + read_cache_file(file_system, &meta_file, "meta", parse, log).unwrap_or_default(), + ); } let data_file = self.file_path(file_system, &self.data_file_name); - let imports_by_module = file_system - .read(&data_file) - .ok() - .and_then(|contents| parse_json_to_map(&contents)) - .unwrap_or_default(); + let imports_by_module = + read_cache_file(file_system, &data_file, "data", parse_json_to_map, log) + .unwrap_or_default(); imports_by_module .into_iter() @@ -165,6 +215,7 @@ impl<'a> Cache<'a> { file_system: &mut dyn FileSystem, mtimes: &HashMap, imports_by_module: &HashMap>, + log: &mut Vec, ) -> GrimpResult<()> { for (file_name, contents) in MARKER_FILES { let path = self.file_path(file_system, file_name); @@ -175,6 +226,10 @@ impl<'a> Cache<'a> { let data_file = self.file_path(file_system, &self.data_file_name); file_system.write(&data_file, &serialize_imports_by_module(imports_by_module))?; + log.push(LogRecord::info( + LOGGER, + format!("Wrote data cache file {data_file}."), + )); for found_package in self.found_packages { let package_mtimes: HashMap<&str, f64> = found_package @@ -188,6 +243,10 @@ impl<'a> Cache<'a> { let meta_file = self.file_path(file_system, &meta_file_name(found_package)); let contents = serde_json::to_string(&package_mtimes).expect("Failed to serialize"); file_system.write(&meta_file, &contents)?; + log.push(LogRecord::info( + LOGGER, + format!("Wrote meta cache file {meta_file}."), + )); } Ok(()) } diff --git a/rust/src/graph_building.rs b/rust/src/graph_building.rs index 7748c06e..872b74fb 100644 --- a/rust/src/graph_building.rs +++ b/rust/src/graph_building.rs @@ -3,10 +3,14 @@ use crate::errors::GrimpResult; use crate::filesystem::real_file_system; use crate::graph::{Graph, GraphWrapper}; use crate::import_scanning::{DirectImport, into_py_err, scan_for_imports_no_py}; +use crate::log_records::LogRecord; use crate::module_finding::{FoundPackage, Module, find_package}; use pyo3::prelude::*; use std::collections::{HashMap, HashSet}; +/// A `LogRecord` as returned to Python: (logger name, logging method name, message). +type PyLogRecord = (&'static str, &'static str, String); + /// Builds an import graph of the supplied packages. /// /// This is called by `grimp.application.usecases.build_graph`. @@ -16,6 +20,9 @@ use std::collections::{HashMap, HashSet}; /// - include_external_packages: whether to include imports of external modules. /// - exclude_type_checking_imports: whether to exclude imports behind TYPE_CHECKING guards. /// - cache_dir: the directory to use for caching, or None for no caching. +/// +/// Returns the graph, and the messages for the caller to log, as Rust can't use Python's logging +/// directly. #[pyfunction] pub fn build_graph( py: Python, @@ -23,7 +30,7 @@ pub fn build_graph( include_external_packages: bool, exclude_type_checking_imports: bool, cache_dir: Option, -) -> PyResult { +) -> PyResult<(GraphWrapper, Vec)> { py.detach(|| { build_graph_no_py( &packages, @@ -32,7 +39,13 @@ pub fn build_graph( cache_dir.as_deref(), ) }) - .map(GraphWrapper::from_graph) + .map(|(graph, log)| { + let log = log + .into_iter() + .map(|record| (record.logger, record.level, record.message)) + .collect(); + (GraphWrapper::from_graph(graph), log) + }) .map_err(|e| into_py_err(py, e)) } @@ -41,14 +54,15 @@ fn build_graph_no_py( include_external_packages: bool, exclude_type_checking_imports: bool, cache_dir: Option<&str>, -) -> GrimpResult { +) -> GrimpResult<(Graph, Vec)> { let mut file_system = real_file_system(); + let mut log = vec![]; // Find the modules. let mut found_packages = HashSet::new(); let mut mtimes = HashMap::new(); for (name, directory) in packages { - let (found_package, package_mtimes) = find_package(name, directory)?; + let (found_package, package_mtimes) = find_package(name, directory, &mut log)?; found_packages.insert(found_package); mtimes.extend(package_mtimes); } @@ -63,7 +77,7 @@ fn build_graph_no_py( ) }); let mut imports_by_module = match &cache { - Some(cache) => cache.read(&*file_system, &mtimes), + Some(cache) => cache.read(&*file_system, &mtimes, &mut log), None => HashMap::new(), }; let modules_to_scan: HashSet = found_packages @@ -82,10 +96,10 @@ fn build_graph_no_py( )?); } if let Some(cache) = &cache { - cache.write(&mut *file_system, &mtimes, &imports_by_module)?; + cache.write(&mut *file_system, &mtimes, &imports_by_module, &mut log)?; } - Ok(assemble_graph(&found_packages, &imports_by_module)) + Ok((assemble_graph(&found_packages, &imports_by_module), log)) } fn assemble_graph( diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 0f5142e3..b98072a2 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -6,6 +6,7 @@ pub mod graph; mod graph_building; pub mod import_parsing; mod import_scanning; +mod log_records; pub mod module_expressions; mod module_finding; diff --git a/rust/src/log_records.rs b/rust/src/log_records.rs new file mode 100644 index 00000000..df9c2e6f --- /dev/null +++ b/rust/src/log_records.rs @@ -0,0 +1,27 @@ +/// A message for the Python layer to log, as Rust can't use Python's `logging` directly. +#[derive(Debug)] +pub struct LogRecord { + /// The name of the Python logger to log to. + pub logger: &'static str, + /// The name of the method of the logger to call, e.g. "info" or "warning". + pub level: &'static str, + pub message: String, +} + +impl LogRecord { + pub fn info(logger: &'static str, message: String) -> Self { + Self { + logger, + level: "info", + message, + } + } + + pub fn warning(logger: &'static str, message: String) -> Self { + Self { + logger, + level: "warning", + message, + } + } +} diff --git a/rust/src/module_finding.rs b/rust/src/module_finding.rs index fc54bd7f..2970c75f 100644 --- a/rust/src/module_finding.rs +++ b/rust/src/module_finding.rs @@ -1,4 +1,5 @@ use crate::errors::GrimpResult; +use crate::log_records::LogRecord; use std::collections::{BTreeSet, HashMap}; use std::fmt; use std::fs; @@ -33,14 +34,19 @@ pub struct FoundPackage { pub namespace_packages: BTreeSet, } +const LOGGER: &str = "grimp.adaptors.modulefinder"; + /// Finds the modules and namespace packages inside a package directory. /// +/// Anything to be logged is added to `log`. +/// /// Also returns the mtime of each module's file (keyed by module name), in the same form as /// Python's `os.path.getmtime`, so that caches written by earlier versions of Grimp (which used /// that) can still be used. pub fn find_package( name: &str, directory: &str, + log: &mut Vec, ) -> GrimpResult<(FoundPackage, HashMap)> { let mut python_files = vec![]; let mut namespace_dirs = vec![]; @@ -49,6 +55,7 @@ pub fn find_package( false, &mut python_files, &mut namespace_dirs, + log, )?; let module_name = |path: &Path| -> String { @@ -99,6 +106,7 @@ fn walk( in_regular_package: bool, python_files: &mut Vec, namespace_dirs: &mut Vec, + log: &mut Vec, ) -> GrimpResult<()> { let mut file_names = vec![]; let mut subdirectories = vec![]; @@ -124,15 +132,25 @@ fn walk( let in_regular_package = in_regular_package || has_init; let python_files_before = python_files.len(); - python_files.extend( - file_names - .iter() - .filter(|name| is_python_file(name)) - .map(|name| directory.join(name)), - ); + for name in &file_names { + // Hidden files, and files like `some.module.py`, aren't treated as Python files. + if name.ends_with(".py") && !name.starts_with('.') { + if name.matches('.').count() == 1 { + python_files.push(directory.join(name)); + } else { + log.push(LogRecord::warning( + LOGGER, + format!( + "Warning: skipping module with too many dots in the name: {}", + directory.join(name).display() + ), + )); + } + } + } for (name, path) in &subdirectories { if is_identifier(name) { - walk(path, in_regular_package, python_files, namespace_dirs)?; + walk(path, in_regular_package, python_files, namespace_dirs, log)?; } } @@ -142,11 +160,6 @@ fn walk( Ok(()) } -/// Files like `some.module.py` and hidden files aren't treated as Python files. -fn is_python_file(name: &str) -> bool { - name.ends_with(".py") && !name.starts_with('.') && name.matches('.').count() == 1 -} - /// Equivalent to Python's `str.isidentifier`. fn is_identifier(name: &str) -> bool { let mut chars = name.chars(); diff --git a/src/grimp/application/usecases.py b/src/grimp/application/usecases.py index aef04930..782e79fa 100644 --- a/src/grimp/application/usecases.py +++ b/src/grimp/application/usecases.py @@ -4,6 +4,7 @@ from __future__ import annotations +import logging from collections.abc import Sequence from typing import cast @@ -57,13 +58,17 @@ def build_graph( if cache_dir is NotSupplied: cache_dir = DEFAULT_CACHE_DIR - rust_graph = rust.build_graph( + rust_graph, log_records = rust.build_graph( packages, # Tolerate a non-boolean (Import Linter has been known to pass None). include_external_packages=bool(include_external_packages), exclude_type_checking_imports=exclude_type_checking_imports, cache_dir=cache_dir, ) + # Rust can't use Python's logging, so it returns what should be logged. + for logger_name, level, message in log_records: + getattr(logging.getLogger(logger_name), level)(message) + return ImportGraph.from_rustgraph(rust_graph) diff --git a/tests/functional/test_caching.py b/tests/functional/test_caching.py index bfc6cca1..6d4a0d13 100644 --- a/tests/functional/test_caching.py +++ b/tests/functional/test_caching.py @@ -1,4 +1,5 @@ import json +import logging import os import shutil import tempfile @@ -146,7 +147,7 @@ def test_data_file_records_version_and_is_lazy(copied_cachingpackage): ] -def test_cache_with_different_version_is_rebuilt(copied_cachingpackage): +def test_cache_with_different_version_is_rebuilt(copied_cachingpackage, caplog): with tempfile.TemporaryDirectory() as cache_dir: build_graph("cachingpackage", cache_dir=cache_dir) @@ -159,9 +160,11 @@ def test_cache_with_different_version_is_rebuilt(copied_cachingpackage): } data_file.write_text(json.dumps(legacy)) - graph = build_graph("cachingpackage", cache_dir=cache_dir) + with caplog.at_level(logging.INFO): + graph = build_graph("cachingpackage", cache_dir=cache_dir) - # The mismatched cache is ignored and rebuilt from source. + # The mismatched cache is ignored (not treated as corrupt) and rebuilt from source. + assert any("different version of Grimp" in record.message for record in caplog.records) assert graph.get_import_details( importer="cachingpackage.two.alpha", imported="cachingpackage.one.alpha", @@ -512,3 +515,59 @@ def test_uses_cache_for_multiple_packages( if include_external_packages: expected_imports.add("externalpackage") assert graph.find_modules_directly_imported_by("anotherpackage.one") == expected_imports + + +class TestCacheLogging: + @pytest.fixture(autouse=True) + def set_log_level(self, caplog): + caplog.set_level(logging.INFO, logger="grimp.adaptors.caching") + + def test_logs_missing_cache_files(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [ + f"No cache file: {cache_dir / 'mypackage.meta.json'}.", + f"No cache file: {_data_file(cache_dir)}.", + f"Wrote data cache file {_data_file(cache_dir)}.", + f"Wrote meta cache file {cache_dir / 'mypackage.meta.json'}.", + ] + + def test_logs_successful_cache_file_reading(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + caplog.clear() + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [ + f"Used cache meta file {cache_dir / 'mypackage.meta.json'}.", + f"Used cache data file {_data_file(cache_dir)}.", + f"Wrote data cache file {_data_file(cache_dir)}.", + f"Wrote meta cache file {cache_dir / 'mypackage.meta.json'}.", + ] + + def test_logs_corrupt_cache_meta_file(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + (cache_dir / "mypackage.meta.json").write_text("INVALID_JSON") + caplog.clear() + caplog.set_level(logging.WARNING, logger="grimp.adaptors.caching") + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [ + f"Could not use corrupt cache file {cache_dir / 'mypackage.meta.json'}.", + ] + + def test_logs_corrupt_cache_data_file(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _data_file(cache_dir).write_text("INVALID_JSON") + caplog.clear() + caplog.set_level(logging.WARNING, logger="grimp.adaptors.caching") + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [f"Could not use corrupt cache file {_data_file(cache_dir)}."] diff --git a/tests/functional/test_finding_modules.py b/tests/functional/test_finding_modules.py index 91e983af..210bb54b 100644 --- a/tests/functional/test_finding_modules.py +++ b/tests/functional/test_finding_modules.py @@ -134,10 +134,16 @@ def test_ignores_invalid_identifier_directories(write_files): } -@pytest.mark.parametrize("extension", ("py", "txt")) -def test_ignores_dotted_python_files(write_files, extension): +@pytest.mark.parametrize( + "extension, should_warn", + ( + ("py", True), + ("txt", False), # Anything other than .py. + ), +) +def test_ignores_dotted_python_files(write_files, caplog, extension, should_warn): # Python files containing dots (other than the one before .py) should not be discovered. - write_files( + root = write_files( { "mypackage/__init__.py": "", "mypackage/foo.py": "", @@ -149,6 +155,13 @@ def test_ignores_dotted_python_files(write_files, extension): graph = build_graph("mypackage", cache_dir=None) assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.bar"} + if should_warn: + assert caplog.messages == [ + "Warning: skipping module with too many dots in the name: " + f"{root / 'mypackage' / 'bar' / f'baz.dotted.{extension}'}" + ] + else: + assert caplog.messages == [] def test_ignores_hidden_directories(write_files): From b971ea9f507c337ef9fccf2837c3b9ed3d63af94 Mon Sep 17 00:00:00 2001 From: Peter Byfield Date: Thu, 19 Nov 2026 08:12:06 +0100 Subject: [PATCH 9/9] Remove unused timer and dependencies These were already unused before build_graph was replaced: nothing read settings.TIMER, and no Rust code used serde or serde_yaml. Assistant-model: Claude Code --- AGENTS.md | 2 +- rust/Cargo.lock | 22 ------------- rust/Cargo.toml | 2 -- src/grimp/adaptors/timing.py | 8 ----- src/grimp/application/ports/timing.py | 47 --------------------------- src/grimp/main.py | 2 -- tests/adaptors/__init__.py | 0 tests/adaptors/timing.py | 35 -------------------- 8 files changed, 1 insertion(+), 117 deletions(-) delete mode 100644 src/grimp/adaptors/timing.py delete mode 100644 src/grimp/application/ports/timing.py delete mode 100644 tests/adaptors/__init__.py delete mode 100644 tests/adaptors/timing.py diff --git a/AGENTS.md b/AGENTS.md index f6a1636d..9e6df288 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,7 +12,7 @@ hybrid Python/Rust project: the public API and orchestration are in Python - `domain/` — core types (`valueobjects.py`, `analysis.py`), no dependencies on other layers. - `application/` — use cases and ports (`usecases.py`, `graph.py`, `config.py`). -- `adaptors/` — implementations of application ports (package finding, timing). +- `adaptors/` — implementations of application ports (package finding). - `main.py` — top-level entry point (`grimp.build_graph`), wires adaptors to use cases. - `exceptions.py` — shared exceptions, importable from any layer. diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 3b988f13..dcd20291 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -23,9 +23,7 @@ dependencies = [ "ruff_python_parser", "ruff_source_file", "rustc-hash", - "serde", "serde_json", - "serde_yaml", "slotmap", "string-interner", "tap", @@ -857,7 +855,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", - "serde_derive", ] [[package]] @@ -893,19 +890,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.14.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "siphasher" version = "1.0.3" @@ -1068,12 +1052,6 @@ dependencies = [ "rand", ] -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "version_check" version = "0.9.5" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 40ad1e3e..899f7b7d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,9 +24,7 @@ const_format = "0.2.34" ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } -serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.137", features = ["float_roundtrip"] } -serde_yaml = "0.9" encoding_rs = "0.8.35" blake2 = "0.10.6" unicode-ident = "1.0.24" diff --git a/src/grimp/adaptors/timing.py b/src/grimp/adaptors/timing.py deleted file mode 100644 index 44413fb4..00000000 --- a/src/grimp/adaptors/timing.py +++ /dev/null @@ -1,8 +0,0 @@ -import time - -from grimp.application.ports.timing import Timer - - -class SystemClockTimer(Timer): - def get_current_time(self) -> float: - return time.time() diff --git a/src/grimp/application/ports/timing.py b/src/grimp/application/ports/timing.py deleted file mode 100644 index 5837a3b7..00000000 --- a/src/grimp/application/ports/timing.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -import abc -from types import TracebackType -from typing import Self - - -class Timer(abc.ABC): - """ - Context manager to allow easy timing of events. - - This is an abstraction that needs to be implemented using a subclass - that implements the get_current_time method. - - Usage: - - with Timer() as timer: - do_something() - print(timer.duration_in_s) - """ - - def __init__(self) -> None: - # We use a stack so context managers can be nested. - self._start_stack: list[float] = [] - - def __enter__(self) -> Self: - self._start_stack.append(self.get_current_time()) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - end = self.get_current_time() - start = self._start_stack.pop() - self.duration_in_s = int(end - start) - - @abc.abstractmethod - def get_current_time(self) -> float: - """ - Return the time in seconds since the epoch as a floating point number. - - See https://docs.python.org/3/library/time.html#time.time - """ - raise NotImplementedError diff --git a/src/grimp/main.py b/src/grimp/main.py index 9878d3f6..7ad4aca5 100644 --- a/src/grimp/main.py +++ b/src/grimp/main.py @@ -1,11 +1,9 @@ __all__ = ["build_graph"] from .adaptors.packagefinder import ImportLibPackageFinder -from .adaptors.timing import SystemClockTimer from .application.config import settings from .application.usecases import build_graph settings.configure( PACKAGE_FINDER=ImportLibPackageFinder(), - TIMER=SystemClockTimer(), ) diff --git a/tests/adaptors/__init__.py b/tests/adaptors/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/adaptors/timing.py b/tests/adaptors/timing.py deleted file mode 100644 index 30f42ca0..00000000 --- a/tests/adaptors/timing.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations - -from types import TracebackType - -from grimp.application.ports.timing import Timer - - -class FakeTimer(Timer): - ARBITRARY_SECONDS_SINCE_EPOCH = 1_000_000 - - def __init__(self) -> None: - super().__init__() - self._current_time = float(self.ARBITRARY_SECONDS_SINCE_EPOCH) - self._tick_duration = 1 - self._increment = 0 - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._tick() - super().__exit__(exc_type, exc_val, exc_tb) - - def get_current_time(self) -> float: - return self._current_time - - def setup(self, tick_duration: int, increment: int) -> None: - self._tick_duration = tick_duration - self._increment = increment - - def _tick(self) -> None: - self._current_time += self._tick_duration - self._tick_duration += self._increment