diff --git a/.importlinter b/.importlinter index c58f0d92..4c5187c4 100644 --- a/.importlinter +++ b/.importlinter @@ -17,7 +17,5 @@ ignore_imports = ; Grimp doesn't understand extension modules: it thinks grimp._rustgrimp ; is an object within grimp, rather than a module in its own right, ; so we ignore these imports here. - grimp.adaptors.caching -> grimp grimp.application.graph -> grimp - grimp.adaptors.filesystem -> grimp - grimp.application.scanning -> grimp + grimp.application.usecases -> grimp diff --git a/AGENTS.md b/AGENTS.md index 718d6005..9e6df288 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ Grimp builds a queryable graph of the imports within one or more Python packages. It's a hybrid Python/Rust project: the public API and orchestration are in Python -(`src/grimp/`), and the import scanning and graph algorithms are implemented in Rust +(`src/grimp/`), and the graph building and graph algorithms are implemented in Rust (`rust/`) as a PyO3 extension module (`grimp._rustgrimp`), built with maturin. ## Architecture @@ -11,18 +11,18 @@ hybrid Python/Rust project: the public API and orchestration are in Python (see `.importlinter`): - `domain/` — core types (`valueobjects.py`, `analysis.py`), no dependencies on other layers. -- `application/` — use cases and ports (`usecases.py`, `graph.py`, `scanning.py`, `config.py`). -- `adaptors/` — implementations of application ports (filesystem, module/package finding, - caching, timing). +- `application/` — use cases and ports (`usecases.py`, `graph.py`, `config.py`). +- `adaptors/` — implementations of application ports (package finding). - `main.py` — top-level entry point (`grimp.build_graph`), wires adaptors to use cases. - `exceptions.py` — shared exceptions, importable from any layer. Respect this layering: `domain` must not import from `application` or `adaptors`, etc. Run `just lint-python` (which includes `lint-imports`) to check. -The Rust side (`rust/src/`) does the heavy lifting: import parsing (via `ruff`'s parser), -module/import scanning, and graph algorithms (`rust/src/graph/`: pathfinding, cycle -detection, hierarchy queries, etc.). +The Rust side (`rust/src/`) does the heavy lifting. `grimp.build_graph` calls into +`rust/src/graph_building.rs`, which finds the modules, reads and writes the cache, scans the +imports (parsed via `ruff`'s parser) and assembles the graph. It also implements the graph +algorithms (`rust/src/graph/`: pathfinding, cycle detection, hierarchy queries, etc.). ## Setup diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 2182c72b..ed71b2eb 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,12 @@ Changelog ========= +latest +------ + +* Speed up ``build_graph`` by building the graph in Rust. This includes finding the modules, reading + and writing the cache, and assembling the graph. The cache files are unchanged. + 3.20 (2026-11-13) ----------------- diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8e192b6e..dcd20291 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -7,6 +7,7 @@ name = "_rustgrimp" version = "0.1.0" dependencies = [ "bimap", + "blake2", "const_format", "derive-new", "encoding_rs", @@ -22,14 +23,12 @@ dependencies = [ "ruff_python_parser", "ruff_source_file", "rustc-hash", - "serde", "serde_json", - "serde_yaml", "slotmap", "string-interner", "tap", "thiserror", - "unindent", + "unicode-ident", ] [[package]] @@ -95,6 +94,24 @@ version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" version = "1.12.3" @@ -187,6 +204,16 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "derive-new" version = "0.7.0" @@ -209,6 +236,17 @@ dependencies = [ "syn", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + [[package]] name = "either" version = "1.16.0" @@ -236,6 +274,16 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "get-size-derive2" version = "0.10.1" @@ -807,7 +855,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", - "serde_derive", ] [[package]] @@ -843,19 +890,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "serde_yaml" -version = "0.9.34+deprecated" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" -dependencies = [ - "indexmap 2.14.0", - "itoa", - "ryu", - "serde", - "unsafe-libyaml", -] - [[package]] name = "siphasher" version = "1.0.3" @@ -893,6 +927,12 @@ dependencies = [ "serde", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.118" @@ -957,6 +997,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1006,18 +1052,6 @@ dependencies = [ "rand", ] -[[package]] -name = "unindent" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" - -[[package]] -name = "unsafe-libyaml" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" - [[package]] name = "version_check" version = "0.9.5" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index a82574df..899f7b7d 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,11 +24,10 @@ const_format = "0.2.34" ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.15.20" } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0.137" -serde_yaml = "0.9" -unindent = "0.2.4" +serde_json = { version = "1.0.137", features = ["float_roundtrip"] } encoding_rs = "0.8.35" +blake2 = "0.10.6" +unicode-ident = "1.0.24" pyo3 = "0.29.0" [features] diff --git a/rust/src/caching.rs b/rust/src/caching.rs index cd33562a..4baa3170 100644 --- a/rust/src/caching.rs +++ b/rust/src/caching.rs @@ -1,79 +1,13 @@ use crate::errors::{GrimpError, GrimpResult}; -use crate::filesystem::get_file_system_boxed; -use crate::import_scanning::{DirectImport, imports_by_module_to_py}; -use crate::module_finding::Module; -use pyo3::types::PyAnyMethods; -use pyo3::types::{PyDict, PySet}; -use pyo3::types::{PyDictMethods, PySetMethods}; -use pyo3::{Borrowed, Bound, FromPyObject, PyAny, PyErr, PyResult, Python, pyfunction}; +use crate::filesystem::FileSystem; +use crate::import_scanning::DirectImport; +use crate::log_records::LogRecord; +use crate::module_finding::{FoundPackage, Module}; +use blake2::Blake2bVar; +use blake2::digest::{Update, VariableOutput}; use std::collections::{HashMap, HashSet}; -/// Writes the cache file containing all the imports for a given package. -/// Args: -/// - filename: str -/// - imports_by_module: dict[Module, Set[DirectImport]] -/// - file_system: The file system interface to use. (A BasicFileSystem.) -#[pyfunction] -pub fn write_cache_data_map_file<'py>( - filename: &str, - imports_by_module: Bound<'py, PyDict>, - file_system: Bound<'py, PyAny>, -) -> PyResult<()> { - let mut file_system_boxed = get_file_system_boxed(&file_system)?; - - let ImportsByModule(imports_by_module_rust) = imports_by_module.extract()?; - - let file_contents = serialize_imports_by_module(&imports_by_module_rust); - - file_system_boxed.write(filename, &file_contents)?; - - Ok(()) -} - -/// Reads the cache file containing all the imports for a given package. -/// Args: -/// - filename: str -/// - file_system: The file system interface to use. (A BasicFileSystem.) -/// Returns Dict[Module, Set[DirectImport]] -#[pyfunction] -pub fn read_cache_data_map_file<'py>( - py: Python<'py>, - filename: &str, - file_system: Bound<'py, PyAny>, -) -> PyResult> { - let file_system_boxed = get_file_system_boxed(&file_system)?; - - let file_contents = file_system_boxed.read(filename)?; - - let imports_by_module = parse_json_to_map(&file_contents, filename)?; - - Ok(imports_by_module_to_py(py, imports_by_module)) -} - -/// A newtype wrapper for HashMap> that implements FromPyObject. -pub struct ImportsByModule(pub HashMap>); - -impl<'a, 'py> FromPyObject<'a, 'py> for ImportsByModule { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult { - let py_dict = ob.cast::()?; - let mut imports_by_module_rust = HashMap::new(); - - for (py_key, py_value) in py_dict.iter() { - let module: Module = py_key.extract()?; - let py_set = py_value.cast::()?; - let mut hashset: HashSet = HashSet::new(); - for element in py_set.iter() { - let direct_import: DirectImport = element.extract()?; - hashset.insert(direct_import); - } - imports_by_module_rust.insert(module, hashset); - } - - Ok(ImportsByModule(imports_by_module_rust)) - } -} +const LOGGER: &str = "grimp.adaptors.caching"; /// The version of the data cache file format. /// @@ -113,24 +47,24 @@ fn serialize_imports_by_module( serde_json::to_string(&envelope).expect("Failed to serialize to JSON") } -pub fn parse_json_to_map( - json_str: &str, - filename: &str, -) -> GrimpResult>> { - // Parse into a generic value first, so we can distinguish genuinely corrupt - // JSON from a cache file written by a different format version (e.g. by an - // older Grimp). The latter should be silently rebuilt, not warned about. - let value: serde_json::Value = serde_json::from_str(json_str) - .map_err(|_| GrimpError::CorruptCache(filename.to_string()))?; - - let version = value.get("version").and_then(|v| v.as_u64()); - if version != Some(CACHE_SCHEMA_VERSION as u64) { - return Err(GrimpError::CacheVersionMismatch(filename.to_string())); +/// Why the contents of a cache file can't be used. +enum Unusable { + Corrupt, + /// The file was written by a different version of Grimp. + DifferentVersion, +} + +/// Parses the contents of a data file. +fn parse_json_to_map(json_str: &str) -> Result>, Unusable> { + let value: serde_json::Value = serde_json::from_str(json_str).map_err(|_| Unusable::Corrupt)?; + + if value.get("version").and_then(|v| v.as_u64()) != Some(CACHE_SCHEMA_VERSION as u64) { + return Err(Unusable::DifferentVersion); } let raw_map: HashMap> = serde_json::from_value(value.get("imports_by_module").cloned().unwrap_or_default()) - .map_err(|_| GrimpError::CorruptCache(filename.to_string()))?; + .map_err(|_| Unusable::Corrupt)?; let mut parsed_map: HashMap> = HashMap::new(); @@ -155,3 +89,183 @@ pub fn parse_json_to_map( Ok(parsed_map) } + +/// Reads and parses a cache file, logging what happened. +/// +/// Returns None if the file can't be used. +fn read_cache_file( + file_system: &dyn FileSystem, + file: &str, + kind: &str, + parse: impl FnOnce(&str) -> Result, + log: &mut Vec, +) -> Option { + let unusable = match file_system.read(file) { + Ok(contents) => match parse(&contents) { + Ok(value) => { + log.push(LogRecord::info( + LOGGER, + format!("Used cache {kind} file {file}."), + )); + return Some(value); + } + Err(unusable) => unusable, + }, + Err(GrimpError::FileNotFound(_)) => { + log.push(LogRecord::info(LOGGER, format!("No cache file: {file}."))); + return None; + } + Err(_) => Unusable::Corrupt, + }; + log.push(match unusable { + Unusable::Corrupt => { + LogRecord::warning(LOGGER, format!("Could not use corrupt cache file {file}.")) + } + Unusable::DifferentVersion => LogRecord::info( + LOGGER, + format!("Cache file {file} was written by a different version of Grimp; rebuilding."), + ), + }); + None +} + +/// A cache of the imports of each module, for building a graph. +/// +/// The file names and formats must not change without good reason, so that caches written by +/// earlier versions of Grimp can still be used. +pub struct Cache<'a> { + directory: &'a str, + found_packages: &'a HashSet, + data_file_name: String, +} + +impl<'a> Cache<'a> { + pub fn new( + directory: &'a str, + found_packages: &'a HashSet, + include_external_packages: bool, + exclude_type_checking_imports: bool, + ) -> Self { + // The data file is specific to the packages and options used. + let mut package_names: Vec<&str> = found_packages.iter().map(|p| p.name.as_str()).collect(); + package_names.sort(); + let mut identifier = package_names.join(","); + if include_external_packages { + identifier += ":external"; + } + if exclude_type_checking_imports { + identifier += ":no_type_checking"; + } + // Hash the identifier to avoid filenames that are too long for the filesystem. + let mut hasher = Blake2bVar::new(20).unwrap(); + hasher.update(identifier.as_bytes()); + let mut hash = [0; 20]; + hasher.finalize_variable(&mut hash).unwrap(); + let hex_hash: String = hash.iter().map(|byte| format!("{byte:02x}")).collect(); + + Self { + directory, + found_packages, + data_file_name: format!("{hex_hash}.data.json"), + } + } + + /// Reads the imports of the modules that haven't been modified since they were cached. + /// + /// Anything that can't be read (e.g. the cache doesn't exist yet) is treated as a cache miss. + /// + /// Args: + /// - mtimes: the current modification time of each module, by module name. + /// - log: anything to be logged is added to this. + pub fn read( + &self, + file_system: &dyn FileSystem, + mtimes: &HashMap, + log: &mut Vec, + ) -> HashMap> { + let mut cached_mtimes: HashMap = HashMap::new(); + for found_package in self.found_packages { + let meta_file = self.file_path(file_system, &meta_file_name(found_package)); + let parse = |contents: &str| { + serde_json::from_str::>(contents) + .map_err(|_| Unusable::Corrupt) + }; + cached_mtimes.extend( + read_cache_file(file_system, &meta_file, "meta", parse, log).unwrap_or_default(), + ); + } + + let data_file = self.file_path(file_system, &self.data_file_name); + let imports_by_module = + read_cache_file(file_system, &data_file, "data", parse_json_to_map, log) + .unwrap_or_default(); + + imports_by_module + .into_iter() + .filter(|(module, _)| { + cached_mtimes + .get(&module.name) + .is_some_and(|cached| mtimes.get(&module.name) == Some(cached)) + }) + .collect() + } + + pub fn write( + &self, + file_system: &mut dyn FileSystem, + mtimes: &HashMap, + imports_by_module: &HashMap>, + log: &mut Vec, + ) -> GrimpResult<()> { + for (file_name, contents) in MARKER_FILES { + let path = self.file_path(file_system, file_name); + if !file_system.exists(&path) { + file_system.write(&path, contents)?; + } + } + + let data_file = self.file_path(file_system, &self.data_file_name); + file_system.write(&data_file, &serialize_imports_by_module(imports_by_module))?; + log.push(LogRecord::info( + LOGGER, + format!("Wrote data cache file {data_file}."), + )); + + for found_package in self.found_packages { + let package_mtimes: HashMap<&str, f64> = found_package + .module_files + .iter() + .map(|module_file| { + let name = module_file.module.name.as_str(); + (name, mtimes[name]) + }) + .collect(); + let meta_file = self.file_path(file_system, &meta_file_name(found_package)); + let contents = serde_json::to_string(&package_mtimes).expect("Failed to serialize"); + file_system.write(&meta_file, &contents)?; + log.push(LogRecord::info( + LOGGER, + format!("Wrote meta cache file {meta_file}."), + )); + } + Ok(()) + } + + fn file_path(&self, file_system: &dyn FileSystem, file_name: &str) -> String { + file_system.join(vec![self.directory.to_string(), file_name.to_string()]) + } +} + +fn meta_file_name(found_package: &FoundPackage) -> String { + format!("{}.meta.json", found_package.name) +} + +const MARKER_FILES: [(&str, &str); 2] = [ + (".gitignore", "# Automatically created by Grimp.\n*"), + ( + "CACHEDIR.TAG", + "Signature: 8a477f597d28d172789f06886806bc55\n\ + # This file is a cache directory tag automatically created by Grimp.\n\ + # For information about cache directory tags see https://bford.info/cachedir/", + ), +]; diff --git a/rust/src/errors.rs b/rust/src/errors.rs index b2ba6a25..081a54ce 100644 --- a/rust/src/errors.rs +++ b/rust/src/errors.rs @@ -1,6 +1,6 @@ use crate::exceptions; use pyo3::PyErr; -use pyo3::exceptions::{PyFileNotFoundError, PyUnicodeError, PyValueError}; +use pyo3::exceptions::{PyFileNotFoundError, PyOSError, PyUnicodeError, PyValueError}; use ruff_python_parser::ParseError as RuffParseError; use thiserror::Error; @@ -34,17 +34,14 @@ pub enum GrimpError { parse_error: RuffParseError, }, - #[error("Could not use corrupt cache file {0}.")] - CorruptCache(String), - - #[error("Cache file {0} was written by a different version of Grimp.")] - CacheVersionMismatch(String), - #[error("{0}")] FileNotFound(String), #[error("{0}")] UndecodableFile(String), + + #[error("{0}")] + Io(#[from] std::io::Error), } pub type GrimpResult = Result; @@ -63,13 +60,10 @@ impl From for PyErr { GrimpError::ParseError { line_number, text, .. } => PyErr::new::((line_number, text)), - GrimpError::CorruptCache(_) => exceptions::CorruptCache::new_err(value.to_string()), - GrimpError::CacheVersionMismatch(_) => { - exceptions::CacheVersionMismatch::new_err(value.to_string()) - } GrimpError::FileNotFound(_) => PyFileNotFoundError::new_err(value.to_string()), // Not UnicodeDecodeError, as that can't be created from just a message. GrimpError::UndecodableFile(_) => PyUnicodeError::new_err(value.to_string()), + GrimpError::Io(_) => PyOSError::new_err(value.to_string()), } } } diff --git a/rust/src/exceptions.rs b/rust/src/exceptions.rs index f4824fdb..f25a78cd 100644 --- a/rust/src/exceptions.rs +++ b/rust/src/exceptions.rs @@ -5,8 +5,6 @@ use pyo3::prelude::*; create_exception!(_rustgrimp, ModuleNotPresent, PyException); create_exception!(_rustgrimp, NoSuchContainer, PyException); create_exception!(_rustgrimp, InvalidModuleExpression, PyException); -create_exception!(_rustgrimp, CorruptCache, PyException); -create_exception!(_rustgrimp, CacheVersionMismatch, PyException); // We need to use here `pyclass(extends=PyException)` instead of `create_exception!` // since the exception contains custom data. See: diff --git a/rust/src/filesystem.rs b/rust/src/filesystem.rs index 883aaaf4..bc6b222f 100644 --- a/rust/src/filesystem.rs +++ b/rust/src/filesystem.rs @@ -1,16 +1,10 @@ use crate::errors::{GrimpError, GrimpResult}; -use itertools::Itertools; -use pyo3::exceptions::PyTypeError; -use pyo3::prelude::*; use regex::Regex; -use std::collections::HashMap; -use std::ffi::OsStr; use std::fs; use std::fs::File; use std::io::prelude::*; use std::path::{Path, PathBuf}; -use std::sync::{Arc, LazyLock, Mutex}; -use unindent::unindent; +use std::sync::LazyLock; static ENCODING_RE: LazyLock = LazyLock::new(|| Regex::new(r"^[ \t\f]*#.*?coding[:=][ \t]*([-_.a-zA-Z0-9]+)").unwrap()); @@ -58,8 +52,6 @@ fn lookup_encoding(name: &str) -> Option { } pub trait FileSystem: Send + Sync { - fn sep(&self) -> &str; - fn join(&self, components: Vec) -> String; fn split(&self, file_name: &str) -> (String, String); @@ -68,25 +60,13 @@ pub trait FileSystem: Send + Sync { fn read(&self, file_name: &str) -> GrimpResult; - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()>; + fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()>; } -#[derive(Clone)] -#[pyclass(from_py_object)] -struct RealBasicFileSystem {} - -// Implements a BasicFileSystem (defined in grimp.application.ports.filesystem.BasicFileSystem) -// that actually reads files. -#[pyclass(name = "RealBasicFileSystem")] -pub struct PyRealBasicFileSystem { - inner: RealBasicFileSystem, -} - -impl FileSystem for RealBasicFileSystem { - fn sep(&self) -> &str { - std::path::MAIN_SEPARATOR_STR - } +/// A file system that actually reads and writes files. +struct RealFileSystem {} +impl FileSystem for RealFileSystem { fn join(&self, components: Vec) -> String { let mut path = PathBuf::new(); for component in components { @@ -183,290 +163,16 @@ impl FileSystem for RealBasicFileSystem { } } - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { + fn write(&mut self, file_name: &str, contents: &str) -> GrimpResult<()> { let file_path: PathBuf = file_name.into(); if let Some(patent_dir) = file_path.parent() { fs::create_dir_all(patent_dir)?; } - File::create(file_path)? - .write_all(contents.as_bytes()) - .map_err(Into::into) - } -} - -#[pymethods] -impl PyRealBasicFileSystem { - #[new] - fn new() -> Self { - PyRealBasicFileSystem { - inner: RealBasicFileSystem {}, - } - } - - #[getter] - fn sep(&self) -> &str { - self.inner.sep() - } - - #[pyo3(signature = (*components))] - fn join(&self, components: Vec) -> String { - self.inner.join(components) - } - - fn split(&self, file_name: &str) -> (String, String) { - self.inner.split(file_name) - } - - fn exists(&self, file_name: &str) -> bool { - self.inner.exists(file_name) - } - - fn read(&self, file_name: &str) -> PyResult { - Ok(self.inner.read(file_name)?) - } - - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - self.inner.write(file_name, contents) - } -} - -type FileSystemContents = HashMap; - -#[derive(Clone)] -struct FakeBasicFileSystem { - contents: Arc>, -} - -// Implements BasicFileSystem (defined in grimp.application.ports.filesystem.BasicFileSystem). -#[pyclass(name = "FakeBasicFileSystem")] -pub struct PyFakeBasicFileSystem { - inner: FakeBasicFileSystem, -} - -impl FakeBasicFileSystem { - fn new(contents: Option<&str>, content_map: Option>) -> PyResult { - let mut parsed_contents = match contents { - Some(contents) => parse_indented_file_system_string(contents), - None => HashMap::new(), - }; - if let Some(content_map) = content_map { - let unindented_map: HashMap = content_map - .into_iter() - .map(|(key, val)| (key, unindent(&val).trim().to_string())) - .collect(); - parsed_contents.extend(unindented_map); - }; - Ok(FakeBasicFileSystem { - contents: Arc::new(Mutex::new(parsed_contents)), - }) - } -} - -impl FileSystem for FakeBasicFileSystem { - fn sep(&self) -> &str { - "/" - } - - fn join(&self, components: Vec) -> String { - let sep = self.sep(); - components - .into_iter() - .map(|c| c.trim_end_matches(sep).to_string()) - .join(sep) - } - - fn split(&self, file_name: &str) -> (String, String) { - let path; - let head; - let tail; - if let Some(file_name_without_trailing_slash) = file_name.strip_suffix("/") { - head = Path::new(file_name_without_trailing_slash); - tail = OsStr::new(""); - } else { - path = Path::new(file_name); - head = path.parent().unwrap_or(Path::new("")); - tail = path.file_name().unwrap_or(OsStr::new("")); - } - ( - head.to_str() - .expect("Path components should be valid unicode") - .to_string(), - tail.to_str() - .expect("Path components should be valid unicode") - .to_string(), - ) - } - - /// Checks if a file or directory exists within the file system. - fn exists(&self, file_name: &str) -> bool { - self.contents.lock().unwrap().contains_key(file_name) - } - - fn read(&self, file_name: &str) -> GrimpResult { - let contents = self.contents.lock().unwrap(); - match contents.get(file_name) { - Some(file_contents) => Ok(file_contents.clone()), - None => Err(GrimpError::FileNotFound(format!( - "No such file: {file_name}" - ))), - } - } - - #[allow(unused_variables)] - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - let mut contents_mut = self.contents.lock().unwrap(); - contents_mut.insert(file_name.to_string(), contents.to_string()); + File::create(file_path)?.write_all(contents.as_bytes())?; Ok(()) } } -#[pymethods] -impl PyFakeBasicFileSystem { - #[pyo3(signature = (contents=None, content_map=None))] - #[new] - fn new(contents: Option<&str>, content_map: Option>) -> PyResult { - Ok(PyFakeBasicFileSystem { - inner: FakeBasicFileSystem::new(contents, content_map)?, - }) - } - - #[getter] - fn sep(&self) -> &str { - self.inner.sep() - } - - #[pyo3(signature = (*components))] - fn join(&self, components: Vec) -> String { - self.inner.join(components) - } - - fn split(&self, file_name: &str) -> (String, String) { - self.inner.split(file_name) - } - - /// Checks if a file or directory exists within the file system. - fn exists(&self, file_name: &str) -> bool { - self.inner.exists(file_name) - } - - fn read(&self, file_name: &str) -> PyResult { - Ok(self.inner.read(file_name)?) - } - - fn write(&mut self, file_name: &str, contents: &str) -> PyResult<()> { - self.inner.write(file_name, contents) - } - - // Temporary workaround method for Python tests. - fn convert_to_basic(&self) -> PyResult { - Ok(PyFakeBasicFileSystem { - inner: self.inner.clone(), - }) - } -} - -/// Parses an indented string representing a file system structure -/// into a HashMap where keys are full file paths. -/// See tests.adaptors.filesystem.FakeFileSystem for the API. -fn parse_indented_file_system_string(file_system_string: &str) -> HashMap { - let mut file_paths_map: HashMap = HashMap::new(); - let mut path_stack: Vec = Vec::new(); // Stores current directory path components - let mut first_line = true; // Flag to handle the very first path component - let mut first_line_indent: usize = 0; - // Normalize newlines and split into lines - let buffer = file_system_string.replace("\r\n", "\n"); - let lines: Vec<&str> = buffer.split('\n').collect(); - - let first_line_starts_with_slash = lines[0].trim().starts_with('/'); - for line_raw in lines { - let line = line_raw.trim_end(); // Remove trailing whitespace - if line.is_empty() { - continue; // Skip empty lines - } - let current_indent = - line.chars().take_while(|&c| c.is_whitespace()).count() - first_line_indent; - let trimmed_line = line.trim_start(); - - // Assuming 4 spaces per indentation level for calculating depth - let current_depth = current_indent / 4; - if first_line { - // The first non-empty line sets the base path. - // It might be absolute (/a/b/) or relative (a/b/). - let root_component = trimmed_line.trim_end_matches('/').to_string(); - path_stack.push(root_component); - first_line = false; - first_line_indent = current_indent; - } else { - // Adjust the path_stack based on indentation level - // Pop elements from the stack until we reach the correct parent directory depth - while path_stack.len() > current_depth { - path_stack.pop(); - } - // If the current line is a file, append it to the path for inserting into map, - // then pop it off so that subsequent siblings are correctly handled. - // If it's a directory, append it and it stays on the stack for its children. - let component_to_add = trimmed_line.trim_end_matches('/').to_string(); - if !component_to_add.is_empty() { - // Avoid pushing empty strings due to lines like just "/" - path_stack.push(component_to_add); - } - } - - // Construct the full path - // Join components on the stack. If the first component started with '/', - // ensure the final path also starts with '/'. - let full_path = if !path_stack.is_empty() { - let mut joined = path_stack.join("/"); - // If the original root started with a slash, ensure the final path does too. - // But be careful not to double-slash if a component is e.g. "/root" - if first_line_starts_with_slash && !joined.starts_with('/') { - joined = format!("/{joined}"); - } - joined - } else { - "".to_string() - }; - - // If it's a file (doesn't end with '/'), add it to the map - // A file is not a directory, so its name should be removed from the stack after processing - // so that sibling items are at the correct level. - if !trimmed_line.ends_with('/') { - file_paths_map.insert(full_path, String::new()); // Value can be empty or actual content - if !path_stack.is_empty() { - path_stack.pop(); // Pop the file name off the stack - } - } - } - - // Edge case: If the very first line was a file and it ended up on the stack, it needs to be processed. - // This handles single-file inputs like "myfile.txt" - if !path_stack.is_empty() - && !path_stack - .last() - .expect("path_stack should be non-empty") - .ends_with('/') - && !file_paths_map.contains_key(&path_stack.join("/")) - { - file_paths_map.insert(path_stack.join("/"), String::new()); - } - - file_paths_map -} - -#[allow(clippy::borrowed_box)] -pub fn get_file_system_boxed<'py>( - file_system: &Bound<'py, PyAny>, -) -> PyResult> { - let file_system_boxed: Box; - if let Ok(py_real) = file_system.extract::>() { - file_system_boxed = Box::new(py_real.inner.clone()); - } else if let Ok(py_fake) = file_system.extract::>() { - file_system_boxed = Box::new(py_fake.inner.clone()); - } else { - return Err(PyTypeError::new_err( - "file_system must be an instance of RealBasicFileSystem or FakeBasicFileSystem", - )); - } - - Ok(file_system_boxed) +pub fn real_file_system() -> Box { + Box::new(RealFileSystem {}) } diff --git a/rust/src/graph/mod.rs b/rust/src/graph/mod.rs index 40ff48cb..f5077913 100644 --- a/rust/src/graph/mod.rs +++ b/rust/src/graph/mod.rs @@ -81,6 +81,10 @@ pub struct GraphWrapper { } impl GraphWrapper { + pub fn from_graph(graph: Graph) -> Self { + GraphWrapper { _graph: graph } + } + fn get_visible_module_by_name(&self, name: &str) -> Result<&Module, ModuleNotPresent> { self._graph .get_module_by_name(name) diff --git a/rust/src/graph_building.rs b/rust/src/graph_building.rs new file mode 100644 index 00000000..872b74fb --- /dev/null +++ b/rust/src/graph_building.rs @@ -0,0 +1,142 @@ +use crate::caching::Cache; +use crate::errors::GrimpResult; +use crate::filesystem::real_file_system; +use crate::graph::{Graph, GraphWrapper}; +use crate::import_scanning::{DirectImport, into_py_err, scan_for_imports_no_py}; +use crate::log_records::LogRecord; +use crate::module_finding::{FoundPackage, Module, find_package}; +use pyo3::prelude::*; +use std::collections::{HashMap, HashSet}; + +/// A `LogRecord` as returned to Python: (logger name, logging method name, message). +type PyLogRecord = (&'static str, &'static str, String); + +/// Builds an import graph of the supplied packages. +/// +/// This is called by `grimp.application.usecases.build_graph`. +/// +/// Args: +/// - packages: (name, directory) of each package. A package may have more than one directory. +/// - include_external_packages: whether to include imports of external modules. +/// - exclude_type_checking_imports: whether to exclude imports behind TYPE_CHECKING guards. +/// - cache_dir: the directory to use for caching, or None for no caching. +/// +/// Returns the graph, and the messages for the caller to log, as Rust can't use Python's logging +/// directly. +#[pyfunction] +pub fn build_graph( + py: Python, + packages: Vec<(String, String)>, + include_external_packages: bool, + exclude_type_checking_imports: bool, + cache_dir: Option, +) -> PyResult<(GraphWrapper, Vec)> { + py.detach(|| { + build_graph_no_py( + &packages, + include_external_packages, + exclude_type_checking_imports, + cache_dir.as_deref(), + ) + }) + .map(|(graph, log)| { + let log = log + .into_iter() + .map(|record| (record.logger, record.level, record.message)) + .collect(); + (GraphWrapper::from_graph(graph), log) + }) + .map_err(|e| into_py_err(py, e)) +} + +fn build_graph_no_py( + packages: &[(String, String)], + include_external_packages: bool, + exclude_type_checking_imports: bool, + cache_dir: Option<&str>, +) -> GrimpResult<(Graph, Vec)> { + let mut file_system = real_file_system(); + let mut log = vec![]; + + // Find the modules. + let mut found_packages = HashSet::new(); + let mut mtimes = HashMap::new(); + for (name, directory) in packages { + let (found_package, package_mtimes) = find_package(name, directory, &mut log)?; + found_packages.insert(found_package); + mtimes.extend(package_mtimes); + } + + // Read the imports of each module, from the cache if possible, otherwise from the source. + let cache = cache_dir.map(|directory| { + Cache::new( + directory, + &found_packages, + include_external_packages, + exclude_type_checking_imports, + ) + }); + let mut imports_by_module = match &cache { + Some(cache) => cache.read(&*file_system, &mtimes, &mut log), + None => HashMap::new(), + }; + let modules_to_scan: HashSet = found_packages + .iter() + .flat_map(|found_package| &found_package.module_files) + .map(|module_file| module_file.module.clone()) + .filter(|module| !imports_by_module.contains_key(module)) + .collect(); + if !modules_to_scan.is_empty() { + imports_by_module.extend(scan_for_imports_no_py( + &file_system, + &found_packages, + include_external_packages, + &modules_to_scan, + exclude_type_checking_imports, + )?); + } + if let Some(cache) = &cache { + cache.write(&mut *file_system, &mtimes, &imports_by_module, &mut log)?; + } + + Ok((assemble_graph(&found_packages, &imports_by_module), log)) +} + +fn assemble_graph( + found_packages: &HashSet, + imports_by_module: &HashMap>, +) -> Graph { + let mut graph = Graph::default(); + + for found_package in found_packages { + for namespace_package in &found_package.namespace_packages { + graph.get_or_add_module(&namespace_package.name); + } + } + + let is_external = |module: &str| { + !found_packages.iter().any(|found_package| { + module == found_package.name || module.starts_with(&format!("{}.", found_package.name)) + }) + }; + for (module, direct_imports) in imports_by_module { + let importer = graph.get_or_add_module(&module.name).token(); + for direct_import in direct_imports { + // External modules are squashed. + let imported = if is_external(&direct_import.imported) { + graph.get_or_add_squashed_module(&direct_import.imported) + } else { + graph.get_or_add_module(&direct_import.imported) + } + .token(); + graph.add_detailed_import( + importer, + imported, + direct_import.line_number as u32, + &direct_import.line_contents, + direct_import.is_lazy, + ); + } + } + graph +} diff --git a/rust/src/import_scanning.rs b/rust/src/import_scanning.rs index ca5ae99d..6ceb0b39 100644 --- a/rust/src/import_scanning.rs +++ b/rust/src/import_scanning.rs @@ -1,10 +1,9 @@ use crate::errors::{GrimpError, GrimpResult}; -use crate::filesystem::{FileSystem, get_file_system_boxed}; +use crate::filesystem::FileSystem; +use crate::import_parsing; use crate::module_finding::{FoundPackage, Module}; -use crate::{import_parsing, module_finding}; use itertools::Itertools; use pyo3::prelude::*; -use pyo3::types::{PyDict, PySet}; /// Statically analyses some Python modules for import statements within their shared package. use rayon::prelude::*; use std::collections::{HashMap, HashSet}; @@ -19,44 +18,6 @@ pub struct DirectImport { pub is_lazy: bool, } -impl<'a, 'py> FromPyObject<'a, 'py> for DirectImport { - type Error = PyErr; - - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult { - let importer: String = ob.getattr("importer")?.getattr("name")?.extract()?; - let imported: String = ob.getattr("imported")?.getattr("name")?.extract()?; - let line_number: usize = ob.getattr("line_number")?.extract()?; - let line_contents: String = ob.getattr("line_contents")?.extract()?; - let is_lazy: bool = ob.getattr("is_lazy")?.extract()?; - - Ok(DirectImport { - importer, - imported, - line_number, - line_contents, - is_lazy, - }) - } -} - -fn py_found_packages_to_rust(py_found_packages: &Bound<'_, PyAny>) -> HashSet { - let py_set = py_found_packages - .cast::() - .expect("Expected py_found_packages to be a Python set."); - - let mut rust_found_packages = HashSet::new(); - // Iterate over the elements in the Python set. - for py_found_package_any in py_set.iter() { - // Extract each Python 'FoundPackage' object into a Rust 'FoundPackage' struct. - // Panics if extraction fails, as specified. - let rust_found_package: FoundPackage = py_found_package_any - .extract() - .expect("Failed to extract Python FoundPackage to Rust FoundPackage."); - rust_found_packages.insert(rust_found_package); - } - rust_found_packages -} - fn get_modules_from_found_packages(found_packages: &HashSet) -> HashSet { let mut modules = HashSet::new(); for package in found_packages { @@ -81,7 +42,7 @@ fn module_is_descendant(module_name: &str, potential_ancestor: &str) -> bool { /// Statically analyses the given module and returns a set of Modules that /// it imports. #[allow(clippy::borrowed_box)] -fn scan_for_imports_no_py( +pub fn scan_for_imports_no_py( file_system: &Box, found_packages: &HashSet, include_external_packages: bool, @@ -179,36 +140,6 @@ fn scan_for_imports_no_py_single_module( Ok(imports) } -fn to_py_direct_imports<'a>( - py: Python<'a>, - rust_imports: &HashSet, -) -> Bound<'a, PySet> { - let valueobjects_pymodule = PyModule::import(py, "grimp.domain.valueobjects").unwrap(); - let py_module_class = valueobjects_pymodule.getattr("Module").unwrap(); - let py_direct_import_class = valueobjects_pymodule.getattr("DirectImport").unwrap(); - - let pyset = PySet::empty(py).unwrap(); - - for rust_import in rust_imports { - let importer = py_module_class.call1((&rust_import.importer,)).unwrap(); - let imported = py_module_class.call1((&rust_import.imported,)).unwrap(); - let kwargs = PyDict::new(py); - kwargs.set_item("importer", &importer).unwrap(); - kwargs.set_item("imported", &imported).unwrap(); - kwargs - .set_item("line_number", rust_import.line_number) - .unwrap(); - kwargs - .set_item("line_contents", &rust_import.line_contents) - .unwrap(); - kwargs.set_item("is_lazy", rust_import.is_lazy).unwrap(); - let py_direct_import = py_direct_import_class.call((), Some(&kwargs)).unwrap(); - pyset.add(&py_direct_import).unwrap(); - } - - pyset -} - #[allow(clippy::borrowed_box)] fn _determine_module_filename( module: &Module, @@ -365,94 +296,24 @@ fn _distill_external_module( } } -/// Convert the rust data structure into a Python dict[Module, set[DirectImport]]. -pub fn imports_by_module_to_py( - py: Python, - imports_by_module: HashMap>, -) -> Bound { - let valueobjects_pymodule = PyModule::import(py, "grimp.domain.valueobjects").unwrap(); - let py_module_class = valueobjects_pymodule.getattr("Module").unwrap(); - - let imports_by_module_py = PyDict::new(py); - for (module, imports) in imports_by_module.iter() { - let py_module_instance = py_module_class.call1((module.name.clone(),)).unwrap(); - let py_imports = to_py_direct_imports(py, imports); - imports_by_module_py - .set_item(py_module_instance, py_imports) - .unwrap(); - } - imports_by_module_py -} - -/// Statically analyses the given module and returns a set of Modules that -/// it imports. -/// Python args: -/// -/// - module_files The modules to scan. -/// - found_packages: Set of FoundPackages containing all the modules -/// for analysis. -/// - include_external_packages: Whether to include imports of external modules (i.e. -/// modules not contained in modules_by_package_directory) -/// in the results. -/// - exclude_type_checking_imports: If True, don't include imports behind TYPE_CHECKING guards. -/// - file_system: The file system interface to use. (A BasicFileSystem.) -/// -/// Returns dict[Module, set[DirectImport]]. -#[pyfunction] -pub fn scan_for_imports<'py>( - py: Python<'py>, - module_files: Vec>, - found_packages: Bound<'py, PyAny>, - include_external_packages: bool, - exclude_type_checking_imports: bool, - file_system: Bound<'py, PyAny>, -) -> PyResult> { - let file_system_boxed = get_file_system_boxed(&file_system)?; - let found_packages_rust = py_found_packages_to_rust(&found_packages); - let modules_rust: HashSet = module_files - .iter() - .map(|module_file| { - module_file - .getattr("module") - .unwrap() - .extract::() - .unwrap() - }) - .collect(); - - let imports_by_module_result = py.detach(|| { - scan_for_imports_no_py( - &file_system_boxed, - &found_packages_rust, - include_external_packages, - &modules_rust, - exclude_type_checking_imports, - ) - }); - - match imports_by_module_result { - Err(GrimpError::ParseError { +/// Convert a `GrimpError` to a Python exception, translating syntax errors in the scanned +/// source code to `grimp.exceptions.SourceSyntaxError`. +pub fn into_py_err(py: Python, error: GrimpError) -> PyErr { + match error { + GrimpError::ParseError { module_filename, line_number, text, .. - }) => { + } => { // TODO: define SourceSyntaxError using pyo3. let exceptions_pymodule = PyModule::import(py, "grimp.exceptions").unwrap(); let py_exception_class = exceptions_pymodule.getattr("SourceSyntaxError").unwrap(); let exception = py_exception_class .call1((module_filename, line_number, text)) .unwrap(); - return Err(PyErr::from_value(exception)); - } - Err(e) => { - return Err(e.into()); + PyErr::from_value(exception) } - _ => (), + e => e.into(), } - let imports_by_module = imports_by_module_result.unwrap(); - - let imports_by_module_py = imports_by_module_to_py(py, imports_by_module); - - Ok(imports_by_module_py) } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 000e0df7..b98072a2 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -3,8 +3,10 @@ pub mod errors; pub mod exceptions; mod filesystem; pub mod graph; +mod graph_building; pub mod import_parsing; mod import_scanning; +mod log_records; pub mod module_expressions; mod module_finding; @@ -12,24 +14,14 @@ use pyo3::prelude::*; #[pymodule] mod _rustgrimp { - #[pymodule_export] - use crate::import_scanning::scan_for_imports; - - #[pymodule_export] - use crate::caching::read_cache_data_map_file; - - #[pymodule_export] - use crate::caching::write_cache_data_map_file; - #[pymodule_export] use crate::graph::GraphWrapper; #[pymodule_export] - use crate::filesystem::{PyFakeBasicFileSystem, PyRealBasicFileSystem}; + use crate::graph_building::build_graph; #[pymodule_export] use crate::exceptions::{ - CacheVersionMismatch, CorruptCache, InvalidModuleExpression, ModuleNotPresent, - NoSuchContainer, ParseError, + InvalidModuleExpression, ModuleNotPresent, NoSuchContainer, ParseError, }; } diff --git a/rust/src/log_records.rs b/rust/src/log_records.rs new file mode 100644 index 00000000..df9c2e6f --- /dev/null +++ b/rust/src/log_records.rs @@ -0,0 +1,27 @@ +/// A message for the Python layer to log, as Rust can't use Python's `logging` directly. +#[derive(Debug)] +pub struct LogRecord { + /// The name of the Python logger to log to. + pub logger: &'static str, + /// The name of the method of the logger to call, e.g. "info" or "warning". + pub level: &'static str, + pub message: String, +} + +impl LogRecord { + pub fn info(logger: &'static str, message: String) -> Self { + Self { + logger, + level: "info", + message, + } + } + + pub fn warning(logger: &'static str, message: String) -> Self { + Self { + logger, + level: "warning", + message, + } + } +} diff --git a/rust/src/module_finding.rs b/rust/src/module_finding.rs index 868e3f31..2970c75f 100644 --- a/rust/src/module_finding.rs +++ b/rust/src/module_finding.rs @@ -1,15 +1,19 @@ -use pyo3::{prelude::*, types::PyFrozenSet}; -use std::collections::BTreeSet; +use crate::errors::GrimpResult; +use crate::log_records::LogRecord; +use std::collections::{BTreeSet, HashMap}; use std::fmt; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::UNIX_EPOCH; -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, FromPyObject)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct ModuleFile { pub module: Module, } /// A Python module. /// The string is the importable name, e.g. "mypackage.foo". -#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash, FromPyObject)] +#[derive(Debug, Clone, PartialEq, PartialOrd, Ord, Eq, Hash)] pub struct Module { pub name: String, } @@ -30,49 +34,146 @@ pub struct FoundPackage { pub namespace_packages: BTreeSet, } -/// Implements conversion from a Python 'FoundPackage' object to the Rust 'FoundPackage' struct. -/// It extracts 'name', 'directory', and converts the 'module_files' frozenset into a Rust BTreeSet. -impl<'a, 'py> FromPyObject<'a, 'py> for FoundPackage { - type Error = PyErr; +const LOGGER: &str = "grimp.adaptors.modulefinder"; - fn extract(ob: Borrowed<'a, 'py, PyAny>) -> PyResult { - let name: String = ob.getattr("name")?.extract()?; - // Extract the 'directory' attribute. - let directory: String = ob.getattr("directory")?.extract()?; +/// Finds the modules and namespace packages inside a package directory. +/// +/// Anything to be logged is added to `log`. +/// +/// Also returns the mtime of each module's file (keyed by module name), in the same form as +/// Python's `os.path.getmtime`, so that caches written by earlier versions of Grimp (which used +/// that) can still be used. +pub fn find_package( + name: &str, + directory: &str, + log: &mut Vec, +) -> GrimpResult<(FoundPackage, HashMap)> { + let mut python_files = vec![]; + let mut namespace_dirs = vec![]; + walk( + Path::new(directory), + false, + &mut python_files, + &mut namespace_dirs, + log, + )?; - // Access the 'module_files' attribute. - let module_files_py = ob.getattr("module_files")?; - // Downcast the PyAny object to a PyFrozenSet, as Python 'FrozenSet' maps to 'PyFrozenSet'. - let module_files_frozenset = module_files_py.cast::()?; + let module_name = |path: &Path| -> String { + let relative_path = path.strip_prefix(directory).unwrap(); + let mut components = vec![name.to_string()]; + components.extend( + relative_path + .iter() + .map(|c| c.to_string_lossy().into_owned()), + ); + if components.last().unwrap() == "__init__" { + components.pop(); + } + components.join(".") + }; + + let mut module_files = BTreeSet::new(); + let mut mtimes = HashMap::new(); + for path in &python_files { + let module = Module { + name: module_name(&path.with_extension("")), + }; + mtimes.insert(module.name.clone(), mtime(path)?); + module_files.insert(ModuleFile { module }); + } + let namespace_packages = namespace_dirs + .iter() + .map(|dir| Module { + name: module_name(dir), + }) + .collect(); - let mut module_files = BTreeSet::new(); - // Iterate over the Python frozenset. - for py_module_file_any in module_files_frozenset.iter() { - // Extract each element (PyAny) into a Rust 'ModuleFile'. - let module_file: ModuleFile = py_module_file_any.extract()?; - module_files.insert(module_file); + let found_package = FoundPackage { + name: name.to_string(), + directory: directory.to_string(), + module_files, + namespace_packages, + }; + Ok((found_package, mtimes)) +} + +/// Recursively collects the Python files and namespace directories within a directory. +/// +/// A directory is a namespace directory if it isn't (within) a regular package, but does +/// contain Python files somewhere below it. +fn walk( + directory: &Path, + in_regular_package: bool, + python_files: &mut Vec, + namespace_dirs: &mut Vec, + log: &mut Vec, +) -> GrimpResult<()> { + let mut file_names = vec![]; + let mut subdirectories = vec![]; + for entry in fs::read_dir(directory)? { + let entry = entry?; + let Ok(name) = entry.file_name().into_string() else { + continue; + }; + // Follow symlinks to directories, like os.walk(followlinks=True) did. + let file_type = entry.file_type()?; + if file_type.is_dir() || (file_type.is_symlink() && entry.path().is_dir()) { + subdirectories.push((name, entry.path())); + } else { + file_names.push(name); } + } - // Access the 'namespace_packages' attribute. - let namespace_packages_py = ob.getattr("namespace_packages")?; - // Downcast the PyAny object to a PyFrozenSet, as Python 'FrozenSet' maps to 'PyFrozenSet'. - let namespace_packages_frozenset = namespace_packages_py.cast::()?; + let has_init = file_names.iter().any(|name| name == "__init__.py"); + if in_regular_package && !has_init { + // Don't drill down into directories that aren't packages. + return Ok(()); + } + let in_regular_package = in_regular_package || has_init; - let mut namespace_packages = BTreeSet::new(); - // Iterate over the Python frozenset. - for py_namespace_any in namespace_packages_frozenset.iter() { - // Extract each element (PyAny) into a Rust 'ModuleFile'. - let namespace_package: String = py_namespace_any.extract()?; - namespace_packages.insert(Module { - name: namespace_package, - }); + let python_files_before = python_files.len(); + for name in &file_names { + // Hidden files, and files like `some.module.py`, aren't treated as Python files. + if name.ends_with(".py") && !name.starts_with('.') { + if name.matches('.').count() == 1 { + python_files.push(directory.join(name)); + } else { + log.push(LogRecord::warning( + LOGGER, + format!( + "Warning: skipping module with too many dots in the name: {}", + directory.join(name).display() + ), + )); + } } + } + for (name, path) in &subdirectories { + if is_identifier(name) { + walk(path, in_regular_package, python_files, namespace_dirs, log)?; + } + } - Ok(FoundPackage { - name, - directory, - module_files, - namespace_packages, - }) + if !in_regular_package && python_files.len() > python_files_before { + namespace_dirs.push(directory.to_path_buf()); } + Ok(()) +} + +/// Equivalent to Python's `str.isidentifier`. +fn is_identifier(name: &str) -> bool { + let mut chars = name.chars(); + chars + .next() + .is_some_and(|c| c == '_' || unicode_ident::is_xid_start(c)) + && chars.all(unicode_ident::is_xid_continue) +} + +/// Equivalent to Python's `os.path.getmtime`, including the floating point arithmetic. +fn mtime(path: &Path) -> GrimpResult { + let since_epoch = fs::metadata(path)? + .modified()? + .duration_since(UNIX_EPOCH) + .map_err(std::io::Error::other)?; + Ok(since_epoch.as_secs() as f64 + 1e-9 * since_epoch.subsec_nanos() as f64) } diff --git a/src/grimp/adaptors/caching.py b/src/grimp/adaptors/caching.py deleted file mode 100644 index 84d96057..00000000 --- a/src/grimp/adaptors/caching.py +++ /dev/null @@ -1,235 +0,0 @@ -from __future__ import annotations - -import hashlib -import json -import logging -from typing import Any - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import BasicFileSystem -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - -from ..application.ports.caching import Cache as AbstractCache -from ..application.ports.caching import CacheMiss - -logger = logging.getLogger(__name__) -PrimitiveFormat = dict[str, list[tuple[str, int | None, str]]] - - -class CacheFileNamer: - @classmethod - def make_meta_file_name(cls, found_package: FoundPackage) -> str: - return f"{found_package.name}.meta.json" - - @classmethod - def make_data_file_name( - cls, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, - ) -> str: - identifier = cls.make_data_file_unique_string( - found_packages, include_external_packages, exclude_type_checking_imports - ) - - bytes_identifier = identifier.encode() - # Use a hash algorithm with a limited size to avoid cache filenames that are too long - # the filesystem, which can happen if there are more than a few root packages - # being analyzed. - safe_unicode_identifier = hashlib.blake2b( - bytes_identifier, digest_size=20, usedforsecurity=False - ).hexdigest() - return f"{safe_unicode_identifier}.data.json" - - @classmethod - def make_data_file_unique_string( - cls, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, - ) -> str: - """ - Construct a unique string that identifies the analysis parameters. - - Doesn't need to be safe to use for a filename. - """ - package_names = (p.name for p in found_packages) - csv_packages = ",".join(sorted(package_names)) - include_external_packages_option = ":external" if include_external_packages else "" - exclude_type_checking_imports_option = ( - ":no_type_checking" if exclude_type_checking_imports else "" - ) - return ( - csv_packages + include_external_packages_option + exclude_type_checking_imports_option - ) - - -class Cache(AbstractCache): - DEFAULT_CACHE_DIR = ".grimp_cache" - - def __init__(self, *args: Any, namer: type[CacheFileNamer], **kwargs: Any) -> None: - """ - Don't instantiate Cache directly; use Cache.setup(). - """ - super().__init__(*args, **kwargs) - self._mtime_map: dict[str, float] = {} - self._data_map: dict[Module, set[DirectImport]] = {} - self._namer = namer - - @classmethod - def setup( - cls, - file_system: BasicFileSystem, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool = False, - cache_dir: str | None = None, - namer: type[CacheFileNamer] = CacheFileNamer, - ) -> Cache: - cache = cls( - file_system=file_system, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cls.cache_dir_or_default(cache_dir), - namer=namer, - ) - cache._build_mtime_map() - cache._build_data_map() - assert cache.cache_dir - return cache - - @classmethod - def cache_dir_or_default(cls, cache_dir: str | None) -> str: - return cache_dir or cls.DEFAULT_CACHE_DIR - - def read_imports(self, module_file: ModuleFile) -> set[DirectImport]: - try: - cached_mtime = self._mtime_map[module_file.module.name] - except KeyError as exc: - raise CacheMiss from exc - if cached_mtime != module_file.mtime: - raise CacheMiss - - try: - return self._data_map[module_file.module] - except KeyError as exc: - # While we would expect the module to be in here, - # there's no point in crashing if, for some reason, it's not. - raise CacheMiss from exc - - def write( - self, - imports_by_module: dict[Module, set[DirectImport]], - ) -> None: - self._write_marker_files_if_not_already_there() - # Write data file. - data_cache_filename = self.file_system.join( - self.cache_dir, - self._namer.make_data_file_name( - found_packages=self.found_packages, - include_external_packages=self.include_external_packages, - exclude_type_checking_imports=self.exclude_type_checking_imports, - ), - ) - rust.write_cache_data_map_file( - filename=data_cache_filename, - imports_by_module=imports_by_module, - file_system=self.file_system, - ) - - logger.info(f"Wrote data cache file {data_cache_filename}.") - - # Write meta files. - for found_package in self.found_packages: - meta_filename = self.file_system.join( - self.cache_dir, self._namer.make_meta_file_name(found_package) - ) - mtime_map = { - module_file.module.name: module_file.mtime - for module_file in found_package.module_files - } - serialized_meta = json.dumps(mtime_map) - self.file_system.write(meta_filename, serialized_meta) - logger.info(f"Wrote meta cache file {meta_filename}.") - - def _build_mtime_map(self) -> None: - self._mtime_map = self._read_mtime_map_files() - - def _read_mtime_map_files(self) -> dict[str, float]: - all_mtimes: dict[str, float] = {} - for found_package in self.found_packages: - all_mtimes.update(self._read_mtime_map_file(found_package)) - return all_mtimes - - def _read_mtime_map_file(self, found_package: FoundPackage) -> dict[str, float]: - meta_cache_filename = self.file_system.join( - self.cache_dir, self._namer.make_meta_file_name(found_package) - ) - try: - serialized = self.file_system.read(meta_cache_filename) - except FileNotFoundError: - logger.info(f"No cache file: {meta_cache_filename}.") - return {} - try: - deserialized = json.loads(serialized) - except json.JSONDecodeError: - logger.warning(f"Could not use corrupt cache file {meta_cache_filename}.") - return {} - - logger.info(f"Used cache meta file {meta_cache_filename}.") - return deserialized - - def _build_data_map(self) -> None: - self._data_map = self._read_data_map_file() - - def _read_data_map_file(self) -> dict[Module, set[DirectImport]]: - data_cache_filename = self.file_system.join( - self.cache_dir, - self._namer.make_data_file_name( - found_packages=self.found_packages, - include_external_packages=self.include_external_packages, - exclude_type_checking_imports=self.exclude_type_checking_imports, - ), - ) - try: - imports_by_module = rust.read_cache_data_map_file( - data_cache_filename, self.file_system - ) - except FileNotFoundError: - logger.info(f"No cache file: {data_cache_filename}.") - return {} - except rust.CorruptCache: - logger.warning(f"Could not use corrupt cache file {data_cache_filename}.") - return {} - except rust.CacheVersionMismatch: - logger.info( - f"Cache file {data_cache_filename} was written by a different version of Grimp; " - "rebuilding." - ) - return {} - - logger.info(f"Used cache data file {data_cache_filename}.") - return imports_by_module - - def _build_data_cache_filename(self, found_package: FoundPackage) -> str: - return self.file_system.join(self.cache_dir, f"{found_package.name}.data.json") - - def _write_marker_files_if_not_already_there(self) -> None: - marker_files_info = ( - (".gitignore", "# Automatically created by Grimp.\n*"), - ( - "CACHEDIR.TAG", - ( - "Signature: 8a477f597d28d172789f06886806bc55\n" - "# This file is a cache directory tag automatically created by Grimp.\n" - "# For information about cache directory tags see https://bford.info/cachedir/" - ), - ), - ) - - for filename, contents in marker_files_info: - full_filename = self.file_system.join(self.cache_dir, filename) - if not self.file_system.exists(full_filename): - self.file_system.write(full_filename, contents) diff --git a/src/grimp/adaptors/filesystem.py b/src/grimp/adaptors/filesystem.py deleted file mode 100644 index 0a396cee..00000000 --- a/src/grimp/adaptors/filesystem.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import os -import tokenize -from collections.abc import Iterator - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import AbstractFileSystem, BasicFileSystem - - -class FileSystem(AbstractFileSystem): - """ - Abstraction around file system calls. - """ - - @property - def sep(self) -> str: - return os.sep - - def dirname(self, filename: str) -> str: - return os.path.dirname(filename) - - def walk(self, directory_name: str) -> Iterator[tuple[str, list[str], list[str]]]: - yield from os.walk(directory_name, followlinks=True) - - def join(self, *components: str) -> str: - return os.path.join(*components) - - def split(self, file_name: str) -> tuple[str, str]: - return os.path.split(file_name) - - def read(self, file_name: str) -> str: - # Use tokenize.open to give us a better chance of successfully decoding - # source code in a non-ascii compatible encoding. - with tokenize.open(file_name) as file: - return file.read() - - def exists(self, file_name: str) -> bool: - return os.path.isfile(file_name) - - def get_mtime(self, file_name: str) -> float: - return os.path.getmtime(file_name) - - def write(self, file_name: str, contents: str) -> None: - dirname = os.path.dirname(file_name) - if not os.path.exists(dirname): - os.makedirs(dirname) - with open(file_name, "w") as file: - print(contents, file=file) - - def convert_to_basic(self) -> BasicFileSystem: - return rust.RealBasicFileSystem() diff --git a/src/grimp/adaptors/modulefinder.py b/src/grimp/adaptors/modulefinder.py deleted file mode 100644 index 00f77374..00000000 --- a/src/grimp/adaptors/modulefinder.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Iterable, Set - -from grimp.application.ports import modulefinder -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.domain.valueobjects import Module - -logger = logging.getLogger(__name__) - - -class ModuleFinder(modulefinder.AbstractModuleFinder): - def find_package( - self, package_name: str, package_directory: str, file_system: AbstractFileSystem - ) -> modulefinder.FoundPackage: - self.file_system = file_system - - module_files: list[modulefinder.ModuleFile] = [] - - python_files, namespace_dirs = self._get_python_files_and_namespace_dirs_inside_package( - package_directory - ) - for module_filename in python_files: - module_name = self._module_name_from_filename( - package_name, module_filename, package_directory - ) - module_mtime = self.file_system.get_mtime(module_filename) - module_files.append( - modulefinder.ModuleFile(module=Module(module_name), mtime=module_mtime) - ) - - namespace_packages = frozenset( - { - self._namespace_from_dir(package_name, namespace_dir, package_directory) - for namespace_dir in namespace_dirs - } - ) - - return modulefinder.FoundPackage( - name=package_name, - directory=package_directory, - module_files=frozenset(module_files), - namespace_packages=namespace_packages, - ) - - def _get_python_files_and_namespace_dirs_inside_package( - self, directory: str - ) -> tuple[Iterable[str], Set[str]]: - """ - Search the supplied package directory for Python files and namespaces. - - Return tuple consisting of: - 1. Iterable of Python file names. - 2. Set of namespace directories encountered. - """ - python_files: list[str] = [] - candidate_namespace_dirs: list[str] = [] - portion_dirs: set[str] = set() - - for dirpath, dirs, files in self.file_system.walk(directory): - if self._is_in_portion(dirpath, portion_dirs): - # Are we somewhere inside a non-namespace package? - if "__init__.py" not in files: - # Don't drill down further in this directory. - # (This means we won't include 'orphans' - Python packages deeply nested - # in a package that has already included __init__.py files. - for d in list(dirs): - dirs.remove(d) - continue - elif "__init__.py" in files: - # This directory is a portion (i.e. it has a top-level __init__.py). - portion_dirs.add(dirpath) - else: - # We don't yet know whether this is a namespace dir. It'll only be one if we find - # a Python file somewhere within it. - candidate_namespace_dirs.append(dirpath) - - # Don't include directories that aren't valid identifiers. - dirs_to_remove = [d for d in dirs if self._should_ignore_dir(d)] - for d in dirs_to_remove: - dirs.remove(d) - - python_files.extend( - self.file_system.join(dirpath, filename) - for filename in files - if self._is_python_file(filename, dirpath) - ) - - namespace_dirs = self._determine_namespace_dirs(candidate_namespace_dirs, python_files) - return python_files, namespace_dirs - - def _is_in_portion(self, directory: str, portions: Set[str]) -> bool: - return any(directory.startswith(portion) for portion in portions) - - def _should_ignore_dir(self, directory: str) -> bool: - # TODO: make this configurable. - return not directory.isidentifier() - - def _determine_namespace_dirs( - self, candidates: Iterable[str], python_files: Iterable[str] - ) -> set[str]: - namespace_dirs: set[str] = set() - for candidate in candidates: - candidate_with_trailing_sep = candidate + self.file_system.sep - for python_file in python_files: - if python_file.startswith(candidate_with_trailing_sep): - namespace_dirs.add(candidate) - break - return namespace_dirs - - def _is_python_file(self, filename: str, dirpath: str) -> bool: - """ - Given a filename, return whether it's a Python file. - - Files with extra dots in the name won't be treated as Python files. - - Args: - filename: the filename, excluding the path. - dirpath: the directory path containing the file. - - Returns: - Whether it's a Python file. - """ - # Ignore hidden files. - if filename.startswith("."): - return False - - if not filename.endswith(".py"): - return False - - # Ignore files like some.module.py. - if filename.count(".") > 1: - logger.warning( - "Warning: skipping module with too many dots in the name: " - f"{dirpath}{self.file_system.sep}{filename}" - ) - return False - - return True - - def _module_name_from_filename( - self, package_name: str, filename_and_path: str, package_directory: str - ) -> str: - """ - Args: - package_name: the importable name of the top level package. Could - be namespaced. - filename_and_path: the full name of the Python file. - package_directory: the full path of the top level Python package directory. - - Returns: - Absolute module name for importing. - """ - internal_filename_and_path = filename_and_path[len(package_directory) :] - internal_filename_and_path_without_extension = internal_filename_and_path[1:-3] - components = [ - package_name, - *internal_filename_and_path_without_extension.split(self.file_system.sep), - ] - if components[-1] == "__init__": - components.pop() - return ".".join(components) - - def _namespace_from_dir( - self, package_name: str, namespace_dir: str, package_directory: str - ) -> str: - """ - Args: - package_name: the importable name of the top level package. Could - be namespaced. - namespace_dir: the full name of the namespace directory. - package_directory: the full path of the top level Python package directory. - - Returns: - Absolute module name for importing (string). - """ - parent_of_package_directory = package_directory[: -len(package_name)] - directory_relative_to_parent = namespace_dir[len(parent_of_package_directory) :] - components = directory_relative_to_parent.split(self.file_system.sep) - return ".".join(components) diff --git a/src/grimp/adaptors/packagefinder.py b/src/grimp/adaptors/packagefinder.py index 7da528dd..8f900078 100644 --- a/src/grimp/adaptors/packagefinder.py +++ b/src/grimp/adaptors/packagefinder.py @@ -2,11 +2,11 @@ import importlib.util import logging +import os import sys from importlib.machinery import ModuleSpec from grimp import exceptions -from grimp.application.ports.filesystem import AbstractFileSystem from grimp.application.ports.packagefinder import AbstractPackageFinder from grimp.domain.valueobjects import Module @@ -14,9 +14,7 @@ class ImportLibPackageFinder(AbstractPackageFinder): - def determine_package_directories( - self, package_name: str, file_system: AbstractFileSystem - ) -> set[str]: + def determine_package_directories(self, package_name: str) -> set[str]: # Attempt to locate the package file. spec = importlib.util.find_spec(package_name) if not spec: @@ -27,19 +25,16 @@ def determine_package_directories( if ( spec.has_location and spec.origin - and ( - not self._is_a_package(spec, file_system) or self._has_a_non_namespace_parent(spec) - ) + and (not self._is_a_package(spec) or self._has_a_non_namespace_parent(spec)) ): raise exceptions.NotATopLevelModule assert spec.submodule_search_locations # This should be the case if spec.has_location. return set(spec.submodule_search_locations) - def _is_a_package(self, spec: ModuleSpec, file_system: AbstractFileSystem) -> bool: + def _is_a_package(self, spec: ModuleSpec) -> bool: assert spec.origin - filename = file_system.split(spec.origin)[1] - return filename == "__init__.py" + return os.path.basename(spec.origin) == "__init__.py" def _has_a_non_namespace_parent(self, spec: ModuleSpec) -> bool: module = Module(spec.name) diff --git a/src/grimp/adaptors/timing.py b/src/grimp/adaptors/timing.py deleted file mode 100644 index 44413fb4..00000000 --- a/src/grimp/adaptors/timing.py +++ /dev/null @@ -1,8 +0,0 @@ -import time - -from grimp.application.ports.timing import Timer - - -class SystemClockTimer(Timer): - def get_current_time(self) -> float: - return time.time() diff --git a/src/grimp/application/config.py b/src/grimp/application/config.py index 6bec409e..61eacedd 100644 --- a/src/grimp/application/config.py +++ b/src/grimp/application/config.py @@ -15,10 +15,5 @@ def __getattr__(self, name: str) -> Any: return self._config[name] return super().__getattribute__(name) - def copy(self) -> Settings: - new_instance = self.__class__() - new_instance.configure(**self._config) - return new_instance - settings = Settings() diff --git a/src/grimp/application/graph.py b/src/grimp/application/graph.py index 3c679621..04050b04 100644 --- a/src/grimp/application/graph.py +++ b/src/grimp/application/graph.py @@ -40,6 +40,12 @@ def __init__(self) -> None: self._cached_modules: set[str] | None = None self._rustgraph = rust.Graph() + @classmethod + def from_rustgraph(cls, rustgraph: rust.Graph) -> ImportGraph: + graph = cls() + graph._rustgraph = rustgraph + return graph + # Mechanics # --------- diff --git a/src/grimp/application/ports/caching.py b/src/grimp/application/ports/caching.py deleted file mode 100644 index 46f10997..00000000 --- a/src/grimp/application/ports/caching.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import annotations - -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - -from .filesystem import BasicFileSystem - - -class CacheMiss(Exception): - pass - - -class Cache: - def __init__( - self, - file_system: BasicFileSystem, - include_external_packages: bool, - exclude_type_checking_imports: bool, - found_packages: set[FoundPackage], - cache_dir: str, - ) -> None: - """ - Don't instantiate Cache directly; use Cache.setup(). - """ - self.file_system = file_system - self.found_packages = found_packages - self.include_external_packages = include_external_packages - self.exclude_type_checking_imports = exclude_type_checking_imports - self.cache_dir = cache_dir - - @classmethod - def setup( - cls, - file_system: BasicFileSystem, - found_packages: set[FoundPackage], - *, - include_external_packages: bool, - exclude_type_checking_imports: bool = False, - cache_dir: str | None = None, - ) -> Cache: - cache = cls( - file_system=file_system, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cls.cache_dir_or_default(cache_dir), - ) - return cache - - def read_imports(self, module_file: ModuleFile) -> set[DirectImport]: - raise NotImplementedError - - def write( - self, - imports_by_module: dict[Module, set[DirectImport]], - ) -> None: - raise NotImplementedError - - @classmethod - def cache_dir_or_default(cls, cache_dir: str | None) -> str: - raise NotImplementedError diff --git a/src/grimp/application/ports/filesystem.py b/src/grimp/application/ports/filesystem.py deleted file mode 100644 index 92e5504c..00000000 --- a/src/grimp/application/ports/filesystem.py +++ /dev/null @@ -1,115 +0,0 @@ -from __future__ import annotations - -import abc -from collections.abc import Iterator -from typing import Protocol - - -class AbstractFileSystem(abc.ABC): - """ - Abstraction around file system calls. - """ - - @property - @abc.abstractmethod - def sep(self) -> str: - r""" - Return the file separator for the FileSystem. - - E.G. '/' for UNIX systems and '\\' for Windows systems - """ - - @abc.abstractmethod - def dirname(self, filename: str) -> str: - """ - Return the full path to the directory name of the supplied filename. - - E.g. '/path/to/filename.py' will return '/path/to'. - """ - raise NotImplementedError - - @abc.abstractmethod - def walk(self, directory_name: str) -> Iterator[tuple[str, list[str], list[str]]]: - """ - Given a directory, walk the file system recursively. - - For each directory in the tree rooted at directory top (including top itself), - it yields a 3-tuple (dirpath, dirnames, filenames). - """ - raise NotImplementedError - - @abc.abstractmethod - def join(self, *components: str) -> str: - raise NotImplementedError - - @abc.abstractmethod - def split(self, file_name: str) -> tuple[str, str]: - """ - Split the pathname path into a pair, (head, tail) where tail is the last pathname component - and head is everything leading up to that. The tail part will never contain a slash; - if path ends in a slash, tail will be empty. If there is no slash in path, head will be - empty. If path is empty, both head and tail are empty. Trailing slashes are stripped from - head unless it is the root (one or more slashes only). In all cases, join(head, tail) - returns a path to the same location as path (but the strings may differ). - """ - raise NotImplementedError - - @abc.abstractmethod - def read(self, file_name: str) -> str: - """ - Given a file name, return the contents of the file. - """ - raise NotImplementedError - - @abc.abstractmethod - def exists(self, file_name: str) -> bool: - """ - Return whether a file exists. - """ - raise NotImplementedError - - @abc.abstractmethod - def get_mtime(self, file_name: str) -> float: - """ - Return the mtime of a file. - - Raises FileNotFoundError if the file does not exist. - """ - raise NotImplementedError - - @abc.abstractmethod - def write(self, file_name: str, contents: str) -> None: - """ - Write the contents to a file. - """ - raise NotImplementedError - - @abc.abstractmethod - def convert_to_basic(self) -> BasicFileSystem: - """ - Convert this file system to a BasicFileSystem. - """ - raise NotImplementedError - - -class BasicFileSystem(Protocol): - """ - A more limited file system. - - Having two different file system APIs is an interim approach, allowing us to - implement BasicFileSystem in Rust without needing to implement the full range - of methods used by other parts of Grimp. - - Eventually we'll move the full implementation to Rust, and have a single - FileSystem interface. - """ - - def join(self, *components: str) -> str: ... - - def split(self, file_name: str) -> tuple[str, str]: ... - - def read(self, file_name: str) -> str: ... - - def write(self, file_name: str, contents: str) -> None: ... - - def exists(self, file_name: str) -> bool: ... diff --git a/src/grimp/application/ports/modulefinder.py b/src/grimp/application/ports/modulefinder.py deleted file mode 100644 index 1d59f936..00000000 --- a/src/grimp/application/ports/modulefinder.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import abc -from collections.abc import Set -from dataclasses import dataclass - -from grimp.domain.valueobjects import Module - -from .filesystem import AbstractFileSystem - - -@dataclass(frozen=True, order=True) -class ModuleFile: - module: Module - mtime: float - - -@dataclass(frozen=True, order=True) -class FoundPackage: - """ - Set of modules found under a single package, together with metadata. - """ - - name: str - directory: str - module_files: Set[ModuleFile] - namespace_packages: Set[str] = frozenset() - - -class AbstractModuleFinder(abc.ABC): - """ - Finds Python modules inside a package. - """ - - @abc.abstractmethod - def find_package( - self, package_name: str, package_directory: str, file_system: AbstractFileSystem - ) -> FoundPackage: - """ - Searches the package for all importable Python modules. - """ - raise NotImplementedError diff --git a/src/grimp/application/ports/packagefinder.py b/src/grimp/application/ports/packagefinder.py index 317f5318..09478f47 100644 --- a/src/grimp/application/ports/packagefinder.py +++ b/src/grimp/application/ports/packagefinder.py @@ -2,12 +2,8 @@ import abc -from .filesystem import AbstractFileSystem - class AbstractPackageFinder(abc.ABC): @abc.abstractmethod - def determine_package_directories( - self, package_name: str, file_system: AbstractFileSystem - ) -> set[str]: + def determine_package_directories(self, package_name: str) -> set[str]: raise NotImplementedError diff --git a/src/grimp/application/ports/timing.py b/src/grimp/application/ports/timing.py deleted file mode 100644 index 5837a3b7..00000000 --- a/src/grimp/application/ports/timing.py +++ /dev/null @@ -1,47 +0,0 @@ -from __future__ import annotations - -import abc -from types import TracebackType -from typing import Self - - -class Timer(abc.ABC): - """ - Context manager to allow easy timing of events. - - This is an abstraction that needs to be implemented using a subclass - that implements the get_current_time method. - - Usage: - - with Timer() as timer: - do_something() - print(timer.duration_in_s) - """ - - def __init__(self) -> None: - # We use a stack so context managers can be nested. - self._start_stack: list[float] = [] - - def __enter__(self) -> Self: - self._start_stack.append(self.get_current_time()) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - end = self.get_current_time() - start = self._start_stack.pop() - self.duration_in_s = int(end - start) - - @abc.abstractmethod - def get_current_time(self) -> float: - """ - Return the time in seconds since the epoch as a floating point number. - - See https://docs.python.org/3/library/time.html#time.time - """ - raise NotImplementedError diff --git a/src/grimp/application/scanning.py b/src/grimp/application/scanning.py deleted file mode 100644 index 47b85506..00000000 --- a/src/grimp/application/scanning.py +++ /dev/null @@ -1,30 +0,0 @@ -from __future__ import annotations - -from collections.abc import Collection - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.config import settings -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - - -def scan_imports( - module_files: Collection[ModuleFile], - *, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, -) -> dict[ModuleFile, set[DirectImport]]: - file_system: AbstractFileSystem = settings.FILE_SYSTEM - basic_file_system = file_system.convert_to_basic() - imports_by_module: dict[Module, set[DirectImport]] = rust.scan_for_imports( - module_files=tuple(module_files), - found_packages=found_packages, - # Ensure that the passed exclude_type_checking_imports is definitely a boolean, - # otherwise the Rust class will error. - include_external_packages=bool(include_external_packages), - exclude_type_checking_imports=exclude_type_checking_imports, - file_system=basic_file_system, - ) - return {module_file: imports_by_module[module_file.module] for module_file in module_files} diff --git a/src/grimp/application/usecases.py b/src/grimp/application/usecases.py index 34c4e934..782e79fa 100644 --- a/src/grimp/application/usecases.py +++ b/src/grimp/application/usecases.py @@ -4,18 +4,16 @@ from __future__ import annotations -import itertools -from collections.abc import Iterable, Sequence +import logging +from collections.abc import Sequence from typing import cast +from grimp import _rustgrimp as rust # type: ignore[attr-defined] + from ..application.graph import ImportGraph -from ..application.ports import caching -from ..application.ports.filesystem import AbstractFileSystem, BasicFileSystem -from ..application.ports.modulefinder import AbstractModuleFinder, FoundPackage, ModuleFile -from ..application.ports.packagefinder import AbstractPackageFinder -from ..domain.valueobjects import DirectImport, Module from .config import settings -from .scanning import scan_imports + +DEFAULT_CACHE_DIR = ".grimp_cache" class NotSupplied: @@ -51,49 +49,27 @@ def build_graph( "mypackage", "anotherpackage", "onemore", include_external_packages=True, ) """ - file_system: AbstractFileSystem = settings.FILE_SYSTEM - - found_packages = _find_packages( - file_system=file_system, - package_names=[package_name, *additional_package_names], - ) - - imports_by_module = _scan_packages( - found_packages=found_packages, - file_system=file_system.convert_to_basic(), - include_external_packages=include_external_packages, + package_names = _validate_package_names_are_strings([package_name, *additional_package_names]) + packages = [ + (name, directory) + for name in package_names + for directory in settings.PACKAGE_FINDER.determine_package_directories(name) + ] + if cache_dir is NotSupplied: + cache_dir = DEFAULT_CACHE_DIR + + rust_graph, log_records = rust.build_graph( + packages, + # Tolerate a non-boolean (Import Linter has been known to pass None). + include_external_packages=bool(include_external_packages), exclude_type_checking_imports=exclude_type_checking_imports, cache_dir=cache_dir, ) + # Rust can't use Python's logging, so it returns what should be logged. + for logger_name, level, message in log_records: + getattr(logging.getLogger(logger_name), level)(message) - graph = _assemble_graph(found_packages, imports_by_module) - - return graph - - -def _find_packages( - file_system: AbstractFileSystem, package_names: Sequence[object] -) -> set[FoundPackage]: - package_names = _validate_package_names_are_strings(package_names) - - module_finder: AbstractModuleFinder = settings.MODULE_FINDER - package_finder: AbstractPackageFinder = settings.PACKAGE_FINDER - - found_packages: set[FoundPackage] = set() - - for package_name in package_names: - package_directories = package_finder.determine_package_directories( - package_name=package_name, file_system=file_system - ) - for package_directory in package_directories: - found_package = module_finder.find_package( - package_name=package_name, - package_directory=package_directory, - file_system=file_system, - ) - found_packages.add(found_package) - - return found_packages + return ImportGraph.from_rustgraph(rust_graph) def _validate_package_names_are_strings( @@ -104,107 +80,3 @@ def _validate_package_names_are_strings( msg = f"Package names must be strings, got {name.__class__.__name__}." raise TypeError(msg) return cast("Sequence[str]", package_names) - - -def _scan_packages( - found_packages: set[FoundPackage], - file_system: BasicFileSystem, - *, - include_external_packages: bool, - exclude_type_checking_imports: bool, - cache_dir: str | type[NotSupplied] | None, -) -> dict[Module, set[DirectImport]]: - if cache_dir is not None: - cache_dir_if_supplied = cache_dir if cache_dir != NotSupplied else None - cache: caching.Cache = settings.CACHE_CLASS.setup( - file_system=file_system, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - cache_dir=cache_dir_if_supplied, - ) - - module_files_to_scan = { - module_file - for found_package in found_packages - for module_file in found_package.module_files - } - - imports_by_module_file: dict[ModuleFile, set[DirectImport]] = {} - - if cache_dir is not None: - imports_by_module_file.update(_read_imports_from_cache(module_files_to_scan, cache=cache)) - - remaining_module_files_to_scan = module_files_to_scan.difference(imports_by_module_file) - if remaining_module_files_to_scan: - imports_by_module_file.update( - scan_imports( - remaining_module_files_to_scan, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - ) - - imports_by_module: dict[Module, set[DirectImport]] = { - k.module: v for k, v in imports_by_module_file.items() - } - - if cache_dir is not None: - cache.write(imports_by_module) - - return imports_by_module - - -def _assemble_graph( - found_packages: set[FoundPackage], - imports_by_module: dict[Module, set[DirectImport]], -) -> ImportGraph: - graph: ImportGraph = settings.IMPORT_GRAPH_CLASS() - - for namespace_package in itertools.chain.from_iterable( - found_package.namespace_packages for found_package in found_packages - ): - graph.add_module(namespace_package) - - package_modules = {Module(found_package.name) for found_package in found_packages} - - for module, direct_imports in imports_by_module.items(): - graph.add_module(module.name) - for direct_import in direct_imports: - # Before we add the import, check to see if the imported module is in fact an - # external module, and if so, tell the graph that it is a squashed module. - graph.add_module( - direct_import.imported.name, - is_squashed=_is_external(direct_import.imported, package_modules), - ) - - graph.add_import( - importer=direct_import.importer.name, - imported=direct_import.imported.name, - line_number=direct_import.line_number, - line_contents=direct_import.line_contents, - is_lazy=direct_import.is_lazy, - ) - return graph - - -def _is_external(module: Module, package_modules: set[Module]) -> bool: - return not any( - module.is_descendant_of(package_module) or module == package_module - for package_module in package_modules - ) - - -def _read_imports_from_cache( - module_files: Iterable[ModuleFile], *, cache: caching.Cache -) -> dict[ModuleFile, set[DirectImport]]: - imports_by_module_file: dict[ModuleFile, set[DirectImport]] = {} - for module_file in module_files: - try: - direct_imports = cache.read_imports(module_file) - except caching.CacheMiss: - continue - else: - imports_by_module_file[module_file] = direct_imports - return imports_by_module_file diff --git a/src/grimp/main.py b/src/grimp/main.py index 57d02f47..7ad4aca5 100644 --- a/src/grimp/main.py +++ b/src/grimp/main.py @@ -1,19 +1,9 @@ __all__ = ["build_graph"] -from .adaptors.caching import Cache -from .adaptors.filesystem import FileSystem -from .adaptors.modulefinder import ModuleFinder from .adaptors.packagefinder import ImportLibPackageFinder -from .adaptors.timing import SystemClockTimer from .application.config import settings -from .application.graph import ImportGraph from .application.usecases import build_graph settings.configure( - MODULE_FINDER=ModuleFinder(), - FILE_SYSTEM=FileSystem(), - IMPORT_GRAPH_CLASS=ImportGraph, PACKAGE_FINDER=ImportLibPackageFinder(), - CACHE_CLASS=Cache, - TIMER=SystemClockTimer(), ) diff --git a/tests/adaptors/__init__.py b/tests/adaptors/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/adaptors/filesystem.py b/tests/adaptors/filesystem.py deleted file mode 100644 index 6b9809d3..00000000 --- a/tests/adaptors/filesystem.py +++ /dev/null @@ -1,267 +0,0 @@ -from __future__ import annotations - -from collections.abc import Generator -from typing import Any - -import yaml - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import AbstractFileSystem, BasicFileSystem - -DEFAULT_MTIME = 10000.0 - - -class FakeFileSystem(AbstractFileSystem): - def __init__( - self, - contents: str | None = None, - content_map: dict[str, str] | None = None, - mtime_map: dict[str, float] | None = None, - ) -> None: - """ - Files can be declared as existing in the file system in two different ways, either - in a contents string (which is a quick way of defining a lot of files), or in content_map - (which specifies the actual contents of a file in the file system). For a file to be - treated as existing, it needs to be declared in at least one of these. If it isn't - declared in content_map, the file will behave as an empty file. - - Args: - contents: a string in the following format: - - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - - content_map: A dictionary keyed with filenames, with values that are the contents. - If present in content_map, .read(filename) will return the string. - { - '/path/to/foo/__init__.py': "from . import one", - } - mtime_map: A dictionary keyed with filenames, with values that are the mtimes - i.e. last modified times. - """ - self.contents = self._parse_contents(contents) - self._raw_contents = contents - self.content_map = content_map or {} - self.mtime_map: dict[str, float] = mtime_map or {} - - @property - def sep(self) -> str: - return "/" - - def dirname(self, filename: str) -> str: - """ - Return the full path to the directory name of the supplied filename. - - E.g. '/path/to/filename.py' will return '/path/to'. - """ - return self.split(filename)[0] - - def walk(self, directory_name): - """ - Given a directory, walk the file system recursively. - - For each directory in the tree rooted at directory top (including top itself), - it yields a 3-tuple (dirpath, dirnames, filenames). - """ - # Navigate through the nested structure to find the directory - directory_components = [c for c in directory_name.split("/") if c] - - contents = self.contents - for component in directory_components: - key_with_prefix = "/" + component - if key_with_prefix in contents: - contents = contents[key_with_prefix] - elif component in contents: - contents = contents[component] - else: - return [] - - # If we found a file (None) instead of a directory, return empty - if contents is None: - return [] - - yield from self._walk_contents(contents, containing_directory=directory_name) - - def _walk_contents( - self, directory_contents: dict[str, Any], containing_directory: str - ) -> Generator[tuple[str, list[str], list[str]]]: - directories = [] - files = [] - for key, value in directory_contents.items(): - if value is None: - files.append(key) - else: - directories.append(key) - - yield (containing_directory, directories, files) - - if directories: - for directory in directories: - yield from self._walk_contents( - directory_contents=directory_contents[directory], - containing_directory=self.join(containing_directory, directory), - ) - - def join(self, *components: str) -> str: - return self.sep.join(c.rstrip(self.sep) for c in components) - - def split(self, file_name: str) -> tuple[str, str]: - components = file_name.split(self.sep) - if len(components) == 2: - # Handle case where file is child of the root, i.e. /some-file.txt. - # In this case, to conform with the interface we ensure the leading slash - # is included in the returned head. - components.insert(0, "") - return (self.sep.join(components[:-1]), components[-1]) - - def _parse_contents(self, raw_contents: str | None) -> dict[str, Any]: - """ - Returns the raw contents parsed in the form: - { - "/path": { - "/to": - "/mypackage": { - "__init__.py": None, - "foo": { - "__init__.py": None, - "one.py": None, - "two": { - "__init__.py": None, - "blue.py": None, - "green.py": None, - } - }, - }, - }, - } - } - """ - if raw_contents is None: - return {} - - raw_lines = [line for line in raw_contents.split("\n") if line.strip()] - dedented_lines = self._dedent(raw_lines) - - # Group lines by their root paths - # A root path is a line that starts with "/" and has no indentation - root_path_groups = [] - current_group: list[str] = [] - - for line in dedented_lines: - if line.startswith("/") and not line.startswith(" "): - # This is a new root path - if current_group: - root_path_groups.append(current_group) - current_group = [line] - else: - current_group.append(line) - - if current_group: - root_path_groups.append(current_group) - - # Process each root path group - result: dict[str, Any] = {} - for group in root_path_groups: - # First line is the root path - root_path = group[0].rstrip().rstrip("/") - path_components = [c for c in root_path.split("/") if c] - - # Remaining lines are the file tree - yamlified_lines = [] - for line in group[1:]: - trimmed_line = line.rstrip().rstrip("/") - yamlified_line = trimmed_line + ":" - yamlified_lines.append(yamlified_line) - - yamlified_string = "\n".join(yamlified_lines) - nested_contents = yaml.safe_load(yamlified_string) if yamlified_lines else {} - - # Build the nested structure from path components - group_result = nested_contents - for component in reversed(path_components): - group_result = {"/" + component: group_result} - - # Merge into result - self._deep_merge(result, group_result) - - return result - - def _deep_merge(self, target: dict, source: dict) -> None: - """Merge source dict into target dict recursively.""" - for key, value in source.items(): - if key in target and isinstance(target[key], dict) and isinstance(value, dict): - self._deep_merge(target[key], value) - else: - target[key] = value - - def _dedent(self, lines: list[str]) -> list[str]: - """ - Dedent all lines by the same amount. - """ - first_line = lines[0] - first_line_indent = len(first_line) - len(first_line.lstrip()) - return [line[first_line_indent:] for line in lines] - - def read(self, file_name: str) -> str: - if not self.exists(file_name): - raise FileNotFoundError - try: - file_contents = self.content_map[file_name] - except KeyError: - return "" - raw_lines = [line for line in file_contents.split("\n") if line.strip()] - dedented_lines = self._dedent(raw_lines) - return "\n".join(dedented_lines) - - def exists(self, file_name: str) -> bool: - # The file should exist if it's either declared in contents or in content_map. - if file_name in self.content_map.keys(): - return True - - # Split the file path into components and navigate through nested structure - file_components = [c for c in file_name.split("/") if c] - - contents = self.contents - for i, component in enumerate(file_components): - # First, try with "/" prefix (for the top-level path structure) - key_with_prefix = "/" + component - if key_with_prefix in contents: - contents = contents[key_with_prefix] - # If not found, try without prefix (for files and directories in the tree) - elif component in contents: - contents = contents[component] - else: - return False - - # If we've reached None, it's a file (leaf node) - if contents is None: - return True - - # If we've navigated through all components and haven't hit None, - # it's a directory, not a file - return False - return False - - def get_mtime(self, file_name: str) -> float: - if not self.exists(file_name): - raise FileNotFoundError(f"{file_name} does not exist.") - return self.mtime_map.get(file_name, DEFAULT_MTIME) - - def write(self, file_name: str, contents: str) -> None: - self.content_map[file_name] = contents - self.mtime_map[file_name] = DEFAULT_MTIME - - def convert_to_basic(self) -> BasicFileSystem: - """ - Convert this file system to a BasicFileSystem. - """ - return rust.FakeBasicFileSystem( - contents=self._raw_contents, - content_map=self.content_map, - ) diff --git a/tests/adaptors/modulefinder.py b/tests/adaptors/modulefinder.py deleted file mode 100644 index 50acdd2c..00000000 --- a/tests/adaptors/modulefinder.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.application.ports.modulefinder import AbstractModuleFinder, FoundPackage, ModuleFile - - -class BaseFakeModuleFinder(AbstractModuleFinder): - module_files_by_package_name: dict[str, frozenset[ModuleFile]] = {} - - def find_package( - self, package_name: str, package_directory: str, file_system: AbstractFileSystem - ) -> FoundPackage: - return FoundPackage( - name=package_name, - directory=package_directory, - module_files=self.module_files_by_package_name[package_name], - ) diff --git a/tests/adaptors/packagefinder.py b/tests/adaptors/packagefinder.py deleted file mode 100644 index 81d1b70b..00000000 --- a/tests/adaptors/packagefinder.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping - -from grimp.application.ports.filesystem import AbstractFileSystem -from grimp.application.ports.packagefinder import AbstractPackageFinder - - -class BaseFakePackageFinder(AbstractPackageFinder): - directory_map: Mapping[str, set[str]] = {} - - def determine_package_directories( - self, package_name: str, file_system: AbstractFileSystem - ) -> set[str]: - return self.directory_map[package_name] diff --git a/tests/adaptors/timing.py b/tests/adaptors/timing.py deleted file mode 100644 index 30f42ca0..00000000 --- a/tests/adaptors/timing.py +++ /dev/null @@ -1,35 +0,0 @@ -from __future__ import annotations - -from types import TracebackType - -from grimp.application.ports.timing import Timer - - -class FakeTimer(Timer): - ARBITRARY_SECONDS_SINCE_EPOCH = 1_000_000 - - def __init__(self) -> None: - super().__init__() - self._current_time = float(self.ARBITRARY_SECONDS_SINCE_EPOCH) - self._tick_duration = 1 - self._increment = 0 - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._tick() - super().__exit__(exc_type, exc_val, exc_tb) - - def get_current_time(self) -> float: - return self._current_time - - def setup(self, tick_duration: int, increment: int) -> None: - self._tick_duration = tick_duration - self._increment = increment - - def _tick(self) -> None: - self._current_time += self._tick_duration - self._tick_duration += self._increment diff --git a/tests/config.py b/tests/config.py deleted file mode 100644 index cfae0556..00000000 --- a/tests/config.py +++ /dev/null @@ -1,14 +0,0 @@ -from grimp.application.config import settings - - -class override_settings: # noqa: N801 - def __init__(self, **settings_to_override): - self.settings_to_override = settings_to_override - - def __enter__(self): - self.original_settings = settings.copy() - settings.configure(**self.settings_to_override) - - def __exit__(self, exc_type, exc_val, exc_tb): - for setting_name in self.settings_to_override: - settings.configure(**{setting_name: getattr(self.original_settings, setting_name)}) diff --git a/tests/conftest.py b/tests/conftest.py index 974fa891..45ae0a8f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,10 @@ +import importlib import sys +import textwrap from pathlib import Path +import pytest + ASSETS_PATH = Path(__file__).parent / "assets" TEST_ASSETS = ( @@ -13,3 +17,34 @@ # Add test assets to the Python path. sys.path.extend([str((ASSETS_PATH / path).resolve()) for path in TEST_ASSETS]) + + +@pytest.fixture +def write_files(tmp_path, monkeypatch): + """ + Returns a function that writes files to a temporary directory that is on sys.path, so that the + packages they contain can be found by build_graph. + + The function takes a dict mapping file paths (relative to the temporary directory) to their + contents. It returns the directory that the files were written to. A different directory + (relative to the temporary directory) can be passed as root, which is useful for creating + namespace packages that span several directories. + """ + + modules_before = set(sys.modules) + + def write(files: dict[str, str], root: str = ".") -> Path: + root_directory = tmp_path / root + for relative_path, contents in files.items(): + path = root_directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(contents), encoding="utf-8") + monkeypatch.syspath_prepend(str(root_directory)) + importlib.invalidate_caches() + return root_directory + + yield write + + # Finding a package can import its parents, so don't let them leak between tests. + for module_name in set(sys.modules) - modules_before: + del sys.modules[module_name] diff --git a/tests/functional/test_build_graph_arguments.py b/tests/functional/test_build_graph_arguments.py new file mode 100644 index 00000000..a58bbcb0 --- /dev/null +++ b/tests/functional/test_build_graph_arguments.py @@ -0,0 +1,149 @@ +import re + +import pytest + +from grimp import build_graph + + +@pytest.mark.parametrize("include_external_packages", (True, False)) +def test_happy_path(write_files, include_external_packages): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "import mypackage.foo.two.green\nfrom .. import Something", + "mypackage/foo/two/__init__.py": "", + "mypackage/foo/two/green.py": ( + "import mypackage.foo.two.blue\nfrom external.subpackage import foobar\nimport decimal" + ), + "mypackage/foo/two/blue.py": "", + } + ) + + graph = build_graph( + "mypackage", include_external_packages=include_external_packages, cache_dir=None + ) + + expected_import_map = { + "mypackage": set(), + "mypackage.foo": set(), + "mypackage.foo.one": {"mypackage.foo.two.green", "mypackage"}, + "mypackage.foo.two": set(), + "mypackage.foo.two.green": {"mypackage.foo.two.blue"}, + "mypackage.foo.two.blue": set(), + } + if include_external_packages: + expected_import_map["decimal"] = set() + expected_import_map["external"] = set() + expected_import_map["mypackage.foo.two.green"] |= {"external", "decimal"} + + assert set(expected_import_map.keys()) == graph.modules + for importer, imported_modules in expected_import_map.items(): + assert graph.find_modules_directly_imported_by(importer) == imported_modules + + # Check that the external packages are squashed modules. + if include_external_packages: + for module in ("external", "decimal"): + with pytest.raises( + ValueError, match=re.escape("Cannot find children of a squashed module.") + ): + graph.find_children(module) + + +def test_boolean_additional_package_raises_type_error(): + """ + Tests that a useful error message if build_graph is called + with a boolean as the second argument. + + This is because earlier versions of build_graph took include_external_packages + as the second argument, and it's possible it might have been called + as a positional argument. + """ + with pytest.raises(TypeError, match=re.escape("Package names must be strings, got bool.")): + build_graph("mypackage", True) + + +class TestCacheDir: + @pytest.fixture(autouse=True) + def package(self, write_files, tmp_path, monkeypatch): + write_files({"mypackage/__init__.py": ""}) + # Default caching writes to the current directory. + monkeypatch.chdir(tmp_path) + + def test_supplied(self, tmp_path): + build_graph("mypackage", cache_dir=str(tmp_path / "supplied")) + + assert (tmp_path / "supplied" / "mypackage.meta.json").exists() + assert not (tmp_path / ".grimp_cache").exists() + + def test_not_supplied_uses_default(self, tmp_path): + build_graph("mypackage") + + assert (tmp_path / ".grimp_cache" / "mypackage.meta.json").exists() + + def test_none_disables_caching(self, tmp_path): + build_graph("mypackage", cache_dir=None) + + assert not (tmp_path / ".grimp_cache").exists() + + +def test_forgives_wrong_type_being_passed_to_include_external_packages(write_files): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "import mypackage.foo.two.green\nfrom .. import Something", + } + ) + + graph = build_graph( + "mypackage", + # Note: this should be a bool, but we want to tolerate it, + # as Import Linter currently has a bug where it will pass it as None. + include_external_packages=None, + cache_dir=None, + ) + + assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.foo.one"} + + +def test_namespace_package_passed_as_root(write_files): + # A namespace package can span several directories. + write_files( + { + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "import mypackage\nfrom . import two", + "mypackage/foo/two/__init__.py": "", + "mypackage/foo/two/green.py": "", + "mypackage/foo/two/blue.py": "", + "mypackage/bar/__init__.py": "", + "mypackage/bar/three.py": "", + }, + root="one", + ) + write_files( + { + "mypackage/foobar/__init__.py": "", + "mypackage/foobar/four.py": "", + }, + root="two", + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == { + "mypackage", + "mypackage.foo", + "mypackage.foo.one", + "mypackage.foo.two", + "mypackage.foo.two.green", + "mypackage.foo.two.blue", + "mypackage.bar", + "mypackage.bar.three", + "mypackage.foobar", + "mypackage.foobar.four", + } + assert graph.find_modules_directly_imported_by("mypackage.foo.one") == { + "mypackage", + "mypackage.foo.two", + } diff --git a/tests/functional/test_caching.py b/tests/functional/test_caching.py index 4f966d22..6d4a0d13 100644 --- a/tests/functional/test_caching.py +++ b/tests/functional/test_caching.py @@ -1,5 +1,6 @@ import json import logging +import os import shutil import tempfile from pathlib import Path @@ -186,3 +187,387 @@ def _manipulate_data_file(data_file: Path, snippet: str, replacement: str) -> No with open(data_file, "w") as file: file.write(filedata) + + +# Tests of the cache's behaviour, using real files. +# --------------------------------------------------- + +MYPACKAGE_FILES = { + "mypackage/__init__.py": "", + "mypackage/one.py": "from . import two", + "mypackage/two.py": "from . import three", + "mypackage/three.py": "", +} +ANOTHERPACKAGE_FILES = { + "anotherpackage/__init__.py": "", + "anotherpackage/one.py": "import externalpackage\nfrom . import two", + "anotherpackage/two.py": "", +} +MARKER = " # Inserted by test" + + +@pytest.fixture +def cache_dir(tmp_path): + return tmp_path / "cache" + + +def _data_file(cache_dir: Path) -> Path: + [data_file] = cache_dir.glob("*.data.json") + return data_file + + +def _insert_marker_in_cache(cache_dir: Path) -> None: + """ + Alter the cached line contents of "from . import two", so we can tell if the cache was used. + """ + _manipulate_data_file(_data_file(cache_dir), "from . import two", "from . import two" + MARKER) + + +def _line_contents(graph, importer: str, imported: str) -> str: + [import_details] = graph.get_import_details(importer=importer, imported=imported) + return import_details["line_contents"] + + +class TestCacheFiles: + @pytest.mark.parametrize( + "include_external_packages, exclude_type_checking_imports, expected_data_file_name", + ( + # Blake2B 20-character hash of "bluepackage,greenpackage". + (False, False, "3d0968b3c17583a69a90610a871276fe2110c6f4.data.json"), + # Blake2B 20-character hash of "bluepackage,greenpackage:external". + (True, False, "ff83c6ee50a447a433a0c7a345d09de4b0d82a69.data.json"), + # Blake2B 20-character hash of "bluepackage,greenpackage:no_type_checking". + (False, True, "5e8b57eaed0aabd7826b5b0cc6cb2b7db8c82fb8.data.json"), + # Blake2B 20-character hash of "bluepackage,greenpackage:external:no_type_checking". + (True, True, "91f51d0e59ed9e09cfc3e8a94c9249c4c3ab7184.data.json"), + ), + ) + @pytest.mark.parametrize( + "package_names", (("bluepackage", "greenpackage"), ("greenpackage", "bluepackage")) + ) + def test_file_names( + self, + write_files, + cache_dir, + package_names, + include_external_packages, + exclude_type_checking_imports, + expected_data_file_name, + ): + write_files( + { + "bluepackage/__init__.py": "", + "greenpackage/__init__.py": "", + } + ) + + build_graph( + *package_names, + include_external_packages=include_external_packages, + exclude_type_checking_imports=exclude_type_checking_imports, + cache_dir=str(cache_dir), + ) + + # There's a meta file per package, and a data file that depends on the packages + # (but not the order they were passed in) and options. + assert {path.name for path in cache_dir.iterdir()} == { + ".gitignore", + "CACHEDIR.TAG", + "bluepackage.meta.json", + "greenpackage.meta.json", + expected_data_file_name, + } + + @pytest.mark.parametrize("cache_dir_suffix", ("", "/")) + @pytest.mark.parametrize( + "include_external_packages, expected_data_file_name", + ( + (False, "3d0968b3c17583a69a90610a871276fe2110c6f4.data.json"), + (True, "ff83c6ee50a447a433a0c7a345d09de4b0d82a69.data.json"), + ), + ) + def test_contents( + self, + write_files, + cache_dir, + cache_dir_suffix, + include_external_packages, + expected_data_file_name, + ): + root = write_files( + { + "bluepackage/__init__.py": "", + "bluepackage/one.py": "from . import two\nimport externalpackage", + "bluepackage/two.py": "", + "greenpackage/__init__.py": "", + "greenpackage/one.py": "", + "greenpackage/two.py": "from . import one", + } + ) + + build_graph( + "bluepackage", + "greenpackage", + include_external_packages=include_external_packages, + cache_dir=f"{cache_dir}{cache_dir_suffix}", + ) + + # The meta files contain the mtime of each module. + for package, module_files in ( + ( + "bluepackage", + {"bluepackage": "__init__", "bluepackage.one": "one", "bluepackage.two": "two"}, + ), + ( + "greenpackage", + {"greenpackage": "__init__", "greenpackage.one": "one", "greenpackage.two": "two"}, + ), + ): + assert json.loads((cache_dir / f"{package}.meta.json").read_text()) == { + module: os.path.getmtime(root / package / f"{file_stem}.py") + for module, file_stem in module_files.items() + } + + # The data file contains the imports of each module. + expected_blue_one_imports = {("bluepackage.two", False, 1, "from . import two")} + if include_external_packages: + expected_blue_one_imports.add(("externalpackage", False, 2, "import externalpackage")) + data = json.loads((cache_dir / expected_data_file_name).read_text()) + assert data["version"] == 2 + assert { + module: {tuple(imported) for imported in imports} + for module, imports in data["imports_by_module"].items() + } == { + "bluepackage": set(), + "bluepackage.one": expected_blue_one_imports, + "bluepackage.two": set(), + "greenpackage": set(), + "greenpackage.one": set(), + "greenpackage.two": {("greenpackage.one", False, 1, "from . import one")}, + } + + def test_default_cache_dir(self, write_files, tmp_path, monkeypatch): + write_files({"bluepackage/__init__.py": ""}) + monkeypatch.chdir(tmp_path) + + build_graph("bluepackage") + + assert (tmp_path / ".grimp_cache" / "bluepackage.meta.json").exists() + + def test_adds_marker_files(self, write_files, cache_dir): + write_files({"bluepackage/__init__.py": ""}) + + build_graph("bluepackage", cache_dir=str(cache_dir)) + + assert (cache_dir / ".gitignore").read_text() == "# Automatically created by Grimp.\n*" + assert (cache_dir / "CACHEDIR.TAG").read_text() == ( + "Signature: 8a477f597d28d172789f06886806bc55\n" + "# This file is a cache directory tag automatically created by Grimp.\n" + "# For information about cache directory tags see https://bford.info/cachedir/" + ) + + def test_doesnt_overwrite_marker_files(self, write_files, cache_dir): + write_files({"bluepackage/__init__.py": ""}) + cache_dir.mkdir() + (cache_dir / ".gitignore").write_text("Something else") + + build_graph("bluepackage", cache_dir=str(cache_dir)) + + assert (cache_dir / ".gitignore").read_text() == "Something else" + + +class TestCacheUse: + def test_uses_cache_for_unmodified_module(self, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == ( + "from . import two" + MARKER + ) + + def test_ignores_cache_for_module_with_different_mtime(self, write_files, cache_dir): + root = write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + path = root / "mypackage" / "one.py" + os.utime(path, (path.stat().st_atime, path.stat().st_mtime + 100)) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + # The modified module isn't read from the cache, but the others are. + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + assert _line_contents(graph, "mypackage.two", "mypackage.three") == "from . import three" + + def test_ignores_cache_for_new_module(self, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + write_files({"mypackage/four.py": "from . import one"}) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert graph.direct_import_exists(importer="mypackage.four", imported="mypackage.one") + + def test_ignores_cache_for_module_with_no_cached_mtime(self, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + meta_file = cache_dir / "mypackage.meta.json" + mtimes = json.loads(meta_file.read_text()) + del mtimes["mypackage.one"] + meta_file.write_text(json.dumps(mtimes)) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + + def test_ignores_cache_for_module_missing_from_data(self, write_files, cache_dir): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + data_file = _data_file(cache_dir) + data = json.loads(data_file.read_text()) + del data["imports_by_module"]["mypackage.one"] + data_file.write_text(json.dumps(data)) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert graph.direct_import_exists(importer="mypackage.one", imported="mypackage.two") + + @pytest.mark.parametrize( + "serialized_meta", + ( + "INVALID_JSON", + '{"mypackage.one": ["wrong", "type"]}', + ), + ) + def test_ignores_corrupt_meta_file(self, write_files, cache_dir, serialized_meta): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + (cache_dir / "mypackage.meta.json").write_text(serialized_meta) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + + @pytest.mark.parametrize( + "serialized_data", + ( + "INVALID_JSON", + '{"version": 2, "imports_by_module": {"mypackage.one": [["wrong", "type"]]}}', + ), + ) + def test_ignores_corrupt_data_file(self, write_files, cache_dir, serialized_data): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _data_file(cache_dir).write_text(serialized_data) + + graph = build_graph("mypackage", cache_dir=str(cache_dir)) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + # The cache is repaired. + data = json.loads(_data_file(cache_dir).read_text()) + assert data["imports_by_module"]["mypackage.one"] == [ + ["mypackage.two", False, 1, "from . import two"] + ] + + @pytest.mark.parametrize( + "package_names, options", + ( + (("mypackage", "anotherpackage"), {}), + (("mypackage",), {"include_external_packages": True}), + (("mypackage",), {"exclude_type_checking_imports": True}), + ), + ) + def test_doesnt_use_cache_for_different_packages_or_options( + self, write_files, cache_dir, package_names, options + ): + write_files(MYPACKAGE_FILES | ANOTHERPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _insert_marker_in_cache(cache_dir) + + graph = build_graph(*package_names, cache_dir=str(cache_dir), **options) + + assert _line_contents(graph, "mypackage.one", "mypackage.two") == "from . import two" + + @pytest.mark.parametrize("include_external_packages", (False, True)) + def test_uses_cache_for_multiple_packages( + self, write_files, cache_dir, include_external_packages + ): + write_files(MYPACKAGE_FILES | ANOTHERPACKAGE_FILES) + kwargs = dict( + include_external_packages=include_external_packages, cache_dir=str(cache_dir) + ) + build_graph("mypackage", "anotherpackage", **kwargs) + _insert_marker_in_cache(cache_dir) + + graph = build_graph("mypackage", "anotherpackage", **kwargs) + + expected_line_contents = "from . import two" + MARKER + assert _line_contents(graph, "mypackage.one", "mypackage.two") == expected_line_contents + assert ( + _line_contents(graph, "anotherpackage.one", "anotherpackage.two") + == expected_line_contents + ) + expected_imports = {"anotherpackage.two"} + if include_external_packages: + expected_imports.add("externalpackage") + assert graph.find_modules_directly_imported_by("anotherpackage.one") == expected_imports + + +class TestCacheLogging: + @pytest.fixture(autouse=True) + def set_log_level(self, caplog): + caplog.set_level(logging.INFO, logger="grimp.adaptors.caching") + + def test_logs_missing_cache_files(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [ + f"No cache file: {cache_dir / 'mypackage.meta.json'}.", + f"No cache file: {_data_file(cache_dir)}.", + f"Wrote data cache file {_data_file(cache_dir)}.", + f"Wrote meta cache file {cache_dir / 'mypackage.meta.json'}.", + ] + + def test_logs_successful_cache_file_reading(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + caplog.clear() + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [ + f"Used cache meta file {cache_dir / 'mypackage.meta.json'}.", + f"Used cache data file {_data_file(cache_dir)}.", + f"Wrote data cache file {_data_file(cache_dir)}.", + f"Wrote meta cache file {cache_dir / 'mypackage.meta.json'}.", + ] + + def test_logs_corrupt_cache_meta_file(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + (cache_dir / "mypackage.meta.json").write_text("INVALID_JSON") + caplog.clear() + caplog.set_level(logging.WARNING, logger="grimp.adaptors.caching") + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [ + f"Could not use corrupt cache file {cache_dir / 'mypackage.meta.json'}.", + ] + + def test_logs_corrupt_cache_data_file(self, write_files, cache_dir, caplog): + write_files(MYPACKAGE_FILES) + build_graph("mypackage", cache_dir=str(cache_dir)) + _data_file(cache_dir).write_text("INVALID_JSON") + caplog.clear() + caplog.set_level(logging.WARNING, logger="grimp.adaptors.caching") + + build_graph("mypackage", cache_dir=str(cache_dir)) + + assert caplog.messages == [f"Could not use corrupt cache file {_data_file(cache_dir)}."] diff --git a/tests/functional/test_finding_modules.py b/tests/functional/test_finding_modules.py new file mode 100644 index 00000000..210bb54b --- /dev/null +++ b/tests/functional/test_finding_modules.py @@ -0,0 +1,198 @@ +import pytest + +from grimp import build_graph + +# Files are empty unless stated otherwise. +HAPPY_PATH_FILES = { + "mypackage/__init__.py": "", + "mypackage/not-a-python-file.txt": "", + "mypackage/.hidden": "", + "mypackage/foo/__init__.py": "", + "mypackage/foo/one.py": "", + "mypackage/foo/two/__init__.py": "", + "mypackage/foo/two/green.py": "", + "mypackage/foo/two/blue.py": "", +} + + +def test_happy_path(write_files): + write_files(HAPPY_PATH_FILES) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == { + "mypackage", + "mypackage.foo", + "mypackage.foo.one", + "mypackage.foo.two", + "mypackage.foo.two.green", + "mypackage.foo.two.blue", + } + + +NAMESPACE_FILES = { + "somenamespace/foo/blue/__init__.py": "", + "somenamespace/foo/blue/one.py": "", + "somenamespace/foo/blue/two/__init__.py": "", + "somenamespace/foo/blue/two/alpha.py": "", + "somenamespace/foo/blue/noinitpackage/three.py": "", + "somenamespace/foo/blue/noinitpackage/orphan/__init__.py": "", + "somenamespace/foo/blue/noinitpackage/orphan/four.py": "", + "somenamespace/foo/green/five/__init__.py": "", + "somenamespace/foo/green/five/beta.py": "", + "somenamespace/foo/non_python_directory/six/README.txt": "", +} +MODULES_FOO_BLUE = { + "somenamespace.foo.blue", + "somenamespace.foo.blue.one", + "somenamespace.foo.blue.two", + "somenamespace.foo.blue.two.alpha", +} +MODULES_FOO_GREEN_FIVE = { + "somenamespace.foo.green.five", + "somenamespace.foo.green.five.beta", +} + + +@pytest.mark.parametrize( + "package_name, expected_modules", + [ + ( + "somenamespace", + MODULES_FOO_BLUE + | MODULES_FOO_GREEN_FIVE + | {"somenamespace", "somenamespace.foo", "somenamespace.foo.green"}, + ), + ( + "somenamespace.foo", + MODULES_FOO_BLUE + | MODULES_FOO_GREEN_FIVE + | {"somenamespace.foo", "somenamespace.foo.green"}, + ), + ("somenamespace.foo.blue", MODULES_FOO_BLUE), + ], +) +def test_namespaced_packages(write_files, package_name, expected_modules): + write_files(NAMESPACE_FILES) + + graph = build_graph(package_name, cache_dir=None) + + # Note: namespace directories are only included if they (indirectly) contain Python files. + assert graph.modules == expected_modules + + +def test_ignores_orphaned_python_files(write_files): + # Python files in directories that don't contain an __init__.py should not be discovered. + write_files( + { + "mypackage/__init__.py": "", + "mypackage/two/__init__.py": "", + "mypackage/two/green.py": "", + "mypackage/noinitpackage/green.py": "", + "mypackage/noinitpackage/orphan/__init__.py": "", + "mypackage/noinitpackage/orphan/red.py": "", + } + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.two", "mypackage.two.green"} + + +def test_ignores_invalid_identifier_directories(write_files): + write_files( + { + f"namespacepackage/{path}": "" + for path in ( + "foo/valid_underscore_name/__init__.py", + "foo/valid_underscore_name/mod.py", + "foo/invalid-hyphenated-name/__init__.py", + "foo/invalid-hyphenated-name/mod.py", + "foo/1nvalid-number-prefixed/__init__.py", + "foo/1nvalid-number-prefixed/mod.py", + "foo/valid_non_àscii/__init__.py", + "foo/valid_non_àscii/mod.py", + "foo/invalid_non_😃scii/__init__.py", + "foo/invalid_non_😃scii/mod.py", + "bar/__init__.py", + "bar/mod.py", + ) + } + ) + + graph = build_graph("namespacepackage", cache_dir=None) + + assert graph.modules == { + "namespacepackage", + "namespacepackage.foo", + "namespacepackage.foo.valid_underscore_name", + "namespacepackage.foo.valid_underscore_name.mod", + "namespacepackage.foo.valid_non_àscii", + "namespacepackage.foo.valid_non_àscii.mod", + "namespacepackage.bar", + "namespacepackage.bar.mod", + } + + +@pytest.mark.parametrize( + "extension, should_warn", + ( + ("py", True), + ("txt", False), # Anything other than .py. + ), +) +def test_ignores_dotted_python_files(write_files, caplog, extension, should_warn): + # Python files containing dots (other than the one before .py) should not be discovered. + root = write_files( + { + "mypackage/__init__.py": "", + "mypackage/foo.py": "", + "mypackage/bar/__init__.py": "", + f"mypackage/bar/baz.dotted.{extension}": "", + } + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.foo", "mypackage.bar"} + if should_warn: + assert caplog.messages == [ + "Warning: skipping module with too many dots in the name: " + f"{root / 'mypackage' / 'bar' / f'baz.dotted.{extension}'}" + ] + else: + assert caplog.messages == [] + + +def test_ignores_hidden_directories(write_files): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/two/__init__.py": "", + "mypackage/two/green.py": "", + "mypackage/.hidden/green.py": "", + "mypackage/.hidden/orphan/__init__.py": "", + "mypackage/.hidden/orphan/red.py": "", + } + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.two", "mypackage.two.green"} + + +def test_follows_symlinked_directories(write_files, tmp_path): + write_files( + { + "mypackage/__init__.py": "", + "elsewhere/__init__.py": "", + "elsewhere/green.py": "", + } + ) + (tmp_path / "mypackage" / "linked").symlink_to( + tmp_path / "elsewhere", target_is_directory=True + ) + + graph = build_graph("mypackage", cache_dir=None) + + assert graph.modules == {"mypackage", "mypackage.linked", "mypackage.linked.green"} diff --git a/tests/functional/test_scanning_imports.py b/tests/functional/test_scanning_imports.py new file mode 100644 index 00000000..c84e0d54 --- /dev/null +++ b/tests/functional/test_scanning_imports.py @@ -0,0 +1,946 @@ +""" +Tests of the imports found when building a graph. + +(Ported from unit tests of the scanning of imports, which used an in-memory file system.) +""" + +import textwrap + +import pytest + +from grimp import build_graph +from grimp.domain.valueobjects import DirectImport, Module + + +def _source(code: str) -> str: + """Dedent code, and drop the leading newline so that line numbers start from 1.""" + return textwrap.dedent(code).lstrip("\n") + + +def _direct_imports(graph, importer: str) -> set[DirectImport]: + return { + DirectImport( + importer=Module(details["importer"]), + imported=Module(details["imported"]), + line_number=details["line_number"], + line_contents=details["line_contents"], + is_lazy=details["is_lazy"], + ) + for imported in graph.find_modules_directly_imported_by(importer) + for details in graph.get_import_details(importer=importer, imported=imported) + } + + +FOO_PACKAGE_PATHS = ( + "foo/__init__.py", + "foo/one/__init__.py", + "foo/one/blue.py", + "foo/one/green.py", + "foo/two/__init__.py", + "foo/two/brown.py", + "foo/two/yellow.py", + "foo/three.py", +) + + +def _foo_package_files(contents_by_path: dict[str, str]) -> dict[str, str]: + """ + Files in the foo package, which are empty unless otherwise specified. + """ + files = dict.fromkeys(FOO_PACKAGE_PATHS, "") + files.update({path: _source(contents) for path, contents in contents_by_path.items()}) + return files + + +@pytest.mark.parametrize( + "include_external_packages, expected_result", + ( + ( + False, + { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ) + }, + ), + ( + True, + { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("externalone"), + line_number=2, + line_contents="import externalone", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("externaltwo"), + line_number=3, + line_contents="import externaltwo.subpackage # with comment afterwards.", + ), + }, + ), + ), +) +def test_absolute_imports(write_files, include_external_packages, expected_result): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(""" + import foo.two + import externalone + import externaltwo.subpackage # with comment afterwards. + arbitrary_expression = 1 + """), + "foo/two.py": "", + } + ) + + graph = build_graph( + "foo", + include_external_packages=include_external_packages, + cache_dir=None, + ) + + assert _direct_imports(graph, "foo.one") == expected_result + + +def test_lazy_imports(write_files): + write_files( + _foo_package_files( + { + "foo/one/blue.py": """ + # Absolute no-from + lazy import foo.two + lazy import externalone + lazy import externaltwo.subpackage # with comment afterwards. + arbitrary_expression = 1 + + # Absolute from + lazy from foo.one import green + lazy from foo.two import yellow + if t.TYPE_CHECKING: + lazy from foo import three + lazy from external import one + lazy from external.two import blue # with comment afterwards. + + # Relative from + lazy from . import green + lazy from ..two import yellow + lazy from .. import three + """ + } + ) + ) + + graph = build_graph("foo", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, "foo.one.blue") == { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two"), + line_number=2, + line_contents="lazy import foo.two", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("externalone"), + line_number=3, + line_contents="lazy import externalone", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("externaltwo"), + line_number=4, + line_contents="lazy import externaltwo.subpackage # with comment afterwards.", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=8, + line_contents="lazy from foo.one import green", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=9, + line_contents="lazy from foo.two import yellow", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=11, + line_contents="lazy from foo import three", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=12, + line_contents="lazy from external import one", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=13, + line_contents="lazy from external.two import blue # with comment afterwards.", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=16, + line_contents="lazy from . import green", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=17, + line_contents="lazy from ..two import yellow", + is_lazy=True, + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=18, + line_contents="lazy from .. import three", + is_lazy=True, + ), + } + + +def test_non_ascii(write_files): + write_files( + { + "mypackage/__init__.py": "", + "mypackage/blue.py": _source(""" + from ñon_ascii_变 import * + from . import ñon_ascii_变 + import mypackage.ñon_ascii_变.ラーメン + """), + "mypackage/ñon_ascii_变/__init__.py": "", + "mypackage/ñon_ascii_变/ラーメン.py": "", + } + ) + + graph = build_graph("mypackage", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, "mypackage.blue") == { + DirectImport( + importer=Module("mypackage.blue"), + imported=Module("ñon_ascii_变"), + line_number=1, + line_contents="from ñon_ascii_变 import *", + ), + DirectImport( + importer=Module("mypackage.blue"), + imported=Module("mypackage.ñon_ascii_变"), + line_number=2, + line_contents="from . import ñon_ascii_变", + ), + DirectImport( + importer=Module("mypackage.blue"), + imported=Module("mypackage.ñon_ascii_变.ラーメン"), + line_number=3, + line_contents="import mypackage.ñon_ascii_变.ラーメン", + ), + } + + +def test_single_namespace_package_portion(write_files): + MODULE_ONE = Module("namespace.foo.one") + MODULE_TWO = Module("namespace.foo.two") + MODULE_THREE = Module("namespace.foo.three") + MODULE_FOUR = Module("namespace.foo.four") + MODULE_BLUE = Module("namespace.foo.two.blue") + MODULE_ALPHA = Module("namespace.foo.two.green.alpha") + write_files( + { + "namespace/foo/__init__.py": "", + "namespace/foo/one.py": _source(""" + import namespace.foo.two + from namespace.foo import three + from . import four + """), + "namespace/foo/two/__init__.py": "", + "namespace/foo/three.py": "", + "namespace/foo/four.py": "", + "namespace/foo/two/green/__init__.py": "", + "namespace/foo/two/blue.py": "", + "namespace/foo/two/green/alpha.py": _source(""" + from .. import blue + from ... import three + """), + } + ) + + graph = build_graph("namespace.foo", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, MODULE_ONE.name) == { + DirectImport( + importer=MODULE_ONE, + imported=MODULE_TWO, + line_number=1, + line_contents="import namespace.foo.two", + ), + DirectImport( + importer=MODULE_ONE, + imported=MODULE_THREE, + line_number=2, + line_contents="from namespace.foo import three", + ), + DirectImport( + importer=MODULE_ONE, + imported=MODULE_FOUR, + line_number=3, + line_contents="from . import four", + ), + } + assert _direct_imports(graph, MODULE_ALPHA.name) == { + DirectImport( + importer=MODULE_ALPHA, + imported=MODULE_BLUE, + line_number=1, + line_contents="from .. import blue", + ), + DirectImport( + importer=MODULE_ALPHA, + imported=MODULE_THREE, + line_number=2, + line_contents="from ... import three", + ), + } + + +@pytest.mark.parametrize("include_external_packages", (True, False)) +def test_import_of_portion_not_in_graph(write_files, include_external_packages): + # Include two namespaces, one deeper than the other. + # + # Note: unlike the (in-memory) layout this was originally written against, foo.one can't + # have both a module file and a submodule (blue), so the blue module is namespace.foo.two.blue. + # It's at the same depth, so its relative imports resolve in the same way. + MODULE_FOO_ONE = Module("namespace.foo.one") + MODULE_FOO_BLUE = Module("namespace.foo.two.blue") + write_files( + { + "namespace/foo/__init__.py": "", + "namespace/foo/one.py": _source(""" + import namespace.bar.one.orange + from namespace.yellow import one + from .. import mauve + if t.TYPE_CHECKING: + from ..cyan import one + from ..magenta.one import alpha + """), + "namespace/foo/two/__init__.py": "", + "namespace/foo/two/blue.py": _source(""" + from ... import teal + from ...pink import one + from ...scarlet.one import alpha + """), + "namespace/bar/one/green/__init__.py": "", + "namespace/bar/one/green/alpha.py": "", + } + ) + + graph = build_graph( + "namespace.foo", + "namespace.bar.one.green", + include_external_packages=include_external_packages, + cache_dir=None, + ) + + if include_external_packages: + assert _direct_imports(graph, MODULE_FOO_ONE.name) == { + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.bar.one.orange"), + line_number=1, + line_contents="import namespace.bar.one.orange", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.yellow"), + line_number=2, + line_contents="from namespace.yellow import one", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.mauve"), + line_number=3, + line_contents="from .. import mauve", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.cyan"), + line_number=5, + line_contents="from ..cyan import one", + ), + DirectImport( + importer=MODULE_FOO_ONE, + imported=Module("namespace.magenta"), + line_number=6, + line_contents="from ..magenta.one import alpha", + ), + } + assert _direct_imports(graph, MODULE_FOO_BLUE.name) == { + DirectImport( + importer=MODULE_FOO_BLUE, + imported=Module("namespace.teal"), + line_number=1, + line_contents="from ... import teal", + ), + DirectImport( + importer=MODULE_FOO_BLUE, + imported=Module("namespace.pink"), + line_number=2, + line_contents="from ...pink import one", + ), + DirectImport( + importer=MODULE_FOO_BLUE, + imported=Module("namespace.scarlet"), + line_number=3, + line_contents="from ...scarlet.one import alpha", + ), + } + else: + assert _direct_imports(graph, MODULE_FOO_ONE.name) == set() + assert _direct_imports(graph, MODULE_FOO_BLUE.name) == set() + + +@pytest.mark.parametrize( + "include_external_packages, expected_result", + ( + ( + False, + { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=1, + line_contents="from foo.one import green", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=2, + line_contents="from foo.two import yellow", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=4, + line_contents="from foo import three", + ), + }, + ), + ( + True, + { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.green"), + line_number=1, + line_contents="from foo.one import green", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.two.yellow"), + line_number=2, + line_contents="from foo.two import yellow", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.three"), + line_number=4, + line_contents="from foo import three", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=5, + line_contents="from external import one", + ), + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("external"), + line_number=6, + line_contents="from external.two import blue # with comment afterwards.", + ), + }, + ), + ), +) +def test_absolute_from_imports(write_files, include_external_packages, expected_result): + write_files( + _foo_package_files( + { + "foo/one/blue.py": """ + from foo.one import green + from foo.two import yellow + if t.TYPE_CHECKING: + from foo import three + from external import one + from external.two import blue # with comment afterwards. + arbitrary_expression = 1 + """ + } + ) + ) + + graph = build_graph( + "foo", + include_external_packages=include_external_packages, + cache_dir=None, + ) + + assert _direct_imports(graph, "foo.one.blue") == expected_result + + +@pytest.mark.parametrize("module_to_scan_is_package", (True, False)) +def test_relative_from_imports(write_files, module_to_scan_is_package): + if module_to_scan_is_package: + module_to_scan = Module("foo.one") + module_path = "foo/one/__init__.py" + else: + module_to_scan = Module("foo.one.blue") + module_path = "foo/one/blue.py" + write_files( + _foo_package_files( + { + module_path: """ + from . import green + from ..two import yellow + from .. import three + arbitrary_expression = 1 + """ + } + ) + ) + + graph = build_graph( + "foo", + include_external_packages=False, + exclude_type_checking_imports=True, + cache_dir=None, + ) + + assert _direct_imports(graph, module_to_scan.name) == { + DirectImport( + importer=module_to_scan, + imported=Module("foo.one.green"), + line_number=1, + line_contents="from . import green", + ), + DirectImport( + importer=module_to_scan, + imported=Module("foo.two.yellow"), + line_number=2, + line_contents="from ..two import yellow", + ), + DirectImport( + importer=module_to_scan, + imported=Module("foo.three"), + line_number=3, + line_contents="from .. import three", + ), + } + + +@pytest.mark.parametrize( + "import_source", + ("from .two.yellow import my_function", "from foo.two.yellow import my_function"), +) +def test_trims_to_known_modules(write_files, import_source): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": import_source, + "foo/two/__init__.py": "", + "foo/two/yellow.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two.yellow"), + line_number=1, + line_contents=import_source, + ) + } + + +def test_trims_to_known_modules_within_init_file(write_files): + write_files( + { + "foo/__init__.py": "", + "foo/one/__init__.py": "from .yellow import my_function", + "foo/one/yellow.py": "", + "foo/one/blue/__init__.py": "from .alpha import my_function", + "foo/one/blue/alpha.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.one.yellow"), + line_number=1, + line_contents="from .yellow import my_function", + ) + } + assert _direct_imports(graph, "foo.one.blue") == { + DirectImport( + importer=Module("foo.one.blue"), + imported=Module("foo.one.blue.alpha"), + line_number=1, + line_contents="from .alpha import my_function", + ) + } + + +def test_trims_whitespace_from_start_of_line_contents(write_files): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(""" + def my_function(): + from . import two + """), + "foo/two.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=2, + line_contents="from . import two", + ), + } + + +@pytest.mark.parametrize( + "statement, expected_module_name", + ( + # External packages that share a namespace with an internal module resolve + # to the shallowest component that does not clash with an internal module namespace. + ("import namespace", None), + ("import namespace.foo", None), + ("from namespace import foo", None), + ("import namespace.bar", "namespace.bar"), + ("from namespace import bar", "namespace.bar"), + ("from ... import bar", "namespace.bar"), + ("import namespace.bar.orange", "namespace.bar"), + ("from namespace.bar import orange", "namespace.bar"), + ("from ...bar import orange", "namespace.bar"), + ("import namespace.foo.green", "namespace.foo.green"), + ("from namespace.foo import green", "namespace.foo.green"), + ("from .. import green", "namespace.foo.green"), + ("import namespace.foo.green.alpha", "namespace.foo.green"), + ("from namespace.foo.green import alpha", "namespace.foo.green"), + ("from ..green import alpha", "namespace.foo.green"), + ("import namespace.foo.green.alpha.one", "namespace.foo.green"), + ("from namespace.foo.green.alpha import one", "namespace.foo.green"), + ("from ..green.alpha import one", "namespace.foo.green"), + ), +) +def test_external_package_imports_for_namespace_packages( + write_files, statement, expected_module_name +): + write_files( + { + "namespace/foo/blue/__init__.py": "", + "namespace/foo/blue/alpha.py": statement, + "namespace/foo/blue/beta.py": "", + } + ) + + graph = build_graph("namespace.foo.blue", include_external_packages=True, cache_dir=None) + + module_to_scan = Module("namespace.foo.blue.alpha") + if expected_module_name: + assert _direct_imports(graph, module_to_scan.name) == { + DirectImport( + importer=module_to_scan, + imported=Module(expected_module_name), + line_number=1, + line_contents=statement, + ), + } + else: + assert _direct_imports(graph, module_to_scan.name) == set() + + +@pytest.mark.parametrize("statement", ("import bar.blue", "from bar import blue")) +def test_scans_multiple_packages(write_files, statement): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(f""" + import foo.two + {statement} + import externalone + + arbitrary_expression = 1 + """), + "foo/two.py": "", + "bar/__init__.py": "", + "bar/green.py": "", + "bar/blue.py": "", + } + ) + + graph = build_graph("foo", "bar", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("bar.blue"), + line_number=2, + line_contents=statement, + ), + } + + +@pytest.mark.parametrize("exclude_type_checking_imports", (True, False)) +@pytest.mark.parametrize( + "statement, is_statement_valid", + ( + ("if t.TYPE_CHECKING:", True), + ("if TYPE_CHECKING:", True), + ("if typing.TYPE_CHECKING:", True), + ("if WEIRD_ALIAS.TYPE_CHECKING:", True), + ("if type_checking:", False), + ("while TYPE_CHECKING:", False), + ), +) +def test_exclude_type_checking_imports( + write_files, exclude_type_checking_imports, statement, is_statement_valid +): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(f""" + import foo.two + {statement} + import foo.three + import foo.four + import foo.five + arbitrary_expression = 1 + """), + "foo/two.py": "", + "foo/three.py": "", + "foo/four.py": "", + "foo/five.py": "", + } + ) + + graph = build_graph( + "foo", + include_external_packages=False, + exclude_type_checking_imports=exclude_type_checking_imports, + cache_dir=None, + ) + + if exclude_type_checking_imports and is_statement_valid: + expected_result = { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.five"), + line_number=5, + line_contents="import foo.five", + ), + } + else: + expected_result = { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.three"), + line_number=3, + line_contents="import foo.three", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.four"), + line_number=4, + line_contents="import foo.four", + ), + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.five"), + line_number=5, + line_contents="import foo.five", + ), + } + assert _direct_imports(graph, "foo.one") == expected_result + + +@pytest.mark.parametrize( + "code, expected_imported_modules", + ( + pytest.param( + """ + if TYPE_CHECKING: + import foo.two + else: + import foo.three + """, + {"foo.three"}, + id="else", + ), + pytest.param( + """ + if typing.TYPE_CHECKING: + import foo.two + elif sys.version_info >= (3, 12): + import foo.three + else: + import foo.four + """, + {"foo.three", "foo.four"}, + id="elif-and-else", + ), + pytest.param( + """ + if sys.version_info >= (3, 12): + import foo.two + elif TYPE_CHECKING: + import foo.three + else: + import foo.four + """, + {"foo.two", "foo.four"}, + id="elif-type-checking", + ), + pytest.param( + """ + if TYPE_CHECKING: + if sys.version_info >= (3, 12): + import foo.two + else: + import foo.three + if TYPE_CHECKING: + import foo.four + import foo.five + """, + set(), + id="nested", + ), + ), +) +def test_exclude_type_checking_imports_only_excludes_type_checking_branches( + write_files, code, expected_imported_modules +): + files = {"foo/__init__.py": ""} + files.update({f"foo/{name}.py": "" for name in ("two", "three", "four", "five")}) + files["foo/one.py"] = _source(code) + write_files(files) + + graph = build_graph( + "foo", + include_external_packages=False, + exclude_type_checking_imports=True, + cache_dir=None, + ) + + imported_modules = { + direct_import.imported.name for direct_import in _direct_imports(graph, "foo.one") + } + assert imported_modules == expected_imported_modules + + +def test_t_string_syntax(write_files): + write_files( + { + "foo/__init__.py": "", + "foo/one.py": _source(""" + import foo.two + # t-strings are only valid in Python 3.14+ + hello = t"Hello" + """), + "foo/two.py": "", + } + ) + + graph = build_graph("foo", include_external_packages=False, cache_dir=None) + + assert _direct_imports(graph, "foo.one") == { + DirectImport( + importer=Module("foo.one"), + imported=Module("foo.two"), + line_number=1, + line_contents="import foo.two", + ), + } + + +@pytest.mark.parametrize( + "import_statement", + ( + "from namespace.portion.nonexistent import something", + "from .nonexistent import something", + "from ..nonexistent import something", + ), +) +def test_missing_module_in_dotted_root_package_is_ignored(write_files, import_statement: str): + """ + When a dotted root package (namespace package) contains an import of a missing module + within that same package, and include_external_packages=True, the Rust code should not + panic with an index-out-of-bounds error. + + Regression test for https://github.com/python-grimp/grimp/issues/308. + """ + write_files( + { + "namespace/portion/__init__.py": "", + "namespace/portion/subpackage/__init__.py": "", + "namespace/portion/subpackage/importer.py": import_statement, + } + ) + + graph = build_graph("namespace.portion", include_external_packages=True, cache_dir=None) + + assert _direct_imports(graph, "namespace.portion.subpackage.importer") == set() diff --git a/tests/unit/adaptors/test_caching.py b/tests/unit/adaptors/test_caching.py deleted file mode 100644 index 2b684b89..00000000 --- a/tests/unit/adaptors/test_caching.py +++ /dev/null @@ -1,738 +0,0 @@ -import json -import logging - -import pytest - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.adaptors.caching import Cache, CacheFileNamer -from grimp.application.ports.caching import CacheMiss -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module - - -class SimplisticFileNamer(CacheFileNamer): - """ - Simplistic version of the file namer that makes testing easier. - - Does not base64 encode the data filenames. - """ - - @classmethod - def make_data_file_name( - cls, - found_packages: set[FoundPackage], - include_external_packages: bool, - exclude_type_checking_imports: bool, - ) -> str: - unsafe_name = cls.make_data_file_unique_string( - found_packages, include_external_packages, exclude_type_checking_imports - ) - return f"{unsafe_name}.data.json" - - -class TestCacheFileNamer: - def test_names_meta_cache_per_package(self): - result = CacheFileNamer.make_meta_file_name( - FoundPackage( - name="some-package", - directory="whatever", - module_files=set(), - ) - ) - - assert result == "some-package.meta.json" - - @pytest.mark.parametrize( - "include_external_packages, exclude_type_checking_imports, expected", - ( - # Blake2B 20-character hash of "hyphenated-package,underscore_package". - (False, False, "a857d066514de048b7f94fa8d385e8bd7b048406.data.json"), - # Blake2B 20-character hash "hyphenated-package,underscore_package:external". - ( - True, - False, - "021977b6de56b09810ae52f5c9d067622c1ea30f.data.json", - ), - # Blake2B 20-character hash - # "hyphenated-package,underscore_package:external:no_type_checking". - ( - True, - True, - "4c2deb1d787161187915e159b5a17ea8b27cd0d4.data.json", - ), - # Blake2B 20-character hash "hyphenated-package,underscore_package:no_type_checking". - ( - False, - True, - "815e7686179e2f3f817c130eec1121d53e62ff1c.data.json", - ), - ), - ) - def test_make_data_file_name( - self, include_external_packages, exclude_type_checking_imports, expected - ): - result = CacheFileNamer.make_data_file_name( - found_packages={ - FoundPackage( - name="hyphenated-package", - directory="whatever", - module_files=frozenset(), - ), - FoundPackage( - name="underscore_package", - directory="whatever", - module_files=frozenset(), - ), - }, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - - assert result == expected - - @pytest.mark.parametrize( - "include_external_packages, exclude_type_checking_imports, expected", - ( - (False, False, "hyphenated-package,underscore_package"), - (True, False, "hyphenated-package,underscore_package:external"), - (True, True, "hyphenated-package,underscore_package:external:no_type_checking"), - (False, True, "hyphenated-package,underscore_package:no_type_checking"), - ), - ) - def test_make_data_file_unique_string( - self, include_external_packages, exclude_type_checking_imports, expected - ): - result = CacheFileNamer.make_data_file_unique_string( - found_packages={ - FoundPackage( - name="hyphenated-package", - directory="whatever", - module_files=frozenset(), - ), - FoundPackage( - name="underscore_package", - directory="whatever", - module_files=frozenset(), - ), - }, - include_external_packages=include_external_packages, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - - assert result == expected - - -class TestCache: - SOME_MTIME = 1676645081.4935088 - FILE_SYSTEM = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - new.py - unmodified.py - modified.py - /path/to/anotherpackage/ - __init__.py - new.py - unmodified.py - modified.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.unmodified": {SOME_MTIME}, - "mypackage.foo.modified": {SOME_MTIME} - }}""", - ".grimp_cache/anotherpackage.meta.json": f"""{{ - "anotherpackage.unmodified": {SOME_MTIME}, - "anotherpackage.modified": {SOME_MTIME} - }}""", - ".grimp_cache/mypackage.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], ["brown", false, 22, "import brown"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - ".grimp_cache/mypackage:external.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], - ["brown", false, 22, "import brown"], - ["external", false, 100, "import external"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - ".grimp_cache/anotherpackage,mypackage.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], ["brown", false, 22, "import brown"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ], - "anotherpackage.unmodified": [ - ["purple", false, 11, "import purple"], ["green", false, 22, "import green"] - ], - "anotherpackage.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - ".grimp_cache/anotherpackage,mypackage:external.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.unmodified": [ - ["yellow", false, 11, "import yellow"], - ["brown", false, 22, "import brown"], - ["external", false, 100, "import external"] - ], - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ], - "anotherpackage.unmodified": [ - ["purple", false, 11, "import purple"], - ["green", false, 22, "import green"], - ["anotherexternal", false, 100, "import anotherexternal"] - ], - "anotherpackage.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - }, - ) - MODULE_FILE_UNMODIFIED = ModuleFile( - module=Module("mypackage.foo.unmodified"), mtime=SOME_MTIME - ) - MODULE_FILE_MODIFIED = ModuleFile( - module=Module("mypackage.foo.modified"), mtime=SOME_MTIME + 100.0 - ) - MODULE_FILE_NEW = ModuleFile(module=Module("mypackage.foo.new"), mtime=SOME_MTIME) - - FOUND_PACKAGES = { - FoundPackage( - name="mypackage", - directory="/path/to/mypackage/", - module_files=frozenset( - {MODULE_FILE_MODIFIED, MODULE_FILE_UNMODIFIED, MODULE_FILE_NEW} - ), - ), - } - - @pytest.mark.parametrize( - "include_external_packages, expected_data_file", - ( - (True, ".grimp_cache/mypackage:external.data.json"), - (False, ".grimp_cache/mypackage.data.json"), - ), - ) - def test_logs_successful_cache_file_reading( - self, include_external_packages: bool, expected_data_file: str, caplog - ): - caplog.set_level(logging.INFO, logger=Cache.__module__) - - Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - include_external_packages=include_external_packages, - namer=SimplisticFileNamer, - ) - - assert caplog.messages == [ - "Used cache meta file .grimp_cache/mypackage.meta.json.", - f"Used cache data file {expected_data_file}.", - ] - - def test_logs_missing_cache_files(self, caplog): - caplog.set_level(logging.INFO, logger=Cache.__module__) - - Cache.setup( - file_system=rust.FakeBasicFileSystem(), # No cache files. - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - assert caplog.messages == [ - "No cache file: .grimp_cache/mypackage.meta.json.", - "No cache file: .grimp_cache/mypackage.data.json.", - ] - - @pytest.mark.parametrize("serialized_mtime", ("INVALID_JSON", '["wrong", "type"]')) - def test_logs_corrupt_cache_meta_file_reading(self, serialized_mtime: str, caplog): - caplog.set_level(logging.WARNING, logger=Cache.__module__) - - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {serialized_mtime}, - }}""", - ".grimp_cache/mypackage.data.json": "{}", - }, - ) - Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - assert caplog.messages == [ - "Could not use corrupt cache file .grimp_cache/mypackage.meta.json.", - ] - - def test_logs_corrupt_cache_data_file_reading(self, caplog): - caplog.set_level(logging.WARNING, logger=Cache.__module__) - - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {self.SOME_MTIME - 1} - }}""", - ".grimp_cache/mypackage.data.json": "INVALID JSON", - }, - ) - - Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - assert caplog.messages == [ - "Could not use corrupt cache file .grimp_cache/mypackage.data.json.", - ] - - @pytest.mark.parametrize("include_external_packages", (True, False)) - def test_raises_cache_miss_for_module_with_different_mtime( - self, include_external_packages: bool - ): - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - include_external_packages=include_external_packages, - namer=SimplisticFileNamer, - ) - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_MODIFIED) - - @pytest.mark.parametrize("include_external_packages", (True, False)) - def test_raises_cache_miss_for_module_with_no_cached_mtime( - self, include_external_packages: bool - ): - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=include_external_packages, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_NEW) - - @pytest.mark.parametrize( - "include_external_packages, expected_additional_imports", - ( - (False, set()), - ( - True, - { - DirectImport( - importer=MODULE_FILE_UNMODIFIED.module, - imported=Module("external"), - line_number=100, - line_contents="import external", - ), - }, - ), - ), - ) - def test_uses_cache_for_module_with_same_mtime( - self, include_external_packages, expected_additional_imports - ): - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=include_external_packages, - ) - - assert ( - cache.read_imports(self.MODULE_FILE_UNMODIFIED) - == { - DirectImport( - importer=self.MODULE_FILE_UNMODIFIED.module, - imported=Module("yellow"), - line_number=11, - line_contents="import yellow", - ), - DirectImport( - importer=self.MODULE_FILE_UNMODIFIED.module, - imported=Module("brown"), - line_number=22, - line_contents="import brown", - ), - } - | expected_additional_imports - ) - - def test_raises_cache_miss_for_missing_module_from_data(self): - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - somemodule.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.somemodule": {self.SOME_MTIME} - }}""", - ".grimp_cache/mypackage.data.json": """{}""", - }, - ) - module_file = ModuleFile(module=Module("mypackage.somemodule"), mtime=self.SOME_MTIME) - cache = Cache.setup( - file_system=file_system, - found_packages={ - FoundPackage( - name="mypackage", - directory="/path/to/mypackage/", - module_files=frozenset({module_file}), - ), - }, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(module_file) - - @pytest.mark.parametrize("serialized_mtime", ("INVALID_JSON", '["wrong", "type"]')) - def test_raises_cache_miss_for_corrupt_meta_file(self, serialized_mtime): - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - new.py - unmodified.py - modified.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {serialized_mtime} - }}""", - ".grimp_cache/mypackage.data.json": """{ - "version": 2, - "imports_by_module": { - "mypackage.foo.modified": [ - ["stale", false, 33, "We should not use this cached value."] - ] - } - }""", - }, - ) - cache = Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_MODIFIED) - - @pytest.mark.parametrize("serialized_import", ("INVALID_JSON", '["wrong", "type"]')) - def test_raises_cache_miss_for_corrupt_data_file(self, serialized_import): - file_system = rust.FakeBasicFileSystem( - contents=""" - .grimp_cache/ - mypackage.meta.json - mypackage.data.json - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - new.py - unmodified.py - modified.py - """, - content_map={ - ".grimp_cache/mypackage.meta.json": f"""{{ - "mypackage.foo.modified": {self.SOME_MTIME - 1} - }}""", - ".grimp_cache/mypackage.data.json": f"""{{ - "version": 2, - "imports_by_module": {{ - "mypackage.foo.modified": [ - {serialized_import} - ] - }} - }}""", - }, - ) - cache = Cache.setup( - file_system=file_system, - found_packages=self.FOUND_PACKAGES, - namer=SimplisticFileNamer, - include_external_packages=False, - ) - - with pytest.raises(CacheMiss): - cache.read_imports(self.MODULE_FILE_MODIFIED) - - @pytest.mark.parametrize( - "include_external_packages, expected_additional_imports", - ( - (False, set()), - ( - True, - { - DirectImport( - importer=Module("anotherpackage.unmodified"), - imported=Module("anotherexternal"), - line_number=100, - line_contents="import anotherexternal", - ), - }, - ), - ), - ) - def test_uses_cache_multiple_packages( - self, include_external_packages, expected_additional_imports - ): - module_file = ModuleFile(module=Module("anotherpackage.unmodified"), mtime=self.SOME_MTIME) - cache = Cache.setup( - file_system=self.FILE_SYSTEM, - found_packages=self.FOUND_PACKAGES - | { - FoundPackage( - name="anotherpackage", - directory="/path/to/anotherpackage/", - module_files=frozenset({module_file}), - ), - }, - namer=SimplisticFileNamer, - include_external_packages=include_external_packages, - ) - - result = cache.read_imports(module_file) - - assert ( - result - == { - DirectImport( - importer=module_file.module, - imported=Module("purple"), - line_number=11, - line_contents="import purple", - ), - DirectImport( - importer=module_file.module, - imported=Module("green"), - line_number=22, - line_contents="import green", - ), - } - | expected_additional_imports - ) - - @pytest.mark.parametrize( - "cache_dir", - ( - "/tmp/some-cache-dir", - "/tmp/some-cache-dir/", - None, - ), - ) - @pytest.mark.parametrize( - "include_external_packages, expected_data_file_name", - ( - (False, "blue,green.data.json"), - (True, "blue,green:external.data.json"), - ), - ) - def test_write_to_cache( - self, include_external_packages, expected_data_file_name, cache_dir, caplog - ): - caplog.set_level(logging.INFO, logger=Cache.__module__) - file_system = rust.FakeBasicFileSystem() - blue_one = Module(name="blue.one") - blue_two = Module(name="blue.two") - green_one = Module(name="green.one") - green_two = Module(name="green.two") - mtimes = { - blue_one: 10000.1, - blue_two: 20000.2, - green_one: 30000.3, - green_two: 40000.4, - } - cache = Cache.setup( - file_system=file_system, - cache_dir=cache_dir, - found_packages={ - FoundPackage( - name="blue", - module_files=frozenset( - { - ModuleFile(module=blue_one, mtime=mtimes[blue_one]), - ModuleFile(module=blue_two, mtime=mtimes[blue_two]), - } - ), - directory="-", - ), - FoundPackage( - name="green", - module_files=frozenset( - { - ModuleFile(module=green_one, mtime=mtimes[green_one]), - ModuleFile(module=green_two, mtime=mtimes[green_two]), - } - ), - directory="-", - ), - }, - include_external_packages=include_external_packages, - namer=SimplisticFileNamer, - ) - - cache.write( - imports_by_module={ - blue_one: { - DirectImport( - importer=blue_one, - imported=blue_two, - line_number=11, - line_contents="from . import two", - ), - DirectImport( - importer=blue_one, - imported=Module("externalpackage"), - line_number=22, - line_contents="import externalpackage", - ), - }, - blue_two: set(), - green_one: set(), - green_two: { - DirectImport( - importer=green_two, - imported=green_one, - line_number=33, - line_contents="from . import one", - ), - }, - }, - ) - - # Assert the cache is written afterwards. - expected_cache_dir = cache_dir.rstrip(file_system.sep) if cache_dir else ".grimp_cache" - assert set(caplog.messages) == { - f"No cache file: {expected_cache_dir}/{expected_data_file_name}.", - f"No cache file: {expected_cache_dir}/blue.meta.json.", - f"No cache file: {expected_cache_dir}/green.meta.json.", - f"Wrote data cache file {expected_cache_dir}/{expected_data_file_name}.", - f"Wrote meta cache file {expected_cache_dir}/blue.meta.json.", - f"Wrote meta cache file {expected_cache_dir}/green.meta.json.", - } - expected = { - f"{expected_cache_dir}/blue.meta.json": { - blue_one.name: mtimes[blue_one], - blue_two.name: mtimes[blue_two], - }, - f"{expected_cache_dir}/green.meta.json": { - green_one.name: mtimes[green_one], - green_two.name: mtimes[green_two], - }, - f"{expected_cache_dir}/{expected_data_file_name}": { - "version": 2, - "imports_by_module": { - blue_one.name: { - (blue_two.name, False, 11, "from . import two"), - ("externalpackage", False, 22, "import externalpackage"), - }, - blue_two.name: set(), - green_one.name: set(), - green_two.name: { - (green_one.name, False, 33, "from . import one"), - }, - }, - }, - } - for filename, expected_deserialized in expected.items(): - serialized = file_system.read(filename) - deserialized = _unflake(json.loads(serialized)) - assert deserialized == expected_deserialized - - def test_write_to_cache_adds_marker_files(self): - some_cache_dir = "/tmp/some-cache-dir" - file_system = rust.FakeBasicFileSystem() - cache = Cache.setup( - file_system=file_system, - cache_dir=some_cache_dir, - found_packages=set(), - include_external_packages=False, # Value shouldn't matter. - namer=SimplisticFileNamer, - ) - - cache.write( - imports_by_module={}, - ) - - assert file_system.read(f"{some_cache_dir}/.gitignore") == ( - "# Automatically created by Grimp.\n*" - ) - assert file_system.read(f"{some_cache_dir}/CACHEDIR.TAG") == ( - "Signature: 8a477f597d28d172789f06886806bc55\n" - "# This file is a cache directory tag automatically created by Grimp.\n" - "# For information about cache directory tags see https://bford.info/cachedir/" - ) - - -def _unflake(flakey_data): - # Takes the deserialized data that could come in any - # order and order it consistently, so the test isn't flakey. - non_flakey_data = {} - for key, value in flakey_data.items(): - if isinstance(value, list): - non_flakey_value = {tuple(i) for i in value} - elif isinstance(value, dict): - non_flakey_value = _unflake(value) - else: - non_flakey_value = value - non_flakey_data[key] = non_flakey_value - return non_flakey_data diff --git a/tests/unit/adaptors/test_filesystem.py b/tests/unit/adaptors/test_filesystem.py deleted file mode 100644 index 3526e55c..00000000 --- a/tests/unit/adaptors/test_filesystem.py +++ /dev/null @@ -1,278 +0,0 @@ -from __future__ import annotations - -from copy import copy -from typing import TypeAlias - -import pytest - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application.ports.filesystem import BasicFileSystem -from tests.adaptors.filesystem import FakeFileSystem - - -class _Base: - """ - Tests for methods that AbstractFileSystem and BasicFileSystem share. - """ - - file_system_cls: type[BasicFileSystem] - - @pytest.mark.parametrize("path", ("/path/to", "/path/to/")) - def test_join(self, path): - file_system = self.file_system_cls() - assert "/path/to/mypackage/file.py" == file_system.join(path, "mypackage", "file.py") - - @pytest.mark.parametrize( - "path, expected", - [ - ("", ("", "")), - ("/path", ("/", "path")), - ("some-file", ("", "some-file")), - ("/path/to/mypackage/", ("/path/to/mypackage", "")), - ("/path/to/mypackage/some-file", ("/path/to/mypackage", "some-file")), - ("/path/to/mypackage/some-file.py", ("/path/to/mypackage", "some-file.py")), - ], - ) - def test_split(self, path, expected): - file_system = self.file_system_cls() - assert file_system.split(path) == expected - - @pytest.mark.parametrize( - "file_name, expected", - [ - ("/path/to/mypackage", False), - ("/path/to/mypackage/", False), - ("/path/to/mypackage/readme.txt", True), - ("/path/to/mypackage/foo/one.txt", True), - ("/path/to/mypackage/foo/two/green.txt", True), - ("/path/to/mypackage/bar/blue.txt", True), - ("/path/to/nonexistent.txt", False), - ("/path/to/mypackage/purple.txt", False), - ], - ) - def test_exists_content_only(self, file_name, expected): - file_system = self.file_system_cls( - contents=""" - /path/to/mypackage/ - readme.txt - foo/ - one.txt - two/ - green.txt - bar/ - blue.txt - """ - ) - - assert file_system.exists(file_name) == expected - - @pytest.mark.parametrize( - "file_name, expected", - [ - ("/path/to/", False), - ("/path/to/file.txt", True), - ("/path/to/a/deeper/file.txt", True), - ("/path/to/nonexistent.txt", False), - ], - ) - def test_exists_content_map_only(self, file_name, expected): - file_system = self.file_system_cls( - content_map={ - "/path/to/file.txt": "hello", - "/path/to/a/deeper/file.txt": "hello", - } - ) - - assert file_system.exists(file_name) == expected - - @pytest.mark.parametrize( - "file_name, expected", - [ - ("/path/to/file.txt", True), - ("/path/to/mypackage/foo/two/green.txt", True), - ("/path/to/nonexistent.txt", False), - ], - ) - def test_exists_content_map_and_content(self, file_name, expected): - file_system = self.file_system_cls( - contents=""" - /path/to/mypackage/ - readme.txt - foo/ - one.txt - two/ - green.txt - """, - content_map={ - "/path/to/file.txt": "hello", - "/path/to/a/deeper/file.txt": "hello", - }, - ) - - assert file_system.exists(file_name) == expected - - @pytest.mark.parametrize( - "file_name, expected_contents", - ( - ("/path/to/mypackage/readme.txt", "Lorem"), - ("/path/to/mypackage/foo/one.txt", "Ipsum"), - # Listed in contents, but not in content_map. - ("/path/to/mypackage/foo/two/green.txt", ""), - # Listed in content_map, but not in contents. - ("/path/to/mypackage/foo/two/blue.txt", "Dolor sic"), - ( - "/path/to/mypackage/indented.txt", - "This is indented text.\n We should dedent it.", - ), - ("/path/to/mypackage/nonexistent", FileNotFoundError), - ("/path/to/mypackage/foo/three/nonexistent", FileNotFoundError), - # TODO - should we raise an exception if we attempt to read a directory? - ), - ) - def test_read(self, file_name, expected_contents): - file_system = self.file_system_cls( - contents=""" - /path/to/mypackage/ - readme.txt - foo/ - one.txt - two/ - green.txt - """, - content_map={ - "/path/to/mypackage/readme.txt": "Lorem", - "/path/to/mypackage/foo/one.txt": "Ipsum", - "/path/to/mypackage/foo/two/blue.txt": "Dolor sic", - "/path/to/mypackage/indented.txt": """ - This is indented text. - We should dedent it. - """, - }, - ) - if isinstance(expected_contents, type) and issubclass(expected_contents, Exception): - with pytest.raises(expected_contents): - file_system.read(file_name) - else: - assert file_system.read(file_name) == expected_contents - - def test_write(self): - some_filename, some_contents = "path/to/some-file.txt", "Some contents." - file_system = self.file_system_cls() - - file_system.write(some_filename, some_contents) - - assert file_system.read(some_filename) == some_contents - - -WalkReturn: TypeAlias = tuple[str, list[str], list[str]] - - -class TestFakeFileSystem(_Base): - file_system_cls = FakeFileSystem - - MYPACKAGE: WalkReturn = ("/path/to/mypackage", ["foo"], ["__init__.py"]) - MYPACKAGE_FOO: WalkReturn = ("/path/to/mypackage/foo", ["two"], ["__init__.py", "one.py"]) - MYPACKAGE_FOO_TWO: WalkReturn = ( - "/path/to/mypackage/foo/two", - [], - ["__init__.py", "green.py", "blue.py"], - ) - - @pytest.mark.parametrize( - "directory, expected", - [ - ( - "/path/to/mypackage", - [MYPACKAGE, MYPACKAGE_FOO, MYPACKAGE_FOO_TWO], - ), - ( - "/path/to/mypackage/foo", - [MYPACKAGE_FOO, MYPACKAGE_FOO_TWO], - ), - ( - "/path/to/mypackage/foo/two", - [MYPACKAGE_FOO_TWO], - ), - ( - "/anotherpackage", - [("/anotherpackage", [], ["another.txt"])], - ), - ], - ) - def test_walk(self, directory: str, expected: list[WalkReturn]): - file_system = self.file_system_cls( - """ - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - /anotherpackage/ - another.txt - """ - ) - - result = list(file_system.walk(directory)) - - assert result == expected - - def test_empty_if_directory_does_not_exist(self): - file_system = self.file_system_cls( - """ - /path/to/mypackage/ - __init__.py - """ - ) - assert [] == list(file_system.walk("/path/to/nonexistent/package")) - - def test_dirname(self): - file_system = self.file_system_cls() - assert "/path/to" == file_system.dirname("/path/to/file.txt") - - def test_dirnames_can_be_modified_in_place(self): - """ - From the os.walk docs: - The caller can modify the dirnames list in-place (perhaps using del or slice - assignment), and walk() will only recurse into the subdirectories whose names - remain in dirnames; this can be used to prune the search, impose a specific order - of visiting, or even to inform walk() about directories the caller creates or renames - before it resumes walk() again. - """ - file_system = self.file_system_cls( - """ - /path/to/mypackage/ - foo/ - one.txt - skipme/ - two.txt - dontskip/ - three.txt - bar/ - four.txt - """ - ) - - expected_tuples = [ - ("/path/to/mypackage", ["foo", "bar"], []), - ("/path/to/mypackage/foo", ["skipme", "dontskip"], ["one.txt"]), - ("/path/to/mypackage/foo/dontskip", [], ["three.txt"]), - ("/path/to/mypackage/bar", [], ["four.txt"]), - ] - - actual_tuples = [] - for dirpath, dirs, files in file_system.walk("/path/to/mypackage"): - # Ensure we make a copy of dirs (since we change it). - actual_tuples.append((dirpath, copy(dirs), files)) - if "skipme" in dirs: - dirs.remove("skipme") - continue - - assert expected_tuples == actual_tuples - - -class TestFakeBasicFileSystem(_Base): - file_system_cls = rust.FakeBasicFileSystem diff --git a/tests/unit/adaptors/test_modulefinder.py b/tests/unit/adaptors/test_modulefinder.py deleted file mode 100644 index ad5a84a6..00000000 --- a/tests/unit/adaptors/test_modulefinder.py +++ /dev/null @@ -1,317 +0,0 @@ -import pytest - -from grimp.adaptors.modulefinder import ModuleFinder -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import Module -from tests.adaptors.filesystem import DEFAULT_MTIME, FakeFileSystem - - -def test_happy_path(): - module_finder = ModuleFinder() - SOME_MTIME = 12340000.3 - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - not-a-python-file.txt - .hidden - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - """, - mtime_map={ - "/path/to/mypackage/foo/one.py": SOME_MTIME, - }, - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files=frozenset( - { - ModuleFile(module=Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo.one"), mtime=SOME_MTIME), - ModuleFile(module=Module("mypackage.foo.two"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo.two.green"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo.two.blue"), mtime=DEFAULT_MTIME), - } - ), - ) - - -MODULE_FILES_FOO_BLUE = { - ModuleFile(module=Module("somenamespace.foo.blue"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.blue.one"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.blue.two"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.blue.two.alpha"), mtime=DEFAULT_MTIME), -} -MODULE_FILES_FOO_GREEN_FIVE = { - ModuleFile(module=Module("somenamespace.foo.green.five"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("somenamespace.foo.green.five.beta"), mtime=DEFAULT_MTIME), -} - - -@pytest.mark.parametrize( - "package_name, package_directory, expected", - [ - ( - "somenamespace", - "/path/to/somenamespace", - FoundPackage( - name="somenamespace", - directory="/path/to/somenamespace", - module_files=MODULE_FILES_FOO_BLUE | MODULE_FILES_FOO_GREEN_FIVE, - namespace_packages=frozenset( - { - "somenamespace", - "somenamespace.foo", - "somenamespace.foo.green", - } - ), - ), - ), - ( - "somenamespace.foo", - "/path/to/somenamespace/foo", - FoundPackage( - name="somenamespace.foo", - directory="/path/to/somenamespace/foo", - module_files=MODULE_FILES_FOO_BLUE | MODULE_FILES_FOO_GREEN_FIVE, - namespace_packages=frozenset( - { - "somenamespace.foo", - "somenamespace.foo.green", - } - ), - ), - ), - ( - "somenamespace.foo.blue", - "/path/to/somenamespace/foo/blue", - FoundPackage( - name="somenamespace.foo.blue", - directory="/path/to/somenamespace/foo/blue", - module_files=MODULE_FILES_FOO_BLUE, - ), - ), - ], -) -def test_namespaced_packages(package_name: str, package_directory: str, expected: FoundPackage): - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/somenamespace/ - foo/ - blue/ - __init__.py - one.py - two/ - __init__.py - alpha.py - noinitpackage/ - three.py - orphan/ - __init__.py - four.py - green/ - five/ - __init__.py - beta.py - non_python_directory/ - six/ - README.txt - """ - ) - - result = module_finder.find_package( - package_name=package_name, - package_directory=package_directory, - file_system=file_system, - ) - - assert result == expected - - -def test_ignores_orphaned_python_files(): - # Python files in directories that don't contain an __init__.py should not be discovered. - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - two/ - __init__.py - green.py - noinitpackage/ - green.py - orphan/ - __init__.py - red.py - """ - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files={ - ModuleFile(module=Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.two"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.two.green"), mtime=DEFAULT_MTIME), - }, - ) - - -def test_ignores_invalid_identifier_directories(): - # Python files in directories that don't contain an __init__.py should not be discovered. - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/namespacepackage/ - foo/ - valid_underscore_name/ - __init__.py - mod.py - invalid-hyphenated-name/ - __init__.py - mod.py - 1nvalid-number-prefixed/ - __init__.py - mod.py - valid_non_àscii/ - __init__.py - mod.py - invalid_non_😃scii/ - __init__.py - mod.py - bar/ - __init__.py - mod.py - """ - ) - - result = module_finder.find_package( - package_name="namespacepackage", - package_directory="/path/to/namespacepackage", - file_system=file_system, - ) - - module_files = { - ModuleFile(module=Module(name), mtime=DEFAULT_MTIME) - for name in ( - "namespacepackage.foo.valid_underscore_name", - "namespacepackage.foo.valid_underscore_name.mod", - "namespacepackage.foo.valid_non_àscii", - "namespacepackage.foo.valid_non_àscii.mod", - "namespacepackage.bar", - "namespacepackage.bar.mod", - ) - } - assert result == FoundPackage( - name="namespacepackage", - directory="/path/to/namespacepackage", - module_files=module_files, - namespace_packages=frozenset({"namespacepackage", "namespacepackage.foo"}), - ) - - -@pytest.mark.parametrize( - "extension, should_warn", - ( - ("py", True), - ("txt", False), # Anything other than .py. - ), -) -def test_ignores_dotted_python_files(extension, should_warn, caplog): - # Python files containing dots (other than the one before .py) should not be discovered. - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=f""" - /path/to/mypackage/ - __init__.py - foo.py - bar/ - __init__.py - baz.dotted.{extension} - """ - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files={ - ModuleFile(module=Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.foo"), mtime=DEFAULT_MTIME), - ModuleFile(module=Module("mypackage.bar"), mtime=DEFAULT_MTIME), - }, - ) - if should_warn: - assert caplog.messages == [ - ( - "Warning: skipping module with too many dots in the name: " - f"/path/to/mypackage/bar/baz.dotted.{extension}" - ) - ] - else: - assert caplog.messages == [] - - -def test_ignores_hidden_directories(): - module_finder = ModuleFinder() - - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - two/ - __init__.py - green.py - .hidden/ - green.py - orphan/ - __init__.py - red.py - """ - ) - - result = module_finder.find_package( - package_name="mypackage", - package_directory="/path/to/mypackage", - file_system=file_system, - ) - - assert result == FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files={ - ModuleFile(Module("mypackage"), mtime=DEFAULT_MTIME), - ModuleFile(Module("mypackage.two"), mtime=DEFAULT_MTIME), - ModuleFile(Module("mypackage.two.green"), mtime=DEFAULT_MTIME), - }, - ) diff --git a/tests/unit/adaptors/test_packagefinder.py b/tests/unit/adaptors/test_packagefinder.py index 2f687a29..ce1b150b 100644 --- a/tests/unit/adaptors/test_packagefinder.py +++ b/tests/unit/adaptors/test_packagefinder.py @@ -3,9 +3,7 @@ import pytest from grimp import exceptions -from grimp.adaptors.filesystem import FileSystem from grimp.adaptors.packagefinder import ImportLibPackageFinder -from tests.adaptors.filesystem import FakeFileSystem assets = (Path(__file__).parent.parent.parent / "assets").resolve() @@ -44,9 +42,7 @@ ), ) def test_determine_package_directories(package, expected): - assert ( - ImportLibPackageFinder().determine_package_directories(package, FileSystem()) == expected - ) + assert ImportLibPackageFinder().determine_package_directories(package) == expected @pytest.mark.parametrize( @@ -63,4 +59,4 @@ def test_determine_package_directories_doesnt_support_non_top_level_modules(pack with pytest.raises( exceptions.NotATopLevelModule, ): - ImportLibPackageFinder().determine_package_directories(package, FakeFileSystem()) + ImportLibPackageFinder().determine_package_directories(package) diff --git a/tests/unit/application/test_scanning.py b/tests/unit/application/test_scanning.py deleted file mode 100644 index 7f209442..00000000 --- a/tests/unit/application/test_scanning.py +++ /dev/null @@ -1,1254 +0,0 @@ -from __future__ import annotations - -from collections.abc import Set - -import pytest - -from grimp import _rustgrimp as rust # type: ignore[attr-defined] -from grimp.application import scanning -from grimp.application.ports.modulefinder import FoundPackage, ModuleFile -from grimp.domain.valueobjects import DirectImport, Module -from tests.config import override_settings - - -@pytest.mark.parametrize( - "include_external_packages, expected_result", - ( - ( - False, - { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ) - }, - ), - ( - True, - { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("externalone"), - line_number=2, - line_contents="import externalone", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("externaltwo"), - line_number=3, - line_contents="import externaltwo.subpackage # with comment afterwards.", - ), - }, - ), - ), -) -def test_absolute_imports(include_external_packages, expected_result): - module_file_to_scan = _module_to_module_file(Module("foo.one")) - all_module_files = frozenset({module_file_to_scan, _module_to_module_file(Module("foo.two"))}) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": """ - import foo.two - import externalone - import externaltwo.subpackage # with comment afterwards. - arbitrary_expression = 1 - """ - } - ) - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages={ - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=all_module_files, - ) - }, - include_external_packages=include_external_packages, - exclude_type_checking_imports=False, - ) - - assert {module_file_to_scan: expected_result} == result - - -def test_lazy_imports(): - module_file_to_scan = _module_to_module_file(Module("foo.one.blue")) - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.blue"), - Module("foo.one.green"), - Module("foo.two"), - Module("foo.two.brown"), - Module("foo.two.yellow"), - Module("foo.three"), - } - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - blue.py - green.py - two/ - __init__.py - brown.py - yellow.py - three.py - """, - content_map={ - "/path/to/foo/one/blue.py": """ - # Absolute no-from - lazy import foo.two - lazy import externalone - lazy import externaltwo.subpackage # with comment afterwards. - arbitrary_expression = 1 - - # Absolute from - lazy from foo.one import green - lazy from foo.two import yellow - if t.TYPE_CHECKING: - lazy from foo import three - lazy from external import one - lazy from external.two import blue # with comment afterwards. - - # Relative from - lazy from . import green - lazy from ..two import yellow - lazy from .. import three - """ - }, - ) - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages={ - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - }, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - expected = { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two"), - line_number=2, - line_contents="lazy import foo.two", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("externalone"), - line_number=3, - line_contents="lazy import externalone", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("externaltwo"), - line_number=4, - line_contents="lazy import externaltwo.subpackage # with comment afterwards.", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=8, - line_contents="lazy from foo.one import green", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=9, - line_contents="lazy from foo.two import yellow", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=11, - line_contents="lazy from foo import three", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=12, - line_contents="lazy from external import one", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=13, - line_contents="lazy from external.two import blue # with comment afterwards.", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=16, - line_contents="lazy from . import green", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=17, - line_contents="lazy from ..two import yellow", - is_lazy=True, - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=18, - line_contents="lazy from .. import three", - is_lazy=True, - ), - } - } - assert result == expected - - -def test_non_ascii(): - blue_module = Module("mypackage.blue") - blue_module_file = _module_to_module_file(blue_module) - non_ascii_modules = { - Module("mypackage.ñon_ascii_变"), - Module("mypackage.ñon_ascii_变.ラーメン"), - } - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/mypackage/blue.py": """ - from ñon_ascii_变 import * - from . import ñon_ascii_变 - import mypackage.ñon_ascii_变.ラーメン - """, - "/path/to/mypackage/ñon_ascii_变/__init__.py": "", - "/path/to/mypackage/ñon_ascii_变/ラーメン.py": "", - }, - ) - found_packages = { - FoundPackage( - name="mypackage", - directory="/path/to/mypackage", - module_files=frozenset({blue_module_file}) - | _modules_to_module_files(non_ascii_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {blue_module_file}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - assert result == { - blue_module_file: { - DirectImport( - importer=Module("mypackage.blue"), - imported=Module("ñon_ascii_变"), - line_number=1, - line_contents="from ñon_ascii_变 import *", - ), - DirectImport( - importer=Module("mypackage.blue"), - imported=Module("mypackage.ñon_ascii_变"), - line_number=2, - line_contents="from . import ñon_ascii_变", - ), - DirectImport( - importer=Module("mypackage.blue"), - imported=Module("mypackage.ñon_ascii_变.ラーメン"), - line_number=3, - line_contents="import mypackage.ñon_ascii_变.ラーメン", - ), - } - } - - -def test_single_namespace_package_portion(): - MODULE_FOO = Module("namespace.foo") - MODULE_ONE = Module("namespace.foo.one") - MODULE_TWO = Module("namespace.foo.two") - MODULE_THREE = Module("namespace.foo.three") - MODULE_FOUR = Module("namespace.foo.four") - MODULE_GREEN = Module("namespace.foo.two.green") - MODULE_BLUE = Module("namespace.foo.two.blue") - MODULE_ALPHA = Module("namespace.foo.two.green.alpha") - - all_modules = { - MODULE_FOO, - MODULE_ONE, - MODULE_TWO, - MODULE_THREE, - MODULE_FOUR, - MODULE_GREEN, - MODULE_BLUE, - MODULE_ALPHA, - } - module_one_file = _module_to_module_file(MODULE_ONE) - module_alpha_file = _module_to_module_file(MODULE_ALPHA) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/namespace/foo/one.py": """ - import namespace.foo.two - from namespace.foo import three - from . import four - """, - "/path/to/namespace/foo/two/green/alpha.py": """ - from .. import blue - from ... import three - """, - } - ) - found_packages = { - FoundPackage( - name="namespace.foo", - directory="/path/to/namespace/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_one_file, module_alpha_file}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - assert result == { - module_one_file: { - DirectImport( - importer=MODULE_ONE, - imported=MODULE_TWO, - line_number=1, - line_contents="import namespace.foo.two", - ), - DirectImport( - importer=MODULE_ONE, - imported=MODULE_THREE, - line_number=2, - line_contents="from namespace.foo import three", - ), - DirectImport( - importer=MODULE_ONE, - imported=MODULE_FOUR, - line_number=3, - line_contents="from . import four", - ), - }, - module_alpha_file: { - DirectImport( - importer=MODULE_ALPHA, - imported=MODULE_BLUE, - line_number=1, - line_contents="from .. import blue", - ), - DirectImport( - importer=MODULE_ALPHA, - imported=MODULE_THREE, - line_number=2, - line_contents="from ... import three", - ), - }, - } - - -@pytest.mark.parametrize("include_external_packages", (True, False)) -def test_import_of_portion_not_in_graph(include_external_packages): - # Include two namespaces, one deeper than the other. - MODULE_BAR_ONE_GREEN = Module("namespace.bar.one.green") - MODULE_BAR_ONE_GREEN_ALPHA = Module("namespace.bar.one.green.alpha") - - MODULE_FOO = Module("namespace.foo") - MODULE_FOO_ONE = Module("namespace.foo.one") - MODULE_FOO_BLUE = Module("namespace.foo.one.blue") - module_foo_one_file = _module_to_module_file(MODULE_FOO_ONE) - module_foo_blue_file = _module_to_module_file(MODULE_FOO_BLUE) - all_modules = { - MODULE_FOO, - MODULE_FOO_ONE, - MODULE_FOO_BLUE, - } - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/namespace/foo/one.py": """ - import namespace.bar.one.orange - from namespace.yellow import one - from .. import mauve - if t.TYPE_CHECKING: - from ..cyan import one - from ..magenta.one import alpha - """, - "/path/to/namespace/foo/one/blue.py": """ - from ... import teal - from ...pink import one - from ...scarlet.one import alpha - """, - } - ) - - found_packages = { - FoundPackage( - name="namespace.foo", - directory="/path/to/namespace/foo", - module_files=_modules_to_module_files(all_modules), - ), - FoundPackage( - name="namespace.bar.one.green", - directory="/path/to/namespace/bar/one/green", - module_files=_modules_to_module_files( - { - MODULE_BAR_ONE_GREEN, - MODULE_BAR_ONE_GREEN_ALPHA, - } - ), - ), - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_file, module_foo_blue_file}, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=False, - ) - - if include_external_packages: - assert result == { - module_foo_one_file: { - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.bar.one.orange"), - line_number=1, - line_contents="import namespace.bar.one.orange", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.yellow"), - line_number=2, - line_contents="from namespace.yellow import one", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.mauve"), - line_number=3, - line_contents="from .. import mauve", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.cyan"), - line_number=5, - line_contents="from ..cyan import one", - ), - DirectImport( - importer=MODULE_FOO_ONE, - imported=Module("namespace.magenta"), - line_number=6, - line_contents="from ..magenta.one import alpha", - ), - }, - module_foo_blue_file: { - DirectImport( - importer=MODULE_FOO_BLUE, - imported=Module("namespace.teal"), - line_number=1, - line_contents="from ... import teal", - ), - DirectImport( - importer=MODULE_FOO_BLUE, - imported=Module("namespace.pink"), - line_number=2, - line_contents="from ...pink import one", - ), - DirectImport( - importer=MODULE_FOO_BLUE, - imported=Module("namespace.scarlet"), - line_number=3, - line_contents="from ...scarlet.one import alpha", - ), - }, - } - else: - assert result == { - module_foo_one_file: set(), - module_foo_blue_file: set(), - } - - -@pytest.mark.parametrize( - "include_external_packages, expected_result", - ( - ( - False, - { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=1, - line_contents="from foo.one import green", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=2, - line_contents="from foo.two import yellow", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=4, - line_contents="from foo import three", - ), - }, - ), - ( - True, - { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.green"), - line_number=1, - line_contents="from foo.one import green", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.two.yellow"), - line_number=2, - line_contents="from foo.two import yellow", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.three"), - line_number=4, - line_contents="from foo import three", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=5, - line_contents="from external import one", - ), - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("external"), - line_number=6, - line_contents="from external.two import blue # with comment afterwards.", - ), - }, - ), - ), -) -def test_absolute_from_imports(include_external_packages, expected_result): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.blue"), - Module("foo.one.green"), - Module("foo.two"), - Module("foo.two.brown"), - Module("foo.two.yellow"), - Module("foo.three"), - } - module_foo_one_blue_file = _module_to_module_file(Module("foo.one.blue")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - blue.py - green.py - two/ - __init__.py - brown.py - yellow.py - three.py - """, - content_map={ - "/path/to/foo/one/blue.py": """ - from foo.one import green - from foo.two import yellow - if t.TYPE_CHECKING: - from foo import three - from external import one - from external.two import blue # with comment afterwards. - arbitrary_expression = 1 - """ - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_blue_file}, - found_packages=found_packages, - include_external_packages=include_external_packages, - exclude_type_checking_imports=False, - ) - - assert {module_foo_one_blue_file: expected_result} == result - - -@pytest.mark.parametrize("module_to_scan_is_package", (True, False)) -def test_relative_from_imports(module_to_scan_is_package): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.blue"), - Module("foo.one.green"), - Module("foo.two.brown"), - Module("foo.two.yellow"), - Module("foo.three"), - } - if module_to_scan_is_package: - module_to_scan = Module("foo.one") - module_filename = "/path/to/foo/one/__init__.py" - else: - module_to_scan = Module("foo.one.blue") - module_filename = "/path/to/foo/one/blue.py" - module_file_to_scan = _module_to_module_file(module_to_scan) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - blue.py - green.py - two/ - __init__.py - brown.py - yellow.py - three.py - """, - content_map={ - module_filename: """ - from . import green - from ..two import yellow - from .. import three - arbitrary_expression = 1 - """ - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=True, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=module_to_scan, - imported=Module("foo.one.green"), - line_number=1, - line_contents="from . import green", - ), - DirectImport( - importer=module_to_scan, - imported=Module("foo.two.yellow"), - line_number=2, - line_contents="from ..two import yellow", - ), - DirectImport( - importer=module_to_scan, - imported=Module("foo.three"), - line_number=3, - line_contents="from .. import three", - ), - } - } - - -@pytest.mark.parametrize( - "import_source", - ("from .two.yellow import my_function", "from foo.two.yellow import my_function"), -) -def test_trims_to_known_modules(import_source): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.two"), - Module("foo.two.yellow"), - } - module_file_to_scan = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one.py - two/ - __init__.py - yellow.py - """, - content_map={"/path/to/foo/one.py": import_source}, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two.yellow"), - line_number=1, - line_contents=import_source, - ) - } - } - - -def test_trims_to_known_modules_within_init_file(): - all_modules = { - Module("foo"), - Module("foo.one"), - Module("foo.one.yellow"), - Module("foo.one.blue"), - Module("foo.one.blue.alpha"), - } - module_file_foo_one = _module_to_module_file(Module("foo.one")) - module_file_foo_one_blue = _module_to_module_file(Module("foo.one.blue")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one/ - __init__.py - yellow.py - blue/ - __init__.py - alpha.py - """, - content_map={ - "/path/to/foo/one/__init__.py": "from .yellow import my_function", - "/path/to/foo/one/blue/__init__.py": "from .alpha import my_function", - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_foo_one, module_file_foo_one_blue}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_foo_one: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.one.yellow"), - line_number=1, - line_contents="from .yellow import my_function", - ) - }, - module_file_foo_one_blue: { - DirectImport( - importer=Module("foo.one.blue"), - imported=Module("foo.one.blue.alpha"), - line_number=1, - line_contents="from .alpha import my_function", - ) - }, - } - - -def test_trims_whitespace_from_start_of_line_contents(): - all_modules = {Module("foo"), Module("foo.one"), Module("foo.two")} - module_file_to_scan = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - contents=""" - /path/to/foo/ - __init__.py - one.py - two.py - """, - content_map={ - "/path/to/foo/one.py": """ - def my_function(): - from . import two - """ - }, - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=2, - line_contents="from . import two", - ), - } - } - - -@pytest.mark.parametrize( - "statement, expected_module_name", - ( - # External packages that share a namespace with an internal module resolve - # to the shallowest component that does not clash with an internal module namespace. - ("import namespace", None), - ("import namespace.foo", None), - ("from namespace import foo", None), - ("import namespace.bar", "namespace.bar"), - ("from namespace import bar", "namespace.bar"), - ("from ... import bar", "namespace.bar"), - ("import namespace.bar.orange", "namespace.bar"), - ("from namespace.bar import orange", "namespace.bar"), - ("from ...bar import orange", "namespace.bar"), - ("import namespace.foo.green", "namespace.foo.green"), - ("from namespace.foo import green", "namespace.foo.green"), - ("from .. import green", "namespace.foo.green"), - ("import namespace.foo.green.alpha", "namespace.foo.green"), - ("from namespace.foo.green import alpha", "namespace.foo.green"), - ("from ..green import alpha", "namespace.foo.green"), - ("import namespace.foo.green.alpha.one", "namespace.foo.green"), - ("from namespace.foo.green.alpha import one", "namespace.foo.green"), - ("from ..green.alpha import one", "namespace.foo.green"), - ), -) -def test_external_package_imports_for_namespace_packages(statement, expected_module_name): - module_to_scan = Module("namespace.foo.blue.alpha") - module_file_to_scan = _module_to_module_file(module_to_scan) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/namespace/foo/blue/alpha.py": statement, - } - ) - found_packages = { - FoundPackage( - name="namespace.foo.blue", - directory="/path/to/namespace/foo/blue", - module_files=_modules_to_module_files( - { - Module("namespace.foo.blue"), - module_to_scan, - Module("namespace.foo.blue.beta"), - } - ), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, - ) - - if expected_module_name: - assert { - module_file_to_scan: { - DirectImport( - importer=module_to_scan, - imported=Module(expected_module_name), - line_number=1, - line_contents=statement, - ), - } - } == result - else: - assert {module_file_to_scan: set()} == result - - -@pytest.mark.parametrize("statement", ("import bar.blue", "from bar import blue")) -def test_scans_multiple_packages(statement): - foo_modules = {Module("foo"), Module("foo.one"), Module("foo.two")} - bar_modules = {Module("bar"), Module("bar.green"), Module("bar.blue")} - foo_one_module_file = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": f""" - import foo.two - {statement} - import externalone - - arbitrary_expression = 1 - """ - } - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(foo_modules), - ), - FoundPackage( - name="bar", - directory="/path/to/bar", - module_files=_modules_to_module_files(bar_modules), - ), - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {foo_one_module_file}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert { - foo_one_module_file: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("bar.blue"), - line_number=2, - line_contents=statement, - ), - } - } == result - - -@pytest.mark.parametrize("exclude_type_checking_imports", (True, False)) -@pytest.mark.parametrize( - "statement, is_statement_valid", - ( - ("if t.TYPE_CHECKING:", True), - ("if TYPE_CHECKING:", True), - ("if typing.TYPE_CHECKING:", True), - ("if WEIRD_ALIAS.TYPE_CHECKING:", True), - ("if type_checking:", False), - ("while TYPE_CHECKING:", False), - ), -) -def test_exclude_type_checking_imports( - exclude_type_checking_imports, statement, is_statement_valid -): - all_modules = { - Module("foo.one"), - Module("foo.two"), - Module("foo.three"), - Module("foo.four"), - Module("foo.five"), - } - module_foo_one_file = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": f""" - import foo.two - {statement} - import foo.three - import foo.four - import foo.five - arbitrary_expression = 1 - """ - } - ) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_file}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=exclude_type_checking_imports, - ) - - if exclude_type_checking_imports and is_statement_valid: - expected_result = { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.five"), - line_number=5, - line_contents="import foo.five", - ), - } - else: - expected_result = { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.three"), - line_number=3, - line_contents="import foo.three", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.four"), - line_number=4, - line_contents="import foo.four", - ), - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.five"), - line_number=5, - line_contents="import foo.five", - ), - } - assert {module_foo_one_file: expected_result} == result - - -@pytest.mark.parametrize( - "code, expected_imported_modules", - ( - pytest.param( - """ - if TYPE_CHECKING: - import foo.two - else: - import foo.three - """, - {"foo.three"}, - id="else", - ), - pytest.param( - """ - if typing.TYPE_CHECKING: - import foo.two - elif sys.version_info >= (3, 12): - import foo.three - else: - import foo.four - """, - {"foo.three", "foo.four"}, - id="elif-and-else", - ), - pytest.param( - """ - if sys.version_info >= (3, 12): - import foo.two - elif TYPE_CHECKING: - import foo.three - else: - import foo.four - """, - {"foo.two", "foo.four"}, - id="elif-type-checking", - ), - pytest.param( - """ - if TYPE_CHECKING: - if sys.version_info >= (3, 12): - import foo.two - else: - import foo.three - if TYPE_CHECKING: - import foo.four - import foo.five - """, - set(), - id="nested", - ), - ), -) -def test_exclude_type_checking_imports_only_excludes_type_checking_branches( - code, expected_imported_modules -): - all_modules = {Module(f"foo.{name}") for name in ("one", "two", "three", "four", "five")} - module_foo_one_file = _module_to_module_file(Module("foo.one")) - file_system = rust.FakeBasicFileSystem(content_map={"/path/to/foo/one.py": code}) - found_packages = { - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_foo_one_file}, - found_packages=found_packages, - include_external_packages=False, - exclude_type_checking_imports=True, - ) - - imported_modules = { - direct_import.imported.name for direct_import in result[module_foo_one_file] - } - assert imported_modules == expected_imported_modules - - -def test_t_string_syntax(): - module_file_to_scan = _module_to_module_file(Module("foo.one")) - - all_module_files = frozenset({module_file_to_scan, _module_to_module_file(Module("foo.two"))}) - file_system = rust.FakeBasicFileSystem( - content_map={ - "/path/to/foo/one.py": """ - import foo.two - # t-strings are only valid in Python 3.14+ - hello = t"Hello" - """ - } - ) - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages={ - FoundPackage( - name="foo", - directory="/path/to/foo", - module_files=all_module_files, - ) - }, - include_external_packages=False, - exclude_type_checking_imports=False, - ) - - assert result == { - module_file_to_scan: { - DirectImport( - importer=Module("foo.one"), - imported=Module("foo.two"), - line_number=1, - line_contents="import foo.two", - ), - } - } - - -@pytest.mark.parametrize( - "import_statement", - ( - "from namespace.portion.nonexistent import something", - "from .nonexistent import something", - "from ..nonexistent import something", - ), -) -def test_missing_module_in_dotted_root_package_is_ignored(import_statement: str): - """ - When a dotted root package (namespace package) contains an import of a missing module - within that same package, and include_external_packages=True, the Rust code should not - panic with an index-out-of-bounds error. - - Regression test for https://github.com/python-grimp/grimp/issues/308. - """ - module_to_scan = Module("namespace.portion.subpackage.importer") - module_file_to_scan = _module_to_module_file(module_to_scan) - all_modules = { - Module("namespace.portion"), - Module("namespace.portion.subpackage"), - module_to_scan, - } - file_system = rust.FakeBasicFileSystem( - content_map={"/path/to/namespace/portion/subpackage/importer.py": import_statement}, - ) - found_packages = { - FoundPackage( - name="namespace.portion", - directory="/path/to/namespace/portion", - module_files=_modules_to_module_files(all_modules), - ) - } - - with override_settings(FILE_SYSTEM=file_system): - result = scanning.scan_imports( - {module_file_to_scan}, - found_packages=found_packages, - include_external_packages=True, - exclude_type_checking_imports=False, # Required field, not relevant. - ) - - assert result == {module_file_to_scan: set()} - - -def _module_to_module_file(module: Module) -> ModuleFile: - some_mtime = 100933.4 - return ModuleFile(module=module, mtime=some_mtime) - - -def _modules_to_module_files(modules: Set[Module]) -> frozenset[ModuleFile]: - return frozenset({_module_to_module_file(module) for module in modules}) diff --git a/tests/unit/application/test_usecases.py b/tests/unit/application/test_usecases.py deleted file mode 100644 index 7c3da83e..00000000 --- a/tests/unit/application/test_usecases.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import re -from unittest.mock import sentinel - -import pytest - -from grimp.application import usecases -from grimp.application.ports.caching import Cache -from grimp.application.ports.modulefinder import ModuleFile -from grimp.domain.valueobjects import DirectImport, Module -from tests.adaptors.filesystem import FakeFileSystem -from tests.adaptors.packagefinder import BaseFakePackageFinder -from tests.config import override_settings - - -class TestBuildGraph: - @pytest.mark.parametrize("include_external_packages", (True, False)) - def test_happy_path(self, include_external_packages): - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - """, - content_map={ - "/path/to/mypackage/foo/one.py": ( - "import mypackage.foo.two.green\nfrom .. import Something" - ), - "/path/to/mypackage/foo/two/green.py": "import mypackage.foo.two.blue\n" - "from external.subpackage import foobar\n" - "import decimal", - }, - ) - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = {"mypackage": {"/path/to/mypackage"}} - - with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()): - graph = usecases.build_graph( - "mypackage", include_external_packages=include_external_packages - ) - - expected_import_map = { - "mypackage": set(), - "mypackage.foo": set(), - "mypackage.foo.one": {"mypackage.foo.two.green", "mypackage"}, - "mypackage.foo.two": set(), - "mypackage.foo.two.green": {"mypackage.foo.two.blue"}, - "mypackage.foo.two.blue": set(), - } - if include_external_packages: - expected_import_map["decimal"] = set() - expected_import_map["external"] = set() - expected_import_map["mypackage.foo.two.green"] |= {"external", "decimal"} - - assert set(expected_import_map.keys()) == graph.modules - for importer, imported_modules in expected_import_map.items(): - assert graph.find_modules_directly_imported_by(importer) == imported_modules - - # Check that the external packages are squashed modules. - if include_external_packages: - for module in ("external", "decimal"): - with pytest.raises( - ValueError, match=re.escape("Cannot find children of a squashed module.") - ): - graph.find_children(module) - - def test_boolean_additional_package_raises_type_error(self): - """ - Tests that a useful error message if build_graph is called - with a boolean as the second argument. - - This is because earlier versions of build_graph took include_external_packages - as the second argument, and it's possible it might have been called - as a positional argument. - """ - with pytest.raises(TypeError, match=re.escape("Package names must be strings, got bool.")): - usecases.build_graph("mypackage", True) - - @pytest.mark.parametrize( - "supplied_cache_dir", ("/path/to/somewhere", None, sentinel.not_supplied) - ) - def test_build_graph_respects_cache_dir(self, supplied_cache_dir): - file_system = FakeFileSystem() - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = {"mypackage": {"/path/to/mypackage"}} - - SOME_DEFAULT_CACHE_DIR = ".some_default" - - class AssertingCache(Cache): - @classmethod - def cache_dir_or_default(cls, cache_dir: str | None) -> str: - return cache_dir or SOME_DEFAULT_CACHE_DIR - - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - - # Assertions. - - if supplied_cache_dir is None: - raise RuntimeError("Cache should not be instantiated if caching is disabled.") - - expected_cache_dir = ( - SOME_DEFAULT_CACHE_DIR - if supplied_cache_dir is sentinel.not_supplied - else supplied_cache_dir - ) - assert self.cache_dir == expected_cache_dir - - def read_imports(self, module_file: ModuleFile) -> set[DirectImport]: - return set() - - def write( - self, - imports_by_module: dict[Module, set[DirectImport]], - ) -> None: - pass - - with override_settings( - FILE_SYSTEM=file_system, - PACKAGE_FINDER=FakePackageFinder(), - CACHE_CLASS=AssertingCache, - ): - kwargs = dict( - include_external_packages=True, - ) - if supplied_cache_dir is not sentinel.not_supplied: - kwargs["cache_dir"] = supplied_cache_dir - usecases.build_graph("mypackage", **kwargs) - - def test_forgives_wrong_type_being_passed_to_include_external_packages(self): - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - __init__.py - foo/ - __init__.py - one.py - """, - content_map={ - "/path/to/mypackage/foo/one.py": ( - "import mypackage.foo.two.green\nfrom .. import Something" - ), - }, - ) - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = {"mypackage": {"/path/to/mypackage"}} - - with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()): - graph = usecases.build_graph( - "mypackage", - # Note: this should be a bool, but we want to tolerate it, - # as Import Linter currently has a bug where it will pass it as None. - include_external_packages=False, - ) - - expected_modules = { - "mypackage", - "mypackage.foo", - "mypackage.foo.one", - } - assert expected_modules == graph.modules - - def test_namespace_package_passed_as_root(self): - file_system = FakeFileSystem( - contents=""" - /path/to/mypackage/ - foo/ - __init__.py - one.py - two/ - __init__.py - green.py - blue.py - bar/ - __init__.py - three.py - /different-path/to/mypackage/ - foobar/ - __init__.py - four.py - """, - content_map={ - "/path/to/mypackage/foo/one.py": "import mypackage\nfrom . import two", - }, - ) - - class FakePackageFinder(BaseFakePackageFinder): - directory_map = { - "mypackage": { - "/path/to/mypackage", - "/different-path/to/mypackage", - }, - } - - with override_settings(FILE_SYSTEM=file_system, PACKAGE_FINDER=FakePackageFinder()): - graph = usecases.build_graph("mypackage") - - assert graph.modules == { - "mypackage", - "mypackage.foo", - "mypackage.foo.one", - "mypackage.foo.two", - "mypackage.foo.two.green", - "mypackage.foo.two.blue", - "mypackage.bar", - "mypackage.bar.three", - "mypackage.foobar", - "mypackage.foobar.four", - } - assert graph.find_modules_directly_imported_by("mypackage.foo.one") == { - "mypackage", - "mypackage.foo.two", - } diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py deleted file mode 100644 index dcd47608..00000000 --- a/tests/unit/conftest.py +++ /dev/null @@ -1,15 +0,0 @@ -import pytest - -from grimp.adaptors.modulefinder import ModuleFinder -from grimp.application.graph import ImportGraph -from tests.config import override_settings - - -@pytest.fixture(scope="module", autouse=True) -def configure_unit_tests(): - with override_settings( - IMPORT_GRAPH_CLASS=ImportGraph, - MODULE_FINDER=ModuleFinder(), - FILE_SYSTEM=None, - ): - yield