diff --git a/docs/node-scraper-external/README.md b/docs/node-scraper-external/README.md index f3d67f2e..98bccab4 100644 --- a/docs/node-scraper-external/README.md +++ b/docs/node-scraper-external/README.md @@ -1,7 +1,12 @@ # node-scraper external plugins (example) This directory lives at **`/docs/node-scraper-external`** in the `node-scraper` repo and contains -an example external plugin package you can install in editable mode. +an example external plugin package that demonstrates how to create plugins for node-scraper. + +## Overview + +External plugins are discovered by node-scraper via **Python entry points**. This allows plugins +to be distributed as separate packages and automatically discovered when installed. ## Installation @@ -12,44 +17,126 @@ cd ~/node-scraper source venv/bin/activate pip install -e ./docs/node-scraper-external ``` -You should see `ext-nodescraper-plugins` installed in editable mode. +This installs `ext-nodescraper-plugins` in editable mode and registers the plugin entry points. -## Verify the external package is importable +## Verify Plugin Discovery + +Check that node-scraper discovered the external plugin: ```bash -python - <<'PY' -import ext_nodescraper_plugins -print("ext_nodescraper_plugins loaded from:", ext_nodescraper_plugins.__file__) -PY +node-scraper run-plugins -h ``` -## Run external plugins +You should see `SamplePlugin` listed alongside built-in plugins. -Confirm the CLI sees your external plugin(s): +## Run the Example Plugin ```bash -node-scraper run-plugins -h node-scraper run-plugins SamplePlugin ``` -## Add your own plugins +## How It Works + +### Entry Points -Add new modules under the **`ext_nodescraper_plugins/`** package. Example layout: +Plugins are registered in `pyproject.toml` using entry points: + +```toml +[project.entry-points."nodescraper.plugins"] +SamplePlugin = "ext_nodescraper_plugins.sample.sample_plugin:SamplePlugin" +``` + +When you install the package, Python registers these entry points in the package metadata. +Node-scraper automatically discovers and loads plugins from the `nodescraper.plugins` entry point group. + +### Plugin Structure ``` /docs/node-scraper-external -├─ pyproject.toml -└─ ext_nodescraper_plugins/ - └─ sample/ +├─ pyproject.toml # Package metadata + entry points +└─ ext_nodescraper_plugins/ # Plugin package + └─ sample/ # Plugin module ├─ __init__.py - └─ sample_plugin.py + ├─ sample_plugin.py # Plugin class + ├─ sample_collector.py # Data collector + ├─ sample_analyzer.py # Data analyzer + └─ sample_data.py # Data model +``` + +## Creating Your Own External Plugins + +### Step 1: Create Package Structure + +```bash +mkdir my-plugin-package +cd my-plugin-package +mkdir -p ext_nodescraper_plugins/my_plugin ``` +### Step 2: Create pyproject.toml + +```toml +[project] +name = "my-plugin-package" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["amd-node-scraper"] + +[project.entry-points."nodescraper.plugins"] +MyPlugin = "ext_nodescraper_plugins.my_plugin:MyPlugin" + +[build-system] +requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" ``` -Re-install (editable mode picks up code changes automatically, but if you add new files you may -need to re-run): +### Step 3: Implement Your Plugin + +Create `ext_nodescraper_plugins/my_plugin/__init__.py`: + +```python +from nodescraper.base import InBandDataPlugin, InBandDataCollector +from pydantic import BaseModel + +class MyDataModel(BaseModel): + """Your data model""" + data: dict + +class MyCollector(InBandDataCollector[MyDataModel, None]): + """Your data collector""" + DATA_MODEL = MyDataModel + + def collect_data(self, args=None): + # Collection logic + return MyDataModel(data={}) + +class MyPlugin(InBandDataPlugin[MyDataModel, None, None]): + """Your plugin""" + DATA_MODEL = MyDataModel + COLLECTOR = MyCollector +``` + +### Step 4: Install and Test + ```bash pip install -e . +node-scraper run-plugins -h # Should show MyPlugin +node-scraper run-plugins MyPlugin ``` + +## Adding More Plugins to This Package + +To add additional plugins to this example package: + +1. **Create a new module** under `ext_nodescraper_plugins/` +2. **Register the entry point** in `pyproject.toml`: + ```toml + [project.entry-points."nodescraper.plugins"] + SamplePlugin = "ext_nodescraper_plugins.sample.sample_plugin:SamplePlugin" + AnotherPlugin = "ext_nodescraper_plugins.another:AnotherPlugin" + ``` +3. **Reinstall** to register the new entry point: + ```bash + pip install -e . --force-reinstall --no-deps + ``` diff --git a/docs/node-scraper-external/pyproject.toml b/docs/node-scraper-external/pyproject.toml index b07273ab..45eadcdc 100644 --- a/docs/node-scraper-external/pyproject.toml +++ b/docs/node-scraper-external/pyproject.toml @@ -4,6 +4,9 @@ version = "0.1.0" requires-python = ">=3.10" dependencies = ["node-scraper"] +[project.entry-points."nodescraper.plugins"] +SamplePlugin = "ext_nodescraper_plugins.sample.sample_plugin:SamplePlugin" + [build-system] requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" diff --git a/nodescraper/cli/cli.py b/nodescraper/cli/cli.py index 81c18f19..94a1f0fb 100644 --- a/nodescraper/cli/cli.py +++ b/nodescraper/cli/cli.py @@ -54,13 +54,6 @@ from nodescraper.pluginexecutor import PluginExecutor from nodescraper.pluginregistry import PluginRegistry -try: - import ext_nodescraper_plugins as ext_pkg - - extra_pkgs = [ext_pkg] -except ImportError: - extra_pkgs = [] - def build_parser( plugin_reg: PluginRegistry, @@ -376,7 +369,7 @@ def main(arg_input: Optional[list[str]] = None): if arg_input is None: arg_input = sys.argv[1:] - plugin_reg = PluginRegistry(plugin_pkg=extra_pkgs) + plugin_reg = PluginRegistry() config_reg = ConfigRegistry() parser, plugin_subparser_map = build_parser(plugin_reg, config_reg) diff --git a/nodescraper/pluginregistry.py b/nodescraper/pluginregistry.py index 7d3e24dc..6822d329 100644 --- a/nodescraper/pluginregistry.py +++ b/nodescraper/pluginregistry.py @@ -24,6 +24,7 @@ # ############################################################################### import importlib +import importlib.metadata import inspect import pkgutil import types @@ -45,12 +46,14 @@ def __init__( self, plugin_pkg: Optional[list[types.ModuleType]] = None, load_internal_plugins: bool = True, + load_entry_point_plugins: bool = True, ) -> None: """Initialize the PluginRegistry with optional plugin packages. Args: plugin_pkg (Optional[list[types.ModuleType]], optional): The module to search for plugins in. Defaults to None. load_internal_plugins (bool, optional): Whether internal plugin should be loaded. Defaults to True. + load_entry_point_plugins (bool, optional): Whether to load plugins from entry points. Defaults to True. """ if load_internal_plugins: self.plugin_pkg = [internal_plugins, internal_connections, internal_collators] @@ -70,6 +73,10 @@ def __init__( PluginResultCollator, self.plugin_pkg ) + if load_entry_point_plugins: + entry_point_plugins = self.load_plugins_from_entry_points() + self.plugins.update(entry_point_plugins) + @staticmethod def load_plugins( base_class: type, @@ -104,3 +111,42 @@ def _recurse_pkg(pkg: types.ModuleType, base_class: type) -> None: for pkg in search_modules: _recurse_pkg(pkg, base_class) return registry + + @staticmethod + def load_plugins_from_entry_points() -> dict[str, type]: + """Load plugins registered via entry points. + + Returns: + dict[str, type]: A dictionary mapping plugin names to their classes. + """ + plugins = {} + + try: + # Python 3.10+ supports group parameter + try: + eps = importlib.metadata.entry_points(group="nodescraper.plugins") # type: ignore[call-arg] + except TypeError: + # Python 3.9 - entry_points() returns dict-like object + all_eps = importlib.metadata.entry_points() # type: ignore[assignment] + eps = all_eps.get("nodescraper.plugins", []) # type: ignore[assignment, attr-defined] + + for entry_point in eps: + try: + plugin_class = entry_point.load() # type: ignore[attr-defined] + + if ( + inspect.isclass(plugin_class) + and issubclass(plugin_class, PluginInterface) + and not inspect.isabstract(plugin_class) + ): + if hasattr(plugin_class, "is_valid") and not plugin_class.is_valid(): + continue + + plugins[plugin_class.__name__] = plugin_class + except Exception: + pass + + except Exception: + pass + + return plugins diff --git a/pyproject.toml b/pyproject.toml index 358d9cf2..831d83ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,9 +12,10 @@ classifiers = ["Topic :: Software Development"] dependencies = [ "pydantic>=2.8.2", - "paramiko~=3.5.1", + "paramiko>=3.2.0,<4.0.0", "requests", - "pytz" + "pytz", + "urllib3>=1.26.15,<2.0.0" ] [project.optional-dependencies]