Skip to content

Commit 30c61a9

Browse files
chr-hertelchr-hertel
committed
Adopt index command
1 parent 5d91b31 commit 30c61a9

File tree

6 files changed

+80
-9
lines changed

6 files changed

+80
-9
lines changed

‎demo/config/packages/ai.yaml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,9 @@ ai:
9191
model: 'text-embedding-ada-002'
9292
indexer:
9393
blog:
94-
loader: 'Symfony\AI\Store\Document\Loader\RssFeedLoader'
95-
source: 'https://feeds.feedburner.com/symfony/blog'
94+
input:
95+
loader: 'Symfony\AI\Store\Document\Loader\RssFeedLoader'
96+
source: 'https://feeds.feedburner.com/symfony/blog'
9697
filters:
9798
- 'app.filter.week_of_symfony'
9899
transformers:

‎demo/config/reference.php‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,11 @@
504504
* model?: mixed,
505505
* }>,
506506
* indexer?: array<string, array{ // Default: []
507-
* loader: string, // Service name of loader
507+
* loaders?: array<mixed>,
508+
* input?: array{
509+
* loader?: string, // Reference to loader for source.
510+
* source?: string, // Source identifier (file path, URL, etc.)
511+
* },
508512
* source?: mixed, // Source identifier (file path, URL, etc.) or array of sources // Default: null
509513
* transformers?: list<scalar|null>,
510514
* filters?: list<scalar|null>,

‎src/ai-bundle/config/options.php‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1187,9 +1187,20 @@
11871187
->useAttributeAsKey('name')
11881188
->arrayPrototype()
11891189
->children()
1190-
->stringNode('loader')
1191-
->info('Service name of loader')
1192-
->isRequired()
1190+
->arrayNode('loaders')
1191+
->info('Service names of loader')
1192+
->end()
1193+
->arrayNode('input')
1194+
->children()
1195+
->stringNode('loader')
1196+
->info('Reference to loader for source.')
1197+
->cannotBeEmpty()
1198+
->end()
1199+
->stringNode('source')
1200+
->info('Source identifier (file path, URL, etc.)')
1201+
->cannotBeEmpty()
1202+
->end()
1203+
->end()
11931204
->end()
11941205
->variableNode('source')
11951206
->info('Source identifier (file path, URL, etc.) or array of sources')

‎src/ai-bundle/config/services.php‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@
7171
use Symfony\AI\Store\Command\IndexCommand;
7272
use Symfony\AI\Store\Command\RetrieveCommand;
7373
use Symfony\AI\Store\Command\SetupStoreCommand;
74+
use Symfony\AI\Store\Document\Loader\DocumentCollectionLoader;
75+
use Symfony\AI\Store\Document\Loader\RssFeedLoader;
76+
use Symfony\AI\Store\Document\Loader\TextFileLoader;
7477
use Symfony\Component\ExpressionLanguage\ExpressionLanguage;
7578

7679
return static function (ContainerConfigurator $container): void {
@@ -162,6 +165,17 @@
162165
])
163166
->tag('kernel.event_subscriber')
164167

168+
// store source loader
169+
->set('ai.store.source_loader.document_collection', DocumentCollectionLoader::class)
170+
->tag('ai.store.source_loader', ['supports' => DocumentCollectionLoader::supportedSource()])
171+
->set('ai.store.source_loader.rss', RssFeedLoader::class)
172+
->args([
173+
service('http_client'),
174+
])
175+
->tag('ai.store.source_loader', ['supports' => RssFeedLoader::supportedSource()])
176+
->set('ai.store.source_loader.text_file', TextFileLoader::class)
177+
->tag('ai.store.source_loader', ['supports' => TextFileLoader::supportedSource()])
178+
165179
// tools
166180
->set('ai.toolbox.abstract', Toolbox::class)
167181
->abstract()
@@ -241,6 +255,7 @@
241255
->set('ai.command.index', IndexCommand::class)
242256
->args([
243257
tagged_locator('ai.indexer', 'name'),
258+
tagged_locator('ai.indexer.source', 'indexer'),
244259
])
245260
->tag('console.command')
246261
->set('ai.command.retrieve', RetrieveCommand::class)

‎src/ai-bundle/src/AiBundle.php‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@
107107
use Symfony\AI\Store\Bridge\Weaviate\Store as WeaviateStore;
108108
use Symfony\AI\Store\Distance\DistanceCalculator;
109109
use Symfony\AI\Store\Distance\DistanceStrategy;
110+
use Symfony\AI\Store\Document\Loader;
111+
use Symfony\AI\Store\Document\SourceLoaderInterface;
110112
use Symfony\AI\Store\Document\Vectorizer;
111113
use Symfony\AI\Store\Document\VectorizerInterface;
112114
use Symfony\AI\Store\Indexer;
@@ -312,6 +314,9 @@ public function loadExtension(array $config, ContainerConfigurator $container, C
312314
$builder->registerForAutoconfiguration(ResultConverterInterface::class)
313315
->addTag('ai.platform.result_converter');
314316

317+
$builder->registerForAutoconfiguration(SourceLoaderInterface::class)
318+
->addTag('ai.store.source_loader');
319+
315320
if (!ContainerBuilder::willBeAvailable('symfony/security-core', AuthorizationCheckerInterface::class, ['symfony/ai-bundle'])) {
316321
$builder->removeDefinition('ai.security.is_granted_attribute_listener');
317322
$builder->registerAttributeForAutoconfiguration(
@@ -2063,18 +2068,43 @@ private function processIndexerConfig(int|string $name, array $config, Container
20632068
$filters[] = new Reference($filter);
20642069
}
20652070

2071+
$loaders = [];
2072+
if (isset($config['loaders'])) {
2073+
/** @var class-string<SourceLoaderInterface> $sourceLoader */
2074+
foreach ($config['loaders'] as $sourceLoader) {
2075+
$loaders[$sourceLoader::supportedSource()] = new Reference($sourceLoader);
2076+
}
2077+
} else {
2078+
foreach ($container->findTaggedServiceIds('ai.store.source_loader') as $id => $tags) {
2079+
$loaders[$tags[0]['supports']] = new Reference($id);
2080+
}
2081+
}
2082+
$loader = new Definition(Loader::class, [$loaders]);
2083+
20662084
$definition = new Definition(Indexer::class, [
2067-
new Reference($config['loader']),
2085+
$loader,
20682086
new Reference($config['vectorizer']),
20692087
new Reference($config['store']),
2070-
$config['source'],
20712088
$filters,
20722089
$transformers,
20732090
new Reference('logger', ContainerInterface::IGNORE_ON_INVALID_REFERENCE),
20742091
]);
20752092
$definition->addTag('ai.indexer', ['name' => $name]);
20762093

2094+
// Register input object for ai:store:index command
2095+
if (isset($config['input'])) {
2096+
/** @var class-string<SourceLoaderInterface> $loader */
2097+
$sourceLoader = $config['input']['loader'];
2098+
$sourceId = 'ai.indexer.'.$name.'.input';
2099+
$source = (new Definition($sourceLoader::supportedSource()))
2100+
->addTag('ai.indexer.source', ['indexer' => $name])
2101+
->setFactory([$sourceLoader, 'createSource'])
2102+
->setArguments([$config['input']['source']]);
2103+
$container->setDefinition($sourceId, $source);
2104+
}
2105+
20772106
$serviceId = 'ai.indexer.'.$name;
2107+
$container->setDefinition($serviceId.'.loader', $loader);
20782108
$container->setDefinition($serviceId, $definition);
20792109
$container->registerAliasForArgument($serviceId, IndexerInterface::class, (new Target((string) $name))->getParsedName());
20802110
}

‎src/store/src/Command/IndexCommand.php‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
namespace Symfony\AI\Store\Command;
1313

14+
use Symfony\AI\Store\Document\SourceInterface;
1415
use Symfony\AI\Store\Exception\RuntimeException;
1516
use Symfony\AI\Store\IndexerInterface;
1617
use Symfony\Component\Console\Attribute\AsCommand;
@@ -35,9 +36,11 @@ final class IndexCommand extends Command
3536
{
3637
/**
3738
* @param ServiceLocator<IndexerInterface> $indexers
39+
* @param ServiceLocator<SourceInterface> $sources
3840
*/
3941
public function __construct(
4042
private readonly ServiceLocator $indexers,
43+
private readonly ServiceLocator $sources,
4144
) {
4245
parent::__construct();
4346
}
@@ -88,14 +91,21 @@ protected function execute(InputInterface $input, OutputInterface $output): int
8891

8992
$indexerService = $this->indexers->get($indexer);
9093

94+
// TODO: HOW TO PROVIDE SOURCE VIA ARGUMENT/OPTION?
9195
if (null !== $source) {
9296
$indexerService = $indexerService->withSource($source);
9397
}
9498

99+
if (!$this->sources->has($indexer)) {
100+
throw new RuntimeException(\sprintf('The indexer "%s" has no source configured.', $indexer));
101+
}
102+
103+
$source = $this->sources->get($indexer);
104+
95105
$io->title(\sprintf('Indexing documents using "%s" indexer', $indexer));
96106

97107
try {
98-
$indexerService->index([]);
108+
$indexerService->index($source);
99109

100110
$io->success(\sprintf('Documents indexed successfully using "%s" indexer.', $indexer));
101111
} catch (\Exception $e) {

0 commit comments

Comments
 (0)