diff --git a/package.json b/package.json index d344adfb..0187457f 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,8 @@ "imports": { "~docsCatalog": "./src/docs.json", "#toolsHost": "./dist/server.toolsHost.js", - "#workerEntry": "./dist/server.workerEntry.js" + "#workerEntry": "./dist/server.workerEntry.js", + "#collectionPatternFlyApi": "./dist/collection.patternFlyApi.js" }, "exports": { ".": { diff --git a/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap b/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap new file mode 100644 index 00000000..ed8d6a68 --- /dev/null +++ b/src/__tests__/__snapshots__/collection.patternFlyApi.test.ts.snap @@ -0,0 +1,26 @@ +// Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing + +exports[`collectionCallback should match snapshot for collection result 1`] = ` +{ + "records": [ + { + "data": { + "card": { + "category": "css", + "description": "Card css content", + "displayName": "Card", + "id": "api::v1::components::card::css::0", + "path": "https://main.patternfly-org.pages.dev/api/v1/components/Card/css", + "pathSlug": "card", + "section": "components", + "source": "api", + "version": "v1", + }, + }, + "id": "api::v1::components::card::css::0", + "sourceId": "https://main.patternfly-org.pages.dev/api/v1/components/Card/css", + "sourceType": "api", + }, + ], +} +`; diff --git a/src/__tests__/__snapshots__/collection.patternFlyDocs.test.ts.snap b/src/__tests__/__snapshots__/collection.patternFlyDocs.test.ts.snap index 1da65ab4..5eb438d9 100644 --- a/src/__tests__/__snapshots__/collection.patternFlyDocs.test.ts.snap +++ b/src/__tests__/__snapshots__/collection.patternFlyDocs.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing -exports[`patternFlyDocsCollection should match snapshot for collection result 1`] = ` +exports[`collectionCallback should match snapshot for collection result 1`] = ` { "isFallback": false, "records": [ diff --git a/src/__tests__/__snapshots__/collection.patternFlySchemas.test.ts.snap b/src/__tests__/__snapshots__/collection.patternFlySchemas.test.ts.snap index 36c898a3..592d7f29 100644 --- a/src/__tests__/__snapshots__/collection.patternFlySchemas.test.ts.snap +++ b/src/__tests__/__snapshots__/collection.patternFlySchemas.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://jestjs.io/docs/snapshot-testing -exports[`patternFlySchemasCollection should match snapshot for schema collection result 1`] = ` +exports[`collectionCallback should match snapshot for schema collection result 1`] = ` { "records": [ { diff --git a/src/__tests__/__snapshots__/options.defaults.test.ts.snap b/src/__tests__/__snapshots__/options.defaults.test.ts.snap index d784cfbe..9aff8168 100644 --- a/src/__tests__/__snapshots__/options.defaults.test.ts.snap +++ b/src/__tests__/__snapshots__/options.defaults.test.ts.snap @@ -51,6 +51,17 @@ exports[`options defaults should return specific properties: defaults 1`] = ` "nodeVersion": 22, "nodeVersionPreferred": 22, "patternflyOptions": { + "api": { + "base": "https://main.patternfly-org.pages.dev/api", + "componentPaths": [ + "props", + "css", + ], + "crawlCancelMs": 180000, + "crawlIntervalMs": 43200000, + "enabled": false, + "versions": "https://main.patternfly-org.pages.dev/api/versions", + }, "availableResourceVersions": [ "6.0.0", ], @@ -96,6 +107,12 @@ exports[`options defaults should return specific properties: defaults 1`] = ` "cacheLimit": 100, "expire": 180000, }, + "high": { + "cacheLimit": 50, + }, + "medium": { + "cacheLimit": 25, + }, "readFile": { "cacheErrors": false, "cacheLimit": 50, @@ -139,7 +156,10 @@ exports[`options defaults should return specific properties: defaults 1`] = ` ], "urls": [ "https://patternfly.org", + "https://www.patternfly.org", "https://github.com/patternfly", + "https://www.github.com/patternfly", + "https://main.patternfly-org.pages.dev", "https://raw.githubusercontent.com/patternfly", ], }, diff --git a/src/__tests__/collection.patternFlyApi.test.ts b/src/__tests__/collection.patternFlyApi.test.ts new file mode 100644 index 00000000..f1386894 --- /dev/null +++ b/src/__tests__/collection.patternFlyApi.test.ts @@ -0,0 +1,324 @@ +import { + patternFlyApiCollection, + collectionCallback, + apiSpider, + parsePayload, + isEmptyPayload, + crawler +} from '../collection.patternFlyApi'; +import { processDocsFunction } from '../server.getResources'; + +jest.mock('../server.getResources'); + +// Prefer relaxed typing in tests to focus on behavior over typings +const mockedProcessDocsFunction: any = processDocsFunction as any; + +describe('patternFlyApiCollection', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should return the correct collection name and configuration', () => { + const [name, callback, config] = patternFlyApiCollection(); + + expect(name).toBe('patternfly-api'); + expect(callback).toBeDefined(); + expect(config?.runParallel).toContain('#collection'); + }); +}); + +describe('collectionCallback', () => { + const BASE = 'https://main.patternfly-org.pages.dev/api'; + const VERSIONS = `${BASE}/versions`; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('should generate API records and match McpCollectionResult structure', async () => { + // getVersions to ["v1"] + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: VERSIONS, + resolvedPath: VERSIONS, + isSuccess: true + } + ]) + // crawler to leaf at a component facet path ("props") + .mockResolvedValueOnce([ + { + content: 'Button props content', + path: `${BASE}/v1/components/Button`, + resolvedPath: `${BASE}/v1/components/Button/props`, + isSuccess: true + } + ]); + + const result = await collectionCallback(); + + expect(result).toHaveProperty('records'); + expect(Array.isArray(result.records)).toBe(true); + expect(result.records.length).toBeGreaterThan(0); + + const first: any = result.records[0]; + + // Basic record shape + expect(first).toMatchObject({ + id: expect.stringMatching(/^api::/), + sourceType: 'api' + }); + + // Data entry shape + const keys = Object.keys(first.data as any); + + expect(keys.length).toBe(1); + const key: any = keys[0]; + + expect(first).toMatchObject({ + sourceId: `${BASE}/v1/components/Button/props` + }); + + expect(first.data[key]).toMatchObject({ + displayName: 'Button', + pathSlug: 'button', + source: 'api', + version: 'v1', + section: 'components', + category: 'props', + path: `${BASE}/v1/components/Button/props` + }); + }); + + it('should uses kind "doc" when a facet is not a componentPath', async () => { + // getVersions to ["v1"] + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: VERSIONS, + resolvedPath: VERSIONS, + isSuccess: true + } + ]) + // crawler to leaf with non-component facet ("overview") + .mockResolvedValueOnce([ + { + content: 'Overview content', + path: `${BASE}/v1/components/Card`, + resolvedPath: `${BASE}/v1/components/Card/overview`, + isSuccess: true + } + ]); + + const result = await collectionCallback(); + + expect(result.records.length).toBe(1); + const rec: any = result.records[0]; + + // id encodes version, section, item, kind, and index + expect(rec?.id).toMatch(/^api::v1::components::card::doc::0$/); + + const key: any = rec?.data ? Object.keys(rec.data)[0] : ''; + + expect(key).toBe('card'); + expect(rec?.data?.[key]).toMatchObject({ + displayName: 'Card', + category: 'doc' + }); + }); + + it('should match snapshot for collection result', async () => { + // getVersions to ["v1"] + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: VERSIONS, + resolvedPath: VERSIONS, + isSuccess: true + } + ]) + // crawler returns a single leaf entry (enough to snapshot deterministically here) + .mockResolvedValueOnce([ + { + content: 'Card css content', + path: `${BASE}/v1/components/Card`, + resolvedPath: `${BASE}/v1/components/Card/css`, + isSuccess: true + } + ]); + + const result = await collectionCallback(); + + const snapshotSubset = { + ...result, + records: result.records.slice(0, 3) + }; + + expect(snapshotSubset).toMatchSnapshot(); + }); +}); + +describe('isEmptyPayload', () => { + it('treats {}, [], null, "" as empty (soft-404)', () => { + expect(isEmptyPayload('{}')).toBe(true); + expect(isEmptyPayload('[]')).toBe(true); + expect(isEmptyPayload('null')).toBe(true); + expect(isEmptyPayload('""')).toBe(true); + expect(isEmptyPayload('')).toBe(true); + }); +}); + +describe('parsePayload', () => { + it('parses numeric payloads as non-empty', () => { + expect(parsePayload('42').isEmpty).toBe(false); + }); +}); + +describe('crawler', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('recursively crawls and returns content', async () => { + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: 'https://api.com/versions', + resolvedPath: 'https://api.com/versions', + isSuccess: true + } + ]) + .mockResolvedValueOnce([ + { + content: 'some content', + path: 'https://api.com/v1', + resolvedPath: 'https://api.com/v1', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/versions']); + + expect(res).toHaveLength(1); + expect(res[0]?.content).toBe('some content'); + expect(mockedProcessDocsFunction).toHaveBeenCalledTimes(2); + }); + + it('handles component paths and terminates recursion', async () => { + mockedProcessDocsFunction.mockResolvedValueOnce([ + { + content: JSON.stringify(['item1']), + path: 'https://api.com/v1/props', + resolvedPath: 'https://api.com/v1/props', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/v1/props']); + + expect(res).toHaveLength(1); + expect(res[0]?.path).toBe('https://api.com/v1/props'); + expect(mockedProcessDocsFunction).toHaveBeenCalledTimes(1); + }); + + it('filters out empty payloads', async () => { + mockedProcessDocsFunction.mockResolvedValueOnce([ + { + content: '{}', + path: 'https://api.com/v1/leaf', + resolvedPath: 'https://api.com/v1/leaf', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/v1/leaf']); + + expect(res).toHaveLength(0); + }); + + it('handles recursive arrays and joins URLs correctly', async () => { + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['sub-item']), + path: 'https://api.com/v1', + resolvedPath: 'https://api.com/v1', + isSuccess: true + } + ]) + .mockResolvedValue([ + { + content: 'leaf', + path: 'https://api.com/v1/sub-item', + resolvedPath: 'https://api.com/v1/sub-item', + isSuccess: true + } + ]); + + const res = await crawler(['https://api.com/v1']); + + // It should have called for sub-item AND default componentPaths (props, css) + // but my mock returns 'leaf' for everything else + expect(res.length).toBeGreaterThanOrEqual(1); + expect(mockedProcessDocsFunction).toHaveBeenCalledWith(['https://api.com/v1']); + }); +}); + +describe('apiSpider', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + it('returns [] when getVersions rejects', async () => { + mockedProcessDocsFunction.mockResolvedValueOnce([ + { + content: 'Failed to load', + path: 'https://main.patternfly-org.pages.dev/api/versions', + resolvedPath: 'https://main.patternfly-org.pages.dev/api/versions', + isSuccess: false + } + ]); + + const res = await apiSpider(); + + expect(res).toEqual([]); + }); + + it('returns ApiContent[] with metadata shape', async () => { + mockedProcessDocsFunction + .mockResolvedValueOnce([ + { + content: JSON.stringify(['v1']), + path: 'https://main.patternfly-org.pages.dev/api/versions', + resolvedPath: 'https://main.patternfly-org.pages.dev/api/versions', + isSuccess: true + } + ]) + .mockResolvedValueOnce([ + { + content: 'leaf content', + path: 'https://main.patternfly-org.pages.dev/api/v1', + resolvedPath: 'https://main.patternfly-org.pages.dev/api/v1/section/item/facet', + isSuccess: true + } + ]); + + const res = await apiSpider(); + + expect(res.length).toBeGreaterThan(0); + expect(res[0]).toMatchObject({ + url: 'https://main.patternfly-org.pages.dev/api/v1/section/item/facet', + content: 'leaf content', + semanticContext: { + version: 'v1', + section: 'section', + item: 'item', + facet: 'facet' + } + }); + }); +}); diff --git a/src/__tests__/collection.patternFlyDocs.test.ts b/src/__tests__/collection.patternFlyDocs.test.ts index b66e91ef..af8434d5 100644 --- a/src/__tests__/collection.patternFlyDocs.test.ts +++ b/src/__tests__/collection.patternFlyDocs.test.ts @@ -1,4 +1,4 @@ -import { patternFlyDocsCollection } from '../collection.patternFlyDocs'; +import { patternFlyDocsCollection, collectionCallback } from '../collection.patternFlyDocs'; import { EMBEDDED_DOCS } from '../docs.embedded'; describe('patternFlyDocsCollection', () => { @@ -7,15 +7,21 @@ describe('patternFlyDocsCollection', () => { }); it('should return the correct collection name and configuration', () => { - const [name, , config] = patternFlyDocsCollection(); + const [name, callback, config] = patternFlyDocsCollection(); expect(name).toBe('patternfly-docs'); + expect(callback).toBeDefined(); expect(config?.isRequired).toBe(true); }); +}); + +describe('collectionCallback', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); it('should load documentation records and match McpCollectionResult structure', async () => { - const [, callback] = patternFlyDocsCollection(); - const result = await callback(); + const result = await collectionCallback(); expect(result).toHaveProperty('records'); expect(Array.isArray(result.records)).toBe(true); @@ -36,8 +42,7 @@ describe('patternFlyDocsCollection', () => { // Actually, getPatternFlyDocsCatalog already has a try/catch and uses EMBEDDED_DOCS // For this test, we can just verify that if isFallback is true, it contains entries from EMBEDDED_DOCS - const [, callback] = patternFlyDocsCollection(); - const result = await callback(); + const result = await collectionCallback(); if (result.isFallback) { const embeddedNames = Object.keys(EMBEDDED_DOCS.docs).map(name => name.toLowerCase()); @@ -50,8 +55,7 @@ describe('patternFlyDocsCollection', () => { }); it('should match snapshot for collection result', async () => { - const [, callback] = patternFlyDocsCollection(); - const result = await callback(); + const result = await collectionCallback(); // We only snapshot a subset to avoid giant snapshots if docs.json is large const snapshotSubset = { diff --git a/src/__tests__/collection.patternFlySchemas.test.ts b/src/__tests__/collection.patternFlySchemas.test.ts index 6419feeb..7930c6ec 100644 --- a/src/__tests__/collection.patternFlySchemas.test.ts +++ b/src/__tests__/collection.patternFlySchemas.test.ts @@ -1,4 +1,4 @@ -import { patternFlySchemasCollection } from '../collection.patternFlySchemas'; +import { patternFlySchemasCollection, collectionCallback } from '../collection.patternFlySchemas'; jest.mock('../patternFly.helpers', () => ({ getPatternFlyVersionContext: { @@ -16,15 +16,21 @@ describe('patternFlySchemasCollection', () => { }); it('should return the correct collection name and configuration', () => { - const [name, , config] = patternFlySchemasCollection(); + const [name, callback, config] = patternFlySchemasCollection(); expect(name).toBe('patternfly-component-schemas'); + expect(callback).toBeDefined(); expect(config?.isRequired).toBe(true); }); +}); + +describe('collectionCallback', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); it('should generate schema records for components and match McpCollectionResult structure', async () => { - const [, callback] = patternFlySchemasCollection(); - const result = await callback(); + const result = await collectionCallback(); expect(result).toHaveProperty('records'); expect(Array.isArray(result.records)).toBe(true); @@ -52,8 +58,7 @@ describe('patternFlySchemasCollection', () => { }); it('should manually include the Table component with isSchemasAvailable: false', async () => { - const [, callback] = patternFlySchemasCollection(); - const result = await callback(); + const result = await collectionCallback(); const tableRecord = result.records.find(record => record.sourceId === 'table'); @@ -66,8 +71,7 @@ describe('patternFlySchemasCollection', () => { }); it('should match snapshot for schema collection result', async () => { - const [, callback] = patternFlySchemasCollection(); - const result = await callback(); + const result = await collectionCallback(); expect(result).toMatchSnapshot(); }); diff --git a/src/collection.patternFlyApi.ts b/src/collection.patternFlyApi.ts new file mode 100644 index 00000000..044ab9d0 --- /dev/null +++ b/src/collection.patternFlyApi.ts @@ -0,0 +1,371 @@ +import { + type McpCollection, + type McpCollectionRecord, + type McpCollectionResult +} from './collections'; +import { log } from './logger'; +import { processDocsFunction } from './server.getResources'; +import { memo } from './server.caching'; +import { isPlainObject, joinUrl } from './server.helpers'; +import { + getOptions, + getSessionOptions, + runWithOptions, + runWithSession +} from './options.context'; +import { DEFAULT_OPTIONS } from './options.defaults'; + +/** + * Processed content for API responses. + * + * @property url - The URL of the content. + * @property content - The content itself. + * @property semanticContext - Semantic context of the content. + * @property semanticContext.version - PatternFly version of the content. + * @property semanticContext.section - Section of the content. + * @property semanticContext.item - Item of the content. + * @property semanticContext.facet - Facet of the content. + * @property semanticContext.kind - Kind of the content. + * @property semanticContext.metadata - Remaining metadata, if any, of the content. + */ +interface ApiContent { + url: string; + content: string; + semanticContext: { + version?: string | undefined; + section?: string | undefined; + item?: string | undefined; + facet?: string | undefined; + kind?: string | undefined; + metadata?: string[] | undefined; + } +} + +/** + * API crawler response. + * + * @interface ApiCrawler + * + * @property content - Content retrieved from the API. + * @property path - Initial or relative path used to fetch the content. + * @property resolvedPath - Absolute or resolved path after processing the initial path. + */ +interface ApiCrawler { + content: string; + path: string; + resolvedPath: string; +} + +/** + * API parsed payload response + */ +type ParsePayloadApi = string | number | boolean | null | string[] | Record; + +/** + * API parsed payload response. + * + * @interface ParsePayload + * + * @property isEmpty - Whether the parsed payload is considered empty. + * @property {ParsePayloadApi} payload - Parsed version of the input payload. + */ +interface ParsePayload { + isEmpty: boolean; + payload: ParsePayloadApi; +} + +/** + * Parses the given payload and determines its state and structure. + * + * @param payload - Input payload to be parsed. + * @returns An object containing: + * - `isEmpty`: A boolean indicating whether the parsed payload is considered empty. + * - `payload`: The parsed version of the input payload. If the input is a string + * and can be parsed as JSON without error, the parsed result is returned. + * Otherwise, the trimmed string or original value is provided. + */ +const parsePayload = (payload: unknown): ParsePayload => { + const updatedPayload = typeof payload === 'string' ? payload.trim() : ''; + let isEmpty: boolean; + let parsedPayload: ParsePayloadApi; + + try { + parsedPayload = JSON.parse(updatedPayload); + + if (typeof parsedPayload === 'number') { + isEmpty = false; + } else { + isEmpty = (Array.isArray(parsedPayload) && parsedPayload.length === 0) || + (isPlainObject(parsedPayload) && Object.keys(parsedPayload).length === 0) || + parsedPayload === null; + } + } catch { + parsedPayload = updatedPayload; + isEmpty = updatedPayload.length === 0; + } + + return { isEmpty, payload: parsedPayload }; +}; + +/** + * Memoized version of parsePayload. + */ +parsePayload.memo = memo(parsePayload, DEFAULT_OPTIONS.resourceMemoOptions.default); + +/** + * Determines if the payload is empty. + * + * @param payload - Data to be evaluated for emptiness. + * @returns Returns `true` if the payload is empty, otherwise `false`. + */ +const isEmptyPayload = (payload: unknown) => { + if (typeof payload === 'string') { + const trimmedPayload = payload.trim(); + + return trimmedPayload === '' || trimmedPayload === '{}' || trimmedPayload === '[]' || trimmedPayload === 'null' || trimmedPayload === '""'; + } + + return payload === null || payload === undefined || parsePayload.memo(payload).isEmpty; +}; + +/** + * Memoized version of isEmptyPayload. + */ +isEmptyPayload.memo = memo(isEmptyPayload, DEFAULT_OPTIONS.resourceMemoOptions.default); + +/** + * Recursively crawls a list of URLs. + * + * Resolves paths and fetches content; built specifically around the PatternFly API response structure. + * + * @param urls - The list of URLs to crawl. + * @param [options] - An optional configuration object. + * @returns {Promise} A promise that resolves to an array of processed documents, + * each containing information about the crawling result, status, and content. + */ +const crawler = async (urls: string[], options = getOptions()): Promise => { + const componentPaths = options.patternflyOptions.api.componentPaths; + const settled = await processDocsFunction(urls); + const content: ApiCrawler[] = []; + + for (const res of settled) { + const { isEmpty, payload } = parsePayload.memo(res.content); + + if (res.isSuccess) { + if (Array.isArray(payload)) { + if (componentPaths.some(componentPath => res?.path?.includes(componentPath))) { + if (!isEmpty) { + content.push({ ...res }); + } + continue; + } + + const updatedPayload = [...payload, ...componentPaths].map(path => joinUrl(res.path, path)); + const crawledContent = await crawler(updatedPayload); + + content.push(...crawledContent); + continue; + } + + if (!isEmpty) { + content.push({ ...res }); + } + } + } + + return content; +}; + +/** + * Get and process available API versions. + * + * @param [options=getOptions()] - Configuration options. + * @returns A promise that resolves to an array of processed version URLs. + * + * @throws + */ +const getVersions = async (options = getOptions()) => { + const versionUrl = options.patternflyOptions.api.versions; + const processedVersions = await processDocsFunction([versionUrl]); + const versions: string[] = []; + + if (processedVersions[0]) { + const response = processedVersions[0]; + + if (response.isSuccess) { + const { payload } = parsePayload.memo(response.content); + + if (Array.isArray(payload)) { + versions.push(...payload.map(version => joinUrl(options.patternflyOptions.api.base, version))); + } + } + } + + if (versions.length === 0) { + throw new Error(`No API versions available ${versionUrl}.`); + } + + return versions; +}; + +/** + * Process content metadata from response paths. + * + * @param apiResponses - The list of pre-metadata content. + * @param [options=getOptions()] - Configuration options. + * @returns The list of processed API content with metadata. + */ +const contentMetadata = (apiResponses: ApiCrawler[], options = getOptions()): ApiContent[] => { + const base = options.patternflyOptions.api.base; + const componentPaths = options.patternflyOptions.api.componentPaths; + + return apiResponses.map(({ content, resolvedPath }) => { + const [version, section, item, facet, ...remaining] = resolvedPath.replace(base, '').split('/').filter(Boolean) || []; + const kind = facet && (componentPaths.includes(facet) || remaining.includes(facet)) ? facet : 'doc'; + + return { + url: resolvedPath, + content, + semanticContext: { + version, + section, + item, + facet, + kind, + metadata: (remaining.length && remaining) || undefined + } + }; + }); +}; + +/** + * Memoized version of contentMetadata. + */ +contentMetadata.memo = memo(contentMetadata); + +/** + * Initiate API crawl. + * + * @returns A promise resolving to an array of processed API content entries. + */ +const apiSpider = async (): Promise => { + log.info(`API spider crawl started`); + let seedVersions: string[] = []; + let content: ApiCrawler[] = []; + + try { + seedVersions = await getVersions(); + } catch (err) { + log.warn(`API spider: getVersions failed`, err); + + return []; + } + + if (seedVersions.length) { + try { + content = await crawler(seedVersions); + } catch (err) { + log.warn(`API spider: crawler failed`, err); + + return []; + } + } + + // Review the memo here. It may be better served to tie into crawler, + // like `crawler.memo` as part of the countdown to refresh + const updatedContent = contentMetadata.memo(content); + + log.info( + `API spider crawl completed. ${updatedContent.length} content ${ + (updatedContent.length === 1 && 'entry') || 'entries' + } retrieved.` + ); + + return updatedContent; +}; + +/** + * Async collect and process entries for a collection. + * + * @returns {Promise} Object containing a list of processed records. + */ +const collectionCallback = async (): Promise => { + const entries = await apiSpider(); + const recordsMap: Map = new Map(); + + entries?.forEach((entry, index) => { + const semanticContext = entry.semanticContext || {}; + const name = (semanticContext.item || 'api-entry').toLowerCase(); + const version = (semanticContext.version || 'unknown').toLowerCase(); + const displayName = semanticContext.item || name; + + const id = `api::${version}::${semanticContext.section || ''}::${name}::${semanticContext.kind || ''}::${index}`; + + if (recordsMap.has(id)) { + return; + } + + const adaptedEntry = { + displayName, + description: entry.content || `PatternFly API documentation for ${displayName}`, + pathSlug: name, + category: semanticContext.kind, + section: semanticContext.section || 'components', + source: 'api' as const, + version, + id, + path: entry.url + }; + + const record = { + id, + sourceId: entry.url, + sourceType: 'api' as const, + data: { + [name]: adaptedEntry + } + }; + + recordsMap.set(record.id, record); + }); + + return { records: [...recordsMap.values()] }; +}; + +/** + * Create a PatternFly API collection. + * + * @param options - Global options + * @param session - Session options + * @returns {McpCollection} The collection definition tuple + */ +const patternFlyApiCollection = (options = getOptions(), session = getSessionOptions()): McpCollection => { + const callback: McpCollection[1] = async () => + runWithSession(session, async () => + runWithOptions(options, async () => collectionCallback())); + + return [ + 'patternfly-api', + callback, + { + runParallel: '#collectionPatternFlyApi', + runSchedule: { + cancelMs: options.patternflyOptions.api.crawlCancelMs, + intervalMs: options.patternflyOptions.api.crawlIntervalMs + } + } + ]; +}; + +export { + patternFlyApiCollection, + collectionCallback, + apiSpider, + crawler, + isEmptyPayload, + parsePayload, + type ApiContent, + type ApiCrawler, + type ParsePayload, + type ParsePayloadApi +}; diff --git a/src/collection.patternFlyDocs.ts b/src/collection.patternFlyDocs.ts index 5812ac0c..292a6bf1 100644 --- a/src/collection.patternFlyDocs.ts +++ b/src/collection.patternFlyDocs.ts @@ -1,6 +1,12 @@ import { type McpCollection, type McpCollectionRecord } from './collections'; import { EMBEDDED_DOCS, type PatternFlyMcpDocsCatalog } from './docs.embedded'; import { formatUnknownError, log } from './logger'; +import { + getOptions, + getSessionOptions, + runWithOptions, + runWithSession +} from './options.context'; /** * Lazy load the PatternFly documentation catalog. @@ -26,36 +32,49 @@ const getPatternFlyDocsCatalog = async (): Promise} Object containing a list of processed records. */ -const patternFlyDocsCollection = (): McpCollection => { - const callback = async () => { - const docsCatalog = await getPatternFlyDocsCatalog(); - const catalog = [...Object.entries(docsCatalog.docs)]; - const recordsMap: Map = new Map(); +const collectionCallback = async () => { + const docsCatalog = await getPatternFlyDocsCatalog(); + const catalog = [...Object.entries(docsCatalog.docs)]; + const recordsMap: Map = new Map(); + + catalog.forEach(([name, entries]) => { + const normalizedName = name.toLowerCase(); + const id = `docs::${normalizedName}`; - catalog.forEach(([name, entries]) => { - const normalizedName = name.toLowerCase(); - const id = `docs::${normalizedName}`; + if (recordsMap.has(id)) { + return; + } - if (recordsMap.has(id)) { - return; + const record = { + id, + sourceId: normalizedName, + sourceType: 'local' as const, + data: { + [normalizedName]: entries } + }; - const record = { - id, - sourceId: normalizedName, - sourceType: 'local' as const, - data: { - [normalizedName]: entries - } - }; + recordsMap.set(record.id, record); + }); - recordsMap.set(record.id, record); - }); + return { records: [...recordsMap.values()], isFallback: docsCatalog.isFallback }; +}; - return { records: [...recordsMap.values()], isFallback: docsCatalog.isFallback }; - }; +/** + * Create a PatternFly local embedded docs collection from `docs.json`. + * + * @param options - Global options + * @param session - Session options + * @returns {McpCollection} The collection definition tuple + */ +const patternFlyDocsCollection = (options = getOptions(), session = getSessionOptions()): McpCollection => { + const callback: McpCollection[1] = async () => + runWithSession(session, async () => + runWithOptions(options, async () => collectionCallback())); return [ 'patternfly-docs', @@ -66,4 +85,4 @@ const patternFlyDocsCollection = (): McpCollection => { ]; }; -export { patternFlyDocsCollection }; +export { patternFlyDocsCollection, collectionCallback }; diff --git a/src/collection.patternFlySchemas.ts b/src/collection.patternFlySchemas.ts index 56d26b93..cdbb2e98 100644 --- a/src/collection.patternFlySchemas.ts +++ b/src/collection.patternFlySchemas.ts @@ -3,70 +3,87 @@ import { } from '@patternfly/patternfly-component-schemas/json'; import { type McpCollection, type McpCollectionRecord } from './collections'; import { getPatternFlyVersionContext } from './patternFly.helpers'; +import { + getOptions, + getSessionOptions, + runWithOptions, + runWithSession +} from './options.context'; /** - * Component schemas collection from @patternfly/patternfly-component-schemas. + * Async collect and process entries for a collection. * - * @returns Component schemas collection from @patternfly/patternfly-component-schemas. + * @returns {Promise} Object containing a list of processed records. */ -const patternFlySchemasCollection = (): McpCollection => { - const callback = async () => { - const { latestSchemasVersion } = await getPatternFlyVersionContext.memo(); - const recordsMap: Map = new Map(); - - pfComponentNames.forEach(name => { - const normalizedName = name.toLowerCase(); - const id = `schema::${normalizedName}`; +const collectionCallback = async () => { + const { latestSchemasVersion } = await getPatternFlyVersionContext.memo(); + const recordsMap: Map = new Map(); - if (recordsMap.has(id)) { - return; - } + pfComponentNames.forEach(name => { + const normalizedName = name.toLowerCase(); + const id = `schema::${normalizedName}`; - const record = { - id, - sourceId: normalizedName, - sourceType: 'package' as const, - data: { - [normalizedName]: [ - { - displayName: name, - description: `PatternFly React component: ${name}`, - pathSlug: `schemas-${normalizedName}`, - category: 'react', - section: 'components', - source: 'schemas', - version: latestSchemasVersion, - isSchemasAvailable: true - } - ] - } - }; - - recordsMap.set(record.id, record); - }); + if (recordsMap.has(id)) { + return; + } - if (!recordsMap.has('schema::table')) { - recordsMap.set('schema::table', { - id: 'schema::table', - sourceId: 'table', - sourceType: 'package' as const, - data: { - table: [{ - displayName: 'Table', - description: 'PatternFly React component: table', - pathSlug: 'schemas-table', + const record = { + id, + sourceId: normalizedName, + sourceType: 'package' as const, + data: { + [normalizedName]: [ + { + displayName: name, + description: `PatternFly React component: ${name}`, + pathSlug: `schemas-${normalizedName}`, category: 'react', section: 'components', source: 'schemas', version: latestSchemasVersion, - isSchemasAvailable: false - }] - } - }); - } + isSchemasAvailable: true + } + ] + } + }; + + recordsMap.set(record.id, record); + }); - return { records: [...recordsMap.values()] }; - }; + if (!recordsMap.has('schema::table')) { + recordsMap.set('schema::table', { + id: 'schema::table', + sourceId: 'table', + sourceType: 'package' as const, + data: { + table: [{ + displayName: 'Table', + description: 'PatternFly React component: table', + pathSlug: 'schemas-table', + category: 'react', + section: 'components', + source: 'schemas', + version: latestSchemasVersion, + isSchemasAvailable: false + }] + } + }); + } + + return { records: [...recordsMap.values()] }; +}; + +/** + * Create a PatternFly Component Schemas collection from `@patternfly/patternfly-component-schemas`. + * + * @param options - Global options + * @param session - Session options + * @returns {McpCollection} The collection definition tuple + */ +const patternFlySchemasCollection = (options = getOptions(), session = getSessionOptions()): McpCollection => { + const callback: McpCollection[1] = async () => + runWithSession(session, async () => + runWithOptions(options, async () => collectionCallback())); return [ 'patternfly-component-schemas', @@ -78,5 +95,6 @@ const patternFlySchemasCollection = (): McpCollection => { }; export { - patternFlySchemasCollection + patternFlySchemasCollection, + collectionCallback }; diff --git a/src/options.defaults.ts b/src/options.defaults.ts index 3ee44588..d13f5e21 100644 --- a/src/options.defaults.ts +++ b/src/options.defaults.ts @@ -178,6 +178,12 @@ interface ModeOptions { /** * PatternFly-specific options. * + * @property api PatternFly API. + * @property api.base URL starting base for crawling the PatternFly API. + * @property api.versions URL Get the available PatternFly API versions. Versions are required to crawl. + * @property api.componentPaths List of additional PatternFly API component paths to try. + * @property api.crawlCancelMs Timeout in milliseconds for cancelling the PatternFly API crawl. + * @property api.crawlIntervalMs Interval in milliseconds, during server run, for crawling the PatternFly API. * @property availableResourceVersions List of available PatternFly resource versions to the MCP server. * @property availableSearchVersions List of available PatternFly search versions to the MCP server. * @property availableSchemasVersions List of available PatternFly schema versions to the MCP server. @@ -191,6 +197,14 @@ interface ModeOptions { * - 'lowest': Use the lowest major version found. */ interface PatternFlyOptions { + api: { + base: string; + versions: string; + componentPaths: string[]; + crawlCancelMs: number; + crawlIntervalMs: number; + enabled: boolean; + }, availableResourceVersions: ('6.0.0')[]; availableSearchVersions: ('current' | 'latest' | 'v6')[]; availableSchemasVersions: ('v6')[]; @@ -410,6 +424,12 @@ const RESOURCE_MEMO_OPTIONS = { default: { cacheLimit: 3 }, + medium: { + cacheLimit: 25 + }, + high: { + cacheLimit: 50 + }, fetchUrl: { cacheLimit: 100, expire: 3 * 60 * 1000, // 3 minute sliding cache @@ -462,8 +482,10 @@ const STATS_OPTIONS: StatsOptions = { const WHITELIST_OPTIONS: WhitelistOptions = { urls: [ 'https://patternfly.org', - // 'https://www.patternfly.org', + 'https://www.patternfly.org', 'https://github.com/patternfly', + 'https://www.github.com/patternfly', + 'https://main.patternfly-org.pages.dev', 'https://raw.githubusercontent.com/patternfly' ], protocols: ['http', 'https'] @@ -488,6 +510,18 @@ const CHANNEL_BASENAME = 'pf-mcp'; * Default PatternFly-specific options. */ const PATTERNFLY_OPTIONS: PatternFlyOptions = { + api: { + base: 'https://main.patternfly-org.pages.dev/api', + versions: 'https://main.patternfly-org.pages.dev/api/versions', + componentPaths: [ + 'props', + 'css' + ], + crawlCancelMs: 180_000, // 3 minutes + crawlIntervalMs: 43_200_000, // 12 hours + enabled: false + // concurrency: 4 + }, availableResourceVersions: ['6.0.0'], availableSearchVersions: ['current', 'latest', 'v6'], availableSchemasVersions: ['v6'], diff --git a/src/server.helpers.ts b/src/server.helpers.ts index fd3cd775..2a8a5450 100644 --- a/src/server.helpers.ts +++ b/src/server.helpers.ts @@ -556,6 +556,30 @@ const parseUrl = (url: string, { prefix, normalizeSearchParamKeys = true, isStri return undefined; }; +/** + * Joins multiple URL segments into a single URL string, ensuring no double slashes. + * If `base` is not a valid URL, it's returned as-is. + * + * @param base - The base URL string + * @param parts - Additional path segments to join + * @returns The joined URL string + */ +const joinUrl = (base: string, ...parts: string[]): string => { + if (!isUrl(base)) { + return base; + } + + const url = new URL(base); + + parts.join('/').split('/').filter(Boolean).forEach(part => { + const updatedPathname = url.pathname.endsWith('/') ? url.pathname : `${url.pathname}/`; + + url.pathname = `${updatedPathname}${part}`; + }); + + return url.toString(); +}; + /** * Basic split for URIs to find base and search. * @@ -767,6 +791,7 @@ export { isUrl, isUrlObject, isWhitelistedUrl, + joinUrl, listAllCombinations, listIncrementalCombinations, mergeObjects, diff --git a/tsconfig.json b/tsconfig.json index ab16b5cc..7abbd0df 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -25,7 +25,8 @@ "paths": { "#docsCatalog": ["./src/docs.json"], "#toolsHost": ["./src/server.toolsHost.ts"], - "#workerEntry": ["./src/server.workerEntry.ts"] + "#workerEntry": ["./src/server.workerEntry.ts"], + "#collectionPatternFlyApi": ["./src/collection.patternFlyApi.ts"] } }, "include": [