diff --git a/src/__tests__/collection.patternFlyApi.test.ts b/src/__tests__/collection.patternFlyApi.test.ts index f1386894..8216a30b 100644 --- a/src/__tests__/collection.patternFlyApi.test.ts +++ b/src/__tests__/collection.patternFlyApi.test.ts @@ -76,13 +76,17 @@ describe('collectionCallback', () => { expect(keys.length).toBe(1); const key: any = keys[0]; + expect(key).toBe('button'); + expect(first).toMatchObject({ sourceId: `${BASE}/v1/components/Button/props` }); - expect(first.data[key]).toMatchObject({ + expect(Array.isArray(first.data[key])).toBe(true); + + expect(first.data[key][0]).toMatchObject({ displayName: 'Button', - pathSlug: 'button', + pathSlug: 'components-button-props', source: 'api', version: 'v1', section: 'components', @@ -118,15 +122,15 @@ describe('collectionCallback', () => { const rec: any = result.records[0]; // id encodes version, section, item, kind, and index - expect(rec?.id).toMatch(/^api::v1::components::card::doc::0$/); + expect(rec?.id).toMatch(/^api::v1::components::card::overview$/); const key: any = rec?.data ? Object.keys(rec.data)[0] : ''; expect(key).toBe('card'); - expect(rec?.data?.[key]).toMatchObject({ + expect(rec?.data?.[key]).toContainEqual(expect.objectContaining({ displayName: 'Card', - category: 'doc' - }); + category: 'overview' + })); }); it('should match snapshot for collection result', async () => { diff --git a/src/collection.patternFlyApi.ts b/src/collection.patternFlyApi.ts index 044ab9d0..07f091dd 100644 --- a/src/collection.patternFlyApi.ts +++ b/src/collection.patternFlyApi.ts @@ -14,6 +14,14 @@ import { runWithSession } from './options.context'; import { DEFAULT_OPTIONS } from './options.defaults'; +import { + calculateContentQualityScore, + extractApiDescription, + extractApiDisplayName, + extractApiName, + normalizeSlug +} from './collection.patternFlyApiHelpers'; +import { contentType } from './resource.helpers'; /** * Processed content for API responses. @@ -25,20 +33,27 @@ import { DEFAULT_OPTIONS } from './options.defaults'; * @property semanticContext.section - Section of the content. * @property semanticContext.item - Item of the content. * @property semanticContext.facet - Facet of the content. + * @property semanticContext.detail - Detail of the content. + * @property semanticContext.detailType - Detail type of the content. * @property semanticContext.kind - Kind of the content. + * @property semanticContext.contentType - Content type of the content. * @property semanticContext.metadata - Remaining metadata, if any, of the content. */ interface ApiContent { - url: string; + description: string; + displayName: string; + category: string; content: string; - semanticContext: { - version?: string | undefined; - section?: string | undefined; - item?: string | undefined; - facet?: string | undefined; - kind?: string | undefined; - metadata?: string[] | undefined; - } + contentType: string; + hasQuality: boolean; + id: string; + isDeferred: boolean; + name: string; + path: string; + pathSlug: string; + section: string; + source: string; + version: string; } /** @@ -74,6 +89,36 @@ interface ParsePayload { payload: ParsePayloadApi; } +/** + * Deferred API categories. + * + * @note Minimal PatternFly API data quality threshold + * - Last resort for content that requires additional parsing or should be ignored. + * - A quality threshold still has to be met even if these items are removed + * - Quality metrics need to be updated periodically as API content is added. + * + * - `props`: Deferred in favor of using @patternfly/patternfly-component-schemas. + * - `react`: Quality threshold applied. Some examples still contain low-quality data. + * - `react-demos`: Deferred React demonstration components. + * - `html`: Quality threshold applied. Some examples still contain low-quality data. + * - `html-demos`: Deferred HTML demonstration examples. + * - `text`: Quality threshold applied. Some examples still contain low-quality data. + */ +const DEFERRED_API_CATEGORIES = new Set([ + 'props', + // 'react', + 'react-demos', + // 'html', + 'html-demos' + // 'text', + // 'examples' +]); + +/** + * Min content quality threshold. See {@link calculateContentQualityScore} + */ +const MIN_API_QUALITY_THRESHOLD = 0.95; + /** * Parses the given payload and determines its state and structure. * @@ -144,32 +189,66 @@ isEmptyPayload.memo = memo(isEmptyPayload, DEFAULT_OPTIONS.resourceMemoOptions.d * each containing information about the crawling result, status, and content. */ const crawler = async (urls: string[], options = getOptions()): Promise => { - const componentPaths = options.patternflyOptions.api.componentPaths; + const { componentPaths, traversalPaths } = options.patternflyOptions.api; const settled = await processDocsFunction(urls); const content: ApiCrawler[] = []; for (const res of settled) { + if (!res.isSuccess) { + continue; + } + const { isEmpty, payload } = parsePayload.memo(res.content); - if (res.isSuccess) { - if (Array.isArray(payload)) { - if (componentPaths.some(componentPath => res?.path?.includes(componentPath))) { - if (!isEmpty) { - content.push({ ...res }); - } - continue; + if (Array.isArray(payload)) { + // Terminal Data Arrays (props, css, etc) + if (componentPaths.some(componentPath => res?.path?.endsWith(`/${componentPath}`))) { + if (!isEmpty) { + content.push({ ...res }); } - - const updatedPayload = [...payload, ...componentPaths].map(path => joinUrl(res.path, path)); - const crawledContent = await crawler(updatedPayload); - - content.push(...crawledContent); continue; } - if (!isEmpty) { - content.push({ ...res }); - } + // Traversal & Directory Array Processing + const flattenedPayload: string[] = []; + + payload.forEach(value => { + if (typeof value === 'string') { + flattenedPayload.push(value); + + log.debug(`Collection PatternFly API adding path`, value); + } else if (isPlainObject(value)) { + Object.values(value).forEach(value => { + if (typeof value === 'string') { + flattenedPayload.push(value); + + log.debug(`Collection PatternFly API adding path`, value); + } + }); + } + }); + + const updatedPayload = [...flattenedPayload, ...traversalPaths, ...componentPaths].map(path => joinUrl(res.path, path)); + + log.debug(`Collection PatternFly API Crawling ${updatedPayload.length} path(s)`); + + const crawledContent = await crawler(updatedPayload); + + content.push(...crawledContent); + continue; + } + + // String Payloads (Markdown, HTML, .tsx source code) + if (!isEmpty) { + content.push({ ...res }); + } + + // Probe Traversal Paths on Facet Endpoints (e.g. /react -> /react/examples) + if (!traversalPaths.some(traversalPath => res?.path?.endsWith(`/${traversalPath}`))) { + const traversalUrls = traversalPaths.map(traversalPath => joinUrl(res.path, traversalPath)); + const traversalCrawledContent = await crawler(traversalUrls); + + content.push(...traversalCrawledContent); } } @@ -208,48 +287,13 @@ const getVersions = async (options = getOptions()) => { return versions; }; -/** - * Process content metadata from response paths. - * - * @param apiResponses - The list of pre-metadata content. - * @param [options=getOptions()] - Configuration options. - * @returns The list of processed API content with metadata. - */ -const contentMetadata = (apiResponses: ApiCrawler[], options = getOptions()): ApiContent[] => { - const base = options.patternflyOptions.api.base; - const componentPaths = options.patternflyOptions.api.componentPaths; - - return apiResponses.map(({ content, resolvedPath }) => { - const [version, section, item, facet, ...remaining] = resolvedPath.replace(base, '').split('/').filter(Boolean) || []; - const kind = facet && (componentPaths.includes(facet) || remaining.includes(facet)) ? facet : 'doc'; - - return { - url: resolvedPath, - content, - semanticContext: { - version, - section, - item, - facet, - kind, - metadata: (remaining.length && remaining) || undefined - } - }; - }); -}; - -/** - * Memoized version of contentMetadata. - */ -contentMetadata.memo = memo(contentMetadata); - /** * Initiate API crawl. * * @returns A promise resolving to an array of processed API content entries. */ -const apiSpider = async (): Promise => { - log.info(`API spider crawl started`); +const apiSpider = async (): Promise => { + log.info(`Collection PatternFly API spider crawl started`); let seedVersions: string[] = []; let content: ApiCrawler[] = []; @@ -265,27 +309,87 @@ const apiSpider = async (): Promise => { try { content = await crawler(seedVersions); } catch (err) { - log.warn(`API spider: crawler failed`, err); + log.warn(`Collection PatternFly API spider: crawler failed`, err); return []; } } - // Review the memo here. It may be better served to tie into crawler, - // like `crawler.memo` as part of the countdown to refresh - const updatedContent = contentMetadata.memo(content); - log.info( - `API spider crawl completed. ${updatedContent.length} content ${ - (updatedContent.length === 1 && 'entry') || 'entries' + `Collection PatternFly API spider crawl completed. ${content.length} content ${ + (content.length === 1 && 'entry') || 'entries' } retrieved.` ); - return updatedContent; + return content; }; /** - * Async collect and process entries for a collection. + * Light/Immediate process for content metadata from response paths. + * + * @param crawlerResponse - An entry with pre-metadata content. + * @param [options] - Configuration options. + * @returns The process metadata entry. + */ +const contentMetadata = (crawlerResponse: ApiCrawler, options = getOptions()): ApiContent => { + const { content, resolvedPath } = crawlerResponse; + const { base, componentPaths, traversalPaths } = options.patternflyOptions.api; + + // Relative path after '/api/' + const segments = resolvedPath.replace(base, '').split('/').filter(Boolean); + const [version = 'unknown', section = 'components', rawItem = 'api-entry', rawFacet = 'doc', rawDetailType = '', rawDetail = '', ...remaining] = segments; + + const normalizedVersion = version.toLowerCase(); + const normalizedSection = normalizeSlug(section); + const normalizedItem = normalizeSlug(rawItem); + const normalizedFacet = normalizeSlug(rawFacet); + const normalizedDetailType = normalizeSlug(rawDetailType); + const normalizedDetail = normalizeSlug(rawDetail); + + // Build a category from the normalized facet + const normalizedCategory = [...componentPaths, ...traversalPaths].includes(normalizedFacet) ? normalizedFacet : normalizedFacet; + + // Build hierarchical normalized path slug: e.g. "ai/overview/text" or "components/button/props" + const isDetailSameName = normalizedDetail && normalizedDetail.includes(normalizedItem); + const pathSlug = [ + normalizedSection, + isDetailSameName ? undefined : normalizedItem, + normalizedFacet, + normalizedDetailType, + normalizedDetail, + ...remaining.map(normalizeSlug) + ].filter(Boolean).join('-'); + + const name = extractApiName(normalizedItem, normalizedSection); + + const id = `api::${normalizedVersion}::${normalizedSection}::${normalizedItem}::${normalizedCategory}${normalizedDetailType ? `::${normalizedDetailType}::${normalizedDetail}` : ''}`; + + const displayName = extractApiDisplayName(content, { slug: normalizedItem, kind: normalizedCategory, section: normalizedSection }); + const description = extractApiDescription(content, { displayName, kind: normalizedCategory, detailType: normalizedDetailType }); + + const hasQuality = calculateContentQualityScore(content, { kind: normalizedCategory }) < MIN_API_QUALITY_THRESHOLD; + const isDeferred = DEFERRED_API_CATEGORIES.has(normalizedCategory); + + return { + description, + displayName, + category: normalizedCategory, + content, + contentType: contentType(content), + hasQuality, + id, + isDeferred, + name, + path: resolvedPath, + pathSlug, + section: normalizedSection, + source: 'api' as const, + version: normalizedVersion + }; +}; + +/** + * Async collect and process entries for a collection. Add "conditional" metadata. * * @returns {Promise} Object containing a list of processed records. */ @@ -293,41 +397,30 @@ const collectionCallback = async (): Promise => { const entries = await apiSpider(); const recordsMap: Map = new Map(); - entries?.forEach((entry, index) => { - const semanticContext = entry.semanticContext || {}; - const name = (semanticContext.item || 'api-entry').toLowerCase(); - const version = (semanticContext.version || 'unknown').toLowerCase(); - const displayName = semanticContext.item || name; - - const id = `api::${version}::${semanticContext.section || ''}::${name}::${semanticContext.kind || ''}::${index}`; + for (const entry of entries) { + const { name, isDeferred, hasQuality, ...metadata } = contentMetadata(entry); - if (recordsMap.has(id)) { - return; + if (isDeferred || hasQuality) { + continue; } - const adaptedEntry = { - displayName, - description: entry.content || `PatternFly API documentation for ${displayName}`, - pathSlug: name, - category: semanticContext.kind, - section: semanticContext.section || 'components', - source: 'api' as const, - version, - id, - path: entry.url - }; + if (recordsMap.has(metadata.id)) { + continue; + } const record = { - id, - sourceId: entry.url, + id: metadata.id, + sourceId: metadata.path, sourceType: 'api' as const, data: { - [name]: adaptedEntry + [name]: [{ + ...metadata + }] } }; recordsMap.set(record.id, record); - }); + } return { records: [...recordsMap.values()] }; }; @@ -348,10 +441,9 @@ const patternFlyApiCollection = (options = getOptions(), session = getSessionOpt 'patternfly-api', callback, { - runParallel: '#collectionPatternFlyApi', + // runParallel: '#collectionPatternFlyApi', runSchedule: { - cancelMs: options.patternflyOptions.api.crawlCancelMs, - intervalMs: options.patternflyOptions.api.crawlIntervalMs + ...options.patternflyOptions.api.schedule } } ]; diff --git a/src/collection.patternFlyApiHelpers.ts b/src/collection.patternFlyApiHelpers.ts new file mode 100644 index 00000000..e3004aad --- /dev/null +++ b/src/collection.patternFlyApiHelpers.ts @@ -0,0 +1,389 @@ +import { isJson, isJsonLike } from './resource.helpers'; + +/** + * Detect imports that use the `?raw` query param. + * + * @param str + */ +const isRawImport = (str: string) => + /import\s+[\w*\s{},]+\s+from\s+['"][^'"]+\?raw['"]/i.test(str); + +/** + * Detect a `` tag. + * + * @param str + */ +const hasLiveExample = (str: string) => /]*\/?>/i.test(str); + +/** + * Count the number of `` tags in a given string. + * + * @param str - Input string to search for `` tags. + * @returns `` count found in the input string. + */ +const getLiveExampleCount = (str: string) => + (str.match(/]*\/?>/gi) || []).length; + +/** + * Detect empty code fences with external file references that weren't + * inlined. (e.g., ```ts file = "./ButtonBasic.tsx" \n```) + * + * Considered empty if: + * - A fenced code block with a `file` attribute is specified but no content. + * - A fenced code block with no content inside the block, regardless of attributes or language. + * + * @param str - Input string. + * @returns Returns `true` if the input string contains an empty code fence. + */ +const hasEmptyFileCodeFence = (str: string) => + /```[\w-]*\s+file="[^"]+"\s*\n\s*```/i.test(str) || + /```[\w-]*\s*\n\s*```/.test(str); + +/** + * Calculate a quality score for a PatternFly API response. + * + * @param content - Content to score. + * @param [options] - Function options + * @param [options.baseScore] - Base starting score. + * @param [options.kind] - Used to determine which quality metrics are applied. + * @param [options.qualityReduction] - Amount to reduce the base score for each quality metric. + * @param [options.minCharacters] - Minimum number of characters required to avoid quality reduction. + * @returns The calculated quality score. + */ +const calculateContentQualityScore = ( + content: unknown, + { + baseScore = 1, kind, qualityReduction = 0.03, minCharacters = 150 + }: { baseScore?: number; kind?: undefined | string; qualityReduction?: number; minCharacters?: number } = {} +): number => { + if (content === undefined || content === null) { + return baseScore; + } + + const raw = typeof content === 'number' ? String(content) : content; + + if (typeof raw !== 'string') { + return baseScore; + } + + const trimmed = raw.trim(); + + if (trimmed.length === 0) { + return baseScore; + } + + if (kind === 'examples') { + return baseScore; + } + + let score = baseScore; + + if (isJsonLike(trimmed)) { + const jsonValid = isJson(trimmed); + + if (!jsonValid) { + score -= qualityReduction; + } + } + + if (isRawImport(trimmed)) { + score -= qualityReduction; + } + + if (hasLiveExample(trimmed)) { + score -= qualityReduction * getLiveExampleCount(trimmed); + } + + if (trimmed.length < minCharacters && !trimmed.includes('```') && !hasEmptyFileCodeFence(trimmed)) { + score -= qualityReduction; + } + + if (hasEmptyFileCodeFence(trimmed)) { + score -= qualityReduction; + + if (trimmed.length < minCharacters) { + score -= qualityReduction; + } + } + + return Number(Math.min(1, Math.max(0, score)).toFixed(3)); +}; + +/** + * Transform a string. + * + * @param segment - Input string to normalize. + * @returns Normalized slug. + */ +const normalizeSlug = (segment: string): string => { + let updatedSegment = segment; + + if (/[A-Z]/.test(updatedSegment) && !(/^(ai|css|html|mcp|cli|uxd|ui|api|faq|faqs|aria|rtl)$/i.test(updatedSegment))) { + const split = updatedSegment.split(/(?=[A-Z])/); + + if (split.every(val => /^[A-Z]/.test(val))) { + updatedSegment = split.join('-'); + } + } + + return updatedSegment + .trim() + .toLowerCase() + .replace(/_/g, '-') + .replace(/-+/g, '-'); +}; + +/** + * Format a compound slug into a clean title. + * E.g., 'ai-assisted-development_ai-assisted-code-migration' -> 'AI Assisted Development: AI Assisted Code Migration' + * + * @param slug + * @param section + */ +const formatSlugToTitle = (slug: string, section?: string): string => { + if (!slug) { + return 'PatternFly API'; + } + + const acronyms = ['ai', 'css', 'html', 'mcp', 'cli', 'uxd', 'ui', 'api', 'faq', 'faqs', 'aria', 'rtl']; + const acronymRegex = new RegExp(`^(${acronyms.join('|')})$`, 'i'); + + const cleanSection = section + ? section + .split('-') + .map(wordPhrase => + (acronymRegex.test(wordPhrase) + ? wordPhrase.toUpperCase() + : wordPhrase.charAt(0).toUpperCase() + wordPhrase.slice(1))).join(' ') + : ''; + + // Handle bare generic names like 'overview' + if (slug.toLowerCase() === 'overview' && cleanSection) { + return `${cleanSection} Overview`; + } + + return slug + .split('_') + .map(segment => + segment + .split('-') + .map(word => { + if (acronymRegex.test(word)) { + return word.toUpperCase(); + } + + return word.charAt(0).toUpperCase() + word.slice(1); + }) + .join(' ')) + .join(': '); +}; + +/** + * Generate a display name from metadata. + * + * @param [content] - Optional content string. + * @param [context] - Optional context object for generating a unique name. + * @param [context.slug] - Optional slug used for fallback or secondary formatting of the display name. + * @param [context.kind] - Optional kind of content being processed (e.g., 'props', 'css', or 'doc'). + * @param [context.section] - Optional section name used for refining the display name. + * @returns Extracted or formatted display name for the API item. + */ +const extractApiDisplayName = (content?: string, context: { slug?: string; kind?: string; section?: string; } = {}): string => { + const { slug = '', kind = 'doc', section } = context || {}; + + const trimmed = content?.trim() || ''; + + // Props JSON signature + if (kind === 'props' && trimmed.startsWith('{')) { + try { + const parsed = JSON.parse(trimmed); + + if (parsed.name) { + return parsed.name; + } + } catch {} + } + + // CSS JSON Array signature + if (kind === 'css') { + return slug.includes('CSS') ? formatSlugToTitle(slug, section) : `${formatSlugToTitle(slug, section)} CSS`; + } + + // Markdown H1 signature (# Title) + const h1Match = trimmed.match(/^#\s+([^\r\n]+)/m); + + if (h1Match?.[1]?.trim()) { + const title = h1Match[1].trim(); + + // If the H1 is just "Overview", qualify it with the section + if (title.toLowerCase() === 'overview' && section) { + return formatSlugToTitle('overview', section); + } + + return title; + } + + // Fallback to slug + return formatSlugToTitle(slug, section); +}; + +/** + * Provide a fallback description based on kind/category when no prose is available. + * + * @param displayName - Display name + * @param kind - Category / facet kind + */ +const getApiFallbackDescription = (displayName = '', kind = 'doc'): string => { + switch (kind) { + case 'props': + return `PatternFly React component props and TypeScript interfaces for ${displayName}.`; + case 'css': + return `PatternFly ${ + displayName.toLowerCase().includes('css') ? '' : 'CSS '}variables and tokens for ${displayName}.`; + case 'html': + case 'html-demos': + return `PatternFly HTML examples and markup structure for ${displayName}.`; + case 'react': + case 'react-demos': + return `PatternFly React component examples and demos for ${displayName}.`; + case 'examples': + return `PatternFly ${displayName} examples and demos.`; + default: + return `PatternFly documentation and guidelines for ${displayName}.`; + } +}; + +/** + * Generate a description from metadata. + * + * @param [content] - Optional content. + * @param [context] - Optional context for generating a unique description. + * @param [context.displayName] - Display name. + * @param [context.kind] - Type of content. + * @param [context.detailType] - Alternate to `context.kind`, like "examples". + * @returns A generated description from metadata, or a fallback. + */ +const extractApiDescription = ( + content?: string, + context: { displayName?: string; kind?: string; detailType?: string | undefined } = {} +): string => { + const { displayName = '', kind = 'doc', detailType = '' } = context || {}; + + if (kind === 'props' || kind === 'css') { + return getApiFallbackDescription(displayName, kind); + } + + if (detailType === 'examples') { + return getApiFallbackDescription(displayName, detailType); + } + + if (content) { + // Replace import statements, multiline code blocks + const cleanContent = content + .replace(/import\s+[\s\S]*?from\s+['"][^'"]+['"];?/gm, '') + .replace(/import\s+['"][^'"]+['"];?/gm, '') + .replace(/```[\s\S]*?```/gm, ''); + + // Filter headings, tags, and common HTML attributes + const lines = cleanContent + .split('\n') + .map(line => line.trim()) + .filter(line => + line && + !line.startsWith('import ') && + !line.startsWith('#') && + !line.startsWith('---') && + !line.startsWith('![') && + !line.startsWith('<') && + !line.startsWith('```') && + !line.startsWith('export ') && + !line.startsWith('|') && + !line.startsWith('class=') && + !line.startsWith('className=') && + !line.startsWith('style=') && + !line.startsWith('d="') && + !line.startsWith('viewBox=') && + !/^[A-Za-z]+="(.*)"/.test(line) && + !/^(ts|tsx|js|jsx|html)\s+/i.test(line) && + !line.includes('file="./') && + !line.startsWith('["') && + !line.endsWith(',') && + !/^[A-Za-z0-9]+\./.test(line) && + !/^[A-Z][A-Za-z0-9]+,$/.test(line) && + line.length > 20); + + // Finally, does the copy exist? + if (lines.length > 0 && lines[0]) { + let cleanPara = lines[0] + // Convert HTML links to their inner text + .replace(/]*>(.*?)<\/a>/gi, '$1') + // Remove closing HTML stags + .replace(/<\/[A-Za-z0-9_-]+>/g, '') + // Convert bare tags + .replace(/<([A-Za-z0-9_\s-]+)>/g, '$1') + // Remove remaining complex HTML tags with attributes + .replace(/<[A-Za-z0-9_-]+\b[^>]*\/?>/g, '') + // Replace Markdown inline images: `![alt](url) -> alt` + .replace(/!\[([^\]]*)\]\([^)]+\)/g, '$1') + // Replace Markdown links: `[text](url) -> text` + .replace(/\[([^\]]+)\]\([^)]+\)/g, '$1') + // Replace Markdown reference links: `[text][ref] -> text` + .replace(/\[([^\]]+)\]\[[^\]]*\]/g, '$1') + // Remove Markdown formatting characters (bold, italics, inline code, strikethrough) + .replace(/[*_`~]/g, '') + // Normalize excess whitespace + .replace(/\s+/g, ' ') + .trim(); + + if (cleanPara.endsWith(':')) { + cleanPara = `${cleanPara.slice(0, -1)}.`; + } + + return cleanPara.length > 200 ? `${cleanPara.slice(0, 197)}...` : cleanPara; + } + } + + // Fallback + return getApiFallbackDescription(displayName, kind); +}; + +/** + * Extracts and constructs an API entry name based on the provided item and section. + * + * @param item - Entry base name. + * @param section - Entry section. + * @returns Extracted entry name + */ +const extractApiName = (item: string, section: string): string => { + const normalizedItem = item.trim().toLowerCase(); + const normalizedSection = section.trim().toLowerCase(); + + if (normalizedSection === 'components') { + return normalizedItem; + } + + if (normalizedItem === 'overview') { + return `${normalizedSection}-overview`; + } + + // Prevent double-prefix + if (normalizedItem.startsWith(`${normalizedSection}-`)) { + return normalizedItem; + } + + return `${normalizedSection}-${normalizedItem}`; +}; + +export { + calculateContentQualityScore, + extractApiDescription, + extractApiDisplayName, + extractApiName, + formatSlugToTitle, + getApiFallbackDescription, + getLiveExampleCount, + hasEmptyFileCodeFence, + hasLiveExample, + isRawImport, + normalizeSlug +}; diff --git a/src/collections.ts b/src/collections.ts index 78d3132e..f22e5856 100644 --- a/src/collections.ts +++ b/src/collections.ts @@ -64,7 +64,12 @@ type McpCollection = [ handler: (arg?: unknown) => McpCollectionResult | Promise, _config?: { runParallel?: `#${string}`; - runSchedule?: { cancelMs?: number, intervalMs?: number }; + runSchedule?: { + continueOnError?: boolean; + cancelMs?: number; + intervalMs?: number; + repeat?: number + }; // priority?: number; isRequired?: boolean; // group?: string; diff --git a/src/options.defaults.ts b/src/options.defaults.ts index d13f5e21..dc6f53cb 100644 --- a/src/options.defaults.ts +++ b/src/options.defaults.ts @@ -181,9 +181,12 @@ interface ModeOptions { * @property api PatternFly API. * @property api.base URL starting base for crawling the PatternFly API. * @property api.versions URL Get the available PatternFly API versions. Versions are required to crawl. - * @property api.componentPaths List of additional PatternFly API component paths to try. - * @property api.crawlCancelMs Timeout in milliseconds for cancelling the PatternFly API crawl. - * @property api.crawlIntervalMs Interval in milliseconds, during server run, for crawling the PatternFly API. + * @property api.componentPaths List of additional PatternFly API component paths to try and terminate with expected content. + * @property api.traversalPaths List of additional PatternFly API traversal paths to iteratively try. + * @property api.schedule Schedule for crawling the PatternFly API. See {@link McpCollection} config for details. + * @property api.schedule.continueOnError Continue crawling the PatternFly API on error. + * @property api.schedule.intervalMs Interval in milliseconds, during server run, for crawling the PatternFly API. + * @property api.schedule.repeat Number of times to repeat crawling the PatternFly API. * @property availableResourceVersions List of available PatternFly resource versions to the MCP server. * @property availableSearchVersions List of available PatternFly search versions to the MCP server. * @property availableSchemasVersions List of available PatternFly schema versions to the MCP server. @@ -201,9 +204,13 @@ interface PatternFlyOptions { base: string; versions: string; componentPaths: string[]; - crawlCancelMs: number; - crawlIntervalMs: number; + traversalPaths: string[]; enabled: boolean; + schedule: { + continueOnError: boolean; + intervalMs: number; + repeat: number; + } }, availableResourceVersions: ('6.0.0')[]; availableSearchVersions: ('current' | 'latest' | 'v6')[]; @@ -517,10 +524,15 @@ const PATTERNFLY_OPTIONS: PatternFlyOptions = { 'props', 'css' ], - crawlCancelMs: 180_000, // 3 minutes - crawlIntervalMs: 43_200_000, // 12 hours + traversalPaths: [ + 'examples' + ], + schedule: { + continueOnError: true, + intervalMs: 86_400_000 * 7, // 7 days + repeat: Infinity + }, enabled: false - // concurrency: 4 }, availableResourceVersions: ['6.0.0'], availableSearchVersions: ['current', 'latest', 'v6'], diff --git a/src/options.registry.ts b/src/options.registry.ts index cb87218e..c5103329 100644 --- a/src/options.registry.ts +++ b/src/options.registry.ts @@ -11,6 +11,7 @@ import { patternFlySchemasIndexResource } from './resource.patternFlySchemasInde import { patternFlySchemasTemplateResource } from './resource.patternFlySchemasTemplate'; import { patternFlyDocsCollection } from './collection.patternFlyDocs'; import { patternFlySchemasCollection } from './collection.patternFlySchemas'; +import { patternFlyApiCollection } from './collection.patternFlyApi'; /** * Built-in tools. @@ -44,7 +45,8 @@ const builtinResources: McpResourceCreator[] = [ */ const builtinCollections: McpCollectionCreator[] = [ patternFlyDocsCollection, - patternFlySchemasCollection + patternFlySchemasCollection, + patternFlyApiCollection ]; export { builtinCollections, builtinResources, builtinTools }; diff --git a/src/patternFly.getResources.ts b/src/patternFly.getResources.ts index 0444ad78..de55c40c 100644 --- a/src/patternFly.getResources.ts +++ b/src/patternFly.getResources.ts @@ -1,3 +1,5 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; import { getComponentSchema } from '@patternfly/patternfly-component-schemas/json'; import { memo } from './server.caching'; import { buildSearchString, generateHash } from './server.helpers'; @@ -201,6 +203,48 @@ interface PatternFlyMcpAvailableResources extends PatternFlyVersionContext { */ const patternFlyRecordsRegistry = new Map(); +// TODO: remove this when complete with clean-up +// --- Quick Dump Helper --- +const dumpCollectionsToDisk = ( + merged: unknown, + apiCollection: unknown +) => { + try { + const outputDir = path.resolve(process.cwd(), '.dump'); + + if (!fs.existsSync(outputDir)) { + fs.mkdirSync(outputDir, { recursive: true }); + } + + // 2. API collection + fs.writeFileSync( + path.join(outputDir, 'patternfly-api-collection.dump.json'), + JSON.stringify( + { + 'patternfly-api': apiCollection ?? null + }, + null, + 2 + ), + 'utf-8' + ); + + fs.writeFileSync( + path.join(outputDir, 'patternfly-merged.dump.json'), + JSON.stringify( + merged, + null, + 2 + ), + 'utf-8' + ); + + log.info(`[DUMP] Wrote collection dumps to ${outputDir}`); + } catch (err) { + log.error('[DUMP] Failed to write collection dumps:', err); + } +}; + /** * Set the category display label based on the entry's section and category. * @@ -462,6 +506,20 @@ const mutateKeyWordsMap = ( mutateMap(normalizedKeyword); }; +/** + * Normalizes collection resource names into a uniform slug. + * + * @param key - Raw catalog or collection identifier + * @returns Normalized slug for current resource grouping strategy. + */ +const normalizeKey = (key: string): string => { + if (!key) { + return '__unknown__'; + } + + return key.replace(/[^a-z0-9]/gi, '').toLowerCase(); +}; + /** * Get a multifaceted resources breakdown from PatternFly. * @@ -478,10 +536,12 @@ const getPatternFlyMcpResources = async (contextPathOverride?: string): Promise< const { componentNamesIndex, byVersion: componentNamesByVersion, byDocs: componentNamesByDocs } = componentNames; const originalDocs = patternFlyRecordsRegistry.get('patternfly-docs'); + const apiCollection = patternFlyRecordsRegistry.get('patternfly-api'); const catalog = [ ...originalDocs?.records?.flatMap(({ data }) => Object.entries(data as Record)) || [], - ...Array.from(componentNamesByDocs) + ...Array.from(componentNamesByDocs), + ...apiCollection?.records?.flatMap(({ data }) => Object.entries(data as Record)) || [] ]; const resources = new Map(); @@ -494,7 +554,7 @@ const getPatternFlyMcpResources = async (contextPathOverride?: string): Promise< const rawKeywordsMap: PatternFlyMcpKeywordsMap = new Map(); catalog.forEach(([unifiedName, entries]) => { - const name = unifiedName.toLowerCase(); + const name = normalizeKey(unifiedName); const groupId = generateHash(name); hashIndexMap.set(groupId.toLowerCase(), name); @@ -614,7 +674,7 @@ const getPatternFlyMcpResources = async (contextPathOverride?: string): Promise< const filteredKeywords = filterKeywords(rawKeywordsMap); - return { + const output = { ...versionContext, resources, // @deprecated docsIndex - Under review @@ -636,6 +696,14 @@ const getPatternFlyMcpResources = async (contextPathOverride?: string): Promise< byVersion, byVersionComponentNames: componentNamesByVersion }; + + // TODO: remove this when complete with clean-up + dumpCollectionsToDisk( + { byPath, uriIndex: Object.fromEntries(uriIndexMap) }, + apiCollection?.records?.flatMap(({ data }) => Object.entries(data as Record)) || [] + ); + + return output; }; /** @@ -702,6 +770,8 @@ const setPatternFlyCollection = async ( log.warn('Failed getPatternFlyMcpResources clear.', error); } + // TODO: remove this when dump helper is removed. Intended to force initialize the collection records instead of relying on consuming functionality to rebuild. + getPatternFlyMcpResources.memo(); log.debug(`Merging collection ${name} records. (${collection.records.length})`); } } catch (error) { diff --git a/src/server.collections.ts b/src/server.collections.ts index 31a5015f..cddf0370 100644 --- a/src/server.collections.ts +++ b/src/server.collections.ts @@ -42,7 +42,7 @@ options: GlobalOptions = getOptions()): McpCollectionCreator => () => { * Proxy a collection creator with a deferred task wrapper. * * @param {McpCollectionCreator} creator - Original creator. - * @param {CollectionRunSchedule} runSchedule - Schedule config sourced from the collection's + * @param {NonNullable['runSchedule']} runSchedule - Schedule config sourced from the collection's * `_config.runSchedule`. Provides `cancelMs` and `intervalMs` used to build {@link deferTask}. * @param {GlobalOptions} options - Global options. * @returns {McpCollectionCreator} The proxied creator function.