diff --git a/packages/polymech/src/base/collections.ts b/packages/polymech/src/base/collections.ts new file mode 100644 index 0000000..5a7a6ef --- /dev/null +++ b/packages/polymech/src/base/collections.ts @@ -0,0 +1,467 @@ +import type { CollectionEntry } from 'astro:content'; +import { isFolder } from '@polymech/commons'; +import { parseFrontmatter } from '@astrojs/markdown-remark'; + +// Filter function type +export type CollectionFilter = (entry: CollectionEntry, astroConfig?: any) => boolean; + +// Config interface for collection filters +export interface CollectionFilterConfig { + ENABLE_VALID_FRONTMATTER_CHECK?: boolean; + ENABLE_FOLDER_FILTER?: boolean; + ENABLE_DRAFT_FILTER?: boolean; + ENABLE_TITLE_FILTER?: boolean; + ENABLE_BODY_FILTER?: boolean; + ENABLE_DESCRIPTION_FILTER?: boolean; + ENABLE_IMAGE_FILTER?: boolean; + ENABLE_AUTHOR_FILTER?: boolean; + ENABLE_PUBDATE_FILTER?: boolean; + ENABLE_TAGS_FILTER?: boolean; + ENABLE_FILE_EXTENSION_FILTER?: boolean; + REQUIRED_FIELDS?: string[]; + REQUIRED_TAGS?: string[]; + EXCLUDE_TAGS?: string[]; + FILTER_FUTURE_POSTS?: boolean; + FILTER_OLD_POSTS?: boolean; + OLD_POST_CUTOFF_DAYS?: number; +} + +// Default filters +export const hasValidFrontMatter: CollectionFilter = (entry) => { + // Check if the entry has valid frontmatter + // For MD/MDX files, Astro automatically parses frontmatter + // If data exists and is not empty, consider it valid + if (!entry.data) return false; + + // Check for basic required fields (can be customized) + // At minimum, we expect some data to exist + return typeof entry.data === 'object' && Object.keys(entry.data).length > 0; +}; + +/** + * Advanced frontmatter validation using Astro's parseFrontmatter + * This can be used for more thorough validation of raw markdown content + * @param rawContent - Optional raw markdown content to parse + */ +export const hasValidParsedFrontMatter: CollectionFilter = (entry) => { + try { + // If entry already has parsed data, it's valid + if (entry.data && typeof entry.data === 'object' && Object.keys(entry.data).length > 0) { + return true; + } + + // For entries that might need raw frontmatter parsing + // This is more applicable when working with raw markdown content + // In most Astro cases, entry.data is already parsed + return false; + } catch (error) { + console.warn(`Frontmatter parsing failed for entry ${entry.id}:`, error); + return false; + } +}; + +/** + * Create a filter that validates frontmatter against a schema or validation function + * @param validator - Function that validates the frontmatter data + * @returns Filter function + */ +export function createFrontmatterValidator( + validator: (data: any) => boolean +): CollectionFilter { + return (entry) => { + try { + if (!entry.data) return false; + return validator(entry.data); + } catch (error) { + console.warn(`Frontmatter validation failed for entry ${entry.id}:`, error); + return false; + } + }; +} + +/** + * Create a filter that parses and validates raw markdown content + * Useful for validating files that haven't been processed by Astro yet + * @param rawContentGetter - Function to get raw content for an entry + * @param validator - Optional validator function for the parsed frontmatter + * @returns Filter function + */ +export function createRawFrontmatterValidator( + rawContentGetter: (entry: CollectionEntry) => string, + validator?: (data: any) => boolean +): CollectionFilter { + return (entry) => { + try { + const rawContent = rawContentGetter(entry); + if (!rawContent) return false; + + const parsed = parseFrontmatter(rawContent); + + // Check if frontmatter exists and is valid + if (!parsed.frontmatter || typeof parsed.frontmatter !== 'object') { + return false; + } + + // Apply custom validator if provided + if (validator) { + return validator(parsed.frontmatter); + } + + // Default validation: frontmatter should have at least one property + return Object.keys(parsed.frontmatter).length > 0; + + } catch (error) { + console.warn(`Raw frontmatter parsing failed for entry ${entry.id}:`, error); + return false; + } + }; +} + +/** + * Create a filter that reads and validates frontmatter from the actual file + * Uses the entry's filePath to read the raw file content + * @param validator - Optional validator function for the parsed frontmatter + * @returns Filter function + */ +export function createFileBasedFrontmatterValidator( + validator?: (data: any) => boolean +): CollectionFilter { + return (entry) => { + try { + if (!entry.filePath) { + console.warn(`No filePath available for entry ${entry.id}`); + return false; + } + + // This would require fs import in a Node.js environment + // For now, we'll rely on the entry.data which is already parsed + // In a real implementation, you could use: + // const fs = await import('fs'); + // const rawContent = fs.readFileSync(entry.filePath, 'utf-8'); + // const parsed = parseFrontmatter(rawContent); + + // Fallback to using the already parsed data + if (!entry.data || typeof entry.data !== 'object') { + return false; + } + + if (validator) { + return validator(entry.data); + } + + return Object.keys(entry.data).length > 0; + + } catch (error) { + console.warn(`File-based frontmatter validation failed for entry ${entry.id}:`, error); + return false; + } + }; +} + +export const isNotFolder: CollectionFilter = (entry) => { + // Check if the entry is not a folder + // Use filePath if available, otherwise construct from collection and id + const entryPath = entry.filePath || `src/content/${entry.collection}/${entry.id}`; + return !isFolder(entryPath); +}; + +export const isNotDraft: CollectionFilter = (entry) => { + // Filter out draft entries + return !entry.data?.draft; +}; + +export const hasTitle: CollectionFilter = (entry) => { + // Check if entry has a title and it's not the default "Untitled" + return !!entry.data?.title && + entry.data.title.trim() !== '' && + entry.data.title.trim() !== 'Untitled'; +}; + +export const hasBody: CollectionFilter = (entry) => { + // Check if entry has body content + return !!entry.body && entry.body.trim() !== ''; +}; + +export const hasValidFileExtension: CollectionFilter = (entry) => { + // Check if the entry has a valid markdown/mdx file extension + if (!entry.filePath) return true; // If no filePath, assume it's valid + + const validExtensions = ['.md', '.mdx']; + return validExtensions.some(ext => entry.filePath.endsWith(ext)); +}; + +export const hasImage: CollectionFilter = (entry) => { + // Check if entry has an image defined + return !!(entry.data?.image?.url); +}; + +export const hasDescription: CollectionFilter = (entry) => { + // Check if entry has a non-empty description + return !!entry.data?.description && entry.data.description.trim() !== ''; +}; + +export const hasAuthor: CollectionFilter = (entry) => { + // Check if entry has an author (and it's not the default "Unknown") + return !!entry.data?.author && + entry.data.author.trim() !== '' && + entry.data.author !== 'Unknown'; +}; + +export const hasPubDate: CollectionFilter = (entry) => { + // Check if entry has a valid publication date + if (!entry.data?.pubDate) return false; + + try { + const date = new Date(entry.data.pubDate); + return !isNaN(date.getTime()); + } catch { + return false; + } +}; + +export const hasTags: CollectionFilter = (entry) => { + // Check if entry has tags + return !!(entry.data?.tags && Array.isArray(entry.data.tags) && entry.data.tags.length > 0); +}; + +// Default filters array +export const defaultFilters: CollectionFilter[] = [ + hasValidFrontMatter, + isNotFolder, + isNotDraft, + hasTitle // Include title validation by default to filter out "Untitled" entries +]; + +/** + * Generic filter function for collections + * @param collection - Array of collection entries from getCollection() + * @param filters - Array of filter functions to apply (defaults to defaultFilters) + * @param astroConfig - Optional Astro config object + * @returns Filtered array of collection entries + */ +export function filterCollection( + collection: CollectionEntry[], + filters: CollectionFilter[] = defaultFilters, + astroConfig?: any +): CollectionEntry[] { + return collection.filter(entry => { + // Apply all filters - entry must pass ALL filters to be included + return filters.every(filter => { + try { + return filter(entry, astroConfig); + } catch (error) { + console.warn(`Filter failed for entry ${entry.id}:`, error); + return false; // If filter throws, exclude the entry + } + }); + }); +} + +/** + * Convenience function to create custom filter combinations + * @param baseFilters - Base filters to start with + * @param additionalFilters - Additional filters to add + * @returns Combined filter array + */ +export function combineFilters( + baseFilters: CollectionFilter[] = defaultFilters, + additionalFilters: CollectionFilter[] = [] +): CollectionFilter[] { + return [...baseFilters, ...additionalFilters]; +} + +/** + * Create a filter that checks for specific frontmatter fields + * @param requiredFields - Array of field names that must exist + * @returns Filter function + */ +export function createRequiredFieldsFilter(requiredFields: string[]): CollectionFilter { + return (entry) => { + if (!entry.data) return false; + return requiredFields.every(field => + entry.data[field] !== undefined && entry.data[field] !== null && entry.data[field] !== '' + ); + }; +} + +/** + * Create a filter that checks for specific tags + * @param requiredTags - Array of tags that must be present + * @param matchAll - If true, all tags must be present; if false, at least one tag must be present + * @returns Filter function + */ +export function createTagFilter(requiredTags: string[], matchAll: boolean = false): CollectionFilter { + return (entry) => { + const entryTags = entry.data?.tags || []; + if (!Array.isArray(entryTags)) return false; + + if (matchAll) { + return requiredTags.every(tag => entryTags.includes(tag)); + } else { + return requiredTags.some(tag => entryTags.includes(tag)); + } + }; +} + +/** + * Create a filter based on publication date + * @param beforeDate - Optional date - entries must be published before this date + * @param afterDate - Optional date - entries must be published after this date + * @returns Filter function + */ +export function createDateFilter(beforeDate?: Date, afterDate?: Date): CollectionFilter { + return (entry) => { + const pubDate = entry.data?.pubDate; + if (!pubDate) return false; + + const entryDate = new Date(pubDate); + + if (beforeDate && entryDate > beforeDate) return false; + if (afterDate && entryDate < afterDate) return false; + + return true; + }; +} + +/** + * Create a filter that excludes entries with specific tags + * @param excludeTags - Array of tags to exclude + * @returns Filter function + */ +export function createExcludeTagsFilter(excludeTags: string[]): CollectionFilter { + return (entry) => { + const entryTags = entry.data?.tags || []; + if (!Array.isArray(entryTags)) return true; + + return !excludeTags.some(tag => entryTags.includes(tag)); + }; +} + +/** + * Filter that excludes future posts + */ +export const isNotFuture: CollectionFilter = (entry) => { + const pubDate = entry.data?.pubDate; + if (!pubDate) return true; // If no date, include it + + const entryDate = new Date(pubDate); + const now = new Date(); + + return entryDate <= now; +}; + +/** + * Create a filter that excludes old posts based on cutoff days + * @param cutoffDays - Number of days to consider a post "old" + * @returns Filter function + */ +export function createOldPostFilter(cutoffDays: number): CollectionFilter { + return (entry) => { + const pubDate = entry.data?.pubDate; + if (!pubDate) return true; // If no date, include it + + const entryDate = new Date(pubDate); + const cutoffDate = new Date(); + cutoffDate.setDate(cutoffDate.getDate() - cutoffDays); + + return entryDate > cutoffDate; + }; +} + +/** + * Build filters array based on configuration + * @param config - Collection filter configuration + * @returns Array of filter functions + */ +export function buildFiltersFromConfig(config: CollectionFilterConfig): CollectionFilter[] { + const filters: CollectionFilter[] = []; + + // Add default filters based on config + if (config.ENABLE_VALID_FRONTMATTER_CHECK !== false) { + filters.push(hasValidFrontMatter); + } + + if (config.ENABLE_FOLDER_FILTER !== false) { + filters.push(isNotFolder); + } + + if (config.ENABLE_DRAFT_FILTER !== false) { + filters.push(isNotDraft); + } + + if (config.ENABLE_TITLE_FILTER) { + filters.push(hasTitle); + } + + // Add content validation filters + if (config.ENABLE_BODY_FILTER) { + filters.push(hasBody); + } + + if (config.ENABLE_DESCRIPTION_FILTER) { + filters.push(hasDescription); + } + + if (config.ENABLE_IMAGE_FILTER) { + filters.push(hasImage); + } + + if (config.ENABLE_AUTHOR_FILTER) { + filters.push(hasAuthor); + } + + if (config.ENABLE_PUBDATE_FILTER) { + filters.push(hasPubDate); + } + + if (config.ENABLE_TAGS_FILTER) { + filters.push(hasTags); + } + + if (config.ENABLE_FILE_EXTENSION_FILTER !== false) { + filters.push(hasValidFileExtension); + } + + // Add required fields filter + if (config.REQUIRED_FIELDS && config.REQUIRED_FIELDS.length > 0) { + filters.push(createRequiredFieldsFilter(config.REQUIRED_FIELDS)); + } + + // Add required tags filter + if (config.REQUIRED_TAGS && config.REQUIRED_TAGS.length > 0) { + filters.push(createTagFilter(config.REQUIRED_TAGS, true)); // matchAll = true + } + + // Add exclude tags filter + if (config.EXCLUDE_TAGS && config.EXCLUDE_TAGS.length > 0) { + filters.push(createExcludeTagsFilter(config.EXCLUDE_TAGS)); + } + + // Add future posts filter + if (config.FILTER_FUTURE_POSTS) { + filters.push(isNotFuture); + } + + // Add old posts filter + if (config.FILTER_OLD_POSTS && config.OLD_POST_CUTOFF_DAYS) { + filters.push(createOldPostFilter(config.OLD_POST_CUTOFF_DAYS)); + } + + return filters; +} + +/** + * Convenience function that applies filters based on config + * @param collection - Array of collection entries + * @param config - Collection filter configuration + * @param astroConfig - Optional Astro config + * @returns Filtered collection entries + */ +export function filterCollectionWithConfig( + collection: CollectionEntry[], + config: CollectionFilterConfig, + astroConfig?: any +): CollectionEntry[] { + const filters = buildFiltersFromConfig(config); + return filterCollection(collection, filters, astroConfig); +} diff --git a/packages/polymech/src/base/kbot.ts b/packages/polymech/src/base/kbot.ts index a53afb2..716c230 100644 --- a/packages/polymech/src/base/kbot.ts +++ b/packages/polymech/src/base/kbot.ts @@ -1,10 +1,10 @@ import { get_cached_object, set_cached_object, rm_cached_object } from "@polymech/cache" import { run, OptionsSchema } from "@polymech/kbot-d"; import { resolveVariables } from "@polymech/commons/variables" -import { } from "@polymech/core/objects" import { logger, env } from "./index.js" -import { removeEmptyObjects } from "@/base/objects.js" -import { LLM_CACHE } from "@/config/config.js" +import { removeEmptyObjects } from "./objects.js" + +const LLM_CACHE = true import { TemplateProps, diff --git a/packages/polymech/src/components/MasonryGallery.astro b/packages/polymech/src/components/MasonryGallery.astro index d16469a..26646c6 100644 --- a/packages/polymech/src/components/MasonryGallery.astro +++ b/packages/polymech/src/components/MasonryGallery.astro @@ -46,7 +46,7 @@ export interface Props { const { images = [], glob: globPattern, - maxItems = 50, + maxItems = 150, maxWidth = "300px", maxHeight = "400px", entryPath, @@ -406,42 +406,37 @@ if (groupingFunction) { @touchcancel="isSwiping = false;" > {finalImages.map((image, index) => ( -