polymech/site-library
1
0
Fork 0
generated from polymech/site-template
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

latest

Browse Source
This commit is contained in:
lovebird
2025-04-01 18:33:46 +02:00
parent 55dae7814d
commit 1833ce2c2e
13 changed files with 78 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/base/kbot-templates.ts
+35 -160
View File
@@ -2,6 +2,7 @@ import { IKBotTask } from "@polymech/kbot-d";
import { sync as read } from "@polymech/fs/read";
import { sync as exists } from "@polymech/fs/exists";
import { z } from "zod";
import { logger } from "./index.js";
// Type definitions
const InstructionSchema = z.object({
@@ -18,7 +19,7 @@ export interface TemplateProps extends IKBotTask {
disabled?: boolean;
template?: string;
renderer?: string;
}
const TemplateConfigSchema = z.object({
@@ -28,14 +29,13 @@ const TemplateConfigSchema = z.object({
preferences: z.string(),
mode: z.string(),
filters: z.string().optional(),
overrides: z.record(z.array(z.string())).optional(),
variables: z.record(z.string()).optional()
});
type TemplateConfig = z.infer<typeof TemplateConfigSchema>;
const LLMConfigSchema = z.object({
templates: z.record(TemplateConfigSchema),
options: z.record(TemplateConfigSchema),
instructions: InstructionSetSchema.optional(),
defaults: z.record(z.array(z.string())).optional()
});
@@ -50,120 +50,64 @@ export const enum TemplateContext {
}
// Default configuration
export const DEFAULT_CONFIG: LLMConfig = {
templates: {},
options: {},
instructions: {},
defaults: {
tone: ["formal"],
content: ["spellCheck", "removeEmojis", "removePersonalPrefs", "shorten"],
moderation: ["mafiaFilter", "deprogramming"],
context: ["makerTutorials", "units"],
format: ["markdown"]
}
defaults: {}
};
// Default template options
const DEFAULT_TEMPLATE_OPTIONS = {
mode: "completion",
preferences: "none",
filters: "code"
} as const;
const getConfigPath = (context: TemplateContext): string => {
return `./src/config/templates/${context}.json`;
};
// Load configuration from JSON file
export const load = (context: TemplateContext = TemplateContext.COMMONS): LLMConfig => {
const configPath = `./src/config/templates/${context}.json`;
const configPath = getConfigPath(context);
if (exists(configPath)) {
try {
const content = read(configPath);
if (typeof content === 'string') {
const parsed = JSON.parse(content);
return LLMConfigSchema.parse(parsed);
}
const content = read(configPath, 'json') || {};
return LLMConfigSchema.parse(content)
} catch (error) {
console.error(`Error loading ${context} config:`, error);
logger.error(`Error loading ${context} config:`, error);
}
} else {
logger.error(`Config file ${configPath} not found`);
}
return DEFAULT_CONFIG;
};
// Helper function to build prompt from instruction sets
export const buildPrompt = (
instructions: z.infer<typeof InstructionSetSchema>,
defaults: Record<string, string[]>,
overrides?: Record<string, string[]>
defaults: Record<string, string[]>
): string => {
const getInstructions = (category: string, flags: string[]) => {
const set = instructions[category] || [];
return set.filter(x => flags.includes(x.flag)).map(x => x.text);
};
const merged = Object.keys(instructions).reduce((acc, category) => ({
...acc,
[category]: (overrides?.[category] ?? defaults[category] ?? [])
[category]: defaults[category] ?? []
}), {} as Record<string, string[]>);
return Object.entries(merged)
.flatMap(([category, flags]) => getInstructions(category, flags))
.join("\n");
};
// Prompt system
const PromptSchema = z.object({
template: z.string(),
variables: z.record(z.string()).optional(),
format: z.enum(['text', 'json', 'markdown']).default('text')
format: z.enum(['text', 'json', 'markdown', 'schema']).default('text')
});
type Prompt = z.infer<typeof PromptSchema>;
const PromptRegistrySchema = z.record(PromptSchema);
type PromptRegistry = z.infer<typeof PromptRegistrySchema>;
const DEFAULT_PROMPTS: PromptRegistry = {
seo: {
template: "Analyze the following content and return a JSON object with these fields: keywords (array of max 10 strings), title (string), description (string), tags (array of max 5 strings). Format as valid JSON only.",
format: 'json'
},
keywords: {
template: "Return a list of max. 10 keywords that can be used for SEO purposes, separated by commas (dont comment, just the list) : ",
format: 'text'
},
references: {
template: "Return a list of references (only with links), as Markdown, grouped : Articles, Books, Papers, Youtube, Opensource Designs, ... Dont comment !",
format: 'markdown'
},
brief: {
template: "Create a concise description for SEO meta (around 150-160 characters) from the text below. \n Disregard any links or image references. \n Return only the final meta description, no extra commentary.",
format: 'text'
},
toolsAndHardware: {
template: "Extract the required tools, software hardware from the following tutorial. Return as Markdown chapters (H3) with very short bullet points (not bold), with links, max. 5. per category.",
format: 'markdown'
},
learnedSkills: {
template: "Analyze the following tutorial and identify all the skills that a person would learn or improve by completing this project, modest, humble, simple - dont enflate (sustainable, recylcing, ...), Return as Markdown chapter (H2) with very short bullet points (not bold), max. 5.",
format: 'markdown'
},
local: {
template: "Markdown chapter (h4) with a list of local resources, services & suppliers, max. 5, with links, group by category. dont comment, just the list",
format: 'markdown'
}
}
const renderPrompt = (prompt: Prompt, variables: Record<string, string> = {}): string => {
let result = prompt.template;
for (const [key, value] of Object.entries(variables)) {
result = result.replace(new RegExp(`\\{${key}\\}`, 'g'), value);
}
return result;
}
const createTemplate = (config: LLMConfig, name: string, defaults: Partial<TemplateConfig>, promptKey?: keyof PromptRegistry) => {
const createTemplate = (config: LLMConfig, name: string, defaults: Partial<TemplateConfig>) => {
return (opts: Partial<TemplateConfig> = {}) => {
const template = config.templates[name] || defaults;
const prompt = promptKey ? renderPrompt(DEFAULT_PROMPTS[promptKey], opts.variables) : buildPrompt(
const template = config.options[name] || defaults;
const prompt = buildPrompt(
config.instructions || {},
config.defaults || {},
opts.overrides || template.overrides
)
config.defaults || {}
);
const merged = {
...template,
...opts,
@@ -175,86 +119,17 @@ const createTemplate = (config: LLMConfig, name: string, defaults: Partial<Templ
export const createTemplates = (context: TemplateContext = TemplateContext.COMMONS) => {
const config = load(context);
switch (context) {
case TemplateContext.COMMONS:
return {
seo: createTemplate(config, 'seo', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}, 'seo'),
simple: createTemplate(config, 'simple', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}),
codeSimple: createTemplate(config, 'codeSimple', {
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}),
research: createTemplate(config, 'research', {
router: "openai",
model: "gpt-4.5-preview",
...DEFAULT_TEMPLATE_OPTIONS
})
};
case TemplateContext.HOWTO:
return {
keywords: createTemplate(config, 'keywords', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}, 'keywords'),
brief: createTemplate(config, 'keywords', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}, 'brief'),
references: createTemplate(config, 'references', {
model: "perplexity/sonar-deep-research",
...DEFAULT_TEMPLATE_OPTIONS
}, 'references'),
toolsAndHardware: createTemplate(config, 'toolsAndHardware', {
model: "perplexity/sonar-deep-research",
...DEFAULT_TEMPLATE_OPTIONS
}, 'toolsAndHardware'),
requiredSkills: createTemplate(config, 'requiredSkills', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}),
learnedSkills: createTemplate(config, 'learnedSkills', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
}, 'learnedSkills'),
local: createTemplate(config, 'local', {
model: "perplexity/sonar-deep-research",
...DEFAULT_TEMPLATE_OPTIONS
}, 'local')
};
case TemplateContext.MARKETPLACE:
return {
productDescription: createTemplate(config, 'productDescription', {
router: "openai",
model: "gpt-4o",
...DEFAULT_TEMPLATE_OPTIONS
})
};
default:
return {};
}
return Object.keys(config.options).reduce((acc, name) => ({
...acc,
[name]: createTemplate(config, name, {})
}), {});
};
// Export default templates with default config (commons context)
export const templates = createTemplates(TemplateContext.COMMONS);
// Export types
export type { TemplateConfig, LLMConfig, Prompt, PromptRegistry }
export { InstructionSchema,
InstructionSetSchema,
TemplateConfigSchema,
LLMConfigSchema,
PromptSchema,
PromptRegistrySchema };
export {
InstructionSchema,
InstructionSetSchema,
TemplateConfigSchema,
LLMConfigSchema,
PromptSchema,
PromptRegistrySchema
};
src/base/kbot.ts
+10 -17
View File
@@ -2,25 +2,20 @@ import { get_cached_object, set_cached_object } from "@polymech/cache"
import { run, OptionsSchema } from "@polymech/kbot-d";
import { resolveVariables } from "@polymech/commons/variables"
import { } from "@polymech/core/objects"
import { logger } from "./index.js"
const CACHE = true
import { logger, env } from "./index.js"
import { removeEmptyObjects } from "@/base/objects.js"
import { LLM_CACHE } from "@/config/config.js"
import {
TemplateProps,
TemplateContext,
createTemplates
} from "./kbot-templates.js";
import { env } from "./index.js";
export interface Props extends TemplateProps {
context?: TemplateContext;
}
const removeEmpty = (obj: any) => {
return Object.fromEntries(
Object.entries(obj).filter(([_, v]) => v != null && v !== '')
);
};
export const filter = async (content: string, tpl: string = 'howto', opts: Props = {}) => {
if (!content || content.length < 20) {
return content;
@@ -40,20 +35,20 @@ export const filter = async (content: string, tpl: string = 'howto', opts: Props
filters: [],
tools: []
};
const ca_options = JSON.parse(JSON.stringify(removeEmpty(cache_key_obj)));
const ca_options = JSON.parse(JSON.stringify(removeEmptyObjects(cache_key_obj)));
let cached = null
try {
cached = await get_cached_object({ ca_options }, 'kbot');
} catch (e) {
logger.error(`Failed to get cached object for ${content.substring(0, 20)}`, e);
}
if (cached && CACHE) {
if (cached && LLM_CACHE) {
return cached;
}
logger.info(`kbot: template:${tpl} : context:${context} @ ${options.model} : ${content.substring(0, 20)}`)
const result = await run(options);
if(!result || !result[0]){
logger.error(`No result for ${content.substring(0, 20)}`)
if (!result || !result[0]) {
logger.error(`No result for ${content.substring(0, 20)}`)
return content;
}
if (template.format === 'json') {
@@ -70,9 +65,8 @@ export const filter = async (content: string, tpl: string = 'howto', opts: Props
logger.debug(`caching result for "${content.substring(0, 20)}..."`)
return result[0] as string;
};
export const template_filter = async (text: string, template: string, context: TemplateContext = TemplateContext.COMMONS) => {
if(!text || text.length < 20) {
if (!text || text.length < 20) {
return text;
}
const templates = createTemplates(context);
@@ -96,11 +90,10 @@ export const template_filter = async (text: string, template: string, context: T
});
return ret;
};
export const getFilterOptions = (content: string, template: any, opts: Props = {}) => {
return OptionsSchema().parse({
...template,
prompt: `${template.prompt || ""} : ${content}`,
...opts,
});
};
};
src/components/howtos/Detail.astro
+1 -1
View File
@@ -121,7 +121,7 @@ const EditLink = () => {
<div class="flex flex-wrap gap-2 mb-4">
{
howto.tags.map((tag) => (
<span class="bg-orange-500 text-white text-xs px-3 py-1 rounded-full">
<span class="bg-orange-400 text-white text-xs px-3 py-1 rounded-full">
<Translate>{tag.toUpperCase()}</Translate>
</span>
))
src/config/config.ts
+6 -2
View File
@@ -4,6 +4,10 @@ import { resolve, template } from '@polymech/commons'
import { sync as read } from '@polymech/fs/read'
import { sanitizeUri } from 'micromark-util-sanitize-uri'
// LLM
export const LLM_CACHE = false
export const OSR_ROOT = () => path.resolve(resolve("${OSR_ROOT}"))
export const LOGGING_NAMESPACE = 'polymech-site'
@@ -33,7 +37,7 @@ export const HOWTO_ADD_REFERENCES = true
export const HOWTO_COMPLETE_SKILLS = false
export const HOWTO_LOCAL_RESOURCES = false
export const HOWTO_SEO_LLM = true
export const HOWTO_MAX_ITEMS = 10
export const HOWTO_MAX_ITEMS = 1
export const HOWTO_MIGRATION = () => path.resolve(resolve("./data/last.json"))
export const HOWTO_ROOT_INTERN = () => path.resolve(resolve("./public/resources/howtos"))
@@ -56,7 +60,7 @@ export const DIRECTORY_ADD_REFERENCES = true
export const DIRECTORY_COMPLETE_SKILLS = false
export const DIRECTORY_LOCAL_RESOURCES = false
export const DIRECTORY_SEO_LLM = true
export const DIRECTORY_MAX_ITEMS = 20
export const DIRECTORY_MAX_ITEMS = 0
export const DIRECTORY_MIGRATION = () => path.resolve(resolve("./data/last.json"))
export const DIRECTORY_ROOT_INTERN = () => path.resolve(resolve("./public/resources/directory"))
src/config/templates/commons.json
+2 -7
View File
@@ -1,5 +1,5 @@
{
"templates": {
"options": {
"simple": {
"router": "openai",
"model": "gpt-4o",
@@ -7,11 +7,6 @@
"mode": "completion",
"filters": "code"
},
"codeSimple": {
"model": "gpt-4o",
"preferences": "none",
"mode": "completion"
},
"research": {
"model": "perplexity/sonar-deep-research",
"preferences": "none",
@@ -21,7 +16,7 @@
"instructions": {
"tone": [
{ "flag": "formal", "text": "use a formal tone" },
{ "flag": "friendly", "text": "be friendly and approachable" }
{ "flag": "friendly", "text": "Be friendly and approachable!" }
],
"content": [
{ "flag": "spellCheck", "text": "spell & grammar fix the text," },
src/config/templates/howto.json
+4 -7
View File
@@ -1,5 +1,5 @@
{
"templates": {
"options": {
"keywords": {
"router": "openai",
"model": "gpt-4o",
@@ -25,7 +25,7 @@
"model": "perplexity/sonar-deep-research",
"preferences": "none",
"mode": "completion",
"prompt": "Extract the required tools, software hardware from the following tutorial.Return as Markdown chapters (H3) with very short bullet points (not bold), max. 5 per category.",
"prompt": "Extract the required tools: software, hardware from the following tutorial.Return as Markdown chapters (H3) with very short bullet points (not bold), max. 5 per category.",
"filters": "code"
},
"requiredSkills": {
@@ -59,13 +59,10 @@
{ "flag": "safety", "text": "Include safety considerations and warnings" },
{ "flag": "troubleshooting", "text": "Include common issues and solutions" }
],
"format": [
{ "flag": "stepByStep", "text": "Format as clear step-by-step instructions" },
{ "flag": "checklist", "text": "Include progress checkpoints" }
]
"format": []
},
"defaults": {
"context": ["makerTutorials", "units", "safety", "troubleshooting"],
"format": ["stepByStep", "checklist"]
"format": []
}
}
src/model/howto/howto.ts
+3 -15
View File
@@ -3,14 +3,11 @@ import { findUp } from 'find-up'
import pMap from 'p-map'
import { sanitizeFilename } from "@polymech/fs/utils"
import { execFileSync, execFile } from "child_process";
import { execFile } from "child_process";
import { sync as read } from '@polymech/fs/read'
import { sync as exists } from '@polymech/fs/exists'
import { sync as mkdir } from '@polymech/fs/dir'
import { sync as rm } from '@polymech/fs/remove'
import { sync as write } from '@polymech/fs/write'
import type { Loader, LoaderContext } from 'astro/loaders'
import { resolveVariables } from "@polymech/commons/variables"
export * from './howto-model.js'
export * from '@/base/filters.js'
@@ -27,7 +24,6 @@ import {
HOWTO_FILES_WEB,
HOWTO_FILES_ABS,
HOWTO_FILTER_LLM,
default_image,
HOWTO_ROOT,
HOWTO_GLOB,
@@ -43,11 +39,7 @@ import {
HOWTO_MAX_ITEMS
} from "config/config.js"
import { env, logger } from '@/base/index.js'
import { logger } from '@/base/index.js'
import { applyFilters, default_filters_plain, FilterFunction } from '../../base/filters.js'
import { TemplateContext, buildPrompt, LLMConfig, createTemplates } from '@/base/kbot-templates.js';
import { template_filter } from '@/base/kbot.js'
@@ -57,7 +49,6 @@ import { template_filter } from '@/base/kbot.js'
// Assets
//
/////////////////////////////////////////////////////////////////////////
export const item_path = (item: IHowto) => `${HOWTO_ROOT()}/${item.slug}`
export const asset_local_abs = async (item: IHowto, asset: IImage) => {
const sanitizedFilename = sanitizeFilename(asset.name)
@@ -465,10 +456,7 @@ export function loader(): Loader {
/////////////////////////////////////////////////////////////////////////
export const group_by_cat = (items: IHowto[]) => {
return items.reduce((acc: Record<string, IHowto[]>, item: IHowto) => {
const category = item?.category?.label || 'uncategorized'
if (category === 'uncategorized') {
return acc
}
const category = item?.category?.label || 'Uncategorized'
if (!acc[category]) {
acc[category] = []
}
src/pages/[locale]/howtos/index.astro
+1 -2
View File
@@ -15,6 +15,7 @@ const items = (await getCollection(view)).map(
const groups = group_by_cat(items as any);
const locale = Astro.currentLocale;
const test = {};
console.log(Object.keys(groups))
export async function getStaticPaths() {
return LANGUAGES.map((lang) => ({
@@ -46,8 +47,6 @@ export async function getStaticPaths() {
<ListItem
url={`/${locale}/howtos/${item.slug}`}
title={item.title}
price={item.price}
type={item.type}
alt={item.title}
model={{item:item}}
/>