url meta | cache

.astro/collections/resources.schema.json

@@ -23,7 +23,7 @@
               "format": "unix-time"
             }
           ],
-          "default": "2025-03-29T08:49:25.330Z"
+          "default": "2025-03-29T11:30:06.151Z"
         },
         "description": {
           "type": "string",

.kbot/params.json

@@ -1,9 +1,9 @@
 {
-  "model": "perplexity/sonar-deep-research",
+  "model": "gpt-4o",
   "messages": [
     {
       "role": "user",
-      "content": "Extract the required tools, software hardware from the following tutorial. Return as Markdown chapters (H3) with very short bullet points (not bold), with links, max. 5. per category.\n\nText to process:\nPlastic sheets created with a sheet press need not be square.\n\nHere, we demonstrate how to make an octagonal mold for octagonal plastic sheets. These techniques can be used to create any polygonal mold, such as triangular or hexagonal.\n\nFor further guidance, feel free to contact us with any questions or suggestions regarding the mold.\n\n\nUser Location: Sukawati, Indonesia\n\n### Tools\n- Sheet press set\n- Angle grinder\n- Welding tools and materials\n- Jigsaw\n- Sander (for product finishing)\n- Safety gear: welding helmet, safety glasses, mask\n- Marker\n- Ruler\n\n- Computer with drafting/drawing software\n\n### Materials\n- Steel for mold: 0.79 x 0.79 inches (2x2 cm) galvanized steel (standard size = 19.7 ft)\n- Clean, sorted plastic, type optional\n\nDraw the polygon in your preferred drafting software to obtain precise dimensions for each side. \n\nMeasure your sheet press work area. \n\nOffset by 2 inches (5 cm) inward for space around the mold. \n\nDesign the polygon based on your machine's capacity. \n\nOffset by 0.8 inches (2 cm) inward for the steel thickness. \n\nUse the drawing to measure the mold dimensions. This will aid in calculating the plastic required for sheet production.\n\nAfter outlining and measuring, use an angle grinder to cut the 20mm (0.79 inches) hollow galvanized steel to the specified size.\n\nMark dimensions using an erasable marker for accuracy.\nEmploy an angle grinder for cutting.\nUse a whiteboard marker to draw a 1:1 template of your sheets to align angles and dimensions properly, adjusting as necessary.\n\nUsing a whiteboard marker, draw a full-scale template of your sheets to verify and adjust angles and dimensions as necessary.\n\nWeld all sides to join the ends and form your polygonal sheets.\n\nUse an angle grinder to smooth excess welding material. This is essential to ensure the mold lies flat against the sheets, preventing damage and preventing plastic from leaking if the surface is uneven.\n\n### How to Make a Polygonal Sheet\n\nCalculate the area of your polygon from the drawing to determine the required plastic amount.\n\nRefer to the sheet press tutorial for detailed instructions: [YouTube Video](https://youtu.be/TNG2f_hKc_A)\n\nFeel free to share your experience with us.\n\nSincerely,  \nWedoo Team"
+      "content": "use a formal tone\nspell & grammar fix the text,\nremove emojis\nremove personal preferences or biases\nshorten text if possible but preserve personality\nremove references to preciousplastic, bazar and Discord\nremove any brain/green washing, eg: sustainable, circular, recycling ... inflated prospects\nRewrite the following text to remove any inflated or empty language, sugar-coated filler, and needless repetition. The result should be concise, direct, and preserve only the essential ideas.\nContext: howto tutorials, for makers\nConvert units, from metric to imperial and vice versa (in braces)\ndont comment just return as Markdown\n\nText to process:\nWith your oven preheated at the target temperature for your type of plastic, it's time to pop the filled molds into the heat.\n\nWe're using PLA, with a temperature of 170-180°C. This might be different for your oven also, so start a little on the cool side and work your way up. The [filtered] Download Pack has a good reference poster for some other common types of plastics. For 3D printing plastics, check out the datasheets by your filament supplier."
     },
     {
       "role": "user",

config/config.js

@@ -0,0 +1 @@
+export const HOWTO_ROOT = () => '/howto'; 

src/base/filters.test.ts

@@ -0,0 +1,167 @@
+import './__tests__/test-setup.js';
+import { describe, it, expect } from 'vitest';
+import {
+  shortenUrl,
+  renderLinks,
+  filterBannedPhrases,
+  replaceWords,
+  applyFilters,
+  default_filters_plain,
+  default_filters_markdown,
+  validateLinks
+} from './filters.js';
+
+import { item_path } from '../model/howto.js';
+
+describe('filters', () => {
+  describe('item_path', () => {
+    it('should generate correct path from item', () => {
+      const item = { data: { slug: 'test-slug' } };
+      expect(item_path(item)).toBe('/howto/test-slug');
+    });
+  });
+
+  describe('shortenUrl', () => {
+    it('should remove www. prefix and trailing slashes', () => {
+      expect(shortenUrl('https://www.example.com/path/')).toBe('example.com/path');
+    });
+
+    it('should handle URLs without www. prefix', () => {
+      expect(shortenUrl('https://example.com/path')).toBe('example.com/path');
+    });
+
+    it('should handle invalid URLs gracefully', () => {
+      expect(shortenUrl('invalid-url')).toBe('invalid-url');
+    });
+
+    it('should handle URLs with query parameters', () => {
+      expect(shortenUrl('https://example.com/path?param=value')).toBe('example.com/path?param=value');
+    });
+  });
+
+  describe('renderLinks', () => {
+    it('should render non-blacklisted links', () => {
+      const input = 'Check out https://example.com';
+      const expected = 'Check out <a class="text-orange-600 underline" href="https://example.com" target="_blank" rel="noopener noreferrer">example.com</a>';
+      expect(renderLinks(input)).toBe(expected);
+    });
+
+    it('should replace blacklisted links with empty string', () => {
+      const input = 'Check out https://preciousplastic.com';
+      expect(renderLinks(input)).toBe('Check out ');
+    });
+
+    it('should handle multiple links in text', () => {
+      const input = 'Check out https://example.com and https://preciousplastic.com';
+      const result = renderLinks(input);
+      expect(result).toContain('example.com');
+      expect(result).toContain('and ');
+    });
+  });
+
+  describe('filterBannedPhrases', () => {
+    it('should replace banned words with [filtered]', () => {
+      const input = 'The wizard used magic2';
+      const expected = 'The [filtered] used [filtered]';
+      expect(filterBannedPhrases(input)).toBe(expected);
+    });
+
+    it('should handle case-insensitive matching', () => {
+      const input = 'The WIZARD used MAGIC2';
+      const expected = 'The [filtered] used [filtered]';
+      expect(filterBannedPhrases(input)).toBe(expected);
+    });
+
+    it('should not replace partial matches', () => {
+      const input = 'The wizardry used magic2.0';
+      const expected = 'The wizardry used [filtered].0';
+      expect(filterBannedPhrases(input)).toBe(expected);
+    });
+  });
+
+  describe('replaceWords', () => {
+    it('should replace words according to wordReplaceMap', () => {
+      const input = 'I need a Router for my Car';
+      const expected = 'I need a CNC Router for my tufftuff';
+      expect(replaceWords(input)).toBe(expected);
+    });
+
+    it('should handle multi-word replacements', () => {
+      const input = 'I need a laptop stand';
+      expect(replaceWords(input)).toBe('I need a laptoppie');
+    });
+
+    it('should handle case-insensitive matching', () => {
+      const input = 'I need a ROUTER for my CAR';
+      const expected = 'I need a CNC Router for my tufftuff';
+      expect(replaceWords(input)).toBe(expected);
+    });
+  });
+
+  describe('applyFilters', () => {
+    it('should apply plain text filters in sequence', async () => {
+      const input = 'Check out https://example.com with the wizard Router';
+      const result = await applyFilters(input, default_filters_plain);
+      expect(result).toContain('example.com');
+      expect(result).toContain('[filtered]');
+      expect(result).toContain('CNC Router');
+    });
+
+    it('should apply markdown filters in sequence', async () => {
+      const input = 'Check out [example](https://example.com) with the wizard Router';
+      const result = await applyFilters(input, default_filters_markdown);
+      expect(result).toContain('example');
+      expect(result).toContain('[filtered]');
+      expect(result).toContain('CNC Router');
+    });
+
+    it('should handle empty input', async () => {
+      expect(await applyFilters('')).toBe('');
+    });
+
+    it('should handle custom filter array', async () => {
+      const customFilters = [filterBannedPhrases];
+      const input = 'The wizard used magic2';
+      const expected = 'The [filtered] used [filtered]';
+      expect(await applyFilters(input, customFilters)).toBe(expected);
+    });
+
+    it('should handle markdown links with blacklisted URLs', async () => {
+      const input = 'Check out [example](https://preciousplastic.com)';
+      const result = await applyFilters(input, default_filters_markdown);
+      expect(result).toBe('Check out example');
+    });
+  });
+
+  describe('validateLinks', () => {
+    it('should remove invalid links entirely', async () => {
+      const input = 'Check out [example](https://invalid-url-that-does-not-exist.com)';
+      const result = await validateLinks(input);
+      expect(result).toBe('Check out example');
+    });
+
+    it('should preserve valid links', async () => {
+      const input = 'Check out [example](https://example.com)';
+      const result = await validateLinks(input);
+      expect(result).toBe('Check out [example](https://example.com)');
+    });
+
+    it('should handle multiple links in text', async () => {
+      const input = 'Check out [valid](https://example.com) and [invalid](https://invalid-url-that-does-not-exist.com)';
+      const result = await validateLinks(input);
+      expect(result).toBe('Check out [valid](https://example.com) and invalid');
+    });
+
+    it('should handle links with special characters', async () => {
+      const input = '[special](https://example.com/path?param=value#fragment)';
+      const result = await validateLinks(input);
+      expect(result).toBe('[special](https://example.com/path?param=value#fragment)');
+    });
+
+    it('should handle links with special characters that are invalid', async () => {
+      const input = '[special](https://invalid-url-that-does-not-exist.com/path?param=value#fragment)';
+      const result = await validateLinks(input);
+      expect(result).toBe('special');
+    });
+  });
+}); 

src/base/filters.ts

@@ -0,0 +1,262 @@
+process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';
+
+import { urlCache } from './url-cache.js';
+import { filterMarkdownLinks } from "../base/markdown.js";
+import { meta } from '../base/url.js';
+
+export interface FilterFunction { (text: string): string | Promise<string> }
+
+export const blacklist: readonly string[] = [
+  'precious-plastic',
+  'fair-enough',
+  'mad-plastic-labs',
+  'easymoulds',
+  'plasticpreneur',
+  'sustainable-design-studio',
+  'johannplasto'
+] as const;
+
+export const urlBlacklist: readonly string[] = [
+  "thenounproject.com",
+  "preciousplastic.com",
+  "community.preciousplastic.com",
+  "bazar.preciousplastic.com",
+  "onearmy.earth"
+] as const;
+
+export const bannedWords: readonly string[] = [
+  "wizard",
+  "magic2",
+  "precious plastic",
+  "onearmy"
+] as const;
+
+export const wordReplaceMap: Readonly<Record<string, string>> = {
+  Router: "CNC Router",
+  "laptop stand": "laptoppie",
+  Car: "tufftuff"
+} as const;
+
+/**
+ * Shortens a URL by removing 'www.' prefix and trailing slashes
+ * @param url - The URL to shorten
+ * @returns The shortened URL or the original URL if invalid
+ */
+export const shortenUrl = (url: string): string => {
+  try {
+    const { hostname, pathname, search } = new URL(url);
+    const cleanHost = hostname.replace(/^www\./, '');
+    const cleanPath = pathname.replace(/\/$/, '');
+    return `${cleanHost}${decodeURIComponent(cleanPath)}${search}`;
+  } catch (error) {
+    console.warn(`Invalid URL provided to shortenUrl: ${url}`);
+    return url;
+  }
+};
+
+/**
+ * Gets the domain name from a URL
+ * @param url - The URL to extract domain from
+ * @returns The domain name or empty string if invalid
+ */
+export const getDomain = (url: string): string => {
+  try {
+    const { hostname,  } = new URL(url);
+    return hostname.replace(/^www\./, '');
+  } catch {
+    return '';
+  }
+};
+
+export async function validateUrl(
+  url: string,
+  timeout: number = 22500
+): Promise<boolean> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeout);
+  try {
+    const response = await fetch(url, {
+      signal: controller.signal,
+      redirect: 'follow',
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
+          + 'AppleWebKit/537.36 (KHTML, like Gecko) '
+          + 'Chrome/111.0.0.0 Safari/537.36',
+        'Accept-Language': 'en-US,en;q=0.9',
+        'Accept-Encoding': 'gzip, deflate, br',
+        'Connection': 'keep-alive',
+        'Sec-Fetch-Site': 'none',
+        'Sec-Fetch-Mode': 'navigate',
+        'Sec-Fetch-User': '?1',
+        'Sec-Fetch-Dest': 'document'
+      }
+    });
+
+    if (!response.ok || response.status === 404) {
+      console.log(`URL ${url} is 404`, response);
+      await urlCache.set(url, false);
+      return false;
+    }
+
+    // Get meta information for valid URLs
+    const metaInfo = await meta(url);
+    await urlCache.set(url, true, metaInfo);
+    return true;
+  } catch (error) {
+    console.log(`Error validateUrl ${url}`, error);
+    await urlCache.set(url, false);
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+/**
+ * Validates if a URL is accessible with a timeout
+ * @param url - The URL to validate
+ * @param timeoutMs - Timeout in milliseconds (default: 3500)
+ * @returns Promise resolving to true if link is valid, false otherwise
+ */
+async function validateUrl_0(url: string, timeoutMs: number = 10500): Promise<boolean> {
+  try {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+    const response = await fetch(url, {
+      method: 'HEAD',
+      signal: controller.signal,
+      mode: 'no-cors' // This allows checking cross-origin links
+    });
+
+    clearTimeout(timeoutId);
+
+    // For no-cors mode, we can't check the status, so we assume success if we get a response
+    if (response.type === 'opaque') {
+      return true;
+    }
+
+    // Check if status is in 2xx range
+    return response.ok;
+  } catch (error) {
+    // Handle various error cases
+    if (error instanceof Error) {
+      // AbortError means timeout
+      if (error.name === 'AbortError') {
+        console.warn(`Timeout checking URL: ${url}`);
+        return false;
+      }
+      // Network errors or other fetch errors
+      console.warn(`Error checking URL ${url}: ${error.message}`);
+    }
+    return false;
+  }
+}
+
+/**
+ * Validates links in text and removes invalid ones
+ * @param text - The text containing links to validate
+ * @returns Promise resolving to text with invalid links removed
+ */
+export const validateLinks = async (text: string): Promise<string> => {
+  const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
+  const matches = text.matchAll(urlRegex);
+  let processedText = text;
+
+  for (const match of matches) {
+    const [fullMatch, linkText, url] = match;
+    try {
+      // Check cache first
+      const cachedResult = await urlCache.get(url);
+      if (cachedResult !== null) {
+        if (!cachedResult.isValid) {
+          processedText = processedText.replace(fullMatch, `~~[${linkText}](${url})~~`);
+        }
+        continue;
+      }
+
+      // Encode the URL to handle special characters
+      const encodedUrl = encodeURI(url);
+      const isValid = await validateUrl(encodedUrl);
+
+      // Add strikethrough for invalid links while preserving the link
+      if (!isValid) {
+        processedText = processedText.replace(fullMatch, `~~[${linkText}](${url})~~`);
+      }
+    } catch (error) {
+      // If there's an error checking the link, assume it's invalid
+      await urlCache.set(url, false);
+      processedText = processedText.replace(fullMatch, `~~[${linkText}](${url})~~`);
+    }
+  }
+
+  return processedText;
+};
+
+/**
+ * Renders links in text, replacing blacklisted URLs with "[Link Removed]"
+ * @param text - The text containing URLs to process
+ * @returns Processed text with rendered links
+ */
+export const renderLinks = (text: string): string =>
+  text.replace(/https?:\/\/[^\s<"]+/gi, (url) => {
+    const isBlacklisted = urlBlacklist.some((domain) =>
+      url.toLowerCase().includes(domain.toLowerCase())
+    );
+    if (isBlacklisted) return "";
+
+    const domain = getDomain(url);
+    const displayText = `${domain}: ${shortenUrl(url)}`;
+    return `<a class="text-orange-600 underline" href="${url}" target="_blank" rel="noopener noreferrer">${displayText}</a>`;
+  });
+
+/**
+ * Filters out banned phrases from text
+ * @param text - The text to filter
+ * @returns Text with banned phrases replaced
+ */
+export const filterBannedPhrases = (text: string): string =>
+  bannedWords.reduce(
+    (acc, word) => acc.replace(new RegExp(`\\b${word}\\b`, "gi"), "[filtered]"),
+    text
+  );
+
+/**
+ * Replaces specific words in text according to the wordReplaceMap
+ * @param text - The text to process
+ * @returns Text with words replaced according to the mapping
+ */
+export const replaceWords = (text: string): string =>
+  Object.entries(wordReplaceMap).reduce(
+    (acc, [word, replacement]) =>
+      acc.replace(new RegExp(`\\b${word}\\b`, "gi"), replacement),
+    text
+  );
+
+export const default_filters_plain: FilterFunction[] = [
+  renderLinks,
+  filterBannedPhrases,
+  replaceWords  
+] as const;
+
+export const default_filters_markdown: FilterFunction[] = [
+  (text: string) => filterMarkdownLinks(text, urlBlacklist.map(url => ({ pattern: url, replacement: "" }))),
+  filterBannedPhrases,
+  replaceWords,
+  validateLinks
+] as const;
+
+/**
+ * Applies all filters to the input text in sequence
+ * @param text - The text to filter
+ * @param filters - Array of filter functions to apply
+ * @returns Promise resolving to the filtered text
+ */
+export async function applyFilters(text: string = '', filters: FilterFunction[] = default_filters_plain): Promise<string> {
+  return filters.reduce(
+    async (promise, filterFn) => {
+      const currentText = await promise;
+      return filterFn(currentText);
+    },
+    Promise.resolve(text)
+  )
+}
+

src/base/url-cache.ts

@@ -1,6 +1,7 @@
 import fs from 'fs/promises';
 import path from 'path';
-import { meta } from '../base/url.js';
+import { logger } from './index.js'
+import { meta } from './url.js';
 
 interface CacheEntry {
   isValid: boolean;
@@ -86,8 +87,7 @@ class UrlCache {
   }
 
   async expandUrls(): Promise<void> {
-    await this.loadCache();
-    
+    await this.loadCache();    
     for (const [url, entry] of Object.entries(this.cache)) {
       if (entry.isValid && !entry.meta) {
         try {

src/base/url.ts

@@ -107,21 +107,21 @@ export class PuppeteerUrlChecker implements UrlChecker {
         return { valid: true };
       }
 
-      return { 
-        valid: false, 
-        error: `HTTP ${status}: ${response.statusText()}` 
+      return {
+        valid: false,
+        error: `HTTP ${status}: ${response.statusText()}`
       };
 
     } catch (error) {
       if (error instanceof Error) {
-        return { 
-          valid: false, 
-          error: error.message 
+        return {
+          valid: false,
+          error: error.message
         };
       }
-      return { 
-        valid: false, 
-        error: 'Unknown error occurred' 
+      return {
+        valid: false,
+        error: 'Unknown error occurred'
       };
     }
   }
@@ -151,30 +151,30 @@ export class FetchUrlChecker implements UrlChecker {
       clearTimeout(timeoutId);
 
       if (!response.ok) {
-        return { 
-          valid: false, 
-          error: `HTTP ${response.status}: ${response.statusText}` 
+        return {
+          valid: false,
+          error: `HTTP ${response.status}: ${response.statusText}`
         };
       }
 
       return { valid: true };
     } catch (error) {
       if (error instanceof Error) {
-        return { 
-          valid: false, 
-          error: error.message 
+        return {
+          valid: false,
+          error: error.message
         };
       }
-      return { 
-        valid: false, 
-        error: 'Unknown error occurred' 
+      return {
+        valid: false,
+        error: 'Unknown error occurred'
       };
     }
   }
 }
 
 // Default checker instance
-export const defaultChecker: UrlChecker = new PuppeteerUrlChecker();
+export const defaultChecker: UrlChecker = new FetchUrlChecker();
 
 // Export a convenience function
 export async function checkUrl(url: string, timeout?: number): Promise<UrlCheckResult> {
@@ -192,21 +192,18 @@ export interface MetaResult {
 
 export async function meta(url: string): Promise<MetaResult> {
   try {
-    // Check cache first
     const cached = metaCache.get(url);
     if (cached && Date.now() - cached.timestamp < CACHE_DURATION) {
       return cached.data;
     }
-
-    // Validate URL first
     const urlCheck = await checkUrl(url);
     if (!urlCheck.valid) {
       return { error: urlCheck.error };
     }
-
-    // Get link preview
-    const preview = await getLinkPreview(url) as LinkPreviewResult;
-    
+    const preview = await getLinkPreview(url, {
+      followRedirects: 'follow',
+      timeout: 20000
+    }) as LinkPreviewResult;
     const result: MetaResult = {
       title: preview.title || undefined,
       description: preview.description || undefined,
@@ -215,12 +212,10 @@ export async function meta(url: string): Promise<MetaResult> {
       siteName: preview.siteName || undefined
     };
 
-    // Cache the result
     metaCache.set(url, {
       data: result,
       timestamp: Date.now()
     });
-
     return result;
   } catch (error) {
     if (error instanceof Error) {

src/config/config.ts

@@ -22,7 +22,7 @@ export const I18N_ASSET_PATH = "${SRC_DIR}/${SRC_NAME}-${DST_LANG}${SRC_EXT}"
 
 export const HOWTO_GLOB = '**/config.json'
 export const FILES_WEB = 'https://files.polymech.io/files/machines/howtos/'
-export const HOWTO_FILTER_LLM = false
+export const HOWTO_FILTER_LLM = true
 export const HOWTO_ANNOTATIONS = false
 export const HOWTO_ANNOTATIONS_CACHE = false
 export const HOWTO_COMPLETE_RESOURCES = true

src/model/howto.ts

@@ -15,7 +15,7 @@ export * from './howto-model.js'
 export * from '../base/filters.js'
 
 import { IHowto, IImage, ITag, ITEM_TYPE } from './howto-model.js'
-import { blacklist, default_filters_markdown } from '../base/filters.js'
+import { blacklist, default_filters_markdown, validateLinks } from '../base/filters.js'
 import { download } from './download.js'
 
 import { filter } from "@/base/kbot.js"
@@ -23,6 +23,11 @@ import { slugify } from "@/base/strings.js"
 import type { IAnnotation } from "./annotation.js"
 import { AnnotationMode, generateCacheKey, cacheAnnotation, getCachedAnnotation } from './annotation.js'
 
+import { urlCache } from '../base/url-cache.js';
+
+const NB_ITEMS = 350
+const expandUrls = true
+
 import {
   HOWTO_FILES_WEB,
   HOWTO_FILES_ABS,
@@ -43,7 +48,7 @@ import {
 } from "config/config.js"
 
 
-const NB_ITEMS = 10
+
 
 import { env, logger } from '@/base/index.js'
 
@@ -112,7 +117,7 @@ export const asset_local_rel = async (item: IHowto, asset: IImage) => {
   const sanitizedFilename = sanitizeFilename(asset.name).toLowerCase()
   const asset_path = path.join(HOWTO_ROOT(), item.slug, sanitizedFilename)
   if (exists(asset_path)) {
-    return asset_path//`/resources/howtos/${item.slug}/${sanitizedFilename}`
+    return asset_path
   } else {
     await download(asset.downloadUrl, asset_path)
   }
@@ -160,6 +165,7 @@ const commons = async (text: string): Promise<string> => {
   return await template_filter(text, 'simple', TemplateContext.COMMONS);
 }
 const content = async (str: string, filters: FilterFunction[] = default_filters_plain) => await applyFilters(str, filters)
+
 const to_github = async (item: IHowto) => {
   const itemDir = item_path(item)
   const readmeContent = [
@@ -292,17 +298,18 @@ const to_astro = async (item: IHowto) => {
 const complete = async (item: IHowto) => {
   const configPath = path.join(item_path(item), 'config.json')
   const config = read(configPath, 'json') as IHowto || {}
-  item = { ...item, ...config }
-  
+  // item = { ...item, ...config }
+
   if (!HOWTO_ANNOTATIONS) {
     // return item
   }
-  
   // commons: language, tone, bullshit filter, and a piece of love, just a bit, at least :)
   if (HOWTO_FILTER_LLM) {
     item.description = await commons(item.description || '')
   }
 
+  item.description = await applyFilters(item.description || '', [validateLinks])
+
   // default pass, links, words, formatting, ...  
   item.steps = await pMap(
     item.steps,
@@ -344,10 +351,10 @@ const complete = async (item: IHowto) => {
     write(path.join(item_path(item), 'references.md'), item.references as string)
   }
 
-  if(HOWTO_SEO_LLM){
-    item.brief = await template_filter(item.content, 'brief', TemplateContext.HOWTO); 
+  if (HOWTO_SEO_LLM) {
+    item.brief = await template_filter(item.content, 'brief', TemplateContext.HOWTO);
   }
-
+  // item.content = await applyFilters(item.content || '', [validateLinks])
   await to_github(item)
   // await to_mdx(item)
   // await to_astro(item)
@@ -355,8 +362,9 @@ const complete = async (item: IHowto) => {
   return item
 }
 
-const onStoreItem = async (store: any, ctx: LoaderContext) => {
-  const item = store.data.item as IHowto
+const onStoreItem = async (store: any) => {
+  let item = store.data.item as IHowto
+
   item.steps = item.steps || []
   item.cover_image && (item.cover_image.src = await asset_local_rel(item, item.cover_image))
 
@@ -377,11 +385,13 @@ const onStoreItem = async (store: any, ctx: LoaderContext) => {
     step.images = step.images.filter((image) => asset_local_abs(item, image))
   })
   item.files = await downloadFiles(item.slug, item)
-  await complete(item)
+
+  item = await complete(item)
   const configPath = path.join(item_path(item), 'config.json')
   write(configPath, JSON.stringify(item, null, 2))
   logger.info(`Stored item ${item.slug} at ${configPath}`)
-  return item
+  store.data.item = item
+  return store
 }
 
 export function loader(): Loader {
@@ -394,6 +404,9 @@ export function loader(): Loader {
     generateDigest }: LoaderContext) => {
 
     store.clear()
+    if (expandUrls) {
+      //await urlCache.expandUrls()
+    }
     let items = await raw()
     for (const item of items) {
       const id = item.slug
@@ -405,22 +418,15 @@ export function loader(): Loader {
         components: [],
         item
       }
-      //const parsedData = await parseData({ id, data: data });        
-      const storeItem = {
+
+      let storeItem = {
         digest: await generateDigest(data),
         filePath: id,
         id: `${item.slug}`,
         data: data
       }
 
-      await onStoreItem(storeItem, {
-        logger,
-        watcher,
-        parseData,
-        store,
-        generateDigest
-      } as any)
-
+      storeItem = await onStoreItem(storeItem)
       storeItem.data['config'] = JSON.stringify(storeItem.data, null, 2)
       store.set(storeItem)
     }