mafia filter

This commit is contained in:
lovebird 2025-03-22 18:16:27 +01:00
parent 468509e652
commit 5eb222707e
9 changed files with 63 additions and 70 deletions

View File

@ -23,7 +23,7 @@
"format": "unix-time"
}
],
"default": "2025-03-22T10:51:39.637Z"
"default": "2025-03-22T17:14:37.325Z"
},
"description": {
"type": "string",

File diff suppressed because one or more lines are too long

View File

@ -1,9 +1,9 @@
{
"model": "gpt-4.5-preview",
"model": "gpt-4o",
"messages": [
{
"role": "user",
"content": "\nShort summary, as markdown\n https://www.npmjs.com/package/@plastichub/kbot\n"
"content": "spell check this text, fix any errors, remove emojies, misleading or personal preferences, formal Remove any references to 'precious plastic', 'bazar' or 'discord' but also remove any bias, and commercial interests\n\n Improve for clarity \n\n Context: howto tutorials, for makers \n\n dont comment just return as Markdown : This How-To will show you the way to convert your three phase single direction shredder into a new single phase-reverse function shredder. Which means that you would be able to plug it in anyplace you go. This is quite usefull if you like to go around with your shredder or if your workspace has only single-phase electricity.\n\nThe reverse function is quite cool when your shredder gets jammed.\n\n"
},
{
"role": "user",

View File

@ -5,6 +5,9 @@
},
{
"path": "../polymech-mono/packages"
},
{
"path": "../astro-components"
}
],
"settings": {

File diff suppressed because one or more lines are too long

View File

@ -9,18 +9,16 @@ import Wrapper from "@/components/containers/Wrapper.astro";
import GalleryK from "@/components/polymech/GalleryK.astro";
import { files, forward_slash } from "@polymech/commons";
import pMap from "p-map";
import { filter as language } from "@/base/kbot.js";
import { sync as exists } from "@polymech/fs/exists";
import { sync as read } from "@polymech/fs/read";
import { createMarkdownComponent } from "@/base/index.js";
import { translate } from "@/base/i18n.js";
import { applyFilters, shortenUrl } from "@/model/howto.js";
import {
HOWTO_FILES_WEB,
HOWTO_FILES_ABS,
HOWTO_FILTER_LLM,
I18N_SOURCE_LANGUAGE,
} from "config/config.js";
interface Props {
howto: IHowto;
}
@ -36,7 +34,6 @@ let model_files: any = [...files(howto_abs, "**/**/*.(step|stp)")];
model_files = model_files.map((f) =>
forward_slash(`${howto.slug}/${path.relative(path.resolve(howto_abs), f)}`),
);
const content = async (str: string) =>{
const ret = await translate(
await applyFilters(str),
@ -46,62 +43,6 @@ const content = async (str: string) =>{
return ret
}
const component = async (str: string) => await createMarkdownComponent(await content(str))
const urlBlacklist = ["thenounproject.com","preciousplastic.com"];
const bannedWords = ["wizard", "magic2"];
const wordReplaceMap: Record<string, string> = {
Router: "CNC Router",
"laptop stand": "laptoppie",
};
const nl2br = (text: string): string => text.replace(/\r?\n/g, "<br>");
const shortenUrl = (url: string): string => {
try {
const { hostname, pathname } = new URL(url);
const cleanHost = hostname.replace(/^www\./, '');
const cleanPath = pathname.replace(/\/$/, ''); // remove trailing slash
return `${cleanHost}${decodeURIComponent(cleanPath)}`;
} catch {
// If invalid URL, return as-is
return url;
}
};
// Turns URLs into clickable links, unless blacklisted
const renderLinks = (text: string): string =>
text.replace(/https?:\/\/[^\s<"]+/gi, (url) => {
const isBlacklisted = urlBlacklist.some((domain) =>
url.toLowerCase().includes(domain.toLowerCase()),
);
return isBlacklisted
? "[Link Removed]"
: `<a class="text-orange-600 underline" href="${url}" target="_blank" rel="noopener noreferrer">${shortenUrl(url)}</a>`;
});
const filterBannedPhrases = (text: string): string =>
bannedWords.reduce(
(acc, word) => acc.replace(new RegExp(`\\b${word}\\b`, "gi"), "[filtered]"),
text,
);
const replaceWords = (text: string): string =>
Object.entries(wordReplaceMap).reduce(
(acc, [word, replacement]) =>
acc.replace(new RegExp(`\\b${word}\\b`, "gi"), replacement),
text,
);
const filters = [
renderLinks,
filterBannedPhrases,
replaceWords,
HOWTO_FILTER_LLM ? language : (text: string) => text,
];
async function applyFilters(text: string): Promise<string> {
let filtered = text;
for (const filterFn of filters) {
filtered = await filterFn(filtered);
}
return filtered;
}
const stepsWithFilteredMarkdown = await pMap(
howto.steps,
@ -218,6 +159,30 @@ const authorLinks = (howto?.user?.data.urls || []).filter(
>
<Translate>Browse Files</Translate>
</a>
<!-- Resources -->
{
howto.files.length > 0 && (
<div class="p-8 mb-8">
<h2 class="text-xl font-semibold mb-2">
<Translate>Resources</Translate>
</h2>
<ul class="list-disc list-inside">
{howto.files.map((file) => (
<li>
<a
href={`${file.downloadUrl}`}
target="_blank"
rel="noopener noreferrer"
class="text-orange-700 hover:underline break-words"
>
{file.name}
</a>
</li>
))}
</ul>
</div>
)
}
</section>
<section class="border-t border-gray-200 bg-gray-50 p-8">
<h2 class="text-2xl font-bold text-gray-800 mb-6">

View File

@ -159,6 +159,30 @@ const authorLinks = (howto?.user?.data.urls || []).filter(
>
<Translate>Browse Files</Translate>
</a>
<!-- Resources -->
{
howto.files.length > 0 && (
<div class="p-8 mb-8">
<h2 class="text-xl font-semibold mb-2">
<Translate>Resources</Translate>
</h2>
<ul class="list-disc list-inside">
{howto.files.map((file) => (
<li>
<a
href={`${file.downloadUrl}`}
target="_blank"
rel="noopener noreferrer"
class="text-orange-700 hover:underline break-words"
>
{file.name}
</a>
</li>
))}
</ul>
</div>
)
}
</section>
<section class="border-t border-gray-200 bg-gray-50 p-8">
<h2 class="text-2xl font-bold text-gray-800 mb-6">

View File

@ -10,4 +10,4 @@
Detail.astro:
- [ ] step titles (multilang) as anchors (eg: #step-title-en)
- [ ] make step titles (multilang) as linkable (eg: #step-title-en)

View File

@ -49,6 +49,8 @@ export const ITEM_TYPE = 'howto'
//export const load = () => get(`${HOWTO_ROOT()}/${HOWTO_GLOB}`, HOWTO_ROOT(), ITEM_TYPE)
export const item_path = (item: any) => `${HOWTO_ROOT()}/${item.data.slug}`
const blacklist = ['precious-plastic', 'fair-enough', 'mad-plastic-labs', 'the-flipflopi', 'easymoulds','plasticpreneur'];
const download = async (url, outputPath) => {
const stream = createWriteStream(outputPath);
got.stream(url).pipe(stream);
@ -99,9 +101,8 @@ export const howtos = async () => {
})
howtos = howtos.filter((h) => {
return h.steps.length > 0 &&
h._createdBy !== 'precious-plastic'
})
return h.steps.length > 0 && !blacklist.includes(h._createdBy);
});
return howtos
}