generated from polymech/site-template
link checker fuckery
This commit is contained in:
parent
40f89d623e
commit
6c8e235118
@ -23,7 +23,7 @@
|
||||
"format": "unix-time"
|
||||
}
|
||||
],
|
||||
"default": "2025-03-28T10:59:06.225Z"
|
||||
"default": "2025-03-28T15:15:12.509Z"
|
||||
},
|
||||
"description": {
|
||||
"type": "string",
|
||||
|
||||
File diff suppressed because one or more lines are too long
@ -1,90 +1,390 @@
|
||||
{
|
||||
"https://www.youtube.com/watch?v=4LrrFz802To": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159524716
|
||||
"timestamp": 1743173868160
|
||||
},
|
||||
"https://youtu.be/gxkcffQD3eQ": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159526462
|
||||
"timestamp": 1743173868472
|
||||
},
|
||||
"https://zmorph3d.com/blog/hdpe-cnc-machining-materials-overview/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159605158
|
||||
},
|
||||
"https://www.lijiuplastics.com/how-to-cut-hdpe-sheets-for-smooth-burr-free-results/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159610900
|
||||
},
|
||||
"https://www.xometry.com/capabilities/cnc-machining-service/hdpe/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159617036
|
||||
},
|
||||
"https://kingsunmachining.com/blog/machining-hdpe/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159618539
|
||||
},
|
||||
"https://jesspublib.org/wp-content/uploads/X-Carve-Quick-Start-1.pdf": {
|
||||
"https://composimoldstore.com/mold-making-and-casting-guide/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159619212
|
||||
"timestamp": 1743173869404
|
||||
},
|
||||
"https://easel.inventables.com/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159620810
|
||||
},
|
||||
"https://www.thespruce.com/how-to-build-a-diy-pegboard-5225155": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159623033
|
||||
},
|
||||
"https://homeonharbor.com/2021/10/05/diy-in-a-day-shaker-peg-rail-wall-and-trim/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159626882
|
||||
},
|
||||
"https://www.papernstitchblog.com/diy-peg-rail-organizer/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159632080
|
||||
},
|
||||
"https://thediymommy.com/how-to-build-a-giant-framed-pegboard/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159634611
|
||||
},
|
||||
"https://www.instructables.com/DIY-Modern-Peg-Board-Shelving-System/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159637729
|
||||
},
|
||||
"https://onedelightfulhome.com/blog/2021/3/24/pegrail": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159638901
|
||||
},
|
||||
"https://thediyplan.com/diy-pegboard-wall/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159644187
|
||||
},
|
||||
"https://www.prairielightsbooks.com/book/9781569908150": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159644522
|
||||
},
|
||||
"https://www.youtube.com/watch?v=YkHs3H5Vo4M": {
|
||||
"https://www.sdcpublications.com/Textbooks/Complete-Guide-Mold-Making-SOLIDWORKS/ISBN/978-1-63057-720-9/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159646050
|
||||
"timestamp": 1743173869790
|
||||
},
|
||||
"https://3dprints.electronsmith.com/peg-system/": {
|
||||
"https://www.artmolds.com/blogs/mold-making/how-to-make-molds-the-step-by-step-process": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173870133
|
||||
},
|
||||
"https://scholarworks.uni.edu/cgi/viewcontent.cgi?article=3680\\&context=grp": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159652488
|
||||
"timestamp": 1743173871326
|
||||
},
|
||||
"https://example.com/cnc-machines": {
|
||||
"https://pmc.ncbi.nlm.nih.gov/articles/PMC10489002/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159652660
|
||||
"timestamp": 1743173871584
|
||||
},
|
||||
"https://example.com/laser-cutters": {
|
||||
"https://www.youtube.com/watch?v=_pmyVPpULDY": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159652665
|
||||
"timestamp": 1743173871718
|
||||
},
|
||||
"https://example.com": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173872135
|
||||
},
|
||||
"https://www.youtube.com/watch?v=example": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173872341
|
||||
},
|
||||
"https://example.com/shop": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743173873095
|
||||
},
|
||||
"http://read.nxtbook.com/wiley/plasticsengineering/march2015/coverstory_ecomolding.html": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173873539
|
||||
},
|
||||
"https://www.ptonline.com/articles/15-things-to-know-about-servo-driven-injection-machines": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173874249
|
||||
},
|
||||
"https://www.baumueller.com/en/insights/drive-technology/high-performance-drive-solutions-for-clamping-and-injection-units": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173875980
|
||||
},
|
||||
"https://www.bastamachines.com/en/machines-information/injection-moulding-machines": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173876310
|
||||
},
|
||||
"https://pazard.com/injection-molding-machine-parts/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173879868
|
||||
},
|
||||
"https://qecubete.files.wordpress.com/2014/07/injection-molding-handbook.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173880234
|
||||
},
|
||||
"https://www.target.com/p/injection-molding-handbook-2e-2nd-edition-by-tim-a-osswald-hardcover/-/A-93194461": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173880859
|
||||
},
|
||||
"https://files.hanser.de/Files/Article/ARTK_VWO_9783446407817_0001.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173881107
|
||||
},
|
||||
"http://magnum-mfg.com/Injection_Molding_the_Definitive_Engineering_Guide.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173881468
|
||||
},
|
||||
"https://www.youtube.com/watch?v=OOurvulD-pE": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173881640
|
||||
},
|
||||
"https://www.youtube.com/watch?v=cuF3gjvoSKU": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173881803
|
||||
},
|
||||
"https://ljcreate.com/uk/engineering/injection-moulder-assembly-kit/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173884182
|
||||
},
|
||||
"https://linktr.ee/plastmakers": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173884309
|
||||
},
|
||||
"https://www.instructables.com/Forged-Fabric-Parts-With-3D-Printed-Compression-Mo/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173885556
|
||||
},
|
||||
"https://formlabs.com/blog/compression-molding/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173885984
|
||||
},
|
||||
"https://monroeengineering.com/blog/the-beginners-guide-to-compression-molding/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173886635
|
||||
},
|
||||
"https://www.youtube.com/watch?v=amH8BGj0MHg": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173886718
|
||||
},
|
||||
"https://www.youtube.com/watch?v=GqE93pbV_9I": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743173886805
|
||||
},
|
||||
"https://www.cn-kaiming.com/articles/how-does-servo-hydraulic-injection-molding-machine-work.html": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174159519
|
||||
},
|
||||
"https://www.toraytac.com/media/c3feb206-1398-4e0e-bca6-df7780f11745/tcCurg/TenCate%20Advanced%20Composites/Documents/Technical%20papers/TenCate_chopped_fiber_thermoplastics_compression_molding_technical_paper.pdf": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743174618861
|
||||
},
|
||||
"https://youtu.be/qtZv96ciFIU": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174619220
|
||||
},
|
||||
"https://skatehyena.com/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174619472
|
||||
},
|
||||
"https://www.mcmaster.com/91555A101/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174619894
|
||||
},
|
||||
"https://www.facebook.com/marketplace": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174620277
|
||||
},
|
||||
"https://www.gumtree.com": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174620592
|
||||
},
|
||||
"https://nextdoor.co.uk": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174620908
|
||||
},
|
||||
"https://www.freecycle.org": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174621400
|
||||
},
|
||||
"https://www.energy.gov/eere/bioenergy/articles/revolutionary-recycling-process-transforms-plastic-waste": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174623644
|
||||
},
|
||||
"https://www.barnesandnoble.com/w/understanding-plastics-recycling-2e-natalie-rudolph/1137695178": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174624293
|
||||
},
|
||||
"https://practicalactionpublishing.com/book/1999/small-scale-recycling-of-plastics": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174625158
|
||||
},
|
||||
"https://pmc.ncbi.nlm.nih.gov/articles/PMC2873020/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174625380
|
||||
},
|
||||
"https://www.youtube.com/watch?v=JmXf0JgiLTw": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174625536
|
||||
},
|
||||
"https://www.designboom.com/technology/precious-plastic-dave-hakkens-recycling-workshop-06-02-2014/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174626093
|
||||
},
|
||||
"https://www.example.com/motorcycle-stand": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743174627310
|
||||
},
|
||||
"https://www.instructables.com/Making-a-Plate-From-Recycled-Plastic/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174627933
|
||||
},
|
||||
"https://www.ptonline.com/articles/four-ways-to-tackle-threaded-inserts-for-plastics": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174628601
|
||||
},
|
||||
"https://www.crescentind.com/complete-guide-to-plastic-injection-molding": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174629069
|
||||
},
|
||||
"https://www.ptonline.com/news/injection-molding-gains-an-edge-in-motorcycle-gas-tanks": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174629837
|
||||
},
|
||||
"https://firstmold.com/guides/injection-mold-components/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174630305
|
||||
},
|
||||
"https://geomiq.com/injection-moulding-design-guide/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174630555
|
||||
},
|
||||
"https://www.youtube.com/watch?v=V9msuhMjfKY": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174630739
|
||||
},
|
||||
"https://www.youtube.com/watch?v=23Q-chqUHi8": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174630869
|
||||
},
|
||||
"https://wiki.happylab.at/images/2/23/Plasticpreneur_Installation-and-User-Manual_Plastic-Injection-Machine_V1-11_2021.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174631031
|
||||
},
|
||||
"https://www.fictiv.com/fosmc": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174631668
|
||||
},
|
||||
"https://rusefi.com": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174632300
|
||||
},
|
||||
"https://www.irjet.net/archives/V9/i3/IRJET-V9I3135.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174633570
|
||||
},
|
||||
"https://blog.techdesign.com/fosmc-build-open-source-motorcycle/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743174914250
|
||||
},
|
||||
"https://www.xometry.com/resources/casting/plaster-casting/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175060226
|
||||
},
|
||||
"http://bryanyerian.blogspot.com/2012/05/multi-section-plaster-mold-making-from.html": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175060450
|
||||
},
|
||||
"https://www.armstrongrm.com/pages/rpm.html": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175061136
|
||||
},
|
||||
"https://www.protolabs.com/resources/design-tips/9-mistakes-to-avoid-when-designing-injection-molded-parts/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175061438
|
||||
},
|
||||
"https://polytek.com/tutorial/body-casting-tutorial-hydrogel-mold-cold-cast-bronze-casting": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175062175
|
||||
},
|
||||
"https://ceramicartsnetwork.org/daily/article/how-to-make-thrifty-plaster-molds-and-streamline-your-processes": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175067288
|
||||
},
|
||||
"https://sybridge.com/injection-molding-guide/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175067840
|
||||
},
|
||||
"https://sinotech.com/products/formed-metal-parts/plaster-mold-castings/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175068331
|
||||
},
|
||||
"https://composimold.com/mold-making-and-casting-guide/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743175068915
|
||||
},
|
||||
"https://highwaterclays.com/products/the-mold-making-manual": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175069312
|
||||
},
|
||||
"https://www.barnesandnoble.com/w/the-essential-guide-to-mold-making-slip-casting-andrew-martin/1100907494": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175069908
|
||||
},
|
||||
"https://www.youtube.com/watch?v=vJ6mS74ol8E": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175070063
|
||||
},
|
||||
"https://www.youtube.com/watch?v=9SYBEfq-jt4": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175070202
|
||||
},
|
||||
"https://www.youtube.com/watch?v=cmC7SQwdXMQ": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175070282
|
||||
},
|
||||
"https://programs.sigchi.org/chi/2024/program/content/150482": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175070943
|
||||
},
|
||||
"https://class.textile-academy.org/2022/olatz-pereda/projects/00_04_mold_creation_molding_casting/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175071331
|
||||
},
|
||||
"https://www.aimplas.net/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175161057
|
||||
},
|
||||
"https://www.marescirculares.com/": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743175161247
|
||||
},
|
||||
"https://www.vertidoscero.com/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175161696
|
||||
},
|
||||
"https://investors.coca-colacompany.com/news-events/press-releases/detail/971/breakthrough-technology-takes-plastic-from-the-ocean-and": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175233330
|
||||
},
|
||||
"https://www.valenciaport.com/en/valenciaport-sponsors-the-furniture-made-by-aimplas-with-marine-residues/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175234862
|
||||
},
|
||||
"https://www.patagonia.com/our-footprint/netplus-recycled-fishing-nets.html": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175235558
|
||||
},
|
||||
"https://cordis.europa.eu/article/id/445680-a-mantra-for-treatment-of-plastic-in-our-oceans": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175235863
|
||||
},
|
||||
"https://orbit.dtu.dk/files/278424474/Bertelsen_et_al_2022.pdf": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743175235953
|
||||
},
|
||||
"http://www.circularocean.eu/wp-content/uploads/2017/02/Engineering-Properties-of-Fibres-from-Waste-Fishing-Nets.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175236177
|
||||
},
|
||||
"https://www.youtube.com/watch?v=jAlGgXAdq5A": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175236339
|
||||
},
|
||||
"https://www.youtube.com/watch?v=GEqPjhllYTY": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175236479
|
||||
},
|
||||
"https://ecoembesempresas.com/app/uploads/2024/06/Ecodesign_guide_ENG.pdf": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175236629
|
||||
},
|
||||
"https://www.aimplas.net/technologies/recycling/recovery-marine-litter/": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175236776
|
||||
},
|
||||
"https://www.amazon.it/s?k=power+drill": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175278565
|
||||
},
|
||||
"https://www.amazon.it/s?k=metal+base+hardware": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175278795
|
||||
},
|
||||
"https://www.amazon.it/s?k=plastic+shredder": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175279005
|
||||
},
|
||||
"https://www.amazon.it/s?k=hydraulic+sheet+press": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175279163
|
||||
},
|
||||
"https://www.amazon.it/s?k=cnc+CNC Router": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175279339
|
||||
},
|
||||
"https://www.thingiverse.com/thing:12345": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743175279576
|
||||
},
|
||||
"https://www.amazon.it/s?k=digital+calipers": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175279709
|
||||
},
|
||||
"https://github.com/gnea/grbl": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175280314
|
||||
},
|
||||
"https://www.autodesk.com/products/fusion-360": {
|
||||
"isValid": false,
|
||||
"timestamp": 1743159661050
|
||||
},
|
||||
"https://example.com/sanding-tools": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743159661054
|
||||
"timestamp": 1743175280508
|
||||
},
|
||||
"https://www.amazon.it/s?k=metal+screws": {
|
||||
"isValid": true,
|
||||
"timestamp": 1743175280645
|
||||
}
|
||||
}
|
||||
@ -3,7 +3,7 @@
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Extract the required tools, software hardware from the following tutorial. Return as Markdown chapters (H3) with very short bullet points (not bold), with links, max. 5.\n\nText to process:\n# Tutorial: Building a Mini Press for Compression Moulding\n\nTo construct this mini press, you will need the following:\n\n- Welding machine\n- Access to a laser cutting machine\n- Drilling machine\n- Basic assembly skills\n\n\nUser Location: Liberec, Czechia\n\nAll steps are detailed in the video tutorial. Click the yellow download button above to access the blueprints and CAD files.\n\nThis standard size frame can press sheets measuring 37x37 cm (14.6x14.6 in).\n\nMaximum recommended mold height is 80 mm (3.15 in).\n\nProducts made include:\n\n- Sheets: 37x37 cm (14.6x14.6 in), with thicknesses of 3, 5, 20 mm (0.12, 0.20, 0.79 in)\n- Coasters\n- Clocks\n- Clipboards\n- Sheets later used for CNC cutting: lamp designs, animal models.\n\nComplete Machine:\n\nLaser-cut parts for pressing plates:\n\nExplore upgrades and tips for compression molding on:\n\n[Youtube and Instagram](https://linktr.ee/plastmakers)"
|
||||
"content": "Return a list of references (only with links), as Markdown, grouped : Articles, Books, Papers, Youtube, Opensource Designs, ... Dont comment !\n\nText to process:\nThis guide will instruct you on converting 3D print waste into design objects using a shredder, sheet press, and CNC.\n\n\nUser Location: Calenzano, Italy\n\nFirst, collect all plastic waste from your 3D prints, such as supports or defective prints. Sort the waste by plastic type, such as PLA, PETG, and ABS.\n\nAt this stage, shred the material and gather it. Determine the number of shredding cycles based on the desired size.\n\nNow we place all our chopped plastic into the 39x39 in (1x1 m) mold, insert it into the sheet press, and create the plastic sheet.\n\nWe place our 3.28x3.28 ft sheet, cut into 13.78x13.78 in pieces, under the CNC to create rounded pieces with a diameter of 11.81 in.\n\nThe final step is to drill the plastic discs and attach them to a metal base to complete the stool."
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
|
||||
@ -128,11 +128,11 @@ const DEFAULT_PROMPTS: PromptRegistry = {
|
||||
format: 'text'
|
||||
},
|
||||
references: {
|
||||
template: "Return a list of useful references (only with links), as Markdown, grouped : Articles, Books, Papers, Youtube, Opensource Designs, ... Dont comment !",
|
||||
template: "Return a list of references (only with links), as Markdown, grouped : Articles, Books, Papers, Youtube, Opensource Designs, ... Dont comment !",
|
||||
format: 'markdown'
|
||||
},
|
||||
toolsAndHardware: {
|
||||
template: "Extract the required tools, software hardware from the following tutorial. Return as Markdown chapters (H3) with very short bullet points (not bold), with links, max. 5.",
|
||||
template: "Extract the required tools, software hardware from the following tutorial. Return as Markdown chapters (H3) with very short bullet points (not bold), with links, max. 5. per category.",
|
||||
format: 'markdown'
|
||||
},
|
||||
learnedSkills: {
|
||||
|
||||
@ -3,7 +3,7 @@ import { run, OptionsSchema } from "@polymech/kbot-d";
|
||||
import { resolveVariables } from "@polymech/commons/variables"
|
||||
import { } from "@polymech/core/objects"
|
||||
import { logger } from "./index.js"
|
||||
|
||||
const CACHE = true
|
||||
import {
|
||||
TemplateProps,
|
||||
TemplateContext,
|
||||
@ -47,7 +47,7 @@ export const filter = async (content: string, tpl: string = 'howto', opts: Props
|
||||
} catch (e) {
|
||||
logger.error(`Failed to get cached object for ${content.substring(0, 20)}`, e);
|
||||
}
|
||||
if (cached) {
|
||||
if (cached && CACHE) {
|
||||
return cached;
|
||||
}
|
||||
logger.info(`kbot: template:${tpl} : context:${context} @ ${options.model} : ${content.substring(0, 20)}`)
|
||||
|
||||
@ -11,14 +11,14 @@
|
||||
"model": "perplexity/sonar-deep-research",
|
||||
"preferences": "none",
|
||||
"mode": "completion",
|
||||
"prompt": "Return a list of useful references (only with valid links), as Markdown, grouped : Articles, Books, Papers, Youtube, Opensource Designs, ... Dont comment !",
|
||||
"prompt": "Return a list of useful references, as Markdown, grouped : Articles, Books, Papers, Youtube, Software, Opensource Designs, ... Dont comment!",
|
||||
"filters": "code"
|
||||
},
|
||||
"toolsAndHardware": {
|
||||
"model": "perplexity/sonar-deep-research",
|
||||
"preferences": "none",
|
||||
"mode": "completion",
|
||||
"prompt": "Extract the required tools, software hardware from the following tutorial.Return as Markdown chapters (H3) with very short bullet points (not bold), with links, max. 5.",
|
||||
"prompt": "Extract the required tools, software hardware from the following tutorial.Return as Markdown chapters (H3) with very short bullet points (not bold), max. 5 per category.",
|
||||
"filters": "code"
|
||||
},
|
||||
"requiredSkills": {
|
||||
|
||||
@ -1,9 +1,18 @@
|
||||
process.env['NODE_TLS_REJECT_UNAUTHORIZED'] = '0';
|
||||
|
||||
export * from './howto-model.js'
|
||||
import { HOWTO_ROOT } from "config/config.js";
|
||||
import { filterMarkdownLinks } from "../base/markdown.js";
|
||||
import { check } from 'linkinator';
|
||||
import { linkCache } from './link-cache.js';
|
||||
|
||||
import https from 'node:https';
|
||||
|
||||
const httpsAgent = new https.Agent({
|
||||
// Disables TLS/SSL certificate verification
|
||||
rejectUnauthorized: false
|
||||
});
|
||||
|
||||
interface Item {
|
||||
data: {
|
||||
slug: string;
|
||||
@ -68,6 +77,87 @@ export const getDomain = (url: string): string => {
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
export async function validateUrl(
|
||||
url: string,
|
||||
timeout: number = 22500
|
||||
): Promise<boolean> {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeout);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
signal: controller.signal,
|
||||
redirect: 'follow',
|
||||
|
||||
// A more “real” set of headers:
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
|
||||
+ 'AppleWebKit/537.36 (KHTML, like Gecko) '
|
||||
+ 'Chrome/111.0.0.0 Safari/537.36',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Connection': 'keep-alive',
|
||||
// Some sites also watch for these:
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-User': '?1',
|
||||
'Sec-Fetch-Dest': 'document'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok || response.status === 404) {
|
||||
console.log(`URL ${url} is 404`, response);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(`Error validateUrl ${url}`, error);
|
||||
return false;
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Validates if a URL is accessible with a timeout
|
||||
* @param url - The URL to validate
|
||||
* @param timeoutMs - Timeout in milliseconds (default: 3500)
|
||||
* @returns Promise resolving to true if link is valid, false otherwise
|
||||
*/
|
||||
async function validateUrl_0(url: string, timeoutMs: number = 10500): Promise<boolean> {
|
||||
try {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||||
|
||||
const response = await fetch(url, {
|
||||
method: 'HEAD',
|
||||
signal: controller.signal,
|
||||
mode: 'no-cors' // This allows checking cross-origin links
|
||||
});
|
||||
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
// For no-cors mode, we can't check the status, so we assume success if we get a response
|
||||
if (response.type === 'opaque') {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check if status is in 2xx range
|
||||
return response.ok;
|
||||
} catch (error) {
|
||||
// Handle various error cases
|
||||
if (error instanceof Error) {
|
||||
// AbortError means timeout
|
||||
if (error.name === 'AbortError') {
|
||||
console.warn(`Timeout checking URL: ${url}`);
|
||||
return false;
|
||||
}
|
||||
// Network errors or other fetch errors
|
||||
console.warn(`Error checking URL ${url}: ${error.message}`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates links in text and removes invalid ones
|
||||
* @param text - The text containing links to validate
|
||||
@ -77,7 +167,7 @@ export const validateLinks = async (text: string): Promise<string> => {
|
||||
const urlRegex = /\[([^\]]+)\]\(([^)]+)\)/g;
|
||||
const matches = text.matchAll(urlRegex);
|
||||
let processedText = text;
|
||||
|
||||
|
||||
for (const match of matches) {
|
||||
const [fullMatch, linkText, url] = match;
|
||||
try {
|
||||
@ -85,38 +175,29 @@ export const validateLinks = async (text: string): Promise<string> => {
|
||||
const cachedResult = await linkCache.get(url);
|
||||
if (cachedResult !== null) {
|
||||
if (!cachedResult) {
|
||||
processedText = processedText.replace(fullMatch, `~~${linkText}~~`);
|
||||
processedText = processedText.replace(fullMatch, `~~[${linkText}](${url})~~`);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// For testing purposes, treat example.com URLs as valid
|
||||
if (url.includes('example.com')) {
|
||||
await linkCache.set(url, true);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Encode the URL to handle special characters
|
||||
const encodedUrl = encodeURI(url);
|
||||
console.log(`Checking link: ${encodedUrl}`);
|
||||
const result = await check({ path: encodedUrl,
|
||||
timeout: 2500,
|
||||
});
|
||||
|
||||
const isValid = await validateUrl(encodedUrl);
|
||||
console.log(`Checking link: ${encodedUrl} : ${isValid}`);
|
||||
// Cache the result
|
||||
await linkCache.set(url, result.passed);
|
||||
|
||||
// Add strikethrough for invalid links
|
||||
if (!result.passed) {
|
||||
processedText = processedText.replace(fullMatch, `~~${linkText}~~`);
|
||||
await linkCache.set(url, isValid);
|
||||
|
||||
// Add strikethrough for invalid links while preserving the link
|
||||
if (!isValid) {
|
||||
processedText = processedText.replace(fullMatch, `~~[${linkText}](${url})~~`);
|
||||
}
|
||||
} catch (error) {
|
||||
// If there's an error checking the link, assume it's invalid
|
||||
await linkCache.set(url, false);
|
||||
processedText = processedText.replace(fullMatch, `~~${linkText}~~`);
|
||||
processedText = processedText.replace(fullMatch, `~~[${linkText}](${url})~~`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return processedText;
|
||||
};
|
||||
|
||||
@ -131,7 +212,7 @@ export const renderLinks = (text: string): string =>
|
||||
url.toLowerCase().includes(domain.toLowerCase())
|
||||
);
|
||||
if (isBlacklisted) return "";
|
||||
|
||||
|
||||
const domain = getDomain(url);
|
||||
const displayText = `${domain}: ${shortenUrl(url)}`;
|
||||
return `<a class="text-orange-600 underline" href="${url}" target="_blank" rel="noopener noreferrer">${displayText}</a>`;
|
||||
@ -160,7 +241,7 @@ export const replaceWords = (text: string): string =>
|
||||
text
|
||||
);
|
||||
|
||||
export const default_filters_plain: FilterFunction[] = [
|
||||
export const default_filters_plain: FilterFunction[] = [
|
||||
renderLinks,
|
||||
filterBannedPhrases,
|
||||
replaceWords
|
||||
@ -179,7 +260,7 @@ export const default_filters_markdown: FilterFunction[] = [
|
||||
* @param filters - Array of filter functions to apply
|
||||
* @returns Promise resolving to the filtered text
|
||||
*/
|
||||
export async function applyFilters(text: string = '',filters: FilterFunction[] = default_filters_plain): Promise<string> {
|
||||
export async function applyFilters(text: string = '', filters: FilterFunction[] = default_filters_plain): Promise<string> {
|
||||
return filters.reduce(
|
||||
async (promise, filterFn) => {
|
||||
const currentText = await promise;
|
||||
|
||||
@ -41,6 +41,9 @@ import {
|
||||
HOWTO_ADD_REFERENCES
|
||||
} from "config/config.js"
|
||||
|
||||
|
||||
const NB_ITEMS = 50
|
||||
|
||||
import { env, logger } from '@/base/index.js'
|
||||
|
||||
import { applyFilters, default_filters_plain, FilterFunction } from './filters.js'
|
||||
@ -132,10 +135,8 @@ export const raw = async () => {
|
||||
label: 'uncategorized'
|
||||
}
|
||||
})
|
||||
howtos = howtos.filter((h: IHowto) => {
|
||||
return h.steps.length > 0 && !blacklist.includes(h._createdBy);
|
||||
});
|
||||
howtos = howtos.slice(0, 10)
|
||||
howtos = howtos.filter((h: IHowto) => h.steps.length > 0 && !blacklist.includes(h._createdBy))
|
||||
howtos = howtos.slice(0, NB_ITEMS)
|
||||
return howtos
|
||||
}
|
||||
export const defaults = async (data: any, cwd: string, root: string) => {
|
||||
@ -327,13 +328,13 @@ const complete = async (item: IHowto) => {
|
||||
if (HOWTO_ADD_RESOURCES) {
|
||||
item.keywords = await template_filter(item.content, 'keywords', TemplateContext.HOWTO);
|
||||
item.resources = await template_filter(item.content, 'toolsAndHardware', TemplateContext.HOWTO);
|
||||
item.resources = await applyFilters(item.resources, default_filters_markdown);
|
||||
item.resources = await applyFilters(item.resources, default_filters_markdown);
|
||||
write(path.join(item_path(item), 'resources.md'), item.resources as string)
|
||||
}
|
||||
if (HOWTO_ADD_REFERENCES) {
|
||||
item.keywords = await template_filter(item.content, 'keywords', TemplateContext.HOWTO);
|
||||
item.references = await template_filter(item.content, 'references', TemplateContext.HOWTO);
|
||||
item.references = await applyFilters(item.references, default_filters_markdown);
|
||||
item.references = await applyFilters(item.references, default_filters_markdown);
|
||||
write(path.join(item_path(item), 'references.md'), item.references as string)
|
||||
}
|
||||
await to_github(item)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user