import * as path from 'path' import { sync as exists } from "@plastichub/fs/exists" import { async as move } from "@plastichub/fs/move" import { sync as dir } from "@plastichub/fs/dir" import { sync as write } from "@plastichub/fs/write" import { sync as read } from "@plastichub/fs/read" import { resolve } from "@plastichub/osr-cli-commons/fs" import { Promise as BPromise } from 'bluebird' import { IOptionsSync, IDiscoursePostBaseOptions } from '../../types' import { createContent } from './osrl' const YAML = require('json-to-pretty-yaml') const cheerio = require('cheerio') const findUp = require('find-up') const frontMatter = require('front-matter') const md5 = require('md5') import { imageName, downloadFile } from './download' import { toHTML } from '../markdown' import { isNumber } from '@plastichub/core/primitives' import { defaultConfig, fromJSON, tracking, trackingPath } from './' import { SYNC_TRACK_FILENAME, EDiscourseConfigKey } from '../discourse/constants' import { cacheCategories, cacheTags, cacheTopics, cacheUsers } from '../discourse/cache' import { Discourser, Instance } from '../discourse' import { images_urls } from './commons' import { ISearchPost, ISearchTopic } from "../.." import * as md5 from 'md5' import { IComponentConfig } from '@plastichub/osr-commons' import { marketplaceUrl } from '../osr' import { isValidLibraryComponent, readOSRConfig } from '@plastichub/osr-fs-utils' import { logger } from '../../index' import { forward_slash } from '@plastichub/osr-cli-commons' const CONTENT_TEST = false const SKIP_EXISTING = false export const createPost = async (discourse: Discourser, options: IOptionsSync, content) => { if (!isNumber(options.cat)) { logger.error(`category not a number! ${options.title} `) } let data: any try { data = await discourse.createPost(options.title, content, options.cat as number) } catch (e) { debugger } if (data) { if (data && data.id) { try { options.post_id = data.id; options.topic_id = data.topic_id await discourse.changeOwner(options.post_id, options.topic_id, options.user_name) logger.debug('created topic : ' + options.title + ' : ' + data.id + ' | topic id :' + data.topic_id) return true } catch (e) { logger.error('changing owner ' + options.title + ' failed!', e) } } else { logger.debug('creating ' + options.title + ' failed!', data.errors, data); if (data.errors) { if (data.errors[0] && data.errors[0] === 'Title has already been used') { logger.error('title already used : ' + options.title) } } } } else { return false } } export const updatePost = async (discourse: Discourser, options: IOptionsSync, topic_id, content) => { let data: any try { data = await discourse.updatePost(topic_id, content) logger.debug('update post : ' + options.title + ' : ' + data.id + ' | topic id ' + data.topic_id) } catch (e) { return false } if (data) { if (data && data.id) { try { // logger.debug('change user to ', options.owner); options.post_id = data.id; options.topic_id = data.topic_id await new Promise(f => setTimeout(f, 1000)); await discourse.changeOwner(topic_id, topic_id, options.user_name) return true } catch (e) { logger.debug('changing owner ' + options.title + ' failed!') return false } } else { logger.debug('creating ' + options.title + ' failed!', data.errors) if (data.errors) { if (data.errors[0] && data.errors[0] === 'Title has already been used') { logger.error('title already used : ' + options.title) } return false } } } } const uploadImages = async (content: string, discourse: Discourser, options: IOptionsSync) => { const root = path.resolve(resolve(options.root)) if (!exists(root)) { return false } const track_path = trackingPath(root) const track = tracking(root) const html = toHTML(content) const $ = cheerio.load(html, { xmlMode: true }); const images = images_urls(content) $('img').each(function () { if ($(this).attr('src') && $(this).attr('src').length > 5) { images.push($(this).attr('src')) } }) for await (const image of Object.entries(images)) { const url: string = image[1] if (url.length < 10) { continue } if (url.startsWith('upload:')) { continue } if (options.uploadRemote && url.startsWith('http')) { const contentHash = md5(content).substring(0, 5) const cache_path = path.resolve(resolve('${OSR_CACHE}/discourse-downloads/' + contentHash)) if (!exists(cache_path)) { dir(cache_path) } const image_name = imageName(url) const image_local = path.join(cache_path, image_name) if (!exists(image_local)) { try { await downloadFile(url, cache_path) } catch (e) { continue } } if (!exists(image_local)) { continue } if (!track[url]) { const upped: any = await discourse.uploadFile(options.owner, image_local) const data = upped.data; if (data && data.id) { track[url] = data write(track_path, track) } else { console.error('error uploading image') } } continue } if (options.uploadLocal) { const image_path = path.join(root, url) if (exists(image_path) && (!track[url] || options.cache === false)) { const upped: any = await discourse.uploadFile(options.owner, image_path) const data = upped.data; if (data && data.id) { track[url] = data write(track_path, track) } else { console.error('error uploading image') } } } } return track } const syncFile = async (file: string, options: IOptionsSync) => { const discourse = Instance(null, options.config as EDiscourseConfigKey) let config = fromJSON(file, options) || {} as IComponentConfig const componentDir = path.parse(file).dir // ph3 back sync const rel = forward_slash(path.relative(options.root, componentDir)) const productConfigPath = path.join(options.product_root, rel, 'config.json') let body = await createContent(componentDir, options) let images_track if (options.uploadLocal || options.uploadRemote) { images_track = await uploadImages(body, discourse, options) const image_urls = images_urls(body) image_urls.forEach((i) => { if (images_track[i]) { body = body.replace(i, images_track[i].short_url) } else { logger.warn(`Cant resolve image url : ${i} - ${componentDir} ! Image Upload track invalid`) } }) } logger.debug(`Processing ${componentDir}`); const output = path.join(componentDir, '.osr/discourse_raw.md') let dst = path.resolve(resolve(output)) options.debug && logger.info('Write output to: ', dst) write(dst, body) let post_id, topic_id let dOpts: IDiscoursePostBaseOptions = { ...options, cat: config.forumCategory, id: options.id, owner: config.forumUserId || 1, tags: config.forumTags as string, title: config.name, topic_id: config.forumTopicId, post_id: config.forumPostId } options = { ...options, ...dOpts } const hash = md5(JSON.stringify({ cat: dOpts.cat, tags: dOpts.tags, owner: dOpts.owner, body, title: dOpts.title }, null)) // const cats = await cacheCategories(options, discourse) // const tags = await cacheTags(options, discourse) const users = await cacheUsers(options, discourse) await new Promise(f => setTimeout(f, 1000)); let search = await discourse.search(dOpts.title) await new Promise(f => setTimeout(f, 2000)); let dTopic: ISearchTopic let dPost: ISearchPost if (search && search.posts && search.topics) { search.topics.forEach((t, i) => { if (t.title === dOpts.title) { dTopic = t dPost = search.posts[i] topic_id = dTopic.id post_id = dPost.id } }) } if (!dTopic || !dPost) { console.error('!dTopic || !dPost : cant find ' + dOpts.title) // return } const user = users.find((u) => { return u.id === dOpts.owner }) if (!user) { logger.error('Invalid user : ', dOpts.owner) return false } options.user_name = user.username if (SKIP_EXISTING && hash === config.forumPostHash && config.forumTopicId && config.forumPostId) { return } if (CONTENT_TEST) { return } if (post_id) { if (await updatePost(discourse, options, post_id, body)) { if (topic_id) { await new Promise(f => setTimeout(f, 2000)); await discourse.updateTopic(topic_id, dOpts.cat as number, dOpts.title, dOpts.tags ? dOpts.tags.split(',') : []) } } else { logger.error(`Error updating post ${dOpts.title}`) } } else { if (await createPost(discourse, options, body)) { await new Promise(f => setTimeout(f, 1000)); await discourse.updateTopic(options.topic_id, dOpts.cat as number, dOpts.title, dOpts.tags ? dOpts.tags.split(',') : []) } else { logger.error('Creating post failed !', dOpts.title) } } // const visStatus = await discourse.updateTopicVisibility(topic_id, true) // re-read without defaults config = readOSRConfig(file) config.forumPostHash = hash if (dTopic) { config.forumTopicId = dTopic.id }else if(topic_id){ config.forumTopicId = topic_id } if (dPost) { config.forumPostId = dPost.id }else if(post_id){ config.forumPostId = post_id } write(file, config) //ph3 products if (exists(productConfigPath)) { let pConfig = readOSRConfig(productConfigPath) logger.debug(`Updating product config ${productConfigPath}`) pConfig = { ...config //...pConfig, //... /* forumTopicId:config.forumTopicId, forumPostId:config.forumPostId, forumPostHash: config.forumPostHash */ } write(productConfigPath, pConfig) } return body } export const syncComponent = async (options: IOptionsSync) => { let components = options.srcInfo.FILES.filter(isValidLibraryComponent) //let components = options.srcInfo.FILES.filter((c) => { //components = components.filter((c) => { /* try { const config = readOSRConfig(c) as IComponentConfig if (config) { if (config.forum === false) { return false } // return !config.code && !config.cscartId && !config.steps return !!config.name } return false } catch (error) { logger.error(`Invalid config : ${c}`) } })*/ const skipExisting = options.skip /* [ "C:/Users/mc007/Desktop/osr/osr-machines/shredder/asterix-pp/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/asterix-sm-morren/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/bicycle-shredder/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/idefix/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/obelix/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/pp-v3.3/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/components/shredder_v21-light-ex/config.json", "C:/Users/mc007/Desktop/osr/osr-machines/shredder/components/shredder_v31-light/config.json", ] */ if (skipExisting) { components = components.filter((f) => { const config = readOSRConfig(f) as IComponentConfig if (config.forumPostId && config.forumTopicId) { return false } return true }) } //components = [components[0]] logger.info(`Syncing ${components.length} components`, components) await BPromise.resolve(components).map((f) => { try { return syncFile(f, options) } catch (error) { debugger } }, { concurrency: 1 }) } export const sync = async (options: IOptionsSync) => { return syncComponent(options) }