osr-plastic/osr-discourse-src
Archived
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

DEV: Apply syntax_tree formatting to script/*

Browse Source
This commit is contained in:
David Taylor
2023-01-07 11:53:14 +00:00
parent ff508d1ae5
commit 436b3b392b
143 changed files with 8905 additions and 7353 deletions
Download Patch File Download Diff File Expand all files Collapse all files
script/import_scripts/phpbb3/support/bbcode/markdown_node.rb
+4 -2
View File
@@ -1,7 +1,9 @@
# frozen_string_literal: true
module ImportScripts; end
module ImportScripts::PhpBB3; end
module ImportScripts
end
module ImportScripts::PhpBB3
end
module ImportScripts::PhpBB3::BBCode
LINEBREAK_AUTO = :auto
script/import_scripts/phpbb3/support/bbcode/xml_to_markdown.rb
+62 -45
View File
@@ -1,7 +1,7 @@
# frozen_string_literal: true
require 'nokogiri'
require_relative 'markdown_node'
require "nokogiri"
require_relative "markdown_node"
module ImportScripts::PhpBB3::BBCode
class XmlToMarkdown
@@ -14,7 +14,7 @@ module ImportScripts::PhpBB3::BBCode
@allow_inline_code = opts.fetch(:allow_inline_code, false)
@traditional_linebreaks = opts.fetch(:traditional_linebreaks, false)
@doc = Nokogiri::XML(xml)
@doc = Nokogiri.XML(xml)
@list_stack = []
end
@@ -28,9 +28,9 @@ module ImportScripts::PhpBB3::BBCode
private
IGNORED_ELEMENTS = ["s", "e", "i"]
ELEMENTS_WITHOUT_LEADING_WHITESPACES = ["LIST", "LI"]
ELEMENTS_WITH_HARD_LINEBREAKS = ["B", "I", "U"]
IGNORED_ELEMENTS = %w[s e i]
ELEMENTS_WITHOUT_LEADING_WHITESPACES = %w[LIST LI]
ELEMENTS_WITH_HARD_LINEBREAKS = %w[B I U]
EXPLICIT_LINEBREAK_THRESHOLD = 2
def preprocess_xml
@@ -65,9 +65,7 @@ module ImportScripts::PhpBB3::BBCode
xml_node.children.each { |xml_child| visit(xml_child, md_node || md_parent) }
after_hook = "after_#{xml_node.name}"
if respond_to?(after_hook, include_all: true)
send(after_hook, xml_node, md_node)
end
send(after_hook, xml_node, md_node) if respond_to?(after_hook, include_all: true)
end
def create_node(xml_node, md_parent)
@@ -84,19 +82,15 @@ module ImportScripts::PhpBB3::BBCode
end
def visit_B(xml_node, md_node)
if xml_node.parent&.name != 'B'
md_node.enclosed_with = "**"
end
md_node.enclosed_with = "**" if xml_node.parent&.name != "B"
end
def visit_I(xml_node, md_node)
if xml_node.parent&.name != 'I'
md_node.enclosed_with = "_"
end
md_node.enclosed_with = "_" if xml_node.parent&.name != "I"
end
def visit_U(xml_node, md_node)
if xml_node.parent&.name != 'U'
if xml_node.parent&.name != "U"
md_node.prefix = "[u]"
md_node.postfix = "[/u]"
end
@@ -122,10 +116,7 @@ module ImportScripts::PhpBB3::BBCode
md_node.prefix_linebreaks = md_node.postfix_linebreaks = @list_stack.size == 0 ? 2 : 1
md_node.prefix_linebreak_type = LINEBREAK_HTML if @list_stack.size == 0
@list_stack << {
unordered: xml_node.attribute('type').nil?,
item_count: 0
}
@list_stack << { unordered: xml_node.attribute("type").nil?, item_count: 0 }
end
def after_LIST(xml_node, md_node)
@@ -138,21 +129,21 @@ module ImportScripts::PhpBB3::BBCode
list[:item_count] += 1
indentation = ' ' * 2 * depth
symbol = list[:unordered] ? '*' : "#{list[:item_count]}."
indentation = " " * 2 * depth
symbol = list[:unordered] ? "*" : "#{list[:item_count]}."
md_node.prefix = "#{indentation}#{symbol} "
md_node.postfix_linebreaks = 1
end
def visit_IMG(xml_node, md_node)
md_node.text = +"![](#{xml_node.attribute('src')})"
md_node.text = +"![](#{xml_node.attribute("src")})"
md_node.prefix_linebreaks = md_node.postfix_linebreaks = 2
md_node.skip_children
end
def visit_URL(xml_node, md_node)
original_url = xml_node.attribute('url').to_s
original_url = xml_node.attribute("url").to_s
url = CGI.unescapeHTML(original_url)
url = @url_replacement.call(url) if @url_replacement
@@ -173,7 +164,8 @@ module ImportScripts::PhpBB3::BBCode
def visit_br(xml_node, md_node)
md_node.postfix_linebreaks += 1
if md_node.postfix_linebreaks > 1 && ELEMENTS_WITH_HARD_LINEBREAKS.include?(xml_node.parent&.name)
if md_node.postfix_linebreaks > 1 &&
ELEMENTS_WITH_HARD_LINEBREAKS.include?(xml_node.parent&.name)
md_node.postfix_linebreak_type = LINEBREAK_HARD
end
end
@@ -194,7 +186,8 @@ module ImportScripts::PhpBB3::BBCode
def visit_QUOTE(xml_node, md_node)
if post = quoted_post(xml_node)
md_node.prefix = %Q{[quote="#{post[:username]}, post:#{post[:post_number]}, topic:#{post[:topic_id]}"]\n}
md_node.prefix =
%Q{[quote="#{post[:username]}, post:#{post[:post_number]}, topic:#{post[:topic_id]}"]\n}
md_node.postfix = "\n[/quote]"
elsif username = quoted_username(xml_node)
md_node.prefix = %Q{[quote="#{username}"]\n}
@@ -242,11 +235,11 @@ module ImportScripts::PhpBB3::BBCode
return if size.nil?
if size.between?(1, 99)
md_node.prefix = '<small>'
md_node.postfix = '</small>'
md_node.prefix = "<small>"
md_node.postfix = "</small>"
elsif size.between?(101, 200)
md_node.prefix = '<big>'
md_node.postfix = '</big>'
md_node.prefix = "<big>"
md_node.postfix = "</big>"
end
end
@@ -267,7 +260,8 @@ module ImportScripts::PhpBB3::BBCode
parent_prefix = prefix_from_parent(md_parent)
if parent_prefix && md_node.xml_node_name != "br" && (md_parent.prefix_children || !markdown.empty?)
if parent_prefix && md_node.xml_node_name != "br" &&
(md_parent.prefix_children || !markdown.empty?)
prefix = "#{parent_prefix}#{prefix}"
end
@@ -275,11 +269,21 @@ module ImportScripts::PhpBB3::BBCode
text, prefix, postfix = hoist_whitespaces!(markdown, text, prefix, postfix)
end
add_linebreaks!(markdown, md_node.prefix_linebreaks, md_node.prefix_linebreak_type, parent_prefix)
add_linebreaks!(
markdown,
md_node.prefix_linebreaks,
md_node.prefix_linebreak_type,
parent_prefix,
)
markdown << prefix
markdown << text
markdown << postfix
add_linebreaks!(markdown, md_node.postfix_linebreaks, md_node.postfix_linebreak_type, parent_prefix)
add_linebreaks!(
markdown,
md_node.postfix_linebreaks,
md_node.postfix_linebreak_type,
parent_prefix,
)
end
markdown
@@ -296,9 +300,7 @@ module ImportScripts::PhpBB3::BBCode
end
unless postfix.empty?
if ends_with_whitespace?(text)
postfix = "#{postfix}#{text[-1]}"
end
postfix = "#{postfix}#{text[-1]}" if ends_with_whitespace?(text)
text = text.rstrip
end
@@ -319,16 +321,24 @@ module ImportScripts::PhpBB3::BBCode
if linebreak_type == LINEBREAK_HTML
max_linebreak_count = [existing_linebreak_count, required_linebreak_count - 1].max + 1
required_linebreak_count = max_linebreak_count if max_linebreak_count > EXPLICIT_LINEBREAK_THRESHOLD
required_linebreak_count = max_linebreak_count if max_linebreak_count >
EXPLICIT_LINEBREAK_THRESHOLD
end
return if existing_linebreak_count >= required_linebreak_count
rstrip!(markdown)
alternative_linebreak_start_index = required_linebreak_count > EXPLICIT_LINEBREAK_THRESHOLD ? 1 : 2
alternative_linebreak_start_index =
required_linebreak_count > EXPLICIT_LINEBREAK_THRESHOLD ? 1 : 2
required_linebreak_count.times do |index|
linebreak = linebreak(linebreak_type, index, alternative_linebreak_start_index, required_linebreak_count)
linebreak =
linebreak(
linebreak_type,
index,
alternative_linebreak_start_index,
required_linebreak_count,
)
markdown << (linebreak == "\n" ? prefix.rstrip : prefix) if prefix && index > 0
markdown << linebreak
@@ -336,18 +346,25 @@ module ImportScripts::PhpBB3::BBCode
end
def rstrip!(markdown)
markdown.gsub!(/\s*(?:\\?\n|<br>\n)*\z/, '')
markdown.gsub!(/\s*(?:\\?\n|<br>\n)*\z/, "")
end
def linebreak(linebreak_type, linebreak_index, alternative_linebreak_start_index, required_linebreak_count)
def linebreak(
linebreak_type,
linebreak_index,
alternative_linebreak_start_index,
required_linebreak_count
)
use_alternative_linebreak = linebreak_index >= alternative_linebreak_start_index
is_last_linebreak = linebreak_index + 1 == required_linebreak_count
return "<br>\n" if linebreak_type == LINEBREAK_HTML &&
use_alternative_linebreak && is_last_linebreak
if linebreak_type == LINEBREAK_HTML && use_alternative_linebreak && is_last_linebreak
return "<br>\n"
end
return "\\\n" if linebreak_type == LINEBREAK_HARD ||
@traditional_linebreaks || use_alternative_linebreak
if linebreak_type == LINEBREAK_HARD || @traditional_linebreaks || use_alternative_linebreak
return "\\\n"
end
"\n"
end
script/import_scripts/phpbb3/support/constants.rb
+5 -5
View File
@@ -8,8 +8,8 @@ module ImportScripts::PhpBB3
INACTIVE_MANUAL = 3 # Account deactivated by administrator
INACTIVE_REMIND = 4 # Forced user account reactivation
GROUP_ADMINISTRATORS = 'ADMINISTRATORS'
GROUP_MODERATORS = 'GLOBAL_MODERATORS'
GROUP_ADMINISTRATORS = "ADMINISTRATORS"
GROUP_MODERATORS = "GLOBAL_MODERATORS"
# https://wiki.phpbb.com/Table.phpbb_users
USER_TYPE_NORMAL = 0
@@ -21,9 +21,9 @@ module ImportScripts::PhpBB3
AVATAR_TYPE_REMOTE = 2
AVATAR_TYPE_GALLERY = 3
AVATAR_TYPE_STRING_UPLOADED = 'avatar.driver.upload'
AVATAR_TYPE_STRING_REMOTE = 'avatar.driver.remote'
AVATAR_TYPE_STRING_GALLERY = 'avatar.driver.local'
AVATAR_TYPE_STRING_UPLOADED = "avatar.driver.upload"
AVATAR_TYPE_STRING_REMOTE = "avatar.driver.remote"
AVATAR_TYPE_STRING_GALLERY = "avatar.driver.local"
FORUM_TYPE_CATEGORY = 0
FORUM_TYPE_POST = 1
script/import_scripts/phpbb3/support/settings.rb
+44 -43
View File
@@ -1,13 +1,13 @@
# frozen_string_literal: true
require 'csv'
require 'yaml'
require_relative '../../base'
require "csv"
require "yaml"
require_relative "../../base"
module ImportScripts::PhpBB3
class Settings
def self.load(filename)
yaml = YAML::load_file(filename)
yaml = YAML.load_file(filename)
Settings.new(yaml.deep_stringify_keys.with_indifferent_access)
end
@@ -44,40 +44,41 @@ module ImportScripts::PhpBB3
attr_reader :database
def initialize(yaml)
import_settings = yaml['import']
import_settings = yaml["import"]
@site_name = import_settings['site_name']
@site_name = import_settings["site_name"]
@new_categories = import_settings['new_categories']
@category_mappings = import_settings.fetch('category_mappings', []).to_h { |m| [m[:source_category_id].to_s, m] }
@tag_mappings = import_settings['tag_mappings']
@rank_mapping = import_settings['rank_mapping']
@new_categories = import_settings["new_categories"]
@category_mappings =
import_settings.fetch("category_mappings", []).to_h { |m| [m[:source_category_id].to_s, m] }
@tag_mappings = import_settings["tag_mappings"]
@rank_mapping = import_settings["rank_mapping"]
@import_anonymous_users = import_settings['anonymous_users']
@import_attachments = import_settings['attachments']
@import_private_messages = import_settings['private_messages']
@import_polls = import_settings['polls']
@import_bookmarks = import_settings['bookmarks']
@import_passwords = import_settings['passwords']
@import_likes = import_settings['likes']
@import_anonymous_users = import_settings["anonymous_users"]
@import_attachments = import_settings["attachments"]
@import_private_messages = import_settings["private_messages"]
@import_polls = import_settings["polls"]
@import_bookmarks = import_settings["bookmarks"]
@import_passwords = import_settings["passwords"]
@import_likes = import_settings["likes"]
avatar_settings = import_settings['avatars']
@import_uploaded_avatars = avatar_settings['uploaded']
@import_remote_avatars = avatar_settings['remote']
@import_gallery_avatars = avatar_settings['gallery']
avatar_settings = import_settings["avatars"]
@import_uploaded_avatars = avatar_settings["uploaded"]
@import_remote_avatars = avatar_settings["remote"]
@import_gallery_avatars = avatar_settings["gallery"]
@use_bbcode_to_md = import_settings['use_bbcode_to_md']
@use_bbcode_to_md = import_settings["use_bbcode_to_md"]
@original_site_prefix = import_settings['site_prefix']['original']
@new_site_prefix = import_settings['site_prefix']['new']
@base_dir = import_settings['phpbb_base_dir']
@permalinks = PermalinkSettings.new(import_settings['permalinks'])
@original_site_prefix = import_settings["site_prefix"]["original"]
@new_site_prefix = import_settings["site_prefix"]["new"]
@base_dir = import_settings["phpbb_base_dir"]
@permalinks = PermalinkSettings.new(import_settings["permalinks"])
@username_as_name = import_settings['username_as_name']
@emojis = import_settings.fetch('emojis', [])
@custom_fields = import_settings.fetch('custom_fields', [])
@username_as_name = import_settings["username_as_name"]
@emojis = import_settings.fetch("emojis", [])
@custom_fields = import_settings.fetch("custom_fields", [])
@database = DatabaseSettings.new(yaml['database'])
@database = DatabaseSettings.new(yaml["database"])
end
def prefix(val)
@@ -87,7 +88,7 @@ module ImportScripts::PhpBB3
def trust_level_for_posts(rank, trust_level: 0)
if @rank_mapping.present?
@rank_mapping.each do |key, value|
trust_level = [trust_level, key.gsub('trust_level_', '').to_i].max if rank >= value
trust_level = [trust_level, key.gsub("trust_level_", "").to_i].max if rank >= value
end
end
@@ -106,14 +107,14 @@ module ImportScripts::PhpBB3
attr_reader :batch_size
def initialize(yaml)
@type = yaml['type']
@host = yaml['host']
@port = yaml['port']
@username = yaml['username']
@password = yaml['password']
@schema = yaml['schema']
@table_prefix = yaml['table_prefix']
@batch_size = yaml['batch_size']
@type = yaml["type"]
@host = yaml["host"]
@port = yaml["port"]
@username = yaml["username"]
@password = yaml["password"]
@schema = yaml["schema"]
@table_prefix = yaml["table_prefix"]
@batch_size = yaml["batch_size"]
end
end
@@ -124,10 +125,10 @@ module ImportScripts::PhpBB3
attr_reader :normalization_prefix
def initialize(yaml)
@create_category_links = yaml['categories']
@create_topic_links = yaml['topics']
@create_post_links = yaml['posts']
@normalization_prefix = yaml['prefix']
@create_category_links = yaml["categories"]
@create_topic_links = yaml["topics"]
@create_post_links = yaml["posts"]
@normalization_prefix = yaml["prefix"]
end
end
end
script/import_scripts/phpbb3/support/smiley_processor.rb
+29 -31
View File
@@ -18,15 +18,16 @@ module ImportScripts::PhpBB3
def replace_smilies(text)
# :) is encoded as <!-- s:) --><img src="{SMILIES_PATH}/icon_e_smile.gif" alt=":)" title="Smile" /><!-- s:) -->
text.gsub!(/<!-- s(\S+) --><img src="\{SMILIES_PATH\}\/.+?" alt=".*?" title=".*?" \/><!-- s?\S+ -->/) do
emoji($1)
end
text.gsub!(
/<!-- s(\S+) --><img src="\{SMILIES_PATH\}\/.+?" alt=".*?" title=".*?" \/><!-- s?\S+ -->/,
) { emoji($1) }
end
def emoji(smiley_code)
@smiley_map.fetch(smiley_code) do
smiley = @database.get_smiley(smiley_code)
emoji = upload_smiley(smiley_code, smiley[:smiley_url], smiley_code, smiley[:emotion]) if smiley
emoji =
upload_smiley(smiley_code, smiley[:smiley_url], smiley_code, smiley[:emotion]) if smiley
emoji || smiley_as_text(smiley_code)
end
end
@@ -35,37 +36,34 @@ module ImportScripts::PhpBB3
def add_default_smilies
{
[':D', ':-D', ':grin:'] => ':smiley:',
[':)', ':-)', ':smile:'] => ':slight_smile:',
[';)', ';-)', ':wink:'] => ':wink:',
[':(', ':-(', ':sad:'] => ':frowning:',
[':o', ':-o', ':eek:'] => ':astonished:',
[':shock:'] => ':open_mouth:',
[':?', ':-?', ':???:'] => ':confused:',
['8)', '8-)', ':cool:'] => ':sunglasses:',
[':lol:'] => ':laughing:',
[':x', ':-x', ':mad:'] => ':angry:',
[':P', ':-P', ':razz:'] => ':stuck_out_tongue:',
[':oops:'] => ':blush:',
[':cry:'] => ':cry:',
[':evil:'] => ':imp:',
[':twisted:'] => ':smiling_imp:',
[':roll:'] => ':unamused:',
[':!:'] => ':exclamation:',
[':?:'] => ':question:',
[':idea:'] => ':bulb:',
[':arrow:'] => ':arrow_right:',
[':|', ':-|'] => ':neutral_face:',
[':geek:'] => ':nerd:'
}.each do |smilies, emoji|
smilies.each { |smiley| @smiley_map[smiley] = emoji }
end
%w[:D :-D :grin:] => ":smiley:",
%w[:) :-) :smile:] => ":slight_smile:",
%w[;) ;-) :wink:] => ":wink:",
%w[:( :-( :sad:] => ":frowning:",
%w[:o :-o :eek:] => ":astonished:",
[":shock:"] => ":open_mouth:",
%w[:? :-? :???:] => ":confused:",
%w[8) 8-) :cool:] => ":sunglasses:",
[":lol:"] => ":laughing:",
%w[:x :-x :mad:] => ":angry:",
%w[:P :-P :razz:] => ":stuck_out_tongue:",
[":oops:"] => ":blush:",
[":cry:"] => ":cry:",
[":evil:"] => ":imp:",
[":twisted:"] => ":smiling_imp:",
[":roll:"] => ":unamused:",
[":!:"] => ":exclamation:",
[":?:"] => ":question:",
[":idea:"] => ":bulb:",
[":arrow:"] => ":arrow_right:",
%w[:| :-|] => ":neutral_face:",
[":geek:"] => ":nerd:",
}.each { |smilies, emoji| smilies.each { |smiley| @smiley_map[smiley] = emoji } }
end
def add_configured_smilies(emojis)
emojis.each do |emoji, smilies|
Array.wrap(smilies)
.each { |smiley| @smiley_map[smiley] = ":#{emoji}:" }
Array.wrap(smilies).each { |smiley| @smiley_map[smiley] = ":#{emoji}:" }
end
end
script/import_scripts/phpbb3/support/text_processor.rb
+58 -45
View File
@@ -1,6 +1,6 @@
# frozen_string_literal: true
require_relative 'bbcode/xml_to_markdown'
require_relative "bbcode/xml_to_markdown"
module ImportScripts::PhpBB3
class TextProcessor
@@ -14,7 +14,9 @@ module ImportScripts::PhpBB3
@database = database
@smiley_processor = smiley_processor
@he = HTMLEntities.new
@use_xml_to_markdown = phpbb_config[:phpbb_version].start_with?('3.2') || phpbb_config[:phpbb_version].start_with?('3.3')
@use_xml_to_markdown =
phpbb_config[:phpbb_version].start_with?("3.2") ||
phpbb_config[:phpbb_version].start_with?("3.3")
@settings = settings
@new_site_prefix = settings.new_site_prefix
@@ -25,24 +27,27 @@ module ImportScripts::PhpBB3
if @use_xml_to_markdown
unreferenced_attachments = attachments&.dup
converter = BBCode::XmlToMarkdown.new(
raw,
username_from_user_id: lambda { |user_id| @lookup.find_username_by_import_id(user_id) },
smilie_to_emoji: lambda { |smilie| @smiley_processor.emoji(smilie).dup },
quoted_post_from_post_id: lambda { |post_id| @lookup.topic_lookup_from_imported_post_id(post_id) },
upload_md_from_file: (lambda do |filename, index|
unreferenced_attachments[index] = nil
attachments.fetch(index, filename).dup
end if attachments),
url_replacement: nil,
allow_inline_code: false
)
converter =
BBCode::XmlToMarkdown.new(
raw,
username_from_user_id: lambda { |user_id| @lookup.find_username_by_import_id(user_id) },
smilie_to_emoji: lambda { |smilie| @smiley_processor.emoji(smilie).dup },
quoted_post_from_post_id:
lambda { |post_id| @lookup.topic_lookup_from_imported_post_id(post_id) },
upload_md_from_file:
(
lambda do |filename, index|
unreferenced_attachments[index] = nil
attachments.fetch(index, filename).dup
end if attachments
),
url_replacement: nil,
allow_inline_code: false,
)
text = converter.convert
text.gsub!(@short_internal_link_regexp) do |link|
replace_internal_link(link, $1, $2)
end
text.gsub!(@short_internal_link_regexp) { |link| replace_internal_link(link, $1, $2) }
add_unreferenced_attachments(text, unreferenced_attachments)
else
@@ -50,9 +55,7 @@ module ImportScripts::PhpBB3
text = CGI.unescapeHTML(text)
clean_bbcodes(text)
if @settings.use_bbcode_to_md
text = bbcode_to_md(text)
end
text = bbcode_to_md(text) if @settings.use_bbcode_to_md
process_smilies(text)
process_links(text)
process_lists(text)
@@ -65,11 +68,19 @@ module ImportScripts::PhpBB3
end
def process_post(raw, attachments)
process_raw_text(raw, attachments) rescue raw
begin
process_raw_text(raw, attachments)
rescue StandardError
raw
end
end
def process_private_msg(raw, attachments)
process_raw_text(raw, attachments) rescue raw
begin
process_raw_text(raw, attachments)
rescue StandardError
raw
end
end
protected
@@ -78,10 +89,10 @@ module ImportScripts::PhpBB3
# Many phpbb bbcode tags have a hash attached to them. Examples:
# [url=https&#58;//google&#46;com:1qh1i7ky]click here[/url:1qh1i7ky]
# [quote=&quot;cybereality&quot;:b0wtlzex]Some text.[/quote:b0wtlzex]
text.gsub!(/:(?:\w{5,8})\]/, ']')
text.gsub!(/:(?:\w{5,8})\]/, "]")
# remove color tags
text.gsub!(/\[\/?color(=#?[a-z0-9]*)?\]/i, "")
text.gsub!(%r{\[/?color(=#?[a-z0-9]*)?\]}i, "")
end
def bbcode_to_md(text)
@@ -101,23 +112,19 @@ module ImportScripts::PhpBB3
# Internal forum links can have this forms:
# for topics: <!-- l --><a class="postlink-local" href="https://example.com/forums/viewtopic.php?f=26&amp;t=3412">viewtopic.php?f=26&amp;t=3412</a><!-- l -->
# for posts: <!-- l --><a class="postlink-local" href="https://example.com/forums/viewtopic.php?p=1732#p1732">viewtopic.php?p=1732#p1732</a><!-- l -->
text.gsub!(@long_internal_link_regexp) do |link|
replace_internal_link(link, $1, $2)
end
text.gsub!(@long_internal_link_regexp) { |link| replace_internal_link(link, $1, $2) }
# Some links look like this: <!-- m --><a class="postlink" href="http://www.onegameamonth.com">http://www.onegameamonth.com</a><!-- m -->
text.gsub!(/<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)<\/a><!-- \w -->/i, '[\2](\1)')
text.gsub!(%r{<!-- \w --><a(?:.+)href="(\S+)"(?:.*)>(.+)</a><!-- \w -->}i, '[\2](\1)')
# Replace internal forum links that aren't in the <!-- l --> format
text.gsub!(@short_internal_link_regexp) do |link|
replace_internal_link(link, $1, $2)
end
text.gsub!(@short_internal_link_regexp) { |link| replace_internal_link(link, $1, $2) }
# phpBB shortens link text like this, which breaks our markdown processing:
# [http://answers.yahoo.com/question/index ... 223AAkkPli](http://answers.yahoo.com/question/index?qid=20070920134223AAkkPli)
#
# Work around it for now:
text.gsub!(/\[http(s)?:\/\/(www\.)?/i, '[')
text.gsub!(%r{\[http(s)?://(www\.)?}i, "[")
end
def replace_internal_link(link, import_topic_id, import_post_id)
@@ -144,19 +151,20 @@ module ImportScripts::PhpBB3
# convert list tags to ul and list=1 tags to ol
# list=a is not supported, so handle it like list=1
# list=9 and list=x have the same result as list=1 and list=a
text.gsub!(/\[list\](.*?)\[\/list:u\]/mi) do
$1.gsub(/\[\*\](.*?)\[\/\*:m\]\n*/mi) { "* #{$1}\n" }
text.gsub!(%r{\[list\](.*?)\[/list:u\]}mi) do
$1.gsub(%r{\[\*\](.*?)\[/\*:m\]\n*}mi) { "* #{$1}\n" }
end
text.gsub!(/\[list=.*?\](.*?)\[\/list:o\]/mi) do
$1.gsub(/\[\*\](.*?)\[\/\*:m\]\n*/mi) { "1. #{$1}\n" }
text.gsub!(%r{\[list=.*?\](.*?)\[/list:o\]}mi) do
$1.gsub(%r{\[\*\](.*?)\[/\*:m\]\n*}mi) { "1. #{$1}\n" }
end
end
# This replaces existing [attachment] BBCodes with the corresponding HTML tags for Discourse.
# All attachments that haven't been referenced in the text are appended to the end of the text.
def process_attachments(text, attachments)
attachment_regexp = /\[attachment=([\d])+\]<!-- [\w]+ -->([^<]+)<!-- [\w]+ -->\[\/attachment\]?/i
attachment_regexp =
%r{\[attachment=([\d])+\]<!-- [\w]+ -->([^<]+)<!-- [\w]+ -->\[/attachment\]?}i
unreferenced_attachments = attachments.dup
text.gsub!(attachment_regexp) do
@@ -178,29 +186,34 @@ module ImportScripts::PhpBB3
end
def create_internal_link_regexps(original_site_prefix)
host = original_site_prefix.gsub('.', '\.')
link_regex = "http(?:s)?://#{host}/viewtopic\\.php\\?(?:\\S*)(?:t=(\\d+)|p=(\\d+)(?:#p\\d+)?)(?:[^\\s\\)\\]]*)"
host = original_site_prefix.gsub(".", '\.')
link_regex =
"http(?:s)?://#{host}/viewtopic\\.php\\?(?:\\S*)(?:t=(\\d+)|p=(\\d+)(?:#p\\d+)?)(?:[^\\s\\)\\]]*)"
@long_internal_link_regexp = Regexp.new(%Q|<!-- l --><a(?:.+)href="#{link_regex}"(?:.*)</a><!-- l -->|, Regexp::IGNORECASE)
@long_internal_link_regexp =
Regexp.new(
%Q|<!-- l --><a(?:.+)href="#{link_regex}"(?:.*)</a><!-- l -->|,
Regexp::IGNORECASE,
)
@short_internal_link_regexp = Regexp.new(link_regex, Regexp::IGNORECASE)
end
def process_code(text)
text.gsub!(/<span class="syntax.*?>(.*?)<\/span>/) { "#{$1}" }
text.gsub!(/\[code(=[a-z]*)?\](.*?)\[\/code\]/i) { "[code]\n#{@he.decode($2)}\n[/code]" }
text.gsub!(/<br \/>/, "\n")
text.gsub!(%r{<span class="syntax.*?>(.*?)</span>}) { "#{$1}" }
text.gsub!(%r{\[code(=[a-z]*)?\](.*?)\[/code\]}i) { "[code]\n#{@he.decode($2)}\n[/code]" }
text.gsub!(%r{<br />}, "\n")
text
end
def fix_markdown(text)
text.gsub!(/(\n*\[\/?quote.*?\]\n*)/mi) { |q| "\n#{q.strip}\n" }
text.gsub!(%r{(\n*\[/?quote.*?\]\n*)}mi) { |q| "\n#{q.strip}\n" }
text.gsub!(/^!\[[^\]]*\]\([^\]]*\)$/i) { |img| "\n#{img.strip}\n" } # space out images single on line
text
end
def process_videos(text)
# [YOUTUBE]<id>[/YOUTUBE]
text.gsub(/\[youtube\](.+?)\[\/youtube\]/i) { "\nhttps://www.youtube.com/watch?v=#{$1}\n" }
text.gsub(%r{\[youtube\](.+?)\[/youtube\]}i) { "\nhttps://www.youtube.com/watch?v=#{$1}\n" }
text
end
end