From 23d67d21009964c636cd69c06fb99da8a666af08 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Mon, 5 Jun 2017 18:00:15 +0200 Subject: [PATCH 001/382] Add includes image choice to advanced search ui. --- .../discourse/components/search-advanced-options.js.es6 | 3 ++- config/locales/client.en.yml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 b/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 index 0bc886a817..1ffd1cdaea 100644 --- a/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 +++ b/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 @@ -14,7 +14,7 @@ const REGEXP_MIN_POST_COUNT_PREFIX = /^min_post_count:/ig; const REGEXP_POST_TIME_PREFIX = /^(before|after):/ig; const REGEXP_TAGS_REPLACE = /(^(tags?:|#(?=[a-z0-9\-]+::tag))|::tag\s?$)/ig; -const REGEXP_IN_MATCH = /^in:(posted|watching|tracking|bookmarks|first|pinned|unpinned|wiki|unseen)/ig; +const REGEXP_IN_MATCH = /^in:(posted|watching|tracking|bookmarks|first|pinned|unpinned|wiki|unseen|image)/ig; const REGEXP_SPECIAL_IN_LIKES_MATCH = /^in:likes/ig; const REGEXP_SPECIAL_IN_PRIVATE_MATCH = /^in:private/ig; const REGEXP_SPECIAL_IN_SEEN_MATCH = /^in:seen/ig; @@ -36,6 +36,7 @@ export default Em.Component.extend({ {name: I18n.t('search.advanced.filters.pinned'), value: "pinned"}, {name: I18n.t('search.advanced.filters.unpinned'), value: "unpinned"}, {name: I18n.t('search.advanced.filters.wiki'), value: "wiki"}, + {name: I18n.t('search.advanced.filters.images'), value: "image"}, ], statusOptions: [ {name: I18n.t('search.advanced.statuses.open'), value: "open"}, diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 52b4e5301f..71532ab95d 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -1344,6 +1344,7 @@ en: seen: I've read unseen: I've not read wiki: are wiki + images: includes image statuses: label: Where topics open: are open From 8f7d81fde64a756a227e46746af22b99fa922bbc Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Tue, 6 Jun 2017 14:39:53 +0200 Subject: [PATCH 002/382] Add rspec test for searching posts with images. --- config/locales/client.en.yml | 2 +- spec/components/search_spec.rb | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 71532ab95d..c2268368f5 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -1344,7 +1344,7 @@ en: seen: I've read unseen: I've not read wiki: are wiki - images: includes image + images: includes image(s) statuses: label: Where topics open: are open diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index d215c8835c..069f602065 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -628,6 +628,16 @@ describe Search do end + it 'can find posts with images' do + post_uploaded = Fabricate(:post_with_uploaded_image) + post_with_image_urls = Fabricate(:post_with_image_urls) + Fabricate(:post) + TopicLink.extract_from(post_uploaded) + TopicLink.extract_from(post_with_image_urls) + + expect(Search.execute('in:image').posts.map(&:id).sort).to eq([post_uploaded.id, post_with_image_urls.id].sort) + end + it 'can find by latest' do topic1 = Fabricate(:topic, title: 'I do not like that Sam I am') post1 = Fabricate(:post, topic: topic1) From 76712da16697abc24e78bf5748bbb180172f8d3d Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Wed, 7 Jun 2017 20:13:36 +0200 Subject: [PATCH 003/382] Add backend code for searching posts with images. --- lib/search.rb | 4 ++++ spec/components/search_spec.rb | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/search.rb b/lib/search.rb index 543149b18e..156c30c77a 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -338,6 +338,10 @@ class Search end end + advanced_filter(/in:image/) do |posts| + posts.where("posts.image_url IS NOT NULL") + end + advanced_filter(/category:(.+)/) do |posts,match| exact = false diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index 069f602065..2f9f32b07b 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -632,8 +632,9 @@ describe Search do post_uploaded = Fabricate(:post_with_uploaded_image) post_with_image_urls = Fabricate(:post_with_image_urls) Fabricate(:post) - TopicLink.extract_from(post_uploaded) - TopicLink.extract_from(post_with_image_urls) + + CookedPostProcessor.new(post_uploaded).update_post_image + CookedPostProcessor.new(post_with_image_urls).update_post_image expect(Search.execute('in:image').posts.map(&:id).sort).to eq([post_uploaded.id, post_with_image_urls.id].sort) end From 4c22f3a0e2cf85acc82860ed02fc81d01590e037 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Fri, 9 Jun 2017 13:56:18 +0200 Subject: [PATCH 004/382] Add file extension column to TopicLinks. --- db/migrate/20170609115401_add_extension_to_topic_links.rb | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 db/migrate/20170609115401_add_extension_to_topic_links.rb diff --git a/db/migrate/20170609115401_add_extension_to_topic_links.rb b/db/migrate/20170609115401_add_extension_to_topic_links.rb new file mode 100644 index 0000000000..8360b0ef08 --- /dev/null +++ b/db/migrate/20170609115401_add_extension_to_topic_links.rb @@ -0,0 +1,5 @@ +class AddExtensionToTopicLinks < ActiveRecord::Migration + def change + add_column :topic_links, :extension, :string, limit: 5 + end +end From bf002e0873b75ec5a6f252187b61bbaf299caaa7 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Fri, 9 Jun 2017 13:16:50 +0200 Subject: [PATCH 005/382] Add extraction of image_url for oneboxed images. Fix search by images filter name. --- .../components/search-advanced-options.js.es6 | 16 +++++++++++----- lib/cooked_post_processor.rb | 8 ++++---- lib/search.rb | 2 +- spec/components/cooked_post_processor_spec.rb | 4 ++-- spec/components/search_spec.rb | 2 +- 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 b/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 index 1ffd1cdaea..bc345a5977 100644 --- a/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 +++ b/app/assets/javascripts/discourse/components/search-advanced-options.js.es6 @@ -8,13 +8,13 @@ const REGEXP_CATEGORY_PREFIX = /^(category:|#)/ig; const REGEXP_GROUP_PREFIX = /^group:/ig; const REGEXP_BADGE_PREFIX = /^badge:/ig; const REGEXP_TAGS_PREFIX = /^(tags?:|#(?=[a-z0-9\-]+::tag))/ig; -const REGEXP_IN_PREFIX = /^in:/ig; +const REGEXP_IN_PREFIX = /^(in|with):/ig; const REGEXP_STATUS_PREFIX = /^status:/ig; const REGEXP_MIN_POST_COUNT_PREFIX = /^min_post_count:/ig; const REGEXP_POST_TIME_PREFIX = /^(before|after):/ig; const REGEXP_TAGS_REPLACE = /(^(tags?:|#(?=[a-z0-9\-]+::tag))|::tag\s?$)/ig; -const REGEXP_IN_MATCH = /^in:(posted|watching|tracking|bookmarks|first|pinned|unpinned|wiki|unseen|image)/ig; +const REGEXP_IN_MATCH = /^(in|with):(posted|watching|tracking|bookmarks|first|pinned|unpinned|wiki|unseen|image)/ig; const REGEXP_SPECIAL_IN_LIKES_MATCH = /^in:likes/ig; const REGEXP_SPECIAL_IN_PRIVATE_MATCH = /^in:private/ig; const REGEXP_SPECIAL_IN_SEEN_MATCH = /^in:seen/ig; @@ -23,6 +23,8 @@ const REGEXP_CATEGORY_SLUG = /^(\#[a-zA-Z0-9\-:]+)/ig; const REGEXP_CATEGORY_ID = /^(category:[0-9]+)/ig; const REGEXP_POST_TIME_WHEN = /^(before|after)/ig; +const IN_OPTIONS_MAPPING = {'images': 'with'}; + export default Em.Component.extend({ classNames: ['search-advanced-options'], @@ -36,7 +38,7 @@ export default Em.Component.extend({ {name: I18n.t('search.advanced.filters.pinned'), value: "pinned"}, {name: I18n.t('search.advanced.filters.unpinned'), value: "unpinned"}, {name: I18n.t('search.advanced.filters.wiki'), value: "wiki"}, - {name: I18n.t('search.advanced.filters.images'), value: "image"}, + {name: I18n.t('search.advanced.filters.images'), value: "images"}, ], statusOptions: [ {name: I18n.t('search.advanced.statuses.open'), value: "open"}, @@ -392,13 +394,17 @@ export default Em.Component.extend({ updateSearchTermForIn() { const match = this.filterBlocks(REGEXP_IN_MATCH); const inFilter = this.get('searchedTerms.in'); + let keyword = 'in'; + if(inFilter in IN_OPTIONS_MAPPING) { + keyword = IN_OPTIONS_MAPPING[inFilter]; + } let searchTerm = this.get('searchTerm') || ''; if (inFilter) { if (match.length !== 0) { - searchTerm = searchTerm.replace(match[0], `in:${inFilter}`); + searchTerm = searchTerm.replace(match[0], `${keyword}:${inFilter}`); } else { - searchTerm += ` in:${inFilter}`; + searchTerm += ` ${keyword}:${inFilter}`; } this.set('searchTerm', searchTerm.trim()); diff --git a/lib/cooked_post_processor.rb b/lib/cooked_post_processor.rb index 48f1d38b49..43bb956771 100644 --- a/lib/cooked_post_processor.rb +++ b/lib/cooked_post_processor.rb @@ -76,8 +76,6 @@ class CookedPostProcessor limit_size!(img) convert_to_link!(img) end - - update_post_image end def extract_images @@ -98,8 +96,6 @@ class CookedPostProcessor @doc.css("img[src]") - # minus, emojis @doc.css("img.emoji") - - # minus, image inside oneboxes - oneboxed_images - # minus, images inside quotes @doc.css(".quote img") end @@ -283,6 +279,8 @@ class CookedPostProcessor def update_post_image img = extract_images_for_post.first + return if img.blank? + if img["src"].present? @post.update_column(:image_url, img["src"][0...255]) # post @post.topic.update_column(:image_url, img["src"][0...255]) if @post.is_first_post? # topic @@ -301,6 +299,8 @@ class CookedPostProcessor Oneboxer.onebox(url, args) end + update_post_image + # make sure we grab dimensions for oneboxed images oneboxed_images.each { |img| limit_size!(img) } diff --git a/lib/search.rb b/lib/search.rb index 156c30c77a..b3f886af59 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -338,7 +338,7 @@ class Search end end - advanced_filter(/in:image/) do |posts| + advanced_filter(/with:images/) do |posts| posts.where("posts.image_url IS NOT NULL") end diff --git a/spec/components/cooked_post_processor_spec.rb b/spec/components/cooked_post_processor_spec.rb index cc0f5f4467..dc58e4b855 100644 --- a/spec/components/cooked_post_processor_spec.rb +++ b/spec/components/cooked_post_processor_spec.rb @@ -249,7 +249,7 @@ describe CookedPostProcessor do it "adds a topic image if there's one in the first post" do FastImage.stubs(:size) expect(post.topic.image_url).to eq(nil) - cpp.post_process_images + cpp.update_post_image post.topic.reload expect(post.topic.image_url).to be_present end @@ -262,7 +262,7 @@ describe CookedPostProcessor do it "adds a post image if there's one in the post" do FastImage.stubs(:size) expect(reply.image_url).to eq(nil) - cpp.post_process_images + cpp.update_post_image reply.reload expect(reply.image_url).to be_present end diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index 2f9f32b07b..c535748167 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -636,7 +636,7 @@ describe Search do CookedPostProcessor.new(post_uploaded).update_post_image CookedPostProcessor.new(post_with_image_urls).update_post_image - expect(Search.execute('in:image').posts.map(&:id).sort).to eq([post_uploaded.id, post_with_image_urls.id].sort) + expect(Search.execute('with:images').posts.map(&:id)).to contain_exactly(post_uploaded.id, post_with_image_urls.id) end it 'can find by latest' do From eaf46431d49aebda75a8ef5eba0e2b7e4ede082b Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Mon, 19 Jun 2017 17:09:54 +0200 Subject: [PATCH 006/382] Add extraction of file extension in TopicLink and related rspec tests. --- app/models/topic_link.rb | 4 +++- spec/models/topic_link_spec.rb | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/app/models/topic_link.rb b/app/models/topic_link.rb index 12ae7dcaab..c9abf160c4 100644 --- a/app/models/topic_link.rb +++ b/app/models/topic_link.rb @@ -164,6 +164,7 @@ SQL added_urls << url unless TopicLink.exists?(topic_id: post.topic_id, post_id: post.id, url: url) + file_extension = File.extname(parsed.path)[1..5].downcase unless File.extname(parsed.path).empty? begin TopicLink.create!(post_id: post.id, user_id: post.user_id, @@ -173,7 +174,8 @@ SQL internal: internal, link_topic_id: topic_id, link_post_id: reflected_post.try(:id), - quote: link.is_quote) + quote: link.is_quote, + extension: file_extension) rescue ActiveRecord::RecordNotUnique, PG::UniqueViolation # it's fine end diff --git a/spec/models/topic_link_spec.rb b/spec/models/topic_link_spec.rb index 9ee96750f2..f9f0c9d309 100644 --- a/spec/models/topic_link_spec.rb +++ b/spec/models/topic_link_spec.rb @@ -193,7 +193,7 @@ http://b.com/#{'a'*500} end context "link to a local attachments" do - let(:post) { topic.posts.create(user: user, raw: 'ruby.rb') } + let(:post) { topic.posts.create(user: user, raw: 'ruby.rb') } it "extracts the link" do TopicLink.extract_from(post) @@ -203,9 +203,11 @@ http://b.com/#{'a'*500} # is set to internal expect(link).to be_internal # has the correct url - expect(link.url).to eq("/uploads/default/208/87bb3d8428eb4783.rb") + expect(link.url).to eq("/uploads/default/208/87bb3d8428eb4783.rb?foo=bar") # should not be the reflection expect(link).not_to be_reflection + # should have file extension + expect(link.extension).to eq('rb') end end @@ -224,6 +226,8 @@ http://b.com/#{'a'*500} expect(link.url).to eq("//s3.amazonaws.com/bucket/2104a0211c9ce41ed67989a1ed62e9a394c1fbd1446.rb") # should not be the reflection expect(link).not_to be_reflection + # should have file extension + expect(link.extension).to eq('rb') end end From 67ce4b70a67ccefb03874375bf73e904f9d6cb8e Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Tue, 20 Jun 2017 13:01:31 +0200 Subject: [PATCH 007/382] Add index to extension column in TopicLink. --- db/migrate/20170609115401_add_extension_to_topic_links.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/db/migrate/20170609115401_add_extension_to_topic_links.rb b/db/migrate/20170609115401_add_extension_to_topic_links.rb index 8360b0ef08..fbe8b9c184 100644 --- a/db/migrate/20170609115401_add_extension_to_topic_links.rb +++ b/db/migrate/20170609115401_add_extension_to_topic_links.rb @@ -1,5 +1,6 @@ class AddExtensionToTopicLinks < ActiveRecord::Migration def change add_column :topic_links, :extension, :string, limit: 5 + add_index :topic_links, :extension end end From f87d32ac6d6636912d0b8d559493333f4768abd7 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Tue, 20 Jun 2017 21:20:06 +0200 Subject: [PATCH 008/382] Add backend code for searching by filetypes. --- lib/search.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/search.rb b/lib/search.rb index 543149b18e..f3ea5ccd82 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -456,6 +456,15 @@ class Search )", tags) end + advanced_filter(/filetypes?:([a-zA-Z0-9,\-_]+)/) do |posts, match| + file_extensions = match.split(",") + + posts.where("posts.id IN ( + SELECT post_id FROM topic_links + WHERE extension IN (?) + )", file_extensions) + end + private From aad1c9bef2851d5dcc13ba4dddd40d818eb49424 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Tue, 20 Jun 2017 21:21:56 +0200 Subject: [PATCH 009/382] Add rspec tests for searching by a filetype. --- spec/components/search_spec.rb | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index d215c8835c..c7c2c5609e 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -703,6 +703,24 @@ describe Search do expect(Search.execute('green tags:eggs').posts.map(&:id)).to eq([post2.id]) expect(Search.execute('green tags:plants').posts.size).to eq(0) end + + it "can find posts which contains filetypes" do + # Must be posts with real images + post1 = Fabricate(:post, + raw: "https://www.discourse.org/a/img/favicon.png") + post2 = Fabricate(:post, + raw: "Discourse logo\n"\ + "https://www.discourse.org/a/img/favicon.png\n"\ + "https://www.discourse.org/a/img/trust-1x.jpg") + Fabricate(:post) + + TopicLink.extract_from(post1) + TopicLink.extract_from(post2) + + expect(Search.execute('filetype:jpg').posts.map(&:id)).to eq([post2.id]) + expect(Search.execute('filetype:png').posts.map(&:id)).to contain_exactly(post1.id, post2.id) + expect(Search.execute('logo filetype:jpg').posts.map(&:id)).to eq([post2.id]) + end end it 'can parse complex strings using ts_query helper' do From bb392973ca6b57510d26ad4a6ae44356b84b1233 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Mon, 3 Jul 2017 19:06:54 +0200 Subject: [PATCH 010/382] Add migration with extension column to uploads. --- .../20170609115401_add_extension_to_topic_links.rb | 2 +- db/migrate/20170703115216_add_extension_to_uploads.rb | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 db/migrate/20170703115216_add_extension_to_uploads.rb diff --git a/db/migrate/20170609115401_add_extension_to_topic_links.rb b/db/migrate/20170609115401_add_extension_to_topic_links.rb index fbe8b9c184..7c05a12d1a 100644 --- a/db/migrate/20170609115401_add_extension_to_topic_links.rb +++ b/db/migrate/20170609115401_add_extension_to_topic_links.rb @@ -1,6 +1,6 @@ class AddExtensionToTopicLinks < ActiveRecord::Migration def change - add_column :topic_links, :extension, :string, limit: 5 + add_column :topic_links, :extension, :string, limit: 10 add_index :topic_links, :extension end end diff --git a/db/migrate/20170703115216_add_extension_to_uploads.rb b/db/migrate/20170703115216_add_extension_to_uploads.rb new file mode 100644 index 0000000000..76aa08fa11 --- /dev/null +++ b/db/migrate/20170703115216_add_extension_to_uploads.rb @@ -0,0 +1,11 @@ +class AddExtensionToUploads < ActiveRecord::Migration + def up + add_column :uploads, :extension, :string, limit: 10 + execute "CREATE INDEX index_uploads_on_extension ON uploads(lower(extension))" + end + + def down + remove_column :uploads, :extension + execute "DROP INDEX index_uploads_on_extension" + end +end From f0a674d620587af1cb2dae802d2520827c193af8 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Mon, 3 Jul 2017 19:08:59 +0200 Subject: [PATCH 011/382] Add extraction of upload extension. Add rspec test for search of post with upload by extension. --- app/models/topic_link.rb | 2 +- lib/upload_creator.rb | 1 + spec/components/search_spec.rb | 6 ++++-- spec/fabricators/upload_fabricator.rb | 1 + 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/app/models/topic_link.rb b/app/models/topic_link.rb index c9abf160c4..48c7c279c0 100644 --- a/app/models/topic_link.rb +++ b/app/models/topic_link.rb @@ -164,7 +164,7 @@ SQL added_urls << url unless TopicLink.exists?(topic_id: post.topic_id, post_id: post.id, url: url) - file_extension = File.extname(parsed.path)[1..5].downcase unless File.extname(parsed.path).empty? + file_extension = File.extname(parsed.path)[1..10].downcase unless File.extname(parsed.path).empty? begin TopicLink.create!(post_id: post.id, user_id: post.user_id, diff --git a/lib/upload_creator.rb b/lib/upload_creator.rb index a00b13f08f..503c0fadef 100644 --- a/lib/upload_creator.rb +++ b/lib/upload_creator.rb @@ -73,6 +73,7 @@ class UploadCreator @upload.sha1 = sha1 @upload.url = "" @upload.origin = @opts[:origin][0...1000] if @opts[:origin] + @upload.extension = File.extname(@filename)[1..10] if FileHelper.is_image?(@filename) @upload.width, @upload.height = ImageSizer.resize(*@image_info.size) diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index c7c2c5609e..7faf1bac43 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -712,14 +712,16 @@ describe Search do raw: "Discourse logo\n"\ "https://www.discourse.org/a/img/favicon.png\n"\ "https://www.discourse.org/a/img/trust-1x.jpg") + post_with_upload = Fabricate(:post) + post_with_upload.uploads = [Fabricate(:upload)] Fabricate(:post) TopicLink.extract_from(post1) TopicLink.extract_from(post2) expect(Search.execute('filetype:jpg').posts.map(&:id)).to eq([post2.id]) - expect(Search.execute('filetype:png').posts.map(&:id)).to contain_exactly(post1.id, post2.id) - expect(Search.execute('logo filetype:jpg').posts.map(&:id)).to eq([post2.id]) + expect(Search.execute('filetype:png').posts.map(&:id)).to contain_exactly(post1.id, post2.id, post_with_upload.id) + expect(Search.execute('logo filetype:png').posts.map(&:id)).to eq([post2.id]) end end diff --git a/spec/fabricators/upload_fabricator.rb b/spec/fabricators/upload_fabricator.rb index a2afdf7af0..7d0c7f43be 100644 --- a/spec/fabricators/upload_fabricator.rb +++ b/spec/fabricators/upload_fabricator.rb @@ -6,6 +6,7 @@ Fabricator(:upload) do width 100 height 200 url { sequence(:url) { |n| "/uploads/default/#{n}/1234567890123456.png" } } + extension "png" end Fabricator(:upload_s3, from: :upload) do From 8c445e9f17f6ba51a5698a3e948ad46d339615ee Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Tue, 4 Jul 2017 17:50:08 +0200 Subject: [PATCH 012/382] Fix backend code for searching by a filetype as a combination of uploads and topic links. Add rspec test for extracting file extension in upload. --- app/models/upload.rb | 4 ---- lib/file_store/base_store.rb | 2 +- lib/search.rb | 6 +++++- spec/models/upload_spec.rb | 5 +++++ 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/models/upload.rb b/app/models/upload.rb index d6e1916c72..f0eae0896b 100644 --- a/app/models/upload.rb +++ b/app/models/upload.rb @@ -52,10 +52,6 @@ class Upload < ActiveRecord::Base end end - def extension - File.extname(original_filename) - end - def self.generate_digest(path) Digest::SHA1.file(path).hexdigest end diff --git a/lib/file_store/base_store.rb b/lib/file_store/base_store.rb index b79282fe68..b4c8bfe47c 100644 --- a/lib/file_store/base_store.rb +++ b/lib/file_store/base_store.rb @@ -95,7 +95,7 @@ module FileStore end def get_path_for_upload(upload) - get_path_for("original".freeze, upload.id, upload.sha1, upload.extension) + get_path_for("original".freeze, upload.id, upload.sha1, File.extname(upload.original_filename)) end def get_path_for_optimized_image(optimized_image) diff --git a/lib/search.rb b/lib/search.rb index f3ea5ccd82..159de71b8d 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -462,7 +462,11 @@ class Search posts.where("posts.id IN ( SELECT post_id FROM topic_links WHERE extension IN (?) - )", file_extensions) + UNION + SELECT post_uploads.post_id FROM uploads + JOIN post_uploads ON post_uploads.upload_id = uploads.id + WHERE lower(uploads.extension) IN (?) + )", file_extensions, file_extensions) end private diff --git a/spec/models/upload_spec.rb b/spec/models/upload_spec.rb index c187ddbe67..ceef5bf669 100644 --- a/spec/models/upload_spec.rb +++ b/spec/models/upload_spec.rb @@ -46,6 +46,11 @@ describe Upload do end + it "extracts file extension" do + created_upload = UploadCreator.new(image, image_filename).create_for(user_id) + expect(created_upload.extension).to eq("png") + end + context ".get_from_url" do let(:url) { "/uploads/default/original/3X/1/0/10f73034616a796dfd70177dc54b6def44c4ba6f.png" } let(:upload) { Fabricate(:upload, url: url) } From 5b11391588b2cf17c4c528ef86770e93ea71ba3c Mon Sep 17 00:00:00 2001 From: Matt Farmer Date: Fri, 7 Jul 2017 11:11:43 -0400 Subject: [PATCH 013/382] Add a nil check on the connection before attempting to exec it --- script/import_scripts/base.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index 616a534a5d..5d85af8016 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -203,7 +203,7 @@ class ImportScripts::Base return true end ensure - connection.exec('DROP TABLE import_ids') + connection.exec('DROP TABLE import_ids') unless connection.nil? end def created_user(user) From 0b222493f6a1d5c448074ba415ef38131ad67405 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Mon, 10 Jul 2017 12:06:37 +0100 Subject: [PATCH 014/382] For rake plugin:spec, only load ruby files ending in _spec.rb This matches the default behaviour of rspec --- lib/tasks/plugin.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/plugin.rake b/lib/tasks/plugin.rake index 8e7281ad59..acb8e5b1d7 100644 --- a/lib/tasks/plugin.rake +++ b/lib/tasks/plugin.rake @@ -47,7 +47,7 @@ desc 'run plugin specs' task 'plugin:spec', :plugin do |t, args| args.with_defaults(plugin: "*") ruby = `which ruby`.strip - files = Dir.glob("./plugins/#{args[:plugin]}/spec/**/*.rb") + files = Dir.glob("./plugins/#{args[:plugin]}/spec/**/*_spec.rb") if files.length > 0 sh "LOAD_PLUGINS=1 #{ruby} -S rspec #{files.join(' ')}" else From ba9898c5a9b9fdaa61abeb1c051aa28705a2bb2e Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 10 Jul 2017 12:20:50 -0400 Subject: [PATCH 015/382] FIX: smarter newline handling for tags on line alone Run all of pretty text spec on new engine --- app/assets/javascripts/markdown-it-bundle.js | 1 + .../engines/markdown-it/html_img.js.es6 | 74 ++ spec/components/pretty_text_spec.rb | 806 ++++++++++-------- 3 files changed, 515 insertions(+), 366 deletions(-) create mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/html_img.js.es6 diff --git a/app/assets/javascripts/markdown-it-bundle.js b/app/assets/javascripts/markdown-it-bundle.js index 51fe107f81..3460c3a602 100644 --- a/app/assets/javascripts/markdown-it-bundle.js +++ b/app/assets/javascripts/markdown-it-bundle.js @@ -12,3 +12,4 @@ //= require ./pretty-text/engines/markdown-it/table //= require ./pretty-text/engines/markdown-it/paragraph //= require ./pretty-text/engines/markdown-it/newline +//= require ./pretty-text/engines/markdown-it/html_img diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/html_img.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/html_img.js.es6 new file mode 100644 index 0000000000..8d5b02efa7 --- /dev/null +++ b/app/assets/javascripts/pretty-text/engines/markdown-it/html_img.js.es6 @@ -0,0 +1,74 @@ +// special handling for IMG tags on a line by themeselves +// we always have to handle it as so it is an inline +// see: https://talk.commonmark.org/t/newline-and-img-tags/2511 + +const REGEX = /^\s*$/i; + +function rule(state, startLine, endLine) { + + var nextLine, token, lineText, + pos = state.bMarks[startLine] + state.tShift[startLine], + max = state.eMarks[startLine]; + + // if it's indented more than 3 spaces, it should be a code block + if (state.sCount[startLine] - state.blkIndent >= 4) { return false; } + + if (!state.md.options.html) { return false; } + + if (state.src.charCodeAt(pos) !== 0x3C/* < */) { return false; } + let pos1 = state.src.charCodeAt(pos+1); + if (pos1 !== 73 /* I */ && pos1 !== 105 /* i */) { return false; } + + lineText = state.src.slice(pos, max); + + if (!REGEX.test(lineText)) { + return false; + } + + let lines = []; + lines.push(lineText); + + nextLine = startLine + 1; + for (; nextLine < endLine; nextLine++) { + pos = state.bMarks[nextLine] + state.tShift[nextLine]; + max = state.eMarks[nextLine]; + lineText = state.src.slice(pos, max); + + if (lineText.trim() === "") { + break; + } + + if (!REGEX.test(lineText)) { + break; + } + + lines.push(lineText); + } + + state.line = nextLine; + let oldParentType = state.parentType; + state.parentType = 'paragraph'; + + token = state.push('paragraph_open', 'p', 0); + token.map = [startLine, state.line]; + + token = state.push('inline', '', 0); + token.content = lines.join('\n'); + token.map = [startLine, state.line]; + token.children = []; + + token = state.push('paragraph_close', 'p', -1); + state.parentType = oldParentType; + + return true; +} + + +export function setup(helper) { + + if (!helper.markdownIt) { return; } + + helper.registerPlugin(md=>{ + md.block.ruler.before('html_block', 'html_img', rule, {alt: ['fence']}); + }); +} diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb index 966ce95726..7b7b7036fa 100644 --- a/spec/components/pretty_text_spec.rb +++ b/spec/components/pretty_text_spec.rb @@ -4,6 +4,10 @@ require 'html_normalize' describe PrettyText do + before do + SiteSetting.enable_experimental_markdown_it = true + end + def n(html) HtmlNormalize.normalize(html) end @@ -12,23 +16,15 @@ describe PrettyText do n(PrettyText.cook(*args)) end + # see: https://github.com/sparklemotion/nokogiri/issues/1173 + skip 'allows html entities correctly' do + expect(PrettyText.cook("ℵ£¢")).to eq("

ℵ£¢

") + end + let(:wrapped_image) { "
\nScreen Shot 2014-04-14 at 9.47.10 PM.png966x737 1.47 MB\n
" } let(:wrapped_image_excerpt) { } - describe "Cooking" do - - describe "off topic quoting" do - it "can correctly populate topic title" do - topic = Fabricate(:topic, title: "this is a test topic :slight_smile:") - expected = <
-
This is a test topic slight_smile -
-

ddd

-HTML - expect(PrettyText.cook("[quote=\"EvilTrout, post:2, topic:#{topic.id}\"]ddd\n[/quote]", topic_id: 1)).to match_html expected - end - end + describe "Quoting" do describe "with avatar" do let(:default_avatar) { "//test.localhost/uploads/default/avatars/42d/57c/46ce7ee487/{size}.png" } @@ -38,19 +34,91 @@ HTML User.stubs(:default_template).returns(default_avatar) end - it "produces a quote even with new lines in it" do - expect(PrettyText.cook("[quote=\"#{user.username}, post:123, topic:456, full:true\"]ddd\n[/quote]")).to match_html "" + it "do off topic quoting with emoji unescape" do + + topic = Fabricate(:topic, title: "this is a test topic :slight_smile:") + expected = <<~HTML + + HTML + + expect(cook("[quote=\"EvilTrout, post:2, topic:#{topic.id}\"]\nddd\n[/quote]", topic_id: 1)).to eq(n(expected)) end - it "should produce a quote" do - expect(PrettyText.cook("[quote=\"#{user.username}, post:123, topic:456, full:true\"]ddd[/quote]")).to match_html "" + it "produces a quote even with new lines in it" do + md = <<~MD + [quote="#{user.username}, post:123, topic:456, full:true"] + + ddd + + [/quote] + MD + html = <<~HTML + + HTML + + expect(PrettyText.cook(md)).to eq(html.strip) end + it "trims spaces on quote params" do - expect(PrettyText.cook("[quote=\"#{user.username}, post:555, topic: 666\"]ddd[/quote]")).to match_html "" + md = <<~MD + [quote="#{user.username}, post:555, topic: 666"] + ddd + [/quote] + MD + + html = <<~HTML + + HTML + + expect(PrettyText.cook(md)).to eq(html.strip) end end + it "can handle quote edge cases" do + expect(PrettyText.cook("a\n[quote]\ntest\n[/quote]\n\n\na")).to include('aside') + expect(PrettyText.cook("- a\n[quote]\ntest\n[/quote]\n\n\na")).to include('aside') + expect(PrettyText.cook("[quote]\ntest")).not_to include('aside') + expect(PrettyText.cook("[quote]abc\ntest\n[/quote]")).not_to include('aside') + expect(PrettyText.cook("[quote]\ntest\n[/quote]z")).not_to include('aside') + + nested = <<~QUOTE + [quote] + a + [quote] + b + [/quote] + c + [/quote] + QUOTE + + cooked = PrettyText.cook(nested) + expect(cooked.scan('aside').length).to eq(4) + expect(cooked.scan('quote]').length).to eq(0) + end + describe "with letter avatar" do let(:user) { Fabricate(:user) } @@ -61,22 +129,44 @@ HTML end it "should have correct avatar url" do - expect(PrettyText.cook("[quote=\"#{user.username}, post:123, topic:456, full:true\"]ddd[/quote]")).to include("/forum/letter_avatar_proxy") + md = <<~MD + [quote="#{user.username}, post:123, topic:456, full:true"] + ddd + [/quote] + MD + expect(PrettyText.cook(md)).to include("/forum/letter_avatar_proxy") end end end + end + + describe "Mentions" do it "should handle 3 mentions in a row" do expect(PrettyText.cook('@hello @hello @hello')).to match_html "

@hello @hello @hello

" end - it "should handle group mentions with a hyphen and without" do - expect(PrettyText.cook('@hello @hello-hello')).to match_html "

@hello @hello-hello

" + it "can handle mentions" do + Fabricate(:user, username: "sam") + expect(PrettyText.cook("hi @sam! hi")).to match_html '

hi @sam! hi

' + expect(PrettyText.cook("hi\n@sam")).to eq("

hi
\n@sam

") + end + + it "can handle mentions inside a hyperlink" do + expect(PrettyText.cook(" @inner ")).to match_html '

@inner

' end - it "should sanitize the html" do - expect(PrettyText.cook("")).to match_html "

" + it "can handle mentions inside a hyperlink" do + expect(PrettyText.cook("[link @inner](http://site.com)")).to match_html '

link @inner

' + end + + it "can handle a list of mentions" do + expect(PrettyText.cook("@a,@b")).to match_html('

@a,@b

') + end + + it "should handle group mentions with a hyphen and without" do + expect(PrettyText.cook('@hello @hello-hello')).to match_html "

@hello @hello-hello

" end it 'should allow for @mentions to have punctuation' do @@ -85,13 +175,52 @@ HTML ) end - # see: https://github.com/sparklemotion/nokogiri/issues/1173 - skip 'allows html entities correctly' do - expect(PrettyText.cook("ℵ£¢")).to eq("

ℵ£¢

") + end + + describe "code fences" do + it 'indents code correctly' do + code = <<~MD + X + ``` + # + x + ``` + MD + cooked = PrettyText.cook(code) + + html = <<~HTML +

X

+ 
     #
+             x
+
+ HTML + + expect(cooked).to eq(html.strip) end + it "doesn't replace emoji in code blocks with our emoji sets if emoji is enabled" do + expect(PrettyText.cook("```\n💣`\n```\n")).not_to match(/\:bomb\:/) + end + + it 'can include code class correctly' do + expect(PrettyText.cook("```cpp\ncpp\n```")).to match_html("
cpp\n
") + expect(PrettyText.cook("```\ncpp\n```")).to match_html("
cpp\n
") + expect(PrettyText.cook("```text\ncpp\n```")).to match_html("
cpp\n
") + end + + it 'indents code correctly' do + code = "X\n```\n\n #\n x\n```" + cooked = PrettyText.cook(code) + expect(cooked).to match_html("

X

\n
\n    #\n    x\n
") + end + + it 'does censor code fences' do + SiteSetting.censored_words = 'apple|banana' + expect(PrettyText.cook("# banana")).not_to include('banana') + end end + describe "rel nofollow" do before do SiteSetting.add_rel_nofollow_to_user_content = true @@ -330,7 +459,6 @@ HTML )).to eq("boom") end end - end describe "strip links" do @@ -402,20 +530,55 @@ HTML end end - it 'can escape *' do - expect(PrettyText.cook("***a***a")).to match_html("

aa

") - expect(PrettyText.cook("***\\****a")).to match_html("

*a

") + it 'Is smart about linebreaks and IMG tags' do + raw = <<~MD + a + + + + + + + a + + + - li + + + ``` + test + ``` + + ``` + test + ``` + MD + + html = <<~HTML +

a
+

+

+
+

+

+

+

a

+

+

+
    +
  • li
    • +
+

+

+ 
test
+
+ 
test
+
+ HTML + + expect(PrettyText.cook(raw)).to eq(html.strip) end - it 'can include code class correctly' do - expect(PrettyText.cook("```cpp\ncpp\n```")).to match_html("

cpp
") - end - - it 'indents code correctly' do - code = "X\n```\n\n #\n x\n```" - cooked = PrettyText.cook(code) - expect(cooked).to match_html("

X

\n\n

    #\n    x
") - end it 'can substitute s3 cdn correctly' do SiteSetting.enable_s3_uploads = true @@ -425,19 +588,22 @@ HTML SiteSetting.s3_cdn_url = "https://awesome.cdn" # add extra img tag to ensure it does not blow up - raw = < - - - + raw = <<~HTML + + + + + HTML -HTML + html = <<~HTML +

+
+
+
+

+ HTML - cooked = <


-HTML - - expect(PrettyText.cook(raw)).to match_html(cooked) + expect(PrettyText.cook(raw)).to eq(html.strip) end describe "emoji" do @@ -449,10 +615,6 @@ HTML expect(PrettyText.cook("`💣`")).not_to match(/\:bomb\:/) end - it "doesn't replace emoji in code blocks with our emoji sets if emoji is enabled" do - expect(PrettyText.cook("```\n💣`\n```\n")).not_to match(/\:bomb\:/) - end - it "replaces some glyphs that are not in the emoji range" do expect(PrettyText.cook("☺")).to match(/\:slight_smile\:/) end @@ -470,22 +632,6 @@ HTML end end - describe "tag and category links" do - it "produces tag links" do - Fabricate(:topic, {tags: [Fabricate(:tag, name: 'known')]}) - - cooked = PrettyText.cook(" #unknown::tag #known::tag") - - html = <<~HTML -

#unknown::tag #known

- HTML - - expect(cooked).to match_html(html) - end - - # TODO does it make sense to generate hashtags for tags that are missing in action? - end - describe "custom emoji" do it "replaces the custom emoji" do CustomEmoji.create!(name: 'trout', upload: Fabricate(:upload)) @@ -504,317 +650,245 @@ HTML end end - context "markdown it" do - - before do - SiteSetting.enable_experimental_markdown_it = true - end - - it "replaces skin toned emoji" do - expect(PrettyText.cook("hello 👱🏿‍♀️")).to eq("

hello \":blonde_woman:t6:\"

") - expect(PrettyText.cook("hello 👩‍🎤")).to eq("

hello \":woman_singer:\"

") - expect(PrettyText.cook("hello 👩🏾‍🎓")).to eq("

hello \":woman_student:t5:\"

") - expect(PrettyText.cook("hello 🤷‍♀️")).to eq("

hello \":woman_shrugging:\"

") - end + it "replaces skin toned emoji" do + expect(PrettyText.cook("hello 👱🏿‍♀️")).to eq("

hello \":blonde_woman:t6:\"

") + expect(PrettyText.cook("hello 👩‍🎤")).to eq("

hello \":woman_singer:\"

") + expect(PrettyText.cook("hello 👩🏾‍🎓")).to eq("

hello \":woman_student:t5:\"

") + expect(PrettyText.cook("hello 🤷‍♀️")).to eq("

hello \":woman_shrugging:\"

") + end - it "supports href schemes" do - SiteSetting.allowed_href_schemes = "macappstore|steam" - cooked = cook("[Steam URL Scheme](steam://store/452530)") - expected = '

Steam URL Scheme

' - expect(cooked).to eq(n expected) - end + it "supports href schemes" do + SiteSetting.allowed_href_schemes = "macappstore|steam" + cooked = cook("[Steam URL Scheme](steam://store/452530)") + expected = '

Steam URL Scheme

' + expect(cooked).to eq(n expected) + end - it "supports forbidden schemes" do - SiteSetting.allowed_href_schemes = "macappstore|itunes" - cooked = cook("[Steam URL Scheme](steam://store/452530)") - expected = '

Steam URL Scheme

' - expect(cooked).to eq(n expected) - end + it "supports forbidden schemes" do + SiteSetting.allowed_href_schemes = "macappstore|itunes" + cooked = cook("[Steam URL Scheme](steam://store/452530)") + expected = '

Steam URL Scheme

' + expect(cooked).to eq(n expected) + end - it "produces tag links" do - Fabricate(:topic, {tags: [Fabricate(:tag, name: 'known')]}) + it "produces tag links" do + Fabricate(:topic, {tags: [Fabricate(:tag, name: 'known')]}) - cooked = PrettyText.cook(" #unknown::tag #known::tag") + cooked = PrettyText.cook(" #unknown::tag #known::tag") - html = <<~HTML -

#unknown::tag #known

- HTML + html = <<~HTML +

#unknown::tag #known

+ HTML - expect(cooked).to eq(html.strip) + expect(cooked).to eq(html.strip) - cooked = PrettyText.cook("[`a` #known::tag here](http://somesite.com)") + cooked = PrettyText.cook("[`a` #known::tag here](http://somesite.com)") - html = <<~HTML -

a #known::tag here

- HTML + html = <<~HTML +

a #known::tag here

+ HTML - expect(cooked).to eq(html.strip) + expect(cooked).to eq(html.strip) - cooked = PrettyText.cook("`a` #known::tag here") + cooked = PrettyText.cook("`a` #known::tag here") - expect(cooked).to eq(html.strip) + expect(cooked).to eq(html.strip) - cooked = PrettyText.cook("test #known::tag") - html = <<~HTML -

test #known

- HTML + cooked = PrettyText.cook("test #known::tag") + html = <<~HTML +

test #known

+ HTML - expect(cooked).to eq(html.strip) + expect(cooked).to eq(html.strip) - # ensure it does not fight with the autolinker - expect(PrettyText.cook(' http://somewhere.com/#known')).not_to include('hashtag') - expect(PrettyText.cook(' http://somewhere.com/?#known')).not_to include('hashtag') - expect(PrettyText.cook(' http://somewhere.com/?abc#known')).not_to include('hashtag') - - end - - it "can handle mixed lists" do - # known bug in old md engine - cooked = PrettyText.cook("* a\n\n1. b") - expect(cooked).to match_html("
    \n
  • a
    • \n
    \n
  1. b
    2. \n
") - end - - it "can handle traditional vs non traditional newlines" do - SiteSetting.traditional_markdown_linebreaks = true - expect(PrettyText.cook("1\n2")).to match_html "

1 2

" - - SiteSetting.traditional_markdown_linebreaks = false - expect(PrettyText.cook("1\n2")).to match_html "

1
\n2

" - end - - it "can handle mentions" do - Fabricate(:user, username: "sam") - expect(PrettyText.cook("hi @sam! hi")).to match_html '

hi @sam! hi

' - expect(PrettyText.cook("hi\n@sam")).to eq("

hi
\n@sam

") - end - - it "can handle mentions inside a hyperlink" do - expect(PrettyText.cook(" @inner ")).to match_html '

@inner

' - end - - - it "can handle mentions inside a hyperlink" do - expect(PrettyText.cook("[link @inner](http://site.com)")).to match_html '

link @inner

' - end - - it "can handle a list of mentions" do - expect(PrettyText.cook("@a,@b")).to match_html('

@a,@b

') - end - - it "can handle emoji by name" do - - expected = <:smile::sunny:

-HTML - expect(PrettyText.cook(":smile::sunny:")).to eq(expected.strip) - end - - it "handles emoji boundaries correctly" do - cooked = PrettyText.cook("a,:man:t2:,b") - expected = '

a,:man:t2:,b

' - expect(cooked).to match(expected.strip) - end - - it "can handle emoji by translation" do - expected = '

:wink:

' - expect(PrettyText.cook(";)")).to eq(expected) - end - - it "can handle multiple emojis by translation" do - cooked = PrettyText.cook(":) ;) :)") - expect(cooked.split("img").length-1).to eq(3) - end - - it "handles emoji boundries correctly" do - expect(PrettyText.cook(",:)")).to include("emoji") - expect(PrettyText.cook(":-)\n")).to include("emoji") - expect(PrettyText.cook("a :)")).to include("emoji") - expect(PrettyText.cook(":),")).not_to include("emoji") - expect(PrettyText.cook("abcde ^:;-P")).to include("emoji") - end - - - it 'can include code class correctly' do - expect(PrettyText.cook("```cpp\ncpp\n```")).to match_html("
cpp\n
") - expect(PrettyText.cook("```\ncpp\n```")).to match_html("
cpp\n
") - expect(PrettyText.cook("```text\ncpp\n```")).to match_html("
cpp\n
") - end - - it 'indents code correctly' do - code = "X\n```\n\n #\n x\n```" - cooked = PrettyText.cook(code) - expect(cooked).to match_html("

X

\n
\n    #\n    x\n
") - end - - it 'can censor words correctly' do - SiteSetting.censored_words = 'apple|banana' - expect(PrettyText.cook('yay banana yay')).not_to include('banana') - expect(PrettyText.cook('yay `banana` yay')).not_to include('banana') - expect(PrettyText.cook("yay \n\n```\nbanana\n````\n yay")).not_to include('banana') - expect(PrettyText.cook("# banana")).not_to include('banana') - expect(PrettyText.cook("# banana")).to include("\u25a0\u25a0") - end - - it 'supports typographer' do - SiteSetting.enable_markdown_typographer = true - expect(PrettyText.cook('(tm)')).to eq('

™

') - - SiteSetting.enable_markdown_typographer = false - expect(PrettyText.cook('(tm)')).to eq('

(tm)

') - end - - it 'handles onebox correctly' do - expect(PrettyText.cook("http://a.com\nhttp://b.com").split("onebox").length).to eq(3) - expect(PrettyText.cook("http://a.com\n\nhttp://b.com").split("onebox").length).to eq(3) - expect(PrettyText.cook("a\nhttp://a.com")).to include('onebox') - expect(PrettyText.cook("> http://a.com")).not_to include('onebox') - expect(PrettyText.cook("a\nhttp://a.com a")).not_to include('onebox') - expect(PrettyText.cook("a\nhttp://a.com\na")).to include('onebox') - expect(PrettyText.cook("http://a.com")).to include('onebox') - expect(PrettyText.cook("http://a.com ")).to include('onebox') - expect(PrettyText.cook("http://a.com a")).not_to include('onebox') - expect(PrettyText.cook("- http://a.com")).not_to include('onebox') - expect(PrettyText.cook("")).not_to include('onebox') - expect(PrettyText.cook(" http://a.com")).not_to include('onebox') - expect(PrettyText.cook("a\n http://a.com")).not_to include('onebox') - end - - it "can handle bbcode" do - expect(PrettyText.cook("a[b]b[/b]c")).to eq('

abc

') - expect(PrettyText.cook("a[i]b[/i]c")).to eq('

abc

') - end - - it "can handle quote edge cases" do - expect(PrettyText.cook("a\n[quote]\ntest\n[/quote]\n\n\na")).to include('aside') - expect(PrettyText.cook("- a\n[quote]\ntest\n[/quote]\n\n\na")).to include('aside') - expect(PrettyText.cook("[quote]\ntest")).not_to include('aside') - expect(PrettyText.cook("[quote]abc\ntest\n[/quote]")).not_to include('aside') - expect(PrettyText.cook("[quote]\ntest\n[/quote]z")).not_to include('aside') - - nested = <<~QUOTE - [quote] - a - [quote] - b - [/quote] - c - [/quote] - QUOTE - - cooked = PrettyText.cook(nested) - expect(cooked.scan('aside').length).to eq(4) - expect(cooked.scan('quote]').length).to eq(0) - end - - it "can onebox local topics" do - op = Fabricate(:post) - reply = Fabricate(:post, topic_id: op.topic_id) - - - url = Discourse.base_url + reply.url - quote = create_post(topic_id: op.topic.id, raw: "This is a sample reply with a quote\n\n#{url}") - quote.reload - - expect(quote.cooked).not_to include('[quote') - end - - it "supports tables" do - - markdown = <<~MD - | Tables | Are | Cool | - | ------------- |:-------------:| -----:| - | col 3 is | right-aligned | $1600 | - MD - - expected = <<~HTML - - - - - - - - - - - - - - - -
TablesAreCool
col 3 isright-aligned$1600
- HTML - - expect(PrettyText.cook(markdown)).to eq(expected.strip) - end - - it "do off topic quoting with emoji unescape" do - - topic = Fabricate(:topic, title: "this is a test topic :slight_smile:") - expected = <<~HTML - - HTML - - expect(cook("[quote=\"EvilTrout, post:2, topic:#{topic.id}\"]\nddd\n[/quote]", topic_id: 1)).to eq(n(expected)) - end - - it "supports img bbcode" do - cooked = PrettyText.cook "[img]http://www.image/test.png[/img]" - html = "

" - expect(cooked).to eq(html) - end - - it "provides safety for img bbcode" do - cooked = PrettyText.cook "[img]http://aaa.com[/img]" - html = '

' - expect(cooked).to eq(html) - end - - it "supports email bbcode" do - cooked = PrettyText.cook "[email]sam@sam.com[/email]" - html = '

sam@sam.com

' - expect(cooked).to eq(html) - end - - it "supports url bbcode" do - cooked = PrettyText.cook "[url]http://sam.com[/url]" - html = '

http://sam.com

' - expect(cooked).to eq(html) - end - - it "supports inline code bbcode" do - cooked = PrettyText.cook "Testing [code]codified **stuff** and `more` stuff[/code]" - html = "

Testing codified **stuff** and `more` stuff

" - expect(cooked).to eq(html) - end - - it "supports block code bbcode" do - cooked = PrettyText.cook "[code]\ncodified\n\n\n **stuff** and `more` stuff\n[/code]" - html = "
codified\n\n\n  **stuff** and `more` stuff
" - expect(cooked).to eq(html) - end - - it "support special handling for space in urls" do - cooked = PrettyText.cook "http://testing.com?a%20b" - html = '

http://testing.com?a%20b

' - expect(cooked).to eq(html) - end - - it "supports onebox for decoded urls" do - cooked = PrettyText.cook "http://testing.com?a%50b" - html = '

http://testing.com?aPb

' - expect(cooked).to eq(html) - end + # ensure it does not fight with the autolinker + expect(PrettyText.cook(' http://somewhere.com/#known')).not_to include('hashtag') + expect(PrettyText.cook(' http://somewhere.com/?#known')).not_to include('hashtag') + expect(PrettyText.cook(' http://somewhere.com/?abc#known')).not_to include('hashtag') end + it "can handle mixed lists" do + # known bug in old md engine + cooked = PrettyText.cook("* a\n\n1. b") + expect(cooked).to match_html("
    \n
  • a
    • \n
    \n
  1. b
    2. \n
") + end + + it "can handle traditional vs non traditional newlines" do + SiteSetting.traditional_markdown_linebreaks = true + expect(PrettyText.cook("1\n2")).to match_html "

1 2

" + + SiteSetting.traditional_markdown_linebreaks = false + expect(PrettyText.cook("1\n2")).to match_html "

1
\n2

" + end + + + it "can handle emoji by name" do + + expected = <:smile::sunny:

+HTML + expect(PrettyText.cook(":smile::sunny:")).to eq(expected.strip) + end + + it "handles emoji boundaries correctly" do + cooked = PrettyText.cook("a,:man:t2:,b") + expected = '

a,:man:t2:,b

' + expect(cooked).to match(expected.strip) + end + + it "can handle emoji by translation" do + expected = '

:wink:

' + expect(PrettyText.cook(";)")).to eq(expected) + end + + it "can handle multiple emojis by translation" do + cooked = PrettyText.cook(":) ;) :)") + expect(cooked.split("img").length-1).to eq(3) + end + + it "handles emoji boundries correctly" do + expect(PrettyText.cook(",:)")).to include("emoji") + expect(PrettyText.cook(":-)\n")).to include("emoji") + expect(PrettyText.cook("a :)")).to include("emoji") + expect(PrettyText.cook(":),")).not_to include("emoji") + expect(PrettyText.cook("abcde ^:;-P")).to include("emoji") + end + + it 'can censor words correctly' do + SiteSetting.censored_words = 'apple|banana' + expect(PrettyText.cook('yay banana yay')).not_to include('banana') + expect(PrettyText.cook('yay `banana` yay')).not_to include('banana') + expect(PrettyText.cook("# banana")).not_to include('banana') + expect(PrettyText.cook("# banana")).to include("\u25a0\u25a0") + end + + it 'supports typographer' do + SiteSetting.enable_markdown_typographer = true + expect(PrettyText.cook('(tm)')).to eq('

™

') + + SiteSetting.enable_markdown_typographer = false + expect(PrettyText.cook('(tm)')).to eq('

(tm)

') + end + + it 'handles onebox correctly' do + expect(PrettyText.cook("http://a.com\nhttp://b.com").split("onebox").length).to eq(3) + expect(PrettyText.cook("http://a.com\n\nhttp://b.com").split("onebox").length).to eq(3) + expect(PrettyText.cook("a\nhttp://a.com")).to include('onebox') + expect(PrettyText.cook("> http://a.com")).not_to include('onebox') + expect(PrettyText.cook("a\nhttp://a.com a")).not_to include('onebox') + expect(PrettyText.cook("a\nhttp://a.com\na")).to include('onebox') + expect(PrettyText.cook("http://a.com")).to include('onebox') + expect(PrettyText.cook("http://a.com ")).to include('onebox') + expect(PrettyText.cook("http://a.com a")).not_to include('onebox') + expect(PrettyText.cook("- http://a.com")).not_to include('onebox') + expect(PrettyText.cook("")).not_to include('onebox') + expect(PrettyText.cook(" http://a.com")).not_to include('onebox') + expect(PrettyText.cook("a\n http://a.com")).not_to include('onebox') + end + + it "can handle bbcode" do + expect(PrettyText.cook("a[b]b[/b]c")).to eq('

abc

') + expect(PrettyText.cook("a[i]b[/i]c")).to eq('

abc

') + end + + + it "can onebox local topics" do + op = Fabricate(:post) + reply = Fabricate(:post, topic_id: op.topic_id) + + + url = Discourse.base_url + reply.url + quote = create_post(topic_id: op.topic.id, raw: "This is a sample reply with a quote\n\n#{url}") + quote.reload + + expect(quote.cooked).not_to include('[quote') + end + + it "supports tables" do + + markdown = <<~MD + | Tables | Are | Cool | + | ------------- |:-------------:| -----:| + | col 3 is | right-aligned | $1600 | + MD + + expected = <<~HTML + + + + + + + + + + + + + + + +
TablesAreCool
col 3 isright-aligned$1600
+ HTML + + expect(PrettyText.cook(markdown)).to eq(expected.strip) + end + + + it "supports img bbcode" do + cooked = PrettyText.cook "[img]http://www.image/test.png[/img]" + html = "

" + expect(cooked).to eq(html) + end + + it "provides safety for img bbcode" do + cooked = PrettyText.cook "[img]http://aaa.com[/img]" + html = '

' + expect(cooked).to eq(html) + end + + it "supports email bbcode" do + cooked = PrettyText.cook "[email]sam@sam.com[/email]" + html = '

sam@sam.com

' + expect(cooked).to eq(html) + end + + it "supports url bbcode" do + cooked = PrettyText.cook "[url]http://sam.com[/url]" + html = '

http://sam.com

' + expect(cooked).to eq(html) + end + + it "supports inline code bbcode" do + cooked = PrettyText.cook "Testing [code]codified **stuff** and `more` stuff[/code]" + html = "

Testing codified **stuff** and `more` stuff

" + expect(cooked).to eq(html) + end + + it "supports block code bbcode" do + cooked = PrettyText.cook "[code]\ncodified\n\n\n **stuff** and `more` stuff\n[/code]" + html = "
codified\n\n\n  **stuff** and `more` stuff
" + expect(cooked).to eq(html) + end + + it "support special handling for space in urls" do + cooked = PrettyText.cook "http://testing.com?a%20b" + html = '

http://testing.com?a%20b

' + expect(cooked).to eq(html) + end + + it "supports onebox for decoded urls" do + cooked = PrettyText.cook "http://testing.com?a%50b" + html = '

http://testing.com?aPb

' + expect(cooked).to eq(html) + end + + + it "should sanitize the html" do + expect(PrettyText.cook("")).to eq "" + end + + end From d29a0eeedf23dfd2dd67dcb78aedbc3834078c78 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 10 Jul 2017 12:22:05 -0400 Subject: [PATCH 016/382] allow global shadow for new markdown engine --- config/site_settings.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/site_settings.yml b/config/site_settings.yml index fd487d97a6..6486b7226f 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -495,6 +495,7 @@ posting: enable_experimental_markdown_it: client: true default: false + shadowed_by_global: true traditional_markdown_linebreaks: client: true default: false From dcdd5baf5470cf02bd37f7a70d26896f98d890fa Mon Sep 17 00:00:00 2001 From: Robin Ward Date: Mon, 10 Jul 2017 13:46:30 -0400 Subject: [PATCH 017/382] FIX: Only show search if there are more total posts than the chunk size --- app/assets/javascripts/discourse/widgets/header.js.es6 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/assets/javascripts/discourse/widgets/header.js.es6 b/app/assets/javascripts/discourse/widgets/header.js.es6 index 0dc7716211..9754ad8688 100644 --- a/app/assets/javascripts/discourse/widgets/header.js.es6 +++ b/app/assets/javascripts/discourse/widgets/header.js.es6 @@ -306,8 +306,12 @@ export default createWidget('header', { // If we're viewing a topic, only intercept search if there are cloaked posts if (showSearch && currentPath.match(/^topic\./)) { - showSearch = ($('.topic-post .cooked, .small-action:not(.time-gap)').length < - this.register.lookup('controller:topic').get('model.postStream.stream.length')); + const controller = this.register.lookup('controller:topic'); + const total = controller.get('model.postStream.stream.length') || 0; + const chunkSize = controller.get('model.chunk_size') || 0; + + showSearch = (total > chunkSize) && + $('.topic-post .cooked, .small-action:not(.time-gap)').length < total; } if (state.searchVisible) { From c786700817508b74c25b8789d3c1cd9d701ffcbd Mon Sep 17 00:00:00 2001 From: Robin Ward Date: Mon, 10 Jul 2017 14:20:07 -0400 Subject: [PATCH 018/382] Upgrade onebox for a simple video fix --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index c84ad60a34..a2b7197d26 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -211,7 +211,7 @@ GEM omniauth-twitter (1.3.0) omniauth-oauth (~> 1.1) rack - onebox (1.8.14) + onebox (1.8.15) fast_blank (>= 1.0.0) htmlentities (~> 4.3) moneta (~> 1.0) From 31f7640335ff49e6955e82746b83ac68457d279c Mon Sep 17 00:00:00 2001 From: Robin Ward Date: Mon, 10 Jul 2017 14:31:58 -0400 Subject: [PATCH 019/382] FIX: Allow discourse app to link directly to wizard --- app/assets/javascripts/discourse/lib/url.js.es6 | 1 + 1 file changed, 1 insertion(+) diff --git a/app/assets/javascripts/discourse/lib/url.js.es6 b/app/assets/javascripts/discourse/lib/url.js.es6 index 1275f06f25..1f989f297e 100644 --- a/app/assets/javascripts/discourse/lib/url.js.es6 +++ b/app/assets/javascripts/discourse/lib/url.js.es6 @@ -14,6 +14,7 @@ const SERVER_SIDE_ONLY = [ /^\/raw\//, /^\/posts\/\d+\/raw/, /^\/raw\/\d+/, + /^\/wizard/, /\.rss$/, /\.json$/, ]; From 1e7f0fd8f1529710865f77c38facea3d84a6cf13 Mon Sep 17 00:00:00 2001 From: Bianca Nenciu Date: Sat, 8 Jul 2017 13:21:19 +0100 Subject: [PATCH 020/382] Added events to show and hide the preview pane. --- .../discourse/components/composer-editor.js.es6 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/assets/javascripts/discourse/components/composer-editor.js.es6 b/app/assets/javascripts/discourse/components/composer-editor.js.es6 index fe2ab5aae8..3d6e67747f 100644 --- a/app/assets/javascripts/discourse/components/composer-editor.js.es6 +++ b/app/assets/javascripts/discourse/components/composer-editor.js.es6 @@ -30,6 +30,16 @@ export default Ember.Component.extend({ _setupPreview() { const val = (this.site.mobileView ? false : (this.keyValueStore.get('composer.showPreview') || 'true')); this.set('showPreview', val === 'true'); + + this.appEvents.on('composer:show-preview', () => { + this.set('showPreview', true); + this.keyValueStore.set({ key: 'composer.showPreview', value: true }); + }); + + this.appEvents.on('composer:hide-preview', () => { + this.set('showPreview', false); + this.keyValueStore.set({ key: 'composer.showPreview', value: false }); + }); }, @computed('site.mobileView', 'showPreview') @@ -445,6 +455,8 @@ export default Ember.Component.extend({ @on('willDestroyElement') _composerClosed() { this.appEvents.trigger('composer:will-close'); + this.appEvents.off('composer:show-preview'); + this.appEvents.off('composer:hide-preview'); Ember.run.next(() => { $('#main-outlet').css('padding-bottom', 0); // need to wait a bit for the "slide down" transition of the composer From 375eb290b9caee8cea60830af0c157c8fc920ca5 Mon Sep 17 00:00:00 2001 From: Bianca Nenciu Date: Mon, 10 Jul 2017 23:28:35 +0100 Subject: [PATCH 021/382] Made changes as per review. --- .../discourse/components/composer-editor.js.es6 | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/app/assets/javascripts/discourse/components/composer-editor.js.es6 b/app/assets/javascripts/discourse/components/composer-editor.js.es6 index 3d6e67747f..47e0306fd7 100644 --- a/app/assets/javascripts/discourse/components/composer-editor.js.es6 +++ b/app/assets/javascripts/discourse/components/composer-editor.js.es6 @@ -33,12 +33,10 @@ export default Ember.Component.extend({ this.appEvents.on('composer:show-preview', () => { this.set('showPreview', true); - this.keyValueStore.set({ key: 'composer.showPreview', value: true }); }); this.appEvents.on('composer:hide-preview', () => { this.set('showPreview', false); - this.keyValueStore.set({ key: 'composer.showPreview', value: false }); }); }, @@ -52,6 +50,11 @@ export default Ember.Component.extend({ return showPreview ? I18n.t('composer.hide_preview') : I18n.t('composer.show_preview'); }, + @observes('showPreview') + showPreviewChanged() { + this.keyValueStore.set({ key: 'composer.showPreview', value: this.get('showPreview') }); + }, + @computed markdownOptions() { return { @@ -498,7 +501,6 @@ export default Ember.Component.extend({ togglePreview() { this.toggleProperty('showPreview'); - this.keyValueStore.set({ key: 'composer.showPreview', value: this.get('showPreview') }); }, extraButtons(toolbar) { From c5b1317a15c6f7b7f11ae61eb0d8fc5c03492740 Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Tue, 11 Jul 2017 11:04:29 +0900 Subject: [PATCH 022/382] Remove code that is no longer being used. --- app/models/invite.rb | 10 ---------- spec/models/invite_spec.rb | 19 ------------------- 2 files changed, 29 deletions(-) diff --git a/app/models/invite.rb b/app/models/invite.rb index 9925b335b3..7f7455fa91 100644 --- a/app/models/invite.rb +++ b/app/models/invite.rb @@ -218,16 +218,6 @@ class Invite < ActiveRecord::Base invite end - def self.redeem_from_token(token, email, username=nil, name=nil, topic_id=nil) - invite = Invite.find_by(invite_key: token) - if invite - invite.update_column(:email, email) - invite.topic_invites.create!(invite_id: invite.id, topic_id: topic_id) if topic_id && Topic.find_by_id(topic_id) && !invite.topic_invites.pluck(:topic_id).include?(topic_id) - user = InviteRedeemer.new(invite, username, name).redeem - end - user - end - def resend_invite self.update_columns(created_at: Time.zone.now, updated_at: Time.zone.now) Jobs.enqueue(:invite_email, invite_id: self.id) diff --git a/spec/models/invite_spec.rb b/spec/models/invite_spec.rb index bf6288e3c3..c83f689d0f 100644 --- a/spec/models/invite_spec.rb +++ b/spec/models/invite_spec.rb @@ -465,25 +465,6 @@ describe Invite do end - describe '.redeem_from_token' do - let(:inviter) { Fabricate(:user) } - let(:invite) { Fabricate(:invite, invited_by: inviter, email: 'test@example.com', user_id: nil) } - let(:user) { Fabricate(:user, email: invite.email) } - - it 'redeems the invite from token' do - Invite.redeem_from_token(invite.invite_key, user.email) - invite.reload - expect(invite).to be_redeemed - end - - it 'does not redeem the invite if token does not match' do - Invite.redeem_from_token("bae0071f995bb4b6f756e80b383778b5", user.email) - invite.reload - expect(invite).not_to be_redeemed - end - - end - describe '.rescind_all_invites_from' do it 'removes all invites sent by a user' do user = Fabricate(:user) From f529cb167456b52f9e36c57e714c1aaebf6e566e Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Tue, 11 Jul 2017 11:05:33 +0900 Subject: [PATCH 023/382] Remove validation for invalid interpolation keys. * Our codebase currently supports custom interpolations keys that are not present in the original translation. The proper fix should be to make `TranslateOverride` aware of such keys. --- app/models/translation_override.rb | 12 ------------ config/locales/server.en.yml | 1 - spec/models/translation_override_spec.rb | 13 ------------- 3 files changed, 26 deletions(-) diff --git a/app/models/translation_override.rb b/app/models/translation_override.rb index 11aba82a46..6e73686f9b 100644 --- a/app/models/translation_override.rb +++ b/app/models/translation_override.rb @@ -42,7 +42,6 @@ class TranslationOverride < ActiveRecord::Base original_interpolation_keys = I18nInterpolationKeysFinder.find(original_text) new_interpolation_keys = I18nInterpolationKeysFinder.find(value) missing_keys = (original_interpolation_keys - new_interpolation_keys) - invalid_keys = (original_interpolation_keys | new_interpolation_keys) - original_interpolation_keys if missing_keys.present? self.errors.add(:base, I18n.t( @@ -52,17 +51,6 @@ class TranslationOverride < ActiveRecord::Base return false end - - invalid_keys = (original_interpolation_keys | new_interpolation_keys) - original_interpolation_keys - - if invalid_keys.present? - self.errors.add(:base, I18n.t( - 'activerecord.errors.models.translation_overrides.attributes.value.invalid_interpolation_keys', - keys: invalid_keys.join(', ') - )) - - return false - end end end diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index ca5914b74e..60c7c64026 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -413,7 +413,6 @@ en: translation_overrides: attributes: value: - invalid_interpolation_keys: 'The following interpolation key(s) are invalid: "%{keys}"' missing_interpolation_keys: 'The following interpolation key(s) are missing: "%{keys}"' user_profile: diff --git a/spec/models/translation_override_spec.rb b/spec/models/translation_override_spec.rb index 0a6dc00d57..e331a73e3d 100644 --- a/spec/models/translation_override_spec.rb +++ b/spec/models/translation_override_spec.rb @@ -19,19 +19,6 @@ describe TranslationOverride do )) end end - - describe 'when interpolation keys are invalid' do - it 'should not be valid' do - translation_override = TranslationOverride.upsert!( - I18n.locale, 'some_key', '%{first} %{second} %{third}' - ) - - expect(translation_override.errors.full_messages).to include(I18n.t( - 'activerecord.errors.models.translation_overrides.attributes.value.invalid_interpolation_keys', - keys: 'third' - )) - end - end end end From b605d5d61bac7a1c16a334624ff85ec2e2c539f2 Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Tue, 11 Jul 2017 12:51:12 +0900 Subject: [PATCH 024/382] FIX: Translation should return overrides first before attempting to fallback. https://meta.discourse.org/t/errors-after-migrating-to-a-new-version-1-9-0-beta3/65709/14?u=tgxworld --- lib/i18n/backend/discourse_i18n.rb | 5 ++--- spec/components/discourse_i18n_spec.rb | 13 +++++++++++++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/i18n/backend/discourse_i18n.rb b/lib/i18n/backend/discourse_i18n.rb index ee133d5066..988bf87d5f 100644 --- a/lib/i18n/backend/discourse_i18n.rb +++ b/lib/i18n/backend/discourse_i18n.rb @@ -74,9 +74,8 @@ module I18n existing_translations = super(locale, key, scope, options) overrides = options.dig(:overrides, locale) - if overrides && existing_translations - if options[:count] - + if overrides && !scope&.include?(:models) + if existing_translations && options[:count] remapped_translations = if existing_translations.is_a?(Hash) Hash[existing_translations.map { |k, v| ["#{key}.#{k}", v] }] diff --git a/spec/components/discourse_i18n_spec.rb b/spec/components/discourse_i18n_spec.rb index c3fc16da1a..7db8d9a6cc 100644 --- a/spec/components/discourse_i18n_spec.rb +++ b/spec/components/discourse_i18n_spec.rb @@ -163,6 +163,19 @@ describe I18n::Backend::DiscourseI18n do .to eq('snow is the new queen') end + it "returns override if it exists before falling back" do + I18n.backend.store_translations(:en, got: 'winter') + + expect(I18n.translate('got', default: '')).to eq('winter') + expect(I18n.with_locale(:ru) { I18n.translate('got', default: '') }).to eq('winter') + + TranslationOverride.upsert!('ru', 'got', "summer") + I18n.backend.store_translations(:en, got: 'winter') + + expect(I18n.translate('got', default: '')).to eq('winter') + expect(I18n.with_locale(:ru) { I18n.translate('got', default: '') }).to eq('summer') + end + it 'supports ActiveModel::Naming#human' do Fish = Class.new(ActiveRecord::Base) From 45f4ce379e8ef85a048b2a042ddedcd961add8d0 Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Tue, 11 Jul 2017 13:16:48 +0900 Subject: [PATCH 025/382] Fix broken specs. --- lib/i18n/backend/discourse_i18n.rb | 4 +++- spec/components/discourse_i18n_spec.rb | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/i18n/backend/discourse_i18n.rb b/lib/i18n/backend/discourse_i18n.rb index 988bf87d5f..f940bd8804 100644 --- a/lib/i18n/backend/discourse_i18n.rb +++ b/lib/i18n/backend/discourse_i18n.rb @@ -72,9 +72,11 @@ module I18n # the original translations before applying our overrides. def lookup(locale, key, scope = [], options = {}) existing_translations = super(locale, key, scope, options) + return existing_translations if scope.is_a?(Array) && scope.include?(:models) + overrides = options.dig(:overrides, locale) - if overrides && !scope&.include?(:models) + if overrides if existing_translations && options[:count] remapped_translations = if existing_translations.is_a?(Hash) diff --git a/spec/components/discourse_i18n_spec.rb b/spec/components/discourse_i18n_spec.rb index 7db8d9a6cc..70212a8b0b 100644 --- a/spec/components/discourse_i18n_spec.rb +++ b/spec/components/discourse_i18n_spec.rb @@ -176,7 +176,7 @@ describe I18n::Backend::DiscourseI18n do expect(I18n.with_locale(:ru) { I18n.translate('got', default: '') }).to eq('summer') end - it 'supports ActiveModel::Naming#human' do + it 'does not affect ActiveModel::Naming#human' do Fish = Class.new(ActiveRecord::Base) TranslationOverride.upsert!('en', 'fish', "fake fish") From 3ff9133f857e57f82afed03a6c9b0aaa48f85fa6 Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Tue, 11 Jul 2017 15:07:32 +0900 Subject: [PATCH 026/382] FIX: Only trigger callback for parent transition events. * This fixes a bug where multiple requests were being made to the server whenever the composer is opened. --- app/assets/javascripts/discourse/lib/after-transition.js.es6 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/discourse/lib/after-transition.js.es6 b/app/assets/javascripts/discourse/lib/after-transition.js.es6 index 11fe20b5e4..afa1700e6d 100644 --- a/app/assets/javascripts/discourse/lib/after-transition.js.es6 +++ b/app/assets/javascripts/discourse/lib/after-transition.js.es6 @@ -25,5 +25,8 @@ var transitionEnd = (function() { })(); export default function (element, callback) { - return $(element).on(transitionEnd, callback); + return $(element).on(transitionEnd, event => { + if (event.target !== event.currentTarget) return; + return callback(event); + }); } From 31932813b782c70473a5f759cde1471ee610d828 Mon Sep 17 00:00:00 2001 From: Guo Xiang Tan Date: Tue, 11 Jul 2017 15:26:27 +0900 Subject: [PATCH 027/382] FIX: Wait for CSS transition to end before attempting to focus. https://meta.discourse.org/t/tab-should-work-after-edit-title-or-using-hotkeys/65792 --- .../javascripts/discourse/components/composer-title.js.es6 | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/app/assets/javascripts/discourse/components/composer-title.js.es6 b/app/assets/javascripts/discourse/components/composer-title.js.es6 index 2e37da138e..8841500af8 100644 --- a/app/assets/javascripts/discourse/components/composer-title.js.es6 +++ b/app/assets/javascripts/discourse/components/composer-title.js.es6 @@ -2,6 +2,7 @@ import { default as computed, observes } from 'ember-addons/ember-computed-decor import InputValidation from 'discourse/models/input-validation'; import { load, lookupCache } from 'pretty-text/oneboxer'; import { ajax } from 'discourse/lib/ajax'; +import afterTransition from 'discourse/lib/after-transition'; export default Ember.Component.extend({ classNames: ['title-input'], @@ -10,7 +11,11 @@ export default Ember.Component.extend({ didInsertElement() { this._super(); if (this.get('focusTarget') === 'title') { - this.$('input').putCursorAtEnd(); + const $input = this.$("input"); + + afterTransition(this.$().closest("#reply-control"), () => { + $input.putCursorAtEnd(); + }); } }, From 35fb45749d3a7f90daa77f36a7f668e05e4d8c49 Mon Sep 17 00:00:00 2001 From: Arpit Jalan Date: Tue, 11 Jul 2017 13:20:56 +0530 Subject: [PATCH 028/382] FIX: remove unneeded code --- app/models/invite_redeemer.rb | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/models/invite_redeemer.rb b/app/models/invite_redeemer.rb index df748bf32b..8205cb549e 100644 --- a/app/models/invite_redeemer.rb +++ b/app/models/invite_redeemer.rb @@ -19,9 +19,6 @@ InviteRedeemer = Struct.new(:invite, :username, :name, :password, :user_custom_f # extracted from User cause it is very specific to invites def self.create_user_from_invite(invite, username, name, password=nil, user_custom_fields=nil) - user_exists = User.where(admin: false).find_by_email(invite.email) - return user if user_exists - if username && UsernameValidator.new(username).valid_format? && User.username_available?(username) available_username = username else @@ -94,7 +91,7 @@ InviteRedeemer = Struct.new(:invite, :username, :name, :password, :user_custom_f end def get_existing_user - User.find_by(email: invite.email) + User.where(admin: false).find_by(email: invite.email) end From 3fb1c21dd5fbc17fda9e4c75953d19ed8009798e Mon Sep 17 00:00:00 2001 From: Arpit Jalan Date: Tue, 11 Jul 2017 19:26:33 +0530 Subject: [PATCH 029/382] Fix the build. --- .../javascripts/discourse/components/composer-editor.js.es6 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/discourse/components/composer-editor.js.es6 b/app/assets/javascripts/discourse/components/composer-editor.js.es6 index 47e0306fd7..3373ccd79d 100644 --- a/app/assets/javascripts/discourse/components/composer-editor.js.es6 +++ b/app/assets/javascripts/discourse/components/composer-editor.js.es6 @@ -1,5 +1,5 @@ import userSearch from 'discourse/lib/user-search'; -import { default as computed, on } from 'ember-addons/ember-computed-decorators'; +import { default as computed, on, observes } from 'ember-addons/ember-computed-decorators'; import { linkSeenMentions, fetchUnseenMentions } from 'discourse/lib/link-mentions'; import { linkSeenCategoryHashtags, fetchUnseenCategoryHashtags } from 'discourse/lib/link-category-hashtags'; import { linkSeenTagHashtags, fetchUnseenTagHashtags } from 'discourse/lib/link-tag-hashtag'; From 1163c086a3e8af3de04c94b0f4c5d432e9bd44bd Mon Sep 17 00:00:00 2001 From: Robin Ward Date: Tue, 11 Jul 2017 11:19:35 -0400 Subject: [PATCH 030/382] FIX: Missing `model` binding --- app/assets/javascripts/discourse/templates/topic.hbs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/assets/javascripts/discourse/templates/topic.hbs b/app/assets/javascripts/discourse/templates/topic.hbs index 7cdd7568d9..aa0627534b 100644 --- a/app/assets/javascripts/discourse/templates/topic.hbs +++ b/app/assets/javascripts/discourse/templates/topic.hbs @@ -10,7 +10,7 @@ {{#if model.postStream.loaded}} {{#if model.postStream.firstPostPresent}} - {{#topic-title cancelled="cancelEditingTopic" save="finishedEditingTopic"}} + {{#topic-title cancelled="cancelEditingTopic" save="finishedEditingTopic" model=model}} {{#if editingTopic}} {{#if model.isPrivateMessage}} {{fa-icon "envelope"}} From 3f950a756acf71ee245bed882c71a90b3ba3a949 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 11 Jul 2017 12:13:03 -0400 Subject: [PATCH 031/382] FEATURE: support image dimensions via Markdown image --- .../engines/discourse-markdown-it.js.es6 | 45 +++++++++++++++++++ spec/components/pretty_text_spec.rb | 24 ++++++++++ 2 files changed, 69 insertions(+) diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 index 9b9a971828..4029171cef 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 @@ -99,6 +99,50 @@ function setupHoister(md) { md.renderer.rules.html_raw = renderHoisted; } +const IMG_SIZE_REGEX = /^([1-9]+[0-9]*)x([1-9]+[0-9]*)(,([1-9][0-9]?)%)?$/; +function renderImage(tokens, idx, options, env, slf) { + var token = tokens[idx]; + + let alt = slf.renderInlineAsText(token.children, options, env); + + let split = alt.split('|'); + if (split.length > 1) { + let match; + let info = split.splice(split.length-1)[0]; + + if (match = info.match(IMG_SIZE_REGEX)) { + if (match[1] && match[2]) { + alt = split.join('|'); + + let width = match[1]; + let height = match[2]; + + if (match[4]) { + let percent = parseFloat(match[4]) / 100.0; + width = parseInt(width * percent); + height = parseInt(height * percent); + } + + if (token.attrIndex('width') === -1) { + token.attrs.push(['width', width]); + } + + if (token.attrIndex('height') === -1) { + token.attrs.push(['height', height]); + } + } + + } + } + + token.attrs[token.attrIndex('alt')][1] = alt; + return slf.renderToken(tokens, idx, options); +} + +function setupImageDimensions(md) { + md.renderer.rules.image = renderImage; +} + export function setup(opts, siteSettings, state) { if (opts.setup) { return; @@ -160,6 +204,7 @@ export function setup(opts, siteSettings, state) { setupUrlDecoding(opts.engine); setupHoister(opts.engine); + setupImageDimensions(opts.engine); setupBlockBBCode(opts.engine); setupInlineBBCode(opts.engine); diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb index 7b7b7036fa..3c4089d058 100644 --- a/spec/components/pretty_text_spec.rb +++ b/spec/components/pretty_text_spec.rb @@ -890,5 +890,29 @@ HTML expect(PrettyText.cook("")).to eq "" end + # custom rule used to specify image dimensions via alt tags + describe "image dimensions" do + it "allows title plus dimensions" do + cooked = PrettyText.cook <<~MD + ![title with | title|220x100](http://png.com/my.png) + ![](http://png.com/my.png) + ![|220x100](http://png.com/my.png) + ![stuff](http://png.com/my.png) + ![|220x100,50%](http://png.com/my.png) + MD + + html = <<~HTML +

title with | title
+
+
+ stuff
+

+ HTML + puts cooked + + expect(cooked).to eq(html.strip) + end + end + end From c8e9f4bd3aa2f9abf68d8801bf73e60fd4a596ac Mon Sep 17 00:00:00 2001 From: David Taylor Date: Tue, 11 Jul 2017 17:25:53 +0100 Subject: [PATCH 032/382] Fix failing discourse-details plugin qunit tests --- .../acceptance/details-button-test.js.es6 | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 b/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 index cbf078c279..e429c96e71 100644 --- a/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 +++ b/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 @@ -16,7 +16,7 @@ test('details button', (assert) => { andThen(() => { assert.equal( find(".d-editor-input").val(), - `[details=${I18n.t("composer.details_title")}]${I18n.t("composer.details_text")}[/details]`, + `\n[details=${I18n.t("composer.details_title")}]\n${I18n.t("composer.details_text")}\n[/details]\n`, 'it should contain the right output' ); }); @@ -35,13 +35,13 @@ test('details button', (assert) => { andThen(() => { assert.equal( find(".d-editor-input").val(), - `[details=${I18n.t("composer.details_title")}]This is my title[/details]`, + `\n[details=${I18n.t("composer.details_title")}]\nThis is my title\n[/details]\n`, 'it should contain the right selected output' ); const textarea = findTextarea(); - assert.equal(textarea.selectionStart, 17, 'it should start highlighting at the right position'); - assert.equal(textarea.selectionEnd, 33, 'it should end highlighting at the right position'); + assert.equal(textarea.selectionStart, 19, 'it should start highlighting at the right position'); + assert.equal(textarea.selectionEnd, 35, 'it should end highlighting at the right position'); }); fillIn('.d-editor-input', "Before some text in between After"); @@ -58,13 +58,13 @@ test('details button', (assert) => { andThen(() => { assert.equal( find(".d-editor-input").val(), - `Before [details=${I18n.t("composer.details_title")}]some text in between[/details] After`, + `Before\n[details=${I18n.t("composer.details_title")}]\nsome text in between\n[/details]\nAfter`, 'it should contain the right output' ); const textarea = findTextarea(); - assert.equal(textarea.selectionStart, 24, 'it should start highlighting at the right position'); - assert.equal(textarea.selectionEnd, 44, 'it should end highlighting at the right position'); + assert.equal(textarea.selectionStart, 26, 'it should start highlighting at the right position'); + assert.equal(textarea.selectionEnd, 46, 'it should end highlighting at the right position'); }); fillIn('.d-editor-input', "Before\nsome text in between\nAfter"); @@ -81,12 +81,12 @@ test('details button', (assert) => { andThen(() => { assert.equal( find(".d-editor-input").val(), - `Before\n[details=${I18n.t("composer.details_title")}]some text in between[/details]\nAfter`, + `Before\n\n[details=${I18n.t("composer.details_title")}]\nsome text in between\n[/details]\n\nAfter`, 'it should contain the right output' ); const textarea = findTextarea(); - assert.equal(textarea.selectionStart, 24, 'it should start highlighting at the right position'); - assert.equal(textarea.selectionEnd, 44, 'it should end highlighting at the right position'); + assert.equal(textarea.selectionStart, 26, 'it should start highlighting at the right position'); + assert.equal(textarea.selectionEnd, 46, 'it should end highlighting at the right position'); }); }); From 06366b53794044a10b069cc20cacfd1a1024f724 Mon Sep 17 00:00:00 2001 From: Bianca Nenciu Date: Mon, 20 Mar 2017 18:30:49 +0200 Subject: [PATCH 033/382] latest.json: last topic from first page appears on the second page as well. --- lib/topic_query.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/topic_query.rb b/lib/topic_query.rb index 62e6ec619b..51deb6654f 100644 --- a/lib/topic_query.rb +++ b/lib/topic_query.rb @@ -315,7 +315,7 @@ class TopicQuery if page == 0 (pinned_topics + unpinned_topics)[0...limit] if limit else - offset = (page * per_page) - pinned_topics.count - 1 + offset = (page * per_page) - pinned_topics.count offset = 0 unless offset > 0 unpinned_topics.offset(offset).to_a end From 1c917cc391e6d9b92d7cefed1e1cc1c9704d3ea4 Mon Sep 17 00:00:00 2001 From: Bianca Nenciu Date: Mon, 10 Jul 2017 23:25:29 +0100 Subject: [PATCH 034/382] Added test. --- spec/components/topic_query_spec.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/spec/components/topic_query_spec.rb b/spec/components/topic_query_spec.rb index 4d542032e4..e3cb344d94 100644 --- a/spec/components/topic_query_spec.rb +++ b/spec/components/topic_query_spec.rb @@ -67,6 +67,35 @@ describe TopicQuery do end + context "prioritize_pinned_topics" do + + it "does the pagination correctly" do + + num_topics = 15 + per_page = 3 + + topics = [] + (num_topics - 1).downto(0).each do |i| + topics[i] = Fabricate(:topic) + end + + topic_query = TopicQuery.new(user) + results = topic_query.send(:default_results) + + expect(topic_query.prioritize_pinned_topics(results, { + :per_page => per_page, + :page => 0 + })).to eq(topics[0...per_page]) + + expect(topic_query.prioritize_pinned_topics(results, { + :per_page => per_page, + :page => 1 + })).to eq(topics[per_page...num_topics]) + + end + + end + context 'bookmarks' do it "filters and returns bookmarks correctly" do post = Fabricate(:post) From 5d139e461ca37f282fd51c564e945e9a59c4b647 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 11 Jul 2017 14:43:24 -0400 Subject: [PATCH 035/382] FIX: multi option poll not working (new engine) --- .../engines/markdown-it/bbcode-block.js.es6 | 40 ++++++++----------- .../lib/discourse-markdown/poll.js.es6 | 14 +++---- plugins/poll/spec/lib/pretty_text_spec.rb | 30 +++++++++++++- spec/components/pretty_text_spec.rb | 1 - 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6 index 51050f9206..db00edbea7 100644 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6 @@ -11,6 +11,8 @@ function trailingSpaceOnly(src, start, max) { return true; } +const ATTR_REGEX = /(([a-z0-9]*)\s*=)/ig; + // parse a tag [test a=1 b=2] to a data structure // {tag: "test", attrs={a: "1", b: "2"} export function parseBBCodeTag(src, start, max, multiline) { @@ -71,31 +73,23 @@ export function parseBBCodeTag(src, start, max, multiline) { // trivial parser that is going to have to be rewritten at some point if (raw) { - // reading a key 0, reading a val = 1 - let readingKey = true; - let startSplit = 0; - let key; + let match, key, val; - for(i=0; i 0) { - val = val.replace(/^["'](.*)["']$/, '$1'); - attrs[key] = val; - } - } - readingKey = !readingKey; - startSplit = i+1; + while(match = ATTR_REGEX.exec(raw)) { + if (key) { + val = raw.slice(attrs[key],match.index) || ''; + val = val.trim(); + val = val.replace(/^["'](.*)["']$/, '$1'); + attrs[key] = val; } + key = match[2] || '_default'; + attrs[key] = match.index + match[0].length; + } + + if (key) { + val = raw.slice(attrs[key]); + val = val.replace(/^["'](.*)["']$/, '$1'); + attrs[key] = val; } } diff --git a/plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6 b/plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6 index b7b41648e1..65de3ce659 100644 --- a/plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6 +++ b/plugins/poll/assets/javascripts/lib/discourse-markdown/poll.js.es6 @@ -127,7 +127,7 @@ const rule = { WHITELISTED_ATTRIBUTES.forEach(name => { if (attrs[name]) { - attributes[DATA_PREFIX + name] = attrs[name]; + attributes.push([DATA_PREFIX + name, attrs[name]]); } }); @@ -136,9 +136,9 @@ const rule = { } // we might need these values later... - let min = parseInt(attributes[DATA_PREFIX + "min"], 10); - let max = parseInt(attributes[DATA_PREFIX + "max"], 10); - let step = parseInt(attributes[DATA_PREFIX + "step"], 10); + let min = parseInt(attrs["min"], 10); + let max = parseInt(attrs["max"], 10); + let step = parseInt(attrs["step"], 10); let header = []; @@ -157,7 +157,7 @@ const rule = { header.push(token); // generate the options when the type is "number" - if (attributes[DATA_PREFIX + "type"] === "number") { + if (attrs["type"] === "number") { // default values if (isNaN(min)) { min = 1; } if (isNaN(max)) { max = md.options.discourse.pollMaximumOptions; } @@ -172,7 +172,7 @@ const rule = { header.push(token); for (let o = min; o <= max; o += step) { - token = new state.Token('list_item_open', '', 1); + token = new state.Token('list_item_open', 'li', 1); items.push([token, String(o)]); header.push(token); @@ -180,7 +180,7 @@ const rule = { token.content = String(o); header.push(token); - token = new state.Token('list_item_close', '', -1); + token = new state.Token('list_item_close', 'li', -1); header.push(token); } token = new state.Token('bullet_item_close', '', -1); diff --git a/plugins/poll/spec/lib/pretty_text_spec.rb b/plugins/poll/spec/lib/pretty_text_spec.rb index 417a2989f4..083a9b2d9f 100644 --- a/plugins/poll/spec/lib/pretty_text_spec.rb +++ b/plugins/poll/spec/lib/pretty_text_spec.rb @@ -12,6 +12,34 @@ describe PrettyText do SiteSetting.enable_experimental_markdown_it = true end + it 'supports multi choice polls' do + cooked = PrettyText.cook <<~MD + [poll type=multiple min=1 max=3 public=true] + * option 1 + * option 2 + * option 3 + [/poll] + MD + + expect(cooked).to include('class="poll"') + expect(cooked).to include('data-poll-status="open"') + expect(cooked).to include('data-poll-name="poll"') + expect(cooked).to include('data-poll-type="multiple"') + expect(cooked).to include('data-poll-min="1"') + expect(cooked).to include('data-poll-max="3"') + expect(cooked).to include('data-poll-public="true"') + end + + it 'can dynamically generate a poll' do + + cooked = PrettyText.cook <<~MD + [poll type=number min=1 max=20 step=1] + [/poll] + MD + + expect(cooked.scan(' +
    diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb index 3c4089d058..ea6c749d5d 100644 --- a/spec/components/pretty_text_spec.rb +++ b/spec/components/pretty_text_spec.rb @@ -908,7 +908,6 @@ HTML stuff

    HTML - puts cooked expect(cooked).to eq(html.strip) end From 98e03b04b50624110572558c49acaa520f28d7ed Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 11 Jul 2017 16:48:25 -0400 Subject: [PATCH 036/382] Don't depend on imports for md extensions --- .../engines/discourse-markdown-it.js.es6 | 9 +++ .../markdown-it/category-hashtag.js.es6 | 4 +- .../engines/markdown-it/emoji.js.es6 | 3 +- .../engines/markdown-it/helpers.js.es6 | 60 ------------------- lib/pretty_text.rb | 4 ++ lib/pretty_text/shims.js | 2 +- 6 files changed, 16 insertions(+), 66 deletions(-) diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 index 4029171cef..ac80029cfd 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 @@ -143,11 +143,16 @@ function setupImageDimensions(md) { md.renderer.rules.image = renderImage; } +let Helpers; + export function setup(opts, siteSettings, state) { if (opts.setup) { return; } + // we got to require this late cause bundle is not loaded in pretty-text + Helpers = Helpers || requirejs('pretty-text/engines/markdown-it/helpers'); + opts.markdownIt = true; let optionCallbacks = []; @@ -190,6 +195,10 @@ export function setup(opts, siteSettings, state) { delete opts[entry]; }); + copy.helpers = { + textReplace: Helpers.textReplace + }; + opts.discourse = copy; getOptions.f = () => opts.discourse; diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 index b2c1e8ab44..79d57002a6 100644 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 @@ -1,5 +1,3 @@ -import { textReplace } from 'pretty-text/engines/markdown-it/helpers'; - function addHashtag(buffer, matches, state) { const options = state.md.options.discourse; const [hashtag, slug] = matches; @@ -99,7 +97,7 @@ export function setup(helper) { helper.registerPlugin(md=>{ - md.core.ruler.push('category-hashtag', state => textReplace( + md.core.ruler.push('category-hashtag', state => md.options.discourse.helpers.textReplace( state, applyHashtag, true /* skip all links */ )); }); diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 index dd85d24f34..0e4eed203b 100644 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 @@ -1,6 +1,5 @@ import { buildEmojiUrl, isCustomEmoji } from 'pretty-text/emoji'; import { translations } from 'pretty-text/emoji/data'; -import { textReplace } from 'pretty-text/engines/markdown-it/helpers'; const MAX_NAME_LENGTH = 60; @@ -240,7 +239,7 @@ export function setup(helper) { }); helper.registerPlugin((md)=>{ - md.core.ruler.push('emoji', state => textReplace( + md.core.ruler.push('emoji', state => md.options.discourse.helpers.textReplace( state, (c,s)=>applyEmoji(c,s,md.options.discourse.emojiUnicodeReplacer)) ); }); diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6 index b016b5a86c..c689d5c633 100644 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6 @@ -13,66 +13,6 @@ export default null; // emitter: emitter // }); -export function inlineRegexRule(md, options) { - - const start = options.start.charCodeAt(0); - const maxLength = (options.maxLength || 500) + 1; - - return function(state, silent) { - const pos = state.pos; - - if (state.src.charCodeAt(pos) !== start || silent) { - return false; - } - - // test prev - if (pos > 0) { - let prev = state.src.charCodeAt(pos-1); - if (!md.utils.isWhiteSpace(prev) && !md.utils.isPunctChar(String.fromCharCode(prev))) { - return false; - } - } - - // skip if in a link - if (options.skipInLink && state.tokens) { - let i; - for(i=state.tokens.length-1;i>=0;i--) { - let token = state.tokens[i]; - let type = token.type; - if (type === 'link_open' || (type === 'html_inline' && token.content.substr(0,2).toLowerCase() === "")) { - break; - } - } - } - - const substr = state.src.slice(pos, Math.min(pos + maxLength,state.posMax)); - - const matches = options.matcher.exec(substr); - if (!matches) { - return false; - } - - // got to test trailing boundary - const finalPos = pos+matches[0].length; - if (finalPos < state.posMax) { - const trailing = state.src.charCodeAt(finalPos); - if (!md.utils.isSpace(trailing) && !md.utils.isPunctChar(String.fromCharCode(trailing))) { - return false; - } - } - - if (options.emitter(matches, state)) { - state.pos = Math.min(state.posMax, finalPos); - return true; - } - - return false; - - }; -} // based off https://github.com/markdown-it/markdown-it-emoji/blob/master/dist/markdown-it-emoji.js // diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb index 3231301f53..2b9b3489b7 100644 --- a/lib/pretty_text.rb +++ b/lib/pretty_text.rb @@ -81,6 +81,10 @@ module PrettyText ctx_load(ctx, "vendor/assets/javascripts/lodash.js") ctx_load_manifest(ctx, "pretty-text-bundle.js") + if SiteSetting.enable_experimental_markdown_it + ctx_load_manifest(ctx, "markdown-it-bundle.js") + end + root_path = "#{Rails.root}/app/assets/javascripts/" apply_es6_file(ctx, root_path, "discourse/lib/utilities") diff --git a/lib/pretty_text/shims.js b/lib/pretty_text/shims.js index afb1a199a3..b3efe6cdf5 100644 --- a/lib/pretty_text/shims.js +++ b/lib/pretty_text/shims.js @@ -8,7 +8,7 @@ __emojiUnicodeReplacer = null; __setUnicode = function(replacements) { require('pretty-text/engines/discourse-markdown/emoji').setUnicodeReplacements(replacements); - + let unicodeRegexp = new RegExp(Object.keys(replacements).sort().reverse().join("|"), "g"); __emojiUnicodeReplacer = function(text) { From 1091d101ef7147f06ec7626136177bda15bc08c4 Mon Sep 17 00:00:00 2001 From: David Taylor Date: Tue, 11 Jul 2017 20:09:43 +0100 Subject: [PATCH 037/382] Load plugins for qunit travis tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5eaaec0902..e31fd1e742 100644 --- a/.travis.yml +++ b/.travis.yml @@ -62,4 +62,4 @@ install: - bash -c "if [ '$RAILS_MASTER' == '0' ]; then bundle install --without development --deployment --retry=3 --jobs=3; fi" script: - - bash -c "if [ '$QUNIT_RUN' == '0' ]; then bundle exec rspec && bundle exec rake plugin:spec; else bundle exec rake qunit:test['200000']; fi" + - bash -c "if [ '$QUNIT_RUN' == '0' ]; then bundle exec rspec && bundle exec rake plugin:spec; else LOAD_PLUGINS=1 bundle exec rake qunit:test['200000']; fi" From 2808e3f63eeaa049ef0b7779d4572c5777db335e Mon Sep 17 00:00:00 2001 From: David Taylor Date: Tue, 11 Jul 2017 22:43:33 +0100 Subject: [PATCH 038/382] Fix for the last broken discourse-details qunit test --- .../acceptance/details-button-test.js.es6 | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 b/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 index e429c96e71..61ffac2152 100644 --- a/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 +++ b/plugins/discourse-details/test/javascripts/acceptance/details-button-test.js.es6 @@ -58,7 +58,7 @@ test('details button', (assert) => { andThen(() => { assert.equal( find(".d-editor-input").val(), - `Before\n[details=${I18n.t("composer.details_title")}]\nsome text in between\n[/details]\nAfter`, + `Before \n[details=${I18n.t("composer.details_title")}]\nsome text in between\n[/details]\n After`, 'it should contain the right output' ); @@ -67,12 +67,12 @@ test('details button', (assert) => { assert.equal(textarea.selectionEnd, 46, 'it should end highlighting at the right position'); }); - fillIn('.d-editor-input', "Before\nsome text in between\nAfter"); + fillIn('.d-editor-input', "Before \nsome text in between\n After"); andThen(() => { const textarea = findTextarea(); - textarea.selectionStart = 7; - textarea.selectionEnd = 28; + textarea.selectionStart = 8; + textarea.selectionEnd = 29; }); click('button.options'); @@ -81,12 +81,12 @@ test('details button', (assert) => { andThen(() => { assert.equal( find(".d-editor-input").val(), - `Before\n\n[details=${I18n.t("composer.details_title")}]\nsome text in between\n[/details]\n\nAfter`, + `Before \n\n[details=${I18n.t("composer.details_title")}]\nsome text in between\n[/details]\n\n After`, 'it should contain the right output' ); const textarea = findTextarea(); - assert.equal(textarea.selectionStart, 26, 'it should start highlighting at the right position'); - assert.equal(textarea.selectionEnd, 46, 'it should end highlighting at the right position'); + assert.equal(textarea.selectionStart, 27, 'it should start highlighting at the right position'); + assert.equal(textarea.selectionEnd, 47, 'it should end highlighting at the right position'); }); }); From 9e91d137e3c8590de1a299782c24c097b6a66a58 Mon Sep 17 00:00:00 2001 From: Jeff Atwood Date: Tue, 11 Jul 2017 15:04:04 -0700 Subject: [PATCH 039/382] switch to h4 for invite body title copy --- config/locales/server.en.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 60c7c64026..11de85f357 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1764,7 +1764,7 @@ en: text_body_template: | %{invitee_name} invited you to a discussion - > **%{topic_title}** + > #### %{topic_title} > > %{topic_excerpt} @@ -1782,7 +1782,7 @@ en: text_body_template: | %{invitee_name} invited you to a discussion - > **%{topic_title}** + > #### %{topic_title} > > %{topic_excerpt} @@ -2414,7 +2414,7 @@ en: invited_to_private_message_body: | %{username} invited you to a message - > **%{topic_title}** + > #### %{topic_title} > > %{topic_excerpt} @@ -2425,7 +2425,6 @@ en: invited_to_topic_body: | %{username} invited you to a discussion - > **%{topic_title}** > > %{topic_excerpt} From f585f2cca5c891dc6fc91ee60f32acbcf41727ad Mon Sep 17 00:00:00 2001 From: Jeff Atwood Date: Tue, 11 Jul 2017 15:05:07 -0700 Subject: [PATCH 040/382] switch to H4 for invite body title copy --- config/locales/server.en.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 11de85f357..67a2dce790 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -2425,6 +2425,7 @@ en: invited_to_topic_body: | %{username} invited you to a discussion + > #### %{topic_title} > > %{topic_excerpt} From a6dff79c2cf68da725dc3c9971b29ec8f909763c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Wed, 12 Jul 2017 11:06:28 +0200 Subject: [PATCH 041/382] change log level to info when failing to download a hotlinked image --- app/jobs/regular/pull_hotlinked_images.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/jobs/regular/pull_hotlinked_images.rb b/app/jobs/regular/pull_hotlinked_images.rb index ed158bef13..c38d9a55e2 100644 --- a/app/jobs/regular/pull_hotlinked_images.rb +++ b/app/jobs/regular/pull_hotlinked_images.rb @@ -58,7 +58,7 @@ module Jobs log(:info, "Failed to pull hotlinked image for post: #{post_id}: #{src} - Image is bigger than #{@max_size}") end else - log(:error, "There was an error while downloading '#{src}' locally for post: #{post_id}") + log(:info, "There was an error while downloading '#{src}' locally for post: #{post_id}") end end # have we successfully downloaded that file? From 5be9bee2307dd517c26e6ef269471aceba5d5acf Mon Sep 17 00:00:00 2001 From: Jeff Atwood Date: Wed, 12 Jul 2017 04:52:14 -0700 Subject: [PATCH 042/382] safe to default to read only off during backups --- config/site_settings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/site_settings.yml b/config/site_settings.yml index 6486b7226f..9816e822ce 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -1114,7 +1114,7 @@ legal: backups: readonly_mode_during_backup: - default: true + default: false allow_restore: default: false maximum_backups: From 677267ae786bae753655039d7c2e3deba58b0077 Mon Sep 17 00:00:00 2001 From: Jakub Macina Date: Thu, 6 Jul 2017 19:11:32 +0200 Subject: [PATCH 043/382] Add onceoff job for uploads migration of column extension. Simplify filetype search and related rspec tests. --- app/jobs/onceoff/migrate_upload_extensions.rb | 11 +++++++++++ lib/file_store/base_store.rb | 2 +- lib/search.rb | 8 ++++---- spec/components/search_spec.rb | 12 +++++------- 4 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 app/jobs/onceoff/migrate_upload_extensions.rb diff --git a/app/jobs/onceoff/migrate_upload_extensions.rb b/app/jobs/onceoff/migrate_upload_extensions.rb new file mode 100644 index 0000000000..38bcd3a1ea --- /dev/null +++ b/app/jobs/onceoff/migrate_upload_extensions.rb @@ -0,0 +1,11 @@ +module Jobs + + class MigrateUploadExtensions < Jobs::Onceoff + def execute_onceoff(args) + Upload.find_each do |upload| + upload.extension = File.extname(upload.original_filename)[1..10] + upload.save + end + end + end +end diff --git a/lib/file_store/base_store.rb b/lib/file_store/base_store.rb index b4c8bfe47c..7b17fb2d85 100644 --- a/lib/file_store/base_store.rb +++ b/lib/file_store/base_store.rb @@ -95,7 +95,7 @@ module FileStore end def get_path_for_upload(upload) - get_path_for("original".freeze, upload.id, upload.sha1, File.extname(upload.original_filename)) + get_path_for("original".freeze, upload.id, upload.sha1, "."+upload.extension) end def get_path_for_optimized_image(optimized_image) diff --git a/lib/search.rb b/lib/search.rb index 159de71b8d..5a6c1c2d78 100644 --- a/lib/search.rb +++ b/lib/search.rb @@ -457,16 +457,16 @@ class Search end advanced_filter(/filetypes?:([a-zA-Z0-9,\-_]+)/) do |posts, match| - file_extensions = match.split(",") + file_extensions = match.split(",").map(&:downcase) posts.where("posts.id IN ( SELECT post_id FROM topic_links - WHERE extension IN (?) + WHERE extension IN (:file_extensions) UNION SELECT post_uploads.post_id FROM uploads JOIN post_uploads ON post_uploads.upload_id = uploads.id - WHERE lower(uploads.extension) IN (?) - )", file_extensions, file_extensions) + WHERE lower(uploads.extension) IN (:file_extensions) + )", {file_extensions: file_extensions}) end private diff --git a/spec/components/search_spec.rb b/spec/components/search_spec.rb index 7faf1bac43..c09e3f8d43 100644 --- a/spec/components/search_spec.rb +++ b/spec/components/search_spec.rb @@ -705,21 +705,19 @@ describe Search do end it "can find posts which contains filetypes" do - # Must be posts with real images post1 = Fabricate(:post, - raw: "https://www.discourse.org/a/img/favicon.png") + raw: "http://example.com/image.png") post2 = Fabricate(:post, raw: "Discourse logo\n"\ - "https://www.discourse.org/a/img/favicon.png\n"\ - "https://www.discourse.org/a/img/trust-1x.jpg") - post_with_upload = Fabricate(:post) - post_with_upload.uploads = [Fabricate(:upload)] + "http://example.com/logo.png\n"\ + "http://example.com/vector_image.svg") + post_with_upload = Fabricate(:post, uploads: [Fabricate(:upload)]) Fabricate(:post) TopicLink.extract_from(post1) TopicLink.extract_from(post2) - expect(Search.execute('filetype:jpg').posts.map(&:id)).to eq([post2.id]) + expect(Search.execute('filetype:svg').posts).to eq([post2]) expect(Search.execute('filetype:png').posts.map(&:id)).to contain_exactly(post1.id, post2.id, post_with_upload.id) expect(Search.execute('logo filetype:png').posts.map(&:id)).to eq([post2.id]) end From 76981605fa10975e2e7af457e2f6a31909e0c811 Mon Sep 17 00:00:00 2001 From: Arpit Jalan Date: Wed, 12 Jul 2017 15:31:10 +0530 Subject: [PATCH 044/382] FIX: don't raise error when inviting existing user to private topic via email https://meta.discourse.org/t/inviting-existing-user-to-a-private-topic-message-via-email-shows-error-message/65994 --- .../discourse/controllers/invite.js.es6 | 79 +++++++++++-------- app/models/topic.rb | 24 ++++-- config/locales/client.en.yml | 1 + spec/models/topic_spec.rb | 2 +- 4 files changed, 65 insertions(+), 41 deletions(-) diff --git a/app/assets/javascripts/discourse/controllers/invite.js.es6 b/app/assets/javascripts/discourse/controllers/invite.js.es6 index f9d4d52dcd..90020f449d 100644 --- a/app/assets/javascripts/discourse/controllers/invite.js.es6 +++ b/app/assets/javascripts/discourse/controllers/invite.js.es6 @@ -13,6 +13,7 @@ export default Ember.Controller.extend(ModalFunctionality, { hasCustomMessage: false, customMessage: null, inviteIcon: "envelope", + invitingExistingUserToTopic: false, @computed('isMessage', 'invitingToTopic') title(isMessage, invitingToTopic) { @@ -25,9 +26,10 @@ export default Ember.Controller.extend(ModalFunctionality, { } }, - isAdmin: function(){ + @computed + isAdmin() { return Discourse.User.currentProp("admin"); - }.property(), + }, @computed('isAdmin', 'emailOrUsername', 'invitingToTopic', 'isPrivateTopic', 'model.groupNames', 'model.saving', 'model.details.can_invite_to') disabled(isAdmin, emailOrUsername, invitingToTopic, isPrivateTopic, groupNames, saving, can_invite_to) { @@ -46,29 +48,32 @@ export default Ember.Controller.extend(ModalFunctionality, { return false; }, - disabledCopyLink: function() { - if (this.get('hasCustomMessage')) return true; - if (this.get('model.saving')) return true; - if (Ember.isEmpty(this.get('emailOrUsername'))) return true; - const emailOrUsername = this.get('emailOrUsername').trim(); + @computed('isAdmin', 'emailOrUsername', 'model.saving', 'isPrivateTopic', 'model.groupNames', 'hasCustomMessage') + disabledCopyLink(isAdmin, emailOrUsername, saving, isPrivateTopic, groupNames, hasCustomMessage) { + if (hasCustomMessage) return true; + if (saving) return true; + if (Ember.isEmpty(emailOrUsername)) return true; + const email = emailOrUsername.trim(); // email must be valid - if (!emailValid(emailOrUsername)) return true; + if (!emailValid(email)) return true; // normal users (not admin) can't invite users to private topic via email - if (!this.get('isAdmin') && this.get('isPrivateTopic') && emailValid(emailOrUsername)) return true; + if (!isAdmin && isPrivateTopic && emailValid(email)) return true; // when inviting to private topic via email, group name must be specified - if (this.get('isPrivateTopic') && Ember.isEmpty(this.get('model.groupNames')) && emailValid(emailOrUsername)) return true; + if (isPrivateTopic && Ember.isEmpty(groupNames) && emailValid(email)) return true; return false; - }.property('emailOrUsername', 'model.saving', 'isPrivateTopic', 'model.groupNames', 'hasCustomMessage'), + }, - buttonTitle: function() { - return this.get('model.saving') ? 'topic.inviting' : 'topic.invite_reply.action'; - }.property('model.saving'), + @computed('model.saving') + buttonTitle(saving) { + return saving ? 'topic.inviting' : 'topic.invite_reply.action'; + }, // We are inviting to a topic if the model isn't the current user. // The current user would mean we are inviting to the forum in general. - invitingToTopic: function() { - return this.get('model') !== this.currentUser; - }.property('model'), + @computed('model') + invitingToTopic(model) { + return model !== this.currentUser; + }, @computed('model', 'model.details.can_invite_via_email') canInviteViaEmail(model, can_invite_via_email) { @@ -91,9 +96,7 @@ export default Ember.Controller.extend(ModalFunctionality, { isMessage: Em.computed.equal('model.archetype', 'private_message'), // Allow Existing Members? (username autocomplete) - allowExistingMembers: function() { - return this.get('invitingToTopic'); - }.property('invitingToTopic'), + allowExistingMembers: Ember.computed.alias('invitingToTopic'), // Show Groups? (add invited user to private group) @computed('isAdmin', 'emailOrUsername', 'isPrivateTopic', 'isMessage', 'invitingToTopic', 'canInviteViaEmail') @@ -141,29 +144,34 @@ export default Ember.Controller.extend(ModalFunctionality, { } }, - showGroupsClass: function() { - return this.get('isPrivateTopic') ? 'required' : 'optional'; - }.property('isPrivateTopic'), + @computed('isPrivateTopic') + showGroupsClass(isPrivateTopic) { + return isPrivateTopic ? 'required' : 'optional'; + }, groupFinder(term) { return Group.findAll({search: term, ignore_automatic: true}); }, - successMessage: function() { + @computed('isMessage', 'emailOrUsername', 'invitingExistingUserToTopic') + successMessage(isMessage, emailOrUsername, invitingExistingUserToTopic) { if (this.get('hasGroups')) { return I18n.t('topic.invite_private.success_group'); - } else if (this.get('isMessage')) { + } else if (isMessage) { return I18n.t('topic.invite_private.success'); - } else if ( emailValid(this.get('emailOrUsername')) ) { - return I18n.t('topic.invite_reply.success_email', { emailOrUsername: this.get('emailOrUsername') }); + } else if (invitingExistingUserToTopic) { + return I18n.t('topic.invite_reply.success_existing_email', { emailOrUsername }); + } else if (emailValid(emailOrUsername)) { + return I18n.t('topic.invite_reply.success_email', { emailOrUsername }); } else { return I18n.t('topic.invite_reply.success_username'); } - }.property('model.inviteLink', 'isMessage', 'emailOrUsername'), + }, - errorMessage: function() { - return this.get('isMessage') ? I18n.t('topic.invite_private.error') : I18n.t('topic.invite_reply.error'); - }.property('isMessage'), + @computed('isMessage') + errorMessage(isMessage) { + return isMessage ? I18n.t('topic.invite_private.error') : I18n.t('topic.invite_reply.error'); + }, @computed('canInviteViaEmail') placeholderKey(canInviteViaEmail) { @@ -172,15 +180,17 @@ export default Ember.Controller.extend(ModalFunctionality, { 'topic.invite_reply.username_placeholder'; }, - customMessagePlaceholder: function() { + @computed + customMessagePlaceholder() { return I18n.t('invite.custom_message_placeholder'); - }.property(), + }, // Reset the modal to allow a new user to be invited. reset() { this.set('emailOrUsername', null); this.set('hasCustomMessage', false); this.set('customMessage', null); + this.set('invitingExistingUserToTopic', false); this.get('model').setProperties({ groupNames: null, error: false, @@ -189,6 +199,7 @@ export default Ember.Controller.extend(ModalFunctionality, { inviteLink: null }); }, + actions: { createInvite() { @@ -230,6 +241,8 @@ export default Ember.Controller.extend(ModalFunctionality, { } else if (this.get('isMessage') && result && result.user) { this.get('model.details.allowed_users').pushObject(Ember.Object.create(result.user)); this.appEvents.trigger('post-stream:refresh'); + } else if (this.get('invitingToTopic') && emailValid(this.get('emailOrUsername').trim()) && result && result.user) { + this.set('invitingExistingUserToTopic', true); } }).catch(onerror); } diff --git a/app/models/topic.rb b/app/models/topic.rb index f9fcfadf52..495459d4fd 100644 --- a/app/models/topic.rb +++ b/app/models/topic.rb @@ -744,9 +744,10 @@ SQL # Invite a user to the topic by username or email. Returns success/failure def invite(invited_by, username_or_email, group_ids=nil, custom_message=nil) + user = User.find_by_username_or_email(username_or_email) + if private_message? # If the user exists, add them to the message. - user = User.find_by_username_or_email(username_or_email) raise UserExists.new I18n.t("topic_invite.user_exists") if user.present? && topic_allowed_users.where(user_id: user.id).exists? if user && topic_allowed_users.create!(user_id: user.id) @@ -764,18 +765,27 @@ SQL end if username_or_email =~ /^.+@.+$/ && Guardian.new(invited_by).can_invite_via_email?(self) - # rate limit topic invite RateLimiter.new(invited_by, "topic-invitations-per-day", SiteSetting.max_topic_invitations_per_day, 1.day.to_i).performed! - # NOTE callers expect an invite object if an invite was sent via email - invite_by_email(invited_by, username_or_email, group_ids, custom_message) + if user.present? + # add existing users + Invite.extend_permissions(self, user, invited_by) + + # Notify the user they've been invited + user.notifications.create(notification_type: Notification.types[:invited_to_topic], + topic_id: id, + post_number: 1, + data: { topic_title: title, + display_username: invited_by.username }.to_json) + return true + else + # NOTE callers expect an invite object if an invite was sent via email + invite_by_email(invited_by, username_or_email, group_ids, custom_message) + end else - # invite existing member to a topic - user = User.find_by_username(username_or_email) raise UserExists.new I18n.t("topic_invite.user_exists") if user.present? && topic_allowed_users.where(user_id: user.id).exists? if user && topic_allowed_users.create!(user_id: user.id) - # rate limit topic invite RateLimiter.new(invited_by, "topic-invitations-per-day", SiteSetting.max_topic_invitations_per_day, 1.day.to_i).performed! # Notify the user they've been invited diff --git a/config/locales/client.en.yml b/config/locales/client.en.yml index 1bb210eb6c..6c80ba69c2 100644 --- a/config/locales/client.en.yml +++ b/config/locales/client.en.yml @@ -1724,6 +1724,7 @@ en: success_email: "We mailed out an invitation to {{emailOrUsername}}. We'll notify you when the invitation is redeemed. Check the invitations tab on your user page to keep track of your invites." success_username: "We've invited that user to participate in this topic." error: "Sorry, we couldn't invite that person. Perhaps they have already been invited? (Invites are rate limited)" + success_existing_email: "A user with email {{emailOrUsername}} already exists. We've invited that user to participate in this topic." login_reply: 'Log In to Reply' diff --git a/spec/models/topic_spec.rb b/spec/models/topic_spec.rb index 5325b9f89d..be8fef6922 100644 --- a/spec/models/topic_spec.rb +++ b/spec/models/topic_spec.rb @@ -1748,7 +1748,7 @@ describe Topic do it "should add user to the group" do expect(Guardian.new(walter).can_see?(group_private_topic)).to be_falsey - expect { group_private_topic.invite(group_manager, walter.email) }.to raise_error(Invite::UserExists) + group_private_topic.invite(group_manager, walter.email) expect(walter.groups).to include(group) expect(Guardian.new(walter).can_see?(group_private_topic)).to be_truthy end From d1d43ff5d9182aac9e307e8f2f6da2578c62a4fd Mon Sep 17 00:00:00 2001 From: Neil Lalonde Date: Wed, 12 Jul 2017 16:00:18 -0400 Subject: [PATCH 045/382] FIX: report discobot messages as system messages instead of user-to-user --- .../lib/discourse_narrative_bot/new_user_narrative.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/discourse-narrative-bot/lib/discourse_narrative_bot/new_user_narrative.rb b/plugins/discourse-narrative-bot/lib/discourse_narrative_bot/new_user_narrative.rb index f39377d076..a0ba45f204 100644 --- a/plugins/discourse-narrative-bot/lib/discourse_narrative_bot/new_user_narrative.rb +++ b/plugins/discourse-narrative-bot/lib/discourse_narrative_bot/new_user_narrative.rb @@ -184,7 +184,8 @@ module DiscourseNarrativeBot opts = { title: I18n.t("#{I18N_KEY}.hello.title", title: SiteSetting.title), target_usernames: @user.username, - archetype: Archetype.private_message + archetype: Archetype.private_message, + subtype: TopicSubtype.system_message, } if @post && From 24db001bfabea8e76c0738eaca600cbf259aa765 Mon Sep 17 00:00:00 2001 From: Neil Lalonde Date: Wed, 12 Jul 2017 16:16:59 -0400 Subject: [PATCH 046/382] FIX: user card contents can go up out of the bounds of the card when user has a title and full name --- app/assets/stylesheets/desktop/user-card.scss | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/assets/stylesheets/desktop/user-card.scss b/app/assets/stylesheets/desktop/user-card.scss index 2d48ec8e00..17b4363eba 100644 --- a/app/assets/stylesheets/desktop/user-card.scss +++ b/app/assets/stylesheets/desktop/user-card.scss @@ -205,17 +205,12 @@ $user_card_background: $secondary; .names { float: left; - height: 60px; - position: relative; width: 45%; span { - position: absolute; - bottom: 0; display: block; width: 250px; } - } .badge-section { From ee470b531788b71c22721562e8bbb846004a9bc7 Mon Sep 17 00:00:00 2001 From: Sam Saffron Date: Wed, 12 Jul 2017 08:13:33 -0400 Subject: [PATCH 047/382] remove old markdown engine work-in-progress --- app/assets/javascripts/markdown-it-bundle.js | 28 +- app/assets/javascripts/pretty-text-bundle.js | 3 - .../engines/discourse-markdown-it.js.es6 | 2 +- .../discourse-markdown/auto-link.js.es6 | 27 - .../bbcode-block.js.es6 | 0 .../bbcode-inline.js.es6 | 2 +- .../engines/discourse-markdown/bbcode.js.es6 | 170 -- .../discourse-markdown/bold-italics.js.es6 | 73 - .../category-hashtag.js.es6 | 112 +- .../discourse-markdown/censored.js.es6 | 48 +- .../engines/discourse-markdown/code.js.es6 | 92 +- .../engines/discourse-markdown/emoji.js.es6 | 327 ++-- .../helpers.js.es6 | 0 .../engines/discourse-markdown/html.js.es6 | 52 - .../html_img.js.es6 | 0 .../discourse-markdown/mentions.js.es6 | 129 +- .../engines/discourse-markdown/newline.js.es6 | 75 +- .../engines/discourse-markdown/onebox.js.es6 | 118 +- .../paragraph.js.es6 | 0 .../engines/discourse-markdown/quote.js.es6 | 83 - .../quotes.js.es6 | 0 .../engines/discourse-markdown/table.js.es6 | 50 +- .../markdown-it/category-hashtag.js.es6 | 104 -- .../engines/markdown-it/censored.js.es6 | 44 - .../engines/markdown-it/code.js.es6 | 51 - .../engines/markdown-it/emoji.js.es6 | 246 --- .../engines/markdown-it/mentions.js.es6 | 88 - .../engines/markdown-it/newline.js.es6 | 53 - .../engines/markdown-it/onebox.js.es6 | 89 - .../engines/markdown-it/table.js.es6 | 31 - .../pretty-text/pretty-text.js.es6 | 25 +- .../common/_discourse_javascript.html.erb | 2 - config/locales/server.ar.yml | 1 - config/locales/server.da.yml | 1 - config/locales/server.de.yml | 1 - config/locales/server.el.yml | 1 - config/locales/server.en.yml | 2 - config/locales/server.es.yml | 2 - config/locales/server.fa_IR.yml | 1 - config/locales/server.fi.yml | 2 - config/locales/server.fr.yml | 1 - config/locales/server.he.yml | 1 - config/locales/server.it.yml | 1 - config/locales/server.ko.yml | 2 - config/locales/server.nl.yml | 1 - config/locales/server.pl_PL.yml | 1 - config/locales/server.pt.yml | 1 - config/locales/server.ro.yml | 1 - config/locales/server.sk.yml | 1 - config/locales/server.sv.yml | 1 - config/locales/server.tr_TR.yml | 1 - config/locales/server.vi.yml | 1 - config/locales/server.zh_CN.yml | 1 - config/locales/server.zh_TW.yml | 1 - config/site_settings.yml | 9 +- lib/pretty_text.rb | 13 +- lib/pretty_text/shims.js | 2 - script/import_scripts/lithium.rb | 1 - spec/components/pretty_text_spec.rb | 2 +- test/javascripts/lib/pretty-text-test.js.es6 | 3 - test/javascripts/models/model-test.js.es6 | 2 +- vendor/assets/javascripts/better_markdown.js | 1518 ----------------- 62 files changed, 639 insertions(+), 3061 deletions(-) delete mode 100644 app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6 rename app/assets/javascripts/pretty-text/engines/{markdown-it => discourse-markdown}/bbcode-block.js.es6 (100%) rename app/assets/javascripts/pretty-text/engines/{markdown-it => discourse-markdown}/bbcode-inline.js.es6 (98%) delete mode 100644 app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6 rename app/assets/javascripts/pretty-text/engines/{markdown-it => discourse-markdown}/helpers.js.es6 (100%) delete mode 100644 app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6 rename app/assets/javascripts/pretty-text/engines/{markdown-it => discourse-markdown}/html_img.js.es6 (100%) rename app/assets/javascripts/pretty-text/engines/{markdown-it => discourse-markdown}/paragraph.js.es6 (100%) delete mode 100644 app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6 rename app/assets/javascripts/pretty-text/engines/{markdown-it => discourse-markdown}/quotes.js.es6 (100%) delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/newline.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6 delete mode 100644 app/assets/javascripts/pretty-text/engines/markdown-it/table.js.es6 delete mode 100644 vendor/assets/javascripts/better_markdown.js diff --git a/app/assets/javascripts/markdown-it-bundle.js b/app/assets/javascripts/markdown-it-bundle.js index 3460c3a602..dcacaf1748 100644 --- a/app/assets/javascripts/markdown-it-bundle.js +++ b/app/assets/javascripts/markdown-it-bundle.js @@ -1,15 +1,15 @@ //= require markdown-it.js -//= require ./pretty-text/engines/markdown-it/helpers -//= require ./pretty-text/engines/markdown-it/mentions -//= require ./pretty-text/engines/markdown-it/quotes -//= require ./pretty-text/engines/markdown-it/emoji -//= require ./pretty-text/engines/markdown-it/onebox -//= require ./pretty-text/engines/markdown-it/bbcode-block -//= require ./pretty-text/engines/markdown-it/bbcode-inline -//= require ./pretty-text/engines/markdown-it/code -//= require ./pretty-text/engines/markdown-it/category-hashtag -//= require ./pretty-text/engines/markdown-it/censored -//= require ./pretty-text/engines/markdown-it/table -//= require ./pretty-text/engines/markdown-it/paragraph -//= require ./pretty-text/engines/markdown-it/newline -//= require ./pretty-text/engines/markdown-it/html_img +//= require ./pretty-text/engines/discourse-markdown/helpers +//= require ./pretty-text/engines/discourse-markdown/mentions +//= require ./pretty-text/engines/discourse-markdown/quotes +//= require ./pretty-text/engines/discourse-markdown/emoji +//= require ./pretty-text/engines/discourse-markdown/onebox +//= require ./pretty-text/engines/discourse-markdown/bbcode-block +//= require ./pretty-text/engines/discourse-markdown/bbcode-inline +//= require ./pretty-text/engines/discourse-markdown/code +//= require ./pretty-text/engines/discourse-markdown/category-hashtag +//= require ./pretty-text/engines/discourse-markdown/censored +//= require ./pretty-text/engines/discourse-markdown/table +//= require ./pretty-text/engines/discourse-markdown/paragraph +//= require ./pretty-text/engines/discourse-markdown/newline +//= require ./pretty-text/engines/discourse-markdown/html_img diff --git a/app/assets/javascripts/pretty-text-bundle.js b/app/assets/javascripts/pretty-text-bundle.js index 691fad8e9a..869631eda1 100644 --- a/app/assets/javascripts/pretty-text-bundle.js +++ b/app/assets/javascripts/pretty-text-bundle.js @@ -3,11 +3,8 @@ //= require ./pretty-text/censored-words //= require ./pretty-text/emoji/data //= require ./pretty-text/emoji -//= require ./pretty-text/engines/discourse-markdown //= require ./pretty-text/engines/discourse-markdown-it -//= require_tree ./pretty-text/engines/discourse-markdown //= require xss.min -//= require better_markdown.js //= require ./pretty-text/xss //= require ./pretty-text/white-lister //= require ./pretty-text/sanitizer diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 index ac80029cfd..5ff68ef795 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 @@ -151,7 +151,7 @@ export function setup(opts, siteSettings, state) { } // we got to require this late cause bundle is not loaded in pretty-text - Helpers = Helpers || requirejs('pretty-text/engines/markdown-it/helpers'); + Helpers = Helpers || requirejs('pretty-text/engines/discourse-markdown/helpers'); opts.markdownIt = true; diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6 deleted file mode 100644 index 78ebe441fd..0000000000 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/auto-link.js.es6 +++ /dev/null @@ -1,27 +0,0 @@ -// This addition handles auto linking of text. When included, it will parse out links and create -// ``s for them. - -const urlReplacerArgs = { - matcher: /^((?:https?:(?:\/{1,3}|[a-z0-9%])|www\d{0,3}[.])(?:[^\s()<>]+|\([^\s()<>]+\))+(?:\([^\s()<>]+\)|[^`!()\[\]{};:'".,<>?«»“”‘’\s]))/, - spaceOrTagBoundary: true, - - emitter(matches) { - const url = matches[1]; - let href = url; - - // Don't autolink a markdown link to something - if (url.match(/\]\[\d$/)) { return; } - - // If we improperly caught a markdown link abort - if (url.match(/\(http/)) { return; } - - if (url.match(/^www/)) { href = "http://" + url; } - return ['a', { href }, url]; - } -}; - -export function setup(helper) { - if (helper.markdownIt) { return; } - helper.inlineRegexp(_.merge({start: 'http'}, urlReplacerArgs)); - helper.inlineRegexp(_.merge({start: 'www'}, urlReplacerArgs)); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-block.js.es6 similarity index 100% rename from app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-block.js.es6 rename to app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-block.js.es6 diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-inline.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6 similarity index 98% rename from app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-inline.js.es6 rename to app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6 index 2184adb7ff..71ce50462a 100644 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/bbcode-inline.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6 @@ -1,4 +1,4 @@ -import { parseBBCodeTag } from 'pretty-text/engines/markdown-it/bbcode-block'; +import { parseBBCodeTag } from 'pretty-text/engines/discourse-markdown/bbcode-block'; function tokanizeBBCode(state, silent, ruler) { diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6 deleted file mode 100644 index d011849752..0000000000 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode.js.es6 +++ /dev/null @@ -1,170 +0,0 @@ -export function register(helper, codeName, args, emitter) { - // Optional second param for args - if (typeof args === "function") { - emitter = args; - args = {}; - } - - helper.replaceBlock({ - start: new RegExp("\\[" + codeName + "(=[^\\[\\]]+)?\\]([\\s\\S]*)", "igm"), - stop: new RegExp("\\[\\/" + codeName + "\\]", "igm"), - emitter(blockContents, matches) { - - - const options = helper.getOptions(); - while (blockContents.length && (typeof blockContents[0] === "string" || blockContents[0] instanceof String)) { - blockContents[0] = String(blockContents[0]).replace(/^\s+/, ''); - if (!blockContents[0].length) { - blockContents.shift(); - } else { - break; - } - } - - let contents = []; - if (blockContents.length) { - const nextContents = blockContents.slice(1); - blockContents = this.processBlock(blockContents[0], nextContents); - - nextContents.forEach(nc => { - blockContents = blockContents.concat(this.processBlock(nc, [])); - }); - - blockContents.forEach(bc => { - if (typeof bc === "string" || bc instanceof String) { - var processed = this.processInline(String(bc)); - if (processed.length) { - contents.push(['p'].concat(processed)); - } - } else { - contents.push(bc); - } - }); - } - if (!args.singlePara && contents.length === 1 && contents[0] instanceof Array && contents[0][0] === "para") { - contents[0].shift(); - contents = contents[0]; - } - const result = emitter(contents, matches[1] ? matches[1].replace(/^=|\"/g, '') : null, options); - return args.noWrap ? result : ['p', result]; - } - }); -}; - -export function builders(helper) { - function replaceBBCode(tag, emitter, opts) { - const start = `[${tag}]`; - const stop = `[/${tag}]`; - - opts = opts || {}; - opts = _.merge(opts, { start, stop, emitter }); - helper.inlineBetween(opts); - - opts = _.merge(opts, { start: start.toUpperCase(), stop: stop.toUpperCase(), emitter }); - helper.inlineBetween(opts); - } - - return { - replaceBBCode, - - register(codeName, args, emitter) { - register(helper, codeName, args, emitter); - }, - - rawBBCode(tag, emitter) { - replaceBBCode(tag, emitter, { rawContents: true }); - }, - - removeEmptyLines(contents) { - const result = []; - for (let i=0; i < contents.length; i++) { - if (contents[i] !== "\n") { result.push(contents[i]); } - } - return result; - }, - - replaceBBCodeParamsRaw(tag, emitter) { - var opts = { - rawContents: true, - emitter(contents) { - const m = /^([^\]]+)\]([\S\s]*)$/.exec(contents); - if (m) { return emitter.call(this, m[1], m[2]); } - } - }; - - helper.inlineBetween(_.merge(opts, { start: "[" + tag + "=", stop: "[/" + tag + "]" })); - - tag = tag.toUpperCase(); - helper.inlineBetween(_.merge(opts, { start: "[" + tag + "=", stop: "[/" + tag + "]" })); - } - }; -} - -export function setup(helper) { - - if (helper.markdownIt) { return; } - - helper.whiteList(['span.bbcode-b', 'span.bbcode-i', 'span.bbcode-u', 'span.bbcode-s']); - - const { replaceBBCode, rawBBCode, removeEmptyLines, replaceBBCodeParamsRaw } = builders(helper); - - replaceBBCode('b', contents => ['span', {'class': 'bbcode-b'}].concat(contents)); - replaceBBCode('i', contents => ['span', {'class': 'bbcode-i'}].concat(contents)); - replaceBBCode('u', contents => ['span', {'class': 'bbcode-u'}].concat(contents)); - replaceBBCode('s', contents => ['span', {'class': 'bbcode-s'}].concat(contents)); - - replaceBBCode('ul', contents => ['ul'].concat(removeEmptyLines(contents))); - replaceBBCode('ol', contents => ['ol'].concat(removeEmptyLines(contents))); - replaceBBCode('li', contents => ['li'].concat(removeEmptyLines(contents))); - - rawBBCode('img', href => ['img', {href}]); - rawBBCode('email', contents => ['a', {href: "mailto:" + contents, 'data-bbcode': true}, contents]); - - replaceBBCode('url', contents => { - if (!Array.isArray(contents)) { return; } - - const first = contents[0]; - if (contents.length === 1 && Array.isArray(first) && first[0] === 'a') { - // single-line bbcode links shouldn't be oneboxed, so we mark this as a bbcode link. - if (typeof first[1] !== 'object') { first.splice(1, 0, {}); } - first[1]['data-bbcode'] = true; - } - return ['concat'].concat(contents); - }); - - replaceBBCodeParamsRaw('url', function(param, contents) { - const url = param.replace(/(^")|("$)/g, ''); - return ['a', {'href': url}].concat(this.processInline(contents)); - }); - - replaceBBCodeParamsRaw("email", function(param, contents) { - return ['a', {href: "mailto:" + param, 'data-bbcode': true}].concat(contents); - }); - - helper.onParseNode(event => { - if (!Array.isArray(event.node)) { return; } - const result = [event.node[0]]; - const nodes = event.node.slice(1); - for (let i = 0; i < nodes.length; i++) { - if (Array.isArray(nodes[i]) && nodes[i][0] === 'concat') { - for (let j = 1; j < nodes[i].length; j++) { result.push(nodes[i][j]); } - } else { - result.push(nodes[i]); - } - } - for (let i = 0; i < result.length; i++) { event.node[i] = result[i]; } - }); - - helper.replaceBlock({ - start: /(\[code\])([\s\S]*)/igm, - stop: /\[\/code\]/igm, - rawContents: true, - - emitter(blockContents) { - const options = helper.getOptions(); - const inner = blockContents.join("\n"); - const defaultCodeLang = options.defaultCodeLang; - return ['p', ['pre', ['code', {'class': `lang-${defaultCodeLang}`}, inner]]]; - } - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6 deleted file mode 100644 index a79a983b08..0000000000 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bold-italics.js.es6 +++ /dev/null @@ -1,73 +0,0 @@ -import guid from 'pretty-text/guid'; - -/** - markdown-js doesn't ensure that em/strong codes are present on word boundaries. - So we create our own handlers here. -**/ - -// From PageDown -const aLetter = /[a-zA-Z0-9\u00aa\u00b5\u00ba\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u02c1\u02c6-\u02d1\u02e0-\u02e4\u02ec\u02ee\u0370-\u0374\u0376-\u0377\u037a-\u037d\u0386\u0388-\u038a\u038c\u038e-\u03a1\u03a3-\u03f5\u03f7-\u0481\u048a-\u0523\u0531-\u0556\u0559\u0561-\u0587\u05d0-\u05ea\u05f0-\u05f2\u0621-\u064a\u0660-\u0669\u066e-\u066f\u0671-\u06d3\u06d5\u06e5-\u06e6\u06ee-\u06fc\u06ff\u0710\u0712-\u072f\u074d-\u07a5\u07b1\u07c0-\u07ea\u07f4-\u07f5\u07fa\u0904-\u0939\u093d\u0950\u0958-\u0961\u0966-\u096f\u0971-\u0972\u097b-\u097f\u0985-\u098c\u098f-\u0990\u0993-\u09a8\u09aa-\u09b0\u09b2\u09b6-\u09b9\u09bd\u09ce\u09dc-\u09dd\u09df-\u09e1\u09e6-\u09f1\u0a05-\u0a0a\u0a0f-\u0a10\u0a13-\u0a28\u0a2a-\u0a30\u0a32-\u0a33\u0a35-\u0a36\u0a38-\u0a39\u0a59-\u0a5c\u0a5e\u0a66-\u0a6f\u0a72-\u0a74\u0a85-\u0a8d\u0a8f-\u0a91\u0a93-\u0aa8\u0aaa-\u0ab0\u0ab2-\u0ab3\u0ab5-\u0ab9\u0abd\u0ad0\u0ae0-\u0ae1\u0ae6-\u0aef\u0b05-\u0b0c\u0b0f-\u0b10\u0b13-\u0b28\u0b2a-\u0b30\u0b32-\u0b33\u0b35-\u0b39\u0b3d\u0b5c-\u0b5d\u0b5f-\u0b61\u0b66-\u0b6f\u0b71\u0b83\u0b85-\u0b8a\u0b8e-\u0b90\u0b92-\u0b95\u0b99-\u0b9a\u0b9c\u0b9e-\u0b9f\u0ba3-\u0ba4\u0ba8-\u0baa\u0bae-\u0bb9\u0bd0\u0be6-\u0bef\u0c05-\u0c0c\u0c0e-\u0c10\u0c12-\u0c28\u0c2a-\u0c33\u0c35-\u0c39\u0c3d\u0c58-\u0c59\u0c60-\u0c61\u0c66-\u0c6f\u0c85-\u0c8c\u0c8e-\u0c90\u0c92-\u0ca8\u0caa-\u0cb3\u0cb5-\u0cb9\u0cbd\u0cde\u0ce0-\u0ce1\u0ce6-\u0cef\u0d05-\u0d0c\u0d0e-\u0d10\u0d12-\u0d28\u0d2a-\u0d39\u0d3d\u0d60-\u0d61\u0d66-\u0d6f\u0d7a-\u0d7f\u0d85-\u0d96\u0d9a-\u0db1\u0db3-\u0dbb\u0dbd\u0dc0-\u0dc6\u0e01-\u0e30\u0e32-\u0e33\u0e40-\u0e46\u0e50-\u0e59\u0e81-\u0e82\u0e84\u0e87-\u0e88\u0e8a\u0e8d\u0e94-\u0e97\u0e99-\u0e9f\u0ea1-\u0ea3\u0ea5\u0ea7\u0eaa-\u0eab\u0ead-\u0eb0\u0eb2-\u0eb3\u0ebd\u0ec0-\u0ec4\u0ec6\u0ed0-\u0ed9\u0edc-\u0edd\u0f00\u0f20-\u0f29\u0f40-\u0f47\u0f49-\u0f6c\u0f88-\u0f8b\u1000-\u102a\u103f-\u1049\u1050-\u1055\u105a-\u105d\u1061\u1065-\u1066\u106e-\u1070\u1075-\u1081\u108e\u1090-\u1099\u10a0-\u10c5\u10d0-\u10fa\u10fc\u1100-\u1159\u115f-\u11a2\u11a8-\u11f9\u1200-\u1248\u124a-\u124d\u1250-\u1256\u1258\u125a-\u125d\u1260-\u1288\u128a-\u128d\u1290-\u12b0\u12b2-\u12b5\u12b8-\u12be\u12c0\u12c2-\u12c5\u12c8-\u12d6\u12d8-\u1310\u1312-\u1315\u1318-\u135a\u1380-\u138f\u13a0-\u13f4\u1401-\u166c\u166f-\u1676\u1681-\u169a\u16a0-\u16ea\u1700-\u170c\u170e-\u1711\u1720-\u1731\u1740-\u1751\u1760-\u176c\u176e-\u1770\u1780-\u17b3\u17d7\u17dc\u17e0-\u17e9\u1810-\u1819\u1820-\u1877\u1880-\u18a8\u18aa\u1900-\u191c\u1946-\u196d\u1970-\u1974\u1980-\u19a9\u19c1-\u19c7\u19d0-\u19d9\u1a00-\u1a16\u1b05-\u1b33\u1b45-\u1b4b\u1b50-\u1b59\u1b83-\u1ba0\u1bae-\u1bb9\u1c00-\u1c23\u1c40-\u1c49\u1c4d-\u1c7d\u1d00-\u1dbf\u1e00-\u1f15\u1f18-\u1f1d\u1f20-\u1f45\u1f48-\u1f4d\u1f50-\u1f57\u1f59\u1f5b\u1f5d\u1f5f-\u1f7d\u1f80-\u1fb4\u1fb6-\u1fbc\u1fbe\u1fc2-\u1fc4\u1fc6-\u1fcc\u1fd0-\u1fd3\u1fd6-\u1fdb\u1fe0-\u1fec\u1ff2-\u1ff4\u1ff6-\u1ffc\u203f-\u2040\u2054\u2071\u207f\u2090-\u2094\u2102\u2107\u210a-\u2113\u2115\u2119-\u211d\u2124\u2126\u2128\u212a-\u212d\u212f-\u2139\u213c-\u213f\u2145-\u2149\u214e\u2183-\u2184\u2c00-\u2c2e\u2c30-\u2c5e\u2c60-\u2c6f\u2c71-\u2c7d\u2c80-\u2ce4\u2d00-\u2d25\u2d30-\u2d65\u2d6f\u2d80-\u2d96\u2da0-\u2da6\u2da8-\u2dae\u2db0-\u2db6\u2db8-\u2dbe\u2dc0-\u2dc6\u2dc8-\u2dce\u2dd0-\u2dd6\u2dd8-\u2dde\u2e2f\u3005-\u3006\u3031-\u3035\u303b-\u303c\u3041-\u3096\u309d-\u309f\u30a1-\u30fa\u30fc-\u30ff\u3105-\u312d\u3131-\u318e\u31a0-\u31b7\u31f0-\u31ff\u3400-\u4db5\u4e00-\u9fc3\ua000-\ua48c\ua500-\ua60c\ua610-\ua62b\ua640-\ua65f\ua662-\ua66e\ua67f-\ua697\ua717-\ua71f\ua722-\ua788\ua78b-\ua78c\ua7fb-\ua801\ua803-\ua805\ua807-\ua80a\ua80c-\ua822\ua840-\ua873\ua882-\ua8b3\ua8d0-\ua8d9\ua900-\ua925\ua930-\ua946\uaa00-\uaa28\uaa40-\uaa42\uaa44-\uaa4b\uaa50-\uaa59\uac00-\ud7a3\uf900-\ufa2d\ufa30-\ufa6a\ufa70-\ufad9\ufb00-\ufb06\ufb13-\ufb17\ufb1d\ufb1f-\ufb28\ufb2a-\ufb36\ufb38-\ufb3c\ufb3e\ufb40-\ufb41\ufb43-\ufb44\ufb46-\ufbb1\ufbd3-\ufd3d\ufd50-\ufd8f\ufd92-\ufdc7\ufdf0-\ufdfb\ufe33-\ufe34\ufe4d-\ufe4f\ufe70-\ufe74\ufe76-\ufefc\uff10-\uff19\uff21-\uff3a\uff3f\uff41-\uff5a\uff66-\uffbe\uffc2-\uffc7\uffca-\uffcf\uffd2-\uffd7\uffda-\uffdc]/; - - -function unhoist(obj,from,to){ - let unhoisted = 0; - const regex = new RegExp(from, "g"); - - if(_.isArray(obj)){ - for (let i=0; i= 0) { - const newText = this.processInline(text.substring(match.length, finish+1)); - const unhoisted_length = unhoist(newText,hash,match[0]); - const array = typeof tag === "string" ? [tag].concat(newText) : [tag[0], [tag[1]].concat(newText)]; - return [(finish + match.length + 1) - unhoisted_length, array]; - } - }); - } - - replaceMarkdown('***', ['strong','em']); - replaceMarkdown('___', ['strong','em']); - replaceMarkdown('**', 'strong'); - replaceMarkdown('__', 'strong'); - replaceMarkdown('*', 'em'); - replaceMarkdown('_', 'em'); -}; diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6 index c33eeff65f..79d57002a6 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/category-hashtag.js.es6 @@ -1,20 +1,104 @@ +function addHashtag(buffer, matches, state) { + const options = state.md.options.discourse; + const [hashtag, slug] = matches; + const categoryHashtagLookup = options.categoryHashtagLookup; + const result = categoryHashtagLookup && categoryHashtagLookup(slug); + + let token; + + if (result) { + token = new state.Token('link_open', 'a', 1); + token.attrs = [['class', 'hashtag'], ['href', result[0]]]; + token.block = false; + buffer.push(token); + + token = new state.Token('text', '', 0); + token.content = '#'; + buffer.push(token); + + token = new state.Token('span_open', 'span', 1); + token.block = false; + buffer.push(token); + + token = new state.Token('text', '', 0); + token.content = result[1]; + buffer.push(token); + + buffer.push(new state.Token('span_close', 'span', -1)); + + buffer.push(new state.Token('link_close', 'a', -1)); + } else { + + token = new state.Token('span_open', 'span', 1); + token.attrs = [['class', 'hashtag']]; + buffer.push(token); + + token = new state.Token('text', '', 0); + token.content = hashtag; + buffer.push(token); + + token = new state.Token('span_close', 'span', -1); + buffer.push(token); + } +} + +const REGEX = /#([\w-:]{1,101})/gi; + +function allowedBoundary(content, index, utils) { + let code = content.charCodeAt(index); + return (utils.isWhiteSpace(code) || utils.isPunctChar(String.fromCharCode(code))); +} + +function applyHashtag(content, state) { + let result = null, + match, + pos = 0; + + while (match = REGEX.exec(content)) { + // check boundary + if (match.index > 0) { + if (!allowedBoundary(content, match.index-1, state.md.utils)) { + continue; + } + } + + // check forward boundary as well + if (match.index + match[0].length < content.length) { + if (!allowedBoundary(content, match.index + match[0].length, state.md.utils)) { + continue; + } + } + + if (match.index > pos) { + result = result || []; + let token = new state.Token('text', '', 0); + token.content = content.slice(pos, match.index); + result.push(token); + } + + result = result || []; + addHashtag(result, match, state); + + pos = match.index + match[0].length; + } + + if (result && pos < content.length) { + let token = new state.Token('text', '', 0); + token.content = content.slice(pos); + result.push(token); + } + + return result; +} + export function setup(helper) { - if (helper.markdownIt) { return; } + if (!helper.markdownIt) { return; } - helper.inlineRegexp({ - start: '#', - matcher: /^#([\w-:]{1,101})/i, - spaceOrTagBoundary: true, + helper.registerPlugin(md=>{ - emitter(matches) { - const options = helper.getOptions(); - const [hashtag, slug] = matches; - const categoryHashtagLookup = options.categoryHashtagLookup; - const result = categoryHashtagLookup && categoryHashtagLookup(slug); - - return result ? ['a', { class: 'hashtag', href: result[0] }, '#', ["span", {}, result[1]]] - : ['span', { class: 'hashtag' }, hashtag]; - } + md.core.ruler.push('category-hashtag', state => md.options.discourse.helpers.textReplace( + state, applyHashtag, true /* skip all links */ + )); }); } diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 index d3ed549fe0..3d5cb8d931 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/censored.js.es6 @@ -1,18 +1,44 @@ -import { censor } from 'pretty-text/censored-words'; -import { registerOption } from 'pretty-text/pretty-text'; +import { censorFn } from 'pretty-text/censored-words'; -registerOption((siteSettings, opts) => { - opts.features.censored = true; - opts.censoredWords = siteSettings.censored_words; - opts.censoredPattern = siteSettings.censored_pattern; -}); +function recurse(tokens, apply) { + let i; + for(i=0;i { + if (token.content) { + token.content = censor(token.content); + } + }); +} export function setup(helper) { - if (helper.markdownIt) { return; } + if (!helper.markdownIt) { return; } - helper.addPreProcessor(text => { - const options = helper.getOptions(); - return censor(text, options.censoredWords, options.censoredPattern); + helper.registerOptions((opts, siteSettings) => { + opts.censoredWords = siteSettings.censored_words; + opts.censoredPattern = siteSettings.censored_pattern; + }); + + helper.registerPlugin(md => { + const words = md.options.discourse.censoredWords; + const patterns = md.options.discourse.censoredPattern; + + if ((words && words.length > 0) || (patterns && patterns.length > 0)) { + const replacement = String.fromCharCode(9632); + const censor = censorFn(words, patterns, replacement); + md.core.ruler.push('censored', state => censorTree(state, censor)); + } }); } diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6 index 048b27a36d..c8d94967a1 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/code.js.es6 @@ -1,27 +1,38 @@ -import { escape } from 'pretty-text/sanitizer'; -import { registerOption } from 'pretty-text/pretty-text'; +// we need a custom renderer for code blocks cause we have a slightly non compliant +// format with special handling for text and so on -// Support for various code blocks const TEXT_CODE_CLASSES = ["text", "pre", "plain"]; -function codeFlattenBlocks(blocks) { - let result = ""; - blocks.forEach(function(b) { - result += b; - if (b.trailing) { result += b.trailing; } - }); - return result; + +function render(tokens, idx, options, env, slf, md) { + let token = tokens[idx], + info = token.info ? md.utils.unescapeAll(token.info) : '', + langName = md.options.discourse.defaultCodeLang, + className, + escapedContent = md.utils.escapeHtml(token.content); + + if (info) { + // strip off any additional languages + info = info.split(/\s+/g)[0]; + } + + const acceptableCodeClasses = md.options.discourse.acceptableCodeClasses; + if (acceptableCodeClasses && info && acceptableCodeClasses.indexOf(info) !== -1) { + langName = info; + } + + className = TEXT_CODE_CLASSES.indexOf(info) !== -1 ? 'lang-nohighlight' : 'lang-' + langName; + + return `
    ${escapedContent}
    \n`; } -registerOption((siteSettings, opts) => { - opts.features.code = true; - opts.defaultCodeLang = siteSettings.default_code_lang; - opts.acceptableCodeClasses = (siteSettings.highlighted_languages || "").split("|").concat(['auto', 'nohighlight']); -}); - export function setup(helper) { + if (!helper.markdownIt) { return; } - if (helper.markdownIt) { return; } + helper.registerOptions((opts, siteSettings) => { + opts.defaultCodeLang = siteSettings.default_code_lang; + opts.acceptableCodeClasses = (siteSettings.highlighted_languages || "").split("|").concat(['auto', 'nohighlight']); + }); helper.whiteList({ custom(tag, name, value) { @@ -34,50 +45,7 @@ export function setup(helper) { } }); - helper.replaceBlock({ - start: /^`{3}([^\n\[\]]+)?\n?([\s\S]*)?/gm, - stop: /^```$/gm, - withoutLeading: /\[quote/gm, //if leading text contains a quote this should not match - emitter(blockContents, matches) { - const opts = helper.getOptions(); - - let codeLang = opts.defaultCodeLang; - const acceptableCodeClasses = opts.acceptableCodeClasses; - if (acceptableCodeClasses && matches[1] && acceptableCodeClasses.indexOf(matches[1]) !== -1) { - codeLang = matches[1]; - } - - if (TEXT_CODE_CLASSES.indexOf(matches[1]) !== -1) { - return ['p', ['pre', ['code', {'class': 'lang-nohighlight'}, codeFlattenBlocks(blockContents) ]]]; - } else { - return ['p', ['pre', ['code', {'class': 'lang-' + codeLang}, codeFlattenBlocks(blockContents) ]]]; - } - } - }); - - helper.replaceBlock({ - start: /(]*\>)([\s\S]*)/igm, - stop: /<\/pre>/igm, - rawContents: true, - skipIfTradtionalLinebreaks: true, - - emitter(blockContents) { - return ['p', ['pre', codeFlattenBlocks(blockContents)]]; - } - }); - - // Ensure that content in a code block is fully escaped. This way it's not white listed - // and we can use HTML and Javascript examples. - helper.onParseNode(function(event) { - const node = event.node, - path = event.path; - - if (node[0] === 'code') { - const regexp = (path && path[path.length-1] && path[path.length-1][0] && path[path.length-1][0] === "pre") ? - / +$/g : /^ +| +$/g; - - const contents = node[node.length-1]; - node[node.length-1] = escape(contents.replace(regexp,'')); - } + helper.registerPlugin(md=>{ + md.renderer.rules.fence = (tokens,idx,options,env,slf)=>render(tokens,idx,options,env,slf,md); }); } diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6 index 60864a2263..0e4eed203b 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/emoji.js.es6 @@ -1,117 +1,246 @@ -import { registerOption } from 'pretty-text/pretty-text'; import { buildEmojiUrl, isCustomEmoji } from 'pretty-text/emoji'; import { translations } from 'pretty-text/emoji/data'; -let _unicodeReplacements; -let _unicodeRegexp; -export function setUnicodeReplacements(replacements) { - _unicodeReplacements = replacements; - if (replacements) { - // We sort and reverse to match longer emoji sequences first - _unicodeRegexp = new RegExp(Object.keys(replacements).sort().reverse().join("|"), "g"); - } -}; +const MAX_NAME_LENGTH = 60; -function escapeRegExp(s) { - return s.replace(/[-/\\^$*+?.()|[\]{}]/gi, '\\$&'); +let translationTree = null; + +// This allows us to efficiently search for aliases +// We build a data structure that allows us to quickly +// search through our N next chars to see if any match +// one of our alias emojis. +// +function buildTranslationTree() { + let tree = []; + let lastNode; + + Object.keys(translations).forEach(function(key){ + let i; + let node = tree; + + for(i=0;i 0) { + let prev = content.charCodeAt(pos-1); + if (!state.md.utils.isSpace(prev) && !state.md.utils.isPunctChar(String.fromCharCode(prev))) { + return; } } - return true; + + pos++; + if (content.charCodeAt(pos) === 58) { + return; + } + + let length = 0; + while(length < MAX_NAME_LENGTH) { + length++; + + if (content.charCodeAt(pos+length) === 58) { + // check for t2-t6 + if (content.substr(pos+length+1, 3).match(/t[2-6]:/)) { + length += 3; + } + break; + } + + if (pos+length > content.length) { + return; + } + } + + if (length === MAX_NAME_LENGTH) { + return; + } + + return content.substr(pos, length); } -registerOption((siteSettings, opts, state) => { - opts.features.emoji = !!siteSettings.enable_emoji; - opts.emojiSet = siteSettings.emoji_set || ""; - opts.customEmoji = state.customEmoji; -}); +// straight forward :smile: to emoji image +function getEmojiTokenByName(name, state) { + + let info; + if (info = imageFor(name, state.md.options.discourse)) { + let token = new state.Token('emoji', 'img', 0); + token.attrs = [['src', info.url], + ['title', info.title], + ['class', info.classes], + ['alt', info.title]]; + + return token; + } +} + +function getEmojiTokenByTranslation(content, pos, state) { + + translationTree = translationTree || buildTranslationTree(); + + let currentTree = translationTree; + + let i; + let search = true; + let found = false; + let start = pos; + + while(search) { + + search = false; + let code = content.charCodeAt(pos); + + for (i=0;i 0) { + let leading = content.charAt(start-1); + if (!state.md.utils.isSpace(leading.charCodeAt(0)) && !state.md.utils.isPunctChar(leading)) { + return; + } + } + + // check trailing for punct or space + if (pos < content.length) { + let trailing = content.charCodeAt(pos); + if (!state.md.utils.isSpace(trailing)){ + return; + } + } + + let token = getEmojiTokenByName(found, state); + if (token) { + return { pos, token }; + } +} + +function applyEmoji(content, state, emojiUnicodeReplacer) { + let i; + let result = null; + let contentToken = null; + + let start = 0; + + if (emojiUnicodeReplacer) { + content = emojiUnicodeReplacer(content); + } + + let endToken = content.length; + + for (i=0; i0) { + contentToken = new state.Token('text', '', 0); + contentToken.content = content.slice(start,i); + result.push(contentToken); + } + + result.push(token); + endToken = start = i + offset; + } + } + + if (endToken < content.length) { + contentToken = new state.Token('text', '', 0); + contentToken.content = content.slice(endToken); + result.push(contentToken); + } + + return result; +} export function setup(helper) { - if (helper.markdownIt) { return; } + if (!helper.markdownIt) { return; } - helper.whiteList('img.emoji'); - - function imageFor(code) { - code = code.toLowerCase(); - const opts = helper.getOptions(); - const url = buildEmojiUrl(code, opts); - if (url) { - const title = `:${code}:`; - const classes = isCustomEmoji(code, opts) ? "emoji emoji-custom" : "emoji"; - return ['img', { href: url, title, 'class': classes, alt: title }]; - } - } - - const translationsWithColon = {}; - Object.keys(translations).forEach(t => { - if (t[0] === ':') { - translationsWithColon[t] = translations[t]; - } else { - const replacement = translations[t]; - helper.inlineReplace(t, (token, match, prev) => { - return checkPrev(prev) ? imageFor(replacement) : token; - }); - } - }); - const translationColonRegexp = new RegExp(Object.keys(translationsWithColon).map(t => `(${escapeRegExp(t)})`).join("|")); - - helper.registerInline(':', (text, match, prev) => { - const endPos = text.indexOf(':', 1); - const firstSpace = text.search(/\s/); - if (!checkPrev(prev)) { return; } - - // If there is no trailing colon, check our translations that begin with colons - if (endPos === -1 || (firstSpace !== -1 && endPos > firstSpace)) { - translationColonRegexp.lastIndex = 0; - const m = translationColonRegexp.exec(text); - if (m && m[0] && text.indexOf(m[0]) === 0) { - // Check outer edge - const lastChar = text.charAt(m[0].length); - if (lastChar && !/\s/.test(lastChar)) return; - const contents = imageFor(translationsWithColon[m[0]]); - if (contents) { - return [m[0].length, contents]; - } - } - return; - } - - let between; - const emojiNameMatch = text.match(/(?:.*?)(:(?!:).?[\w-]*(?::t\d)?:)/); - if (emojiNameMatch) { - between = emojiNameMatch[0].slice(1, -1); - } else { - between = text.slice(1, -1); - } - - const contents = imageFor(between); - if (contents) { - return [text.indexOf(between, 1) + between.length + 1, contents]; - } + helper.registerOptions((opts, siteSettings, state)=>{ + opts.features.emoji = !!siteSettings.enable_emoji; + opts.emojiSet = siteSettings.emoji_set || ""; + opts.customEmoji = state.customEmoji; }); - helper.addPreProcessor(text => { - if (_unicodeReplacements) { - _unicodeRegexp.lastIndex = 0; - - let m; - while ((m = _unicodeRegexp.exec(text)) !== null) { - let replacement = ":" + _unicodeReplacements[m[0]] + ":"; - const before = text.charAt(m.index-1); - if (!/\B/.test(before)) { - replacement = "\u200b" + replacement; - } - text = text.replace(m[0], replacement); - } - } - return text; + helper.registerPlugin((md)=>{ + md.core.ruler.push('emoji', state => md.options.discourse.helpers.textReplace( + state, (c,s)=>applyEmoji(c,s,md.options.discourse.emojiUnicodeReplacer)) + ); }); } diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/helpers.js.es6 similarity index 100% rename from app/assets/javascripts/pretty-text/engines/markdown-it/helpers.js.es6 rename to app/assets/javascripts/pretty-text/engines/discourse-markdown/helpers.js.es6 diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6 deleted file mode 100644 index 1d8f21a205..0000000000 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/html.js.es6 +++ /dev/null @@ -1,52 +0,0 @@ -const BLOCK_TAGS = ['address', 'article', 'aside', 'audio', 'blockquote', 'canvas', 'dd', 'details', - 'div', 'dl', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', - 'h4', 'h5', 'h6', 'header', 'hgroup', 'hr', 'iframe', 'noscript', 'ol', 'output', - 'p', 'pre', 'section', 'table', 'tfoot', 'ul', 'video', 'summary']; - -function splitAtLast(tag, block, next, first) { - const endTag = ``; - let endTagIndex = first ? block.indexOf(endTag) : block.lastIndexOf(endTag); - - if (endTagIndex !== -1) { - endTagIndex += endTag.length; - - const trailing = block.substr(endTagIndex).replace(/^\s+/, ''); - if (trailing.length) { - next.unshift(trailing); - } - - return [ block.substr(0, endTagIndex) ]; - } -}; - -export function setup(helper) { - - if (helper.markdownIt) { return; } - - // If a row begins with HTML tags, don't parse it. - helper.registerBlock('html', function(block, next) { - let split, pos; - - // Fix manual blockquote paragraphing even though it's not strictly correct - // PERF NOTE: /\S+
    = 0) { - if(block.substring(0, pos).search(/\s/) === -1) { - split = splitAtLast('blockquote', block, next, true); - if (split) { return this.processInline(split[0]); } - } - } - - const m = /^\s*<\/?([^>]+)\>/.exec(block); - if (m && m[1]) { - const tag = m[1].split(/\s/); - if (tag && tag[0] && BLOCK_TAGS.indexOf(tag[0]) !== -1) { - split = splitAtLast(tag[0], block, next); - if (split) { - if (split.length === 1 && split[0] === block) { return; } - return split; - } - return [ block.toString() ]; - } - } - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/html_img.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/html_img.js.es6 similarity index 100% rename from app/assets/javascripts/pretty-text/engines/markdown-it/html_img.js.es6 rename to app/assets/javascripts/pretty-text/engines/discourse-markdown/html_img.js.es6 diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6 index 84be9e5f32..602af3c15a 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/mentions.js.es6 @@ -1,51 +1,88 @@ -/** - Supports our custom @mention syntax for calling out a user in a post. - It will add a special class to them, and create a link if the user is found in a - local map. -**/ +const regex = /^(\w[\w.-]{0,59})\b/i; + +function applyMentions(state, silent, isWhiteSpace, isPunctChar, mentionLookup, getURL) { + + let pos = state.pos; + + // 64 = @ + if (silent || state.src.charCodeAt(pos) !== 64) { + return false; + } + + if (pos > 0) { + let prev = state.src.charCodeAt(pos-1); + if (!isWhiteSpace(prev) && !isPunctChar(String.fromCharCode(prev))) { + return false; + } + } + + // skip if in a link + if (state.tokens) { + let last = state.tokens[state.tokens.length-1]; + if (last) { + if (last.type === 'link_open') { + return false; + } + if (last.type === 'html_inline' && last.content.substr(0,2) === " { - const node = event.node, - path = event.path; - - if (node[1] && node[1]["class"] === 'mention') { - const parent = path[path.length - 1]; - - // If the parent is an 'a', remove it - if (parent && parent[0] === 'a') { - const name = node[2]; - node.length = 0; - node[0] = "__RAW"; - node[1] = name; - } - } - }); - - helper.inlineRegexp({ - start: '@', - // NOTE: since we can't use SiteSettings here (they loads later in process) - // we are being less strict to account for more cases than allowed - matcher: /^@(\w[\w.-]{0,59})\b/i, - wordBoundary: true, - - emitter(matches) { - const mention = matches[0].trim(); - const name = matches[1]; - const opts = helper.getOptions(); - const mentionLookup = opts.mentionLookup; - - const type = mentionLookup && mentionLookup(name); - if (type === "user") { - return ['a', {'class': 'mention', href: opts.getURL("/u/") + name.toLowerCase()}, mention]; - } else if (type === "group") { - return ['a', {'class': 'mention-group', href: opts.getURL("/groups/") + name}, mention]; - } else { - return ['span', {'class': 'mention'}, mention]; - } - } + helper.registerPlugin(md => { + md.inline.ruler.push('mentions', (state,silent)=> applyMentions( + state, + silent, + md.utils.isWhiteSpace, + md.utils.isPunctChar, + md.options.discourse.mentionLookup, + md.options.discourse.getURL + )); }); } + diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6 index a453445a2c..f1eb2ba759 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/newline.js.es6 @@ -1,30 +1,53 @@ -// Support for the newline behavior in markdown that most expect. Look through all text nodes -// in the tree, replace any new lines with `br`s. +// see: https://github.com/markdown-it/markdown-it/issues/375 +// +// we use a custom paragraph rule cause we have to signal when a +// link starts with a space, so we can bypass a onebox +// this is a freedom patch, so careful, may break on updates + + +function newline(state, silent) { + var token, pmax, max, pos = state.pos; + + if (state.src.charCodeAt(pos) !== 0x0A/* \n */) { return false; } + + pmax = state.pending.length - 1; + max = state.posMax; + + // ' \n' -> hardbreak + // Lookup in pending chars is bad practice! Don't copy to other rules! + // Pending string is stored in concat mode, indexed lookups will cause + // convertion to flat mode. + if (!silent) { + if (pmax >= 0 && state.pending.charCodeAt(pmax) === 0x20) { + if (pmax >= 1 && state.pending.charCodeAt(pmax - 1) === 0x20) { + state.pending = state.pending.replace(/ +$/, ''); + token = state.push('hardbreak', 'br', 0); + } else { + state.pending = state.pending.slice(0, -1); + token = state.push('softbreak', 'br', 0); + } + + } else { + token = state.push('softbreak', 'br', 0); + } + } + + pos++; + + // skip heading spaces for next line + while (pos < max && state.md.utils.isSpace(state.src.charCodeAt(pos))) { + if (token) { + token.leading_space = true; + } + pos++; + } + + state.pos = pos; + return true; +}; export function setup(helper) { - - if (helper.markdownIt) { return; } - - helper.postProcessText((text, event) => { - const { options, insideCounts } = event; - if (options.traditionalMarkdownLinebreaks || (insideCounts.pre > 0)) { return; } - - if (text === "\n") { - // If the tag is just a new line, replace it with a `
    ` - return [['br']]; - } else { - // If the text node contains new lines, perhaps with text between them, insert the - // `
    ` tags. - const split = text.split(/\n+/); - if (split.length) { - const replacement = []; - for (var i=0; i 0) { replacement.push(split[i]); } - if (i !== split.length-1) { replacement.push(['br']); } - } - - return replacement; - } - } + helper.registerPlugin(md => { + md.inline.ruler.at('newline', newline); }); } diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6 index 875321911f..5f04b72b1d 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6 @@ -1,71 +1,89 @@ import { lookupCache } from 'pretty-text/oneboxer'; -// Given a node in the document and its parent, determine whether it is on its own line or not. -function isOnOneLine(link, parent) { - if (!parent) { return false; } - - const siblings = parent.slice(1); - if ((!siblings) || (siblings.length < 1)) { return false; } - - const idx = siblings.indexOf(link); - if (idx === -1) { return false; } - - if (idx > 0) { - const prev = siblings[idx-1]; - if (prev[0] !== 'br') { return false; } +function applyOnebox(state, silent) { + if (silent || !state.tokens || state.tokens.length < 3) { + return; } - if (idx < siblings.length) { - const next = siblings[idx+1]; - if (next && (!((next[0] === 'br') || (typeof next === 'string' && next.trim() === "")))) { return false; } - } + let i; + for(i=1;i { - const node = event.node, - path = event.path; + if (j === 0 && token.leading_space) { + continue; + } else if (j > 0) { + let prevSibling = token.children[j-1]; - // We only care about links - if (node[0] !== 'a') { return; } + if (prevSibling.tag !== 'br' || prevSibling.leading_space) { + continue; + } + } - const parent = path[path.length - 1]; + // look ahead for soft or hard break + let text = token.children[j+1]; + let close = token.children[j+2]; + let lookahead = token.children[j+3]; - // We don't onebox bbcode - if (node[1]['data-bbcode']) { - delete node[1]['data-bbcode']; - return; - } + if (lookahead && lookahead.tag !== 'br') { + continue; + } - // We don't onebox mentions - if (node[1]['class'] === 'mention') { return; } + // check attrs only include a href + let attrs = child["attrs"]; - // Don't onebox links within a list - for (var i=0; i { + md.core.ruler.after('linkify', 'onebox', applyOnebox); }); } diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/paragraph.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/paragraph.js.es6 similarity index 100% rename from app/assets/javascripts/pretty-text/engines/markdown-it/paragraph.js.es6 rename to app/assets/javascripts/pretty-text/engines/discourse-markdown/paragraph.js.es6 diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6 deleted file mode 100644 index e8a4a9a131..0000000000 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/quote.js.es6 +++ /dev/null @@ -1,83 +0,0 @@ -import { register } from 'pretty-text/engines/discourse-markdown/bbcode'; -import { registerOption } from 'pretty-text/pretty-text'; -import { performEmojiUnescape } from 'pretty-text/emoji'; - -registerOption((siteSettings, opts) => { - opts.enableEmoji = siteSettings.enable_emoji; - opts.emojiSet = siteSettings.emoji_set; -}); - - -export function setup(helper) { - - if (helper.markdownIt) { return; } - - register(helper, 'quote', {noWrap: true, singlePara: true}, (contents, bbParams, options) => { - - const params = {'class': 'quote'}; - let username = null; - const opts = helper.getOptions(); - - if (bbParams) { - const paramsSplit = bbParams.split(/\,\s*/); - username = paramsSplit[0]; - - paramsSplit.forEach(function(p,i) { - if (i > 0) { - var assignment = p.split(':'); - if (assignment[0] && assignment[1]) { - const escaped = helper.escape(assignment[0]); - // don't escape attributes, makes no sense - if (escaped === assignment[0]) { - params['data-' + assignment[0]] = helper.escape(assignment[1].trim()); - } - } - } - }); - } - - let avatarImg; - const postNumber = parseInt(params['data-post'], 10); - const topicId = parseInt(params['data-topic'], 10); - - if (options.lookupAvatarByPostNumber) { - // client-side, we can retrieve the avatar from the post - avatarImg = options.lookupAvatarByPostNumber(postNumber, topicId); - } else if (options.lookupAvatar) { - // server-side, we need to lookup the avatar from the username - avatarImg = options.lookupAvatar(username); - } - - // If there's no username just return a simple quote - if (!username) { - return ['p', ['aside', params, ['blockquote'].concat(contents)]]; - } - - const header = ['div', {'class': 'title'}, - ['div', {'class': 'quote-controls'}], - avatarImg ? ['__RAW', avatarImg] : "", - username ? `${username}:` : "" ]; - - if (options.topicId && postNumber && options.getTopicInfo && topicId !== options.topicId) { - const topicInfo = options.getTopicInfo(topicId); - if (topicInfo) { - var href = topicInfo.href; - if (postNumber > 0) { href += "/" + postNumber; } - // get rid of username said stuff - header.pop(); - - let title = topicInfo.title; - - if (opts.enableEmoji) { - title = performEmojiUnescape(topicInfo.title, { - getURL: opts.getURL, emojiSet: opts.emojiSet - }); - } - - header.push(['a', {'href': href}, title]); - } - } - - return ['aside', params, header, ['blockquote'].concat(contents)]; - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/quotes.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6 similarity index 100% rename from app/assets/javascripts/pretty-text/engines/markdown-it/quotes.js.es6 rename to app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6 diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6 index 1b148e6843..4bb5ef92d6 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/table.js.es6 @@ -1,35 +1,31 @@ -import { registerOption } from 'pretty-text/pretty-text'; - -function tableFlattenBlocks(blocks) { - let result = ""; - - blocks.forEach(b => { - result += b; - if (b.trailing) { result += b.trailing; } - }); - - // bypass newline insertion - return result.replace(/[\n\r]/g, " "); -}; - -registerOption((siteSettings, opts) => { - opts.features.table = !!siteSettings.allow_html_tables; -}); - export function setup(helper) { - if (helper.markdownIt) { return; } + if (!helper.markdownIt) { return; } - helper.whiteList(['table', 'table.md-table', 'tbody', 'thead', 'tr', 'th', 'td']); + // this is built in now + // TODO: sanitizer needs fixing, does not properly support this yet - helper.replaceBlock({ - start: /(]*>)([\S\s]*)/igm, - stop: /<\/table>/igm, - rawContents: true, - priority: 1, + // we need a custom callback for style handling + helper.whiteList({ + custom: function(tag,attr,val) { + if (tag !== 'th' && tag !== 'td') { + return false; + } - emitter(contents) { - return ['table', {"class": "md-table"}, tableFlattenBlocks.apply(this, [contents])]; + if (attr !== 'style') { + return false; + } + + return (val === 'text-align:right' || val === 'text-align:left' || val === 'text-align:center'); } }); + + helper.whiteList([ + 'table', + 'tbody', + 'thead', + 'tr', + 'th', + 'td', + ]); } diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 deleted file mode 100644 index 79d57002a6..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/category-hashtag.js.es6 +++ /dev/null @@ -1,104 +0,0 @@ -function addHashtag(buffer, matches, state) { - const options = state.md.options.discourse; - const [hashtag, slug] = matches; - const categoryHashtagLookup = options.categoryHashtagLookup; - const result = categoryHashtagLookup && categoryHashtagLookup(slug); - - let token; - - if (result) { - token = new state.Token('link_open', 'a', 1); - token.attrs = [['class', 'hashtag'], ['href', result[0]]]; - token.block = false; - buffer.push(token); - - token = new state.Token('text', '', 0); - token.content = '#'; - buffer.push(token); - - token = new state.Token('span_open', 'span', 1); - token.block = false; - buffer.push(token); - - token = new state.Token('text', '', 0); - token.content = result[1]; - buffer.push(token); - - buffer.push(new state.Token('span_close', 'span', -1)); - - buffer.push(new state.Token('link_close', 'a', -1)); - } else { - - token = new state.Token('span_open', 'span', 1); - token.attrs = [['class', 'hashtag']]; - buffer.push(token); - - token = new state.Token('text', '', 0); - token.content = hashtag; - buffer.push(token); - - token = new state.Token('span_close', 'span', -1); - buffer.push(token); - } -} - -const REGEX = /#([\w-:]{1,101})/gi; - -function allowedBoundary(content, index, utils) { - let code = content.charCodeAt(index); - return (utils.isWhiteSpace(code) || utils.isPunctChar(String.fromCharCode(code))); -} - -function applyHashtag(content, state) { - let result = null, - match, - pos = 0; - - while (match = REGEX.exec(content)) { - // check boundary - if (match.index > 0) { - if (!allowedBoundary(content, match.index-1, state.md.utils)) { - continue; - } - } - - // check forward boundary as well - if (match.index + match[0].length < content.length) { - if (!allowedBoundary(content, match.index + match[0].length, state.md.utils)) { - continue; - } - } - - if (match.index > pos) { - result = result || []; - let token = new state.Token('text', '', 0); - token.content = content.slice(pos, match.index); - result.push(token); - } - - result = result || []; - addHashtag(result, match, state); - - pos = match.index + match[0].length; - } - - if (result && pos < content.length) { - let token = new state.Token('text', '', 0); - token.content = content.slice(pos); - result.push(token); - } - - return result; -} - -export function setup(helper) { - - if (!helper.markdownIt) { return; } - - helper.registerPlugin(md=>{ - - md.core.ruler.push('category-hashtag', state => md.options.discourse.helpers.textReplace( - state, applyHashtag, true /* skip all links */ - )); - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6 deleted file mode 100644 index 3d5cb8d931..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/censored.js.es6 +++ /dev/null @@ -1,44 +0,0 @@ -import { censorFn } from 'pretty-text/censored-words'; - -function recurse(tokens, apply) { - let i; - for(i=0;i { - if (token.content) { - token.content = censor(token.content); - } - }); -} - -export function setup(helper) { - - if (!helper.markdownIt) { return; } - - helper.registerOptions((opts, siteSettings) => { - opts.censoredWords = siteSettings.censored_words; - opts.censoredPattern = siteSettings.censored_pattern; - }); - - helper.registerPlugin(md => { - const words = md.options.discourse.censoredWords; - const patterns = md.options.discourse.censoredPattern; - - if ((words && words.length > 0) || (patterns && patterns.length > 0)) { - const replacement = String.fromCharCode(9632); - const censor = censorFn(words, patterns, replacement); - md.core.ruler.push('censored', state => censorTree(state, censor)); - } - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6 deleted file mode 100644 index c8d94967a1..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/code.js.es6 +++ /dev/null @@ -1,51 +0,0 @@ -// we need a custom renderer for code blocks cause we have a slightly non compliant -// format with special handling for text and so on - -const TEXT_CODE_CLASSES = ["text", "pre", "plain"]; - - -function render(tokens, idx, options, env, slf, md) { - let token = tokens[idx], - info = token.info ? md.utils.unescapeAll(token.info) : '', - langName = md.options.discourse.defaultCodeLang, - className, - escapedContent = md.utils.escapeHtml(token.content); - - if (info) { - // strip off any additional languages - info = info.split(/\s+/g)[0]; - } - - const acceptableCodeClasses = md.options.discourse.acceptableCodeClasses; - if (acceptableCodeClasses && info && acceptableCodeClasses.indexOf(info) !== -1) { - langName = info; - } - - className = TEXT_CODE_CLASSES.indexOf(info) !== -1 ? 'lang-nohighlight' : 'lang-' + langName; - - return `
    ${escapedContent}
    \n`; -} - -export function setup(helper) { - if (!helper.markdownIt) { return; } - - helper.registerOptions((opts, siteSettings) => { - opts.defaultCodeLang = siteSettings.default_code_lang; - opts.acceptableCodeClasses = (siteSettings.highlighted_languages || "").split("|").concat(['auto', 'nohighlight']); - }); - - helper.whiteList({ - custom(tag, name, value) { - if (tag === 'code' && name === 'class') { - const m = /^lang\-(.+)$/.exec(value); - if (m) { - return helper.getOptions().acceptableCodeClasses.indexOf(m[1]) !== -1; - } - } - } - }); - - helper.registerPlugin(md=>{ - md.renderer.rules.fence = (tokens,idx,options,env,slf)=>render(tokens,idx,options,env,slf,md); - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 deleted file mode 100644 index 0e4eed203b..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/emoji.js.es6 +++ /dev/null @@ -1,246 +0,0 @@ -import { buildEmojiUrl, isCustomEmoji } from 'pretty-text/emoji'; -import { translations } from 'pretty-text/emoji/data'; - -const MAX_NAME_LENGTH = 60; - -let translationTree = null; - -// This allows us to efficiently search for aliases -// We build a data structure that allows us to quickly -// search through our N next chars to see if any match -// one of our alias emojis. -// -function buildTranslationTree() { - let tree = []; - let lastNode; - - Object.keys(translations).forEach(function(key){ - let i; - let node = tree; - - for(i=0;i 0) { - let prev = content.charCodeAt(pos-1); - if (!state.md.utils.isSpace(prev) && !state.md.utils.isPunctChar(String.fromCharCode(prev))) { - return; - } - } - - pos++; - if (content.charCodeAt(pos) === 58) { - return; - } - - let length = 0; - while(length < MAX_NAME_LENGTH) { - length++; - - if (content.charCodeAt(pos+length) === 58) { - // check for t2-t6 - if (content.substr(pos+length+1, 3).match(/t[2-6]:/)) { - length += 3; - } - break; - } - - if (pos+length > content.length) { - return; - } - } - - if (length === MAX_NAME_LENGTH) { - return; - } - - return content.substr(pos, length); -} - -// straight forward :smile: to emoji image -function getEmojiTokenByName(name, state) { - - let info; - if (info = imageFor(name, state.md.options.discourse)) { - let token = new state.Token('emoji', 'img', 0); - token.attrs = [['src', info.url], - ['title', info.title], - ['class', info.classes], - ['alt', info.title]]; - - return token; - } -} - -function getEmojiTokenByTranslation(content, pos, state) { - - translationTree = translationTree || buildTranslationTree(); - - let currentTree = translationTree; - - let i; - let search = true; - let found = false; - let start = pos; - - while(search) { - - search = false; - let code = content.charCodeAt(pos); - - for (i=0;i 0) { - let leading = content.charAt(start-1); - if (!state.md.utils.isSpace(leading.charCodeAt(0)) && !state.md.utils.isPunctChar(leading)) { - return; - } - } - - // check trailing for punct or space - if (pos < content.length) { - let trailing = content.charCodeAt(pos); - if (!state.md.utils.isSpace(trailing)){ - return; - } - } - - let token = getEmojiTokenByName(found, state); - if (token) { - return { pos, token }; - } -} - -function applyEmoji(content, state, emojiUnicodeReplacer) { - let i; - let result = null; - let contentToken = null; - - let start = 0; - - if (emojiUnicodeReplacer) { - content = emojiUnicodeReplacer(content); - } - - let endToken = content.length; - - for (i=0; i0) { - contentToken = new state.Token('text', '', 0); - contentToken.content = content.slice(start,i); - result.push(contentToken); - } - - result.push(token); - endToken = start = i + offset; - } - } - - if (endToken < content.length) { - contentToken = new state.Token('text', '', 0); - contentToken.content = content.slice(endToken); - result.push(contentToken); - } - - return result; -} - -export function setup(helper) { - - if (!helper.markdownIt) { return; } - - helper.registerOptions((opts, siteSettings, state)=>{ - opts.features.emoji = !!siteSettings.enable_emoji; - opts.emojiSet = siteSettings.emoji_set || ""; - opts.customEmoji = state.customEmoji; - }); - - helper.registerPlugin((md)=>{ - md.core.ruler.push('emoji', state => md.options.discourse.helpers.textReplace( - state, (c,s)=>applyEmoji(c,s,md.options.discourse.emojiUnicodeReplacer)) - ); - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6 deleted file mode 100644 index 602af3c15a..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/mentions.js.es6 +++ /dev/null @@ -1,88 +0,0 @@ -const regex = /^(\w[\w.-]{0,59})\b/i; - -function applyMentions(state, silent, isWhiteSpace, isPunctChar, mentionLookup, getURL) { - - let pos = state.pos; - - // 64 = @ - if (silent || state.src.charCodeAt(pos) !== 64) { - return false; - } - - if (pos > 0) { - let prev = state.src.charCodeAt(pos-1); - if (!isWhiteSpace(prev) && !isPunctChar(String.fromCharCode(prev))) { - return false; - } - } - - // skip if in a link - if (state.tokens) { - let last = state.tokens[state.tokens.length-1]; - if (last) { - if (last.type === 'link_open') { - return false; - } - if (last.type === 'html_inline' && last.content.substr(0,2) === " { - md.inline.ruler.push('mentions', (state,silent)=> applyMentions( - state, - silent, - md.utils.isWhiteSpace, - md.utils.isPunctChar, - md.options.discourse.mentionLookup, - md.options.discourse.getURL - )); - }); -} - diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/newline.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/newline.js.es6 deleted file mode 100644 index f1eb2ba759..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/newline.js.es6 +++ /dev/null @@ -1,53 +0,0 @@ -// see: https://github.com/markdown-it/markdown-it/issues/375 -// -// we use a custom paragraph rule cause we have to signal when a -// link starts with a space, so we can bypass a onebox -// this is a freedom patch, so careful, may break on updates - - -function newline(state, silent) { - var token, pmax, max, pos = state.pos; - - if (state.src.charCodeAt(pos) !== 0x0A/* \n */) { return false; } - - pmax = state.pending.length - 1; - max = state.posMax; - - // ' \n' -> hardbreak - // Lookup in pending chars is bad practice! Don't copy to other rules! - // Pending string is stored in concat mode, indexed lookups will cause - // convertion to flat mode. - if (!silent) { - if (pmax >= 0 && state.pending.charCodeAt(pmax) === 0x20) { - if (pmax >= 1 && state.pending.charCodeAt(pmax - 1) === 0x20) { - state.pending = state.pending.replace(/ +$/, ''); - token = state.push('hardbreak', 'br', 0); - } else { - state.pending = state.pending.slice(0, -1); - token = state.push('softbreak', 'br', 0); - } - - } else { - token = state.push('softbreak', 'br', 0); - } - } - - pos++; - - // skip heading spaces for next line - while (pos < max && state.md.utils.isSpace(state.src.charCodeAt(pos))) { - if (token) { - token.leading_space = true; - } - pos++; - } - - state.pos = pos; - return true; -}; - -export function setup(helper) { - helper.registerPlugin(md => { - md.inline.ruler.at('newline', newline); - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6 deleted file mode 100644 index 5f04b72b1d..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/onebox.js.es6 +++ /dev/null @@ -1,89 +0,0 @@ -import { lookupCache } from 'pretty-text/oneboxer'; - -function applyOnebox(state, silent) { - if (silent || !state.tokens || state.tokens.length < 3) { - return; - } - - let i; - for(i=1;i 0) { - let prevSibling = token.children[j-1]; - - if (prevSibling.tag !== 'br' || prevSibling.leading_space) { - continue; - } - } - - // look ahead for soft or hard break - let text = token.children[j+1]; - let close = token.children[j+2]; - let lookahead = token.children[j+3]; - - if (lookahead && lookahead.tag !== 'br') { - continue; - } - - // check attrs only include a href - let attrs = child["attrs"]; - - if (!attrs || attrs.length !== 1 || attrs[0][0] !== "href") { - continue; - } - - // we already know text matches cause it is an auto link - - if (!close || close.type !== "link_close") { - continue; - } - - // we already determined earlier that 0 0 was href - let cached = lookupCache(attrs[0][1]); - - if (cached) { - // replace link with 2 blank text nodes and inline html for onebox - child.type = 'html_raw'; - child.content = cached; - child.inline = true; - - text.type = 'html_raw'; - text.content = ''; - text.inline = true; - - close.type = 'html_raw'; - close.content = ''; - close.inline = true; - - } else { - // decorate... - attrs.push(["class", "onebox"]); - } - } - } - } - } -} - - -export function setup(helper) { - - if (!helper.markdownIt) { return; } - - helper.registerPlugin(md => { - md.core.ruler.after('linkify', 'onebox', applyOnebox); - }); -} diff --git a/app/assets/javascripts/pretty-text/engines/markdown-it/table.js.es6 b/app/assets/javascripts/pretty-text/engines/markdown-it/table.js.es6 deleted file mode 100644 index 4bb5ef92d6..0000000000 --- a/app/assets/javascripts/pretty-text/engines/markdown-it/table.js.es6 +++ /dev/null @@ -1,31 +0,0 @@ -export function setup(helper) { - - if (!helper.markdownIt) { return; } - - // this is built in now - // TODO: sanitizer needs fixing, does not properly support this yet - - // we need a custom callback for style handling - helper.whiteList({ - custom: function(tag,attr,val) { - if (tag !== 'th' && tag !== 'td') { - return false; - } - - if (attr !== 'style') { - return false; - } - - return (val === 'text-align:right' || val === 'text-align:left' || val === 'text-align:center'); - } - }); - - helper.whiteList([ - 'table', - 'tbody', - 'thead', - 'tr', - 'th', - 'td', - ]); -} diff --git a/app/assets/javascripts/pretty-text/pretty-text.js.es6 b/app/assets/javascripts/pretty-text/pretty-text.js.es6 index f2c4633d20..bfd6eaf904 100644 --- a/app/assets/javascripts/pretty-text/pretty-text.js.es6 +++ b/app/assets/javascripts/pretty-text/pretty-text.js.es6 @@ -1,4 +1,3 @@ -import { cook, setup } from 'pretty-text/engines/discourse-markdown'; import { cook as cookIt, setup as setupIt } from 'pretty-text/engines/discourse-markdown-it'; import { sanitize } from 'pretty-text/sanitizer'; import WhiteLister from 'pretty-text/white-lister'; @@ -25,10 +24,6 @@ export function buildOptions(state) { emojiUnicodeReplacer } = state; - if (!siteSettings.enable_experimental_markdown_it) { - setup(); - } - const features = { 'bold-italics': true, 'auto-link': true, @@ -56,15 +51,10 @@ export function buildOptions(state) { mentionLookup: state.mentionLookup, emojiUnicodeReplacer, allowedHrefSchemes: siteSettings.allowed_href_schemes ? siteSettings.allowed_href_schemes.split('|') : null, - markdownIt: siteSettings.enable_experimental_markdown_it + markdownIt: true }; - if (siteSettings.enable_experimental_markdown_it) { - setupIt(options, siteSettings, state); - } else { - // TODO deprecate this - _registerFns.forEach(fn => fn(siteSettings, options, state)); - } + setupIt(options, siteSettings, state); return options; } @@ -74,22 +64,13 @@ export default class { this.opts = opts || {}; this.opts.features = this.opts.features || {}; this.opts.sanitizer = (!!this.opts.sanitize) ? (this.opts.sanitizer || sanitize) : identity; - // We used to do a failsafe call to setup here - // under new engine we always expect setup to be called by buildOptions. - // setup(); } cook(raw) { if (!raw || raw.length === 0) { return ""; } let result; - - if (this.opts.markdownIt) { - result = cookIt(raw, this.opts); - } else { - result = cook(raw, this.opts); - } - + result = cookIt(raw, this.opts); return result ? result : ""; } diff --git a/app/views/common/_discourse_javascript.html.erb b/app/views/common/_discourse_javascript.html.erb index 1d30e8e296..f9b0146793 100644 --- a/app/views/common/_discourse_javascript.html.erb +++ b/app/views/common/_discourse_javascript.html.erb @@ -42,9 +42,7 @@ Discourse.Environment = '<%= Rails.env %>'; Discourse.SiteSettings = ps.get('siteSettings'); Discourse.LetterAvatarVersion = '<%= LetterAvatar.version %>'; - <%- if SiteSetting.enable_experimental_markdown_it %> Discourse.MarkdownItURL = '<%= asset_url('markdown-it-bundle.js') %>'; - <%- end %> I18n.defaultLocale = '<%= SiteSetting.default_locale %>'; Discourse.start(); Discourse.set('assetVersion','<%= Discourse.assets_digest %>'); diff --git a/config/locales/server.ar.yml b/config/locales/server.ar.yml index bff1fc2adf..ab722fe3f9 100644 --- a/config/locales/server.ar.yml +++ b/config/locales/server.ar.yml @@ -946,7 +946,6 @@ ar: notify_mods_when_user_blocked: "إذا تم حظر المستخدم تلقائيا، وإرسال رسالة الى جميع المشرفين." flag_sockpuppets: "إذا رد أحد المستخدمين جديد إلى موضوع من عنوان IP نفسه باسم المستخدم الجديد الذي بدأ هذا الموضوع، علم كل من مناصبهم كدعاية المحتملين." traditional_markdown_linebreaks: "استعمل السطور التالفه التقليديه في Markdown, التي تتطلب مساحتين بيضاوين للسطور التالفه" - allow_html_tables: "كل الجداول يجب ان تدخل ب لغة ال HTML مثال TABLE , THEAD , TD , TR , TH سوف يأوذن لهم ( تتطلب مراجعة لكل المقالات القديمة )" post_undo_action_window_mins: "عدد الدقائق التي يسمح فيها للأعضاء بالتراجع عن آخر إجراءاتهم على المنشور (إعجاب، اشارة، إلخ...)" must_approve_users: "يجب أن الموظفين يوافق على جميع حسابات المستخدم الجديدة قبل أن يتم السماح لهم للوصول إلى الموقع. تحذير: تمكين هذا لموقع الحية إلغاء وصول المستخدمين الحاليين غير الموظفين!" pending_users_reminder_delay: "نبه المشرفين إذا وجد اعضاء ينتظرون الموافقة لمدة اطول من الساعات ، قم بوضع الخيار -1 لايقاف التنبيهات ." diff --git a/config/locales/server.da.yml b/config/locales/server.da.yml index b8244744d0..ea748fa7e0 100644 --- a/config/locales/server.da.yml +++ b/config/locales/server.da.yml @@ -903,7 +903,6 @@ da: notify_mods_when_user_blocked: "Send en besked til alle moderatorer hvis en bruger blokeres automatisk" flag_sockpuppets: "Hvis en ny bruger svarer på et emne fra den samme IP adresse som den der startede emnet, så rapporter begge at deres indlæg potentielt er spam." traditional_markdown_linebreaks: "Brug traditionelle linjeskift i Markdown, som kræver 2 mellemrum i slutningen af sætningen." - allow_html_tables: "Tillad tabeller at blive oprettet i Markdown med brug af HTML tags. TABLE, THEAD, TD, TR, TH vil blive tilladt (kræver en fuld re-indeksering af gamle indlæg som benytter tabeller) " post_undo_action_window_mins: "Antal minutter som brugere er tilladt at fortryde handlinger på et indlæg (like, flag, etc)." must_approve_users: "Personale skal godkende alle nye bruger konti inden de kan tilgå sitet. ADVARSEL: aktivering af dette for et live site vil medføre en ophævning af adgang for eksisterende ikke-personale brugere." pending_users_reminder_delay: "Underret moderatorer hvis nye brugere har ventet på godkendelse i længere end så mange timer. Skriv -1 for at deaktivere notifikationer." diff --git a/config/locales/server.de.yml b/config/locales/server.de.yml index 024f52ccf2..5792ad524c 100644 --- a/config/locales/server.de.yml +++ b/config/locales/server.de.yml @@ -921,7 +921,6 @@ de: notify_mods_when_user_blocked: "Wenn ein Benutzer automatisch gesperrt wird, sende eine Nachricht an alle Moderatoren." flag_sockpuppets: "Wenn ein neuer Benutzer auf ein Thema antwortet, das von einem anderen neuen Benutzer aber mit der gleichen IP-Adresse begonnen wurde, markiere beide Beiträge als potenziellen Spam." traditional_markdown_linebreaks: "Traditionelle Zeilenumbrüche in Markdown, die zwei nachfolgende Leerzeichen für einen Zeilenumbruch benötigen." - allow_html_tables: "Erlaube es, Tabellen in Markdown mit HTML-Tags einzugeben. TABLE, THEAD, TD, TR, TH werden erlaubt (alle Beiträge mit Tabellen müssen ihr HTML erneuern)" post_undo_action_window_mins: "Minuten, die ein Benutzer hat, um Aktionen auf einen Beitrag rückgängig zu machen (Gefällt mir, Meldung, usw.)." must_approve_users: "Team-Mitglieder müssen alle neuen Benutzerkonten freischalten, bevor diese Zugriff auf die Website erhalten. ACHTUNG: Das Aktivieren dieser Option für eine Live-Site entfernt den Zugriff auch für alle existierenden Benutzer außer für Team-Mitglieder!" pending_users_reminder_delay: "Benachrichtige die Moderatoren, falls neue Benutzer mehr als so viele Stunden auf ihre Genehmigung gewartet haben. Stelle -1 ein, um diese Benachrichtigungen zu deaktivieren." diff --git a/config/locales/server.el.yml b/config/locales/server.el.yml index ed8e3e7554..3ad65a4004 100644 --- a/config/locales/server.el.yml +++ b/config/locales/server.el.yml @@ -890,7 +890,6 @@ el: notify_mods_when_user_blocked: "Εάν ένας χρήστης αυτόματα μπλοκαριστει, στείλε μήνυμα σε όλους τους συντονιστές." flag_sockpuppets: "Εάν ένας νέος χρήστης απαντήσει σε ένα νήμα από την ίδια διεύθυνση ΙP όπως ο νέος χρήστης, ο οποίος ξεκίνησε το νήμα, και οι δυο δημοσιεύσεις τους θα επισημανθούν ως δυνητικά ανεπιθύμητες." traditional_markdown_linebreaks: "Χρήση παραδοσιακών αλλαγών γραμμών στη Markdown, η οποία απαιτεί δύο κενά διαστήματα για μια αλλαγή γραμμής." - allow_html_tables: "Αποδοχή εισδοχής πινάκων στη Markdown με τη χρήση ετικετών HTML. TABLE, THEAD, TD, TR, TH θα μπαίνουν στη λίστα επιτρεπόμενων (απαιτείται πληρής αντιγραφή σε όλες τις αναρτήσεις που περιέχουν πίνακες)" post_undo_action_window_mins: "Αριθμός των λεπτών όπου οι χρήστες δικαιούνται να αναιρέσουν πρόσφατες ενέργειες πάνω σε ένα θέμα (μου αρέσει, επισήμανση, κτλ) " must_approve_users: "Το προσωπικό πρέπει να εγκρίνει όλους τους λογαριασμούς των νέων χρηστών προτού τους επιτραπεί να έχουν πρόσβαση στην ιστοσελίδα. Προειδοποίηση: ενεργοποιώντας το για μια ζωντανή ιστοσελίδα θα έχει ως αποτέλεσμα την ανάκληση για τους υπάρχοντες χρήστες που δεν ανήκουν στο προσωπικό!" pending_users_reminder_delay: "Ειδοποίηση συντονιστών αν καινούργιοι χρήστες περιμένουν για αποδοχή για μεγαλύτερο απο αυτό το χρονικό διάστημα. Όρισέ το στο -1 για να απενεργοποιηθούν οι ειδοποιήσεις." diff --git a/config/locales/server.en.yml b/config/locales/server.en.yml index 67a2dce790..57baa880b4 100644 --- a/config/locales/server.en.yml +++ b/config/locales/server.en.yml @@ -1024,9 +1024,7 @@ en: flag_sockpuppets: "If a new user replies to a topic from the same IP address as the new user who started the topic, flag both of their posts as potential spam." traditional_markdown_linebreaks: "Use traditional linebreaks in Markdown, which require two trailing spaces for a linebreak." - enable_experimental_markdown_it: "Enable the experimental markdown.it CommonMark engine, WARNING: some plugins may not work correctly" enable_markdown_typographer: "Use basic typography rules to improve text readability of paragraphs of text, replaces (c) (tm) etc, with symbols, reduces number of question marks and so on" - allow_html_tables: "Allow tables to be entered in Markdown using HTML tags. TABLE, THEAD, TD, TR, TH will be whitelisted (requires full rebake on all old posts containing tables)" post_undo_action_window_mins: "Number of minutes users are allowed to undo recent actions on a post (like, flag, etc)." must_approve_users: "Staff must approve all new user accounts before they are allowed to access the site. WARNING: enabling this for a live site will revoke access for existing non-staff users!" pending_users_reminder_delay: "Notify moderators if new users have been waiting for approval for longer than this many hours. Set to -1 to disable notifications." diff --git a/config/locales/server.es.yml b/config/locales/server.es.yml index 1dbe6baf1f..775899a0f7 100644 --- a/config/locales/server.es.yml +++ b/config/locales/server.es.yml @@ -932,9 +932,7 @@ es: notify_mods_when_user_blocked: "Si un usuario es bloqueado automáticamente, enviar un mensaje a todos los moderadores." flag_sockpuppets: "Si un nuevo usuario responde a un tema desde la misma dirección de IP que el nuevo usuario que inició el tema, reportar los posts de los dos como spam en potencia." traditional_markdown_linebreaks: "Utiliza saltos de línea tradicionales en Markdown, que requieren dos espacios al final para un salto de línea." - enable_experimental_markdown_it: "Habilitar el motor experimental CommonMark markdown.it, ADVERTENCIA: algunos plugins podrían no funcionar correctamente." enable_markdown_typographer: "Utilice reglas básicas de tipografía para mejorar la legibilidad de texto de los párrafos de texto, reemplaza (c) (tm) etc, con símbolos, reduce el número de signos de interrogación y así sucesivamente" - allow_html_tables: "Permitir la inserción de tablas en Markdown usando etiquetas HTML. Se permitirá usar TABLE, THEAD, TD, TR o TH (requiere un rebake completo para los posts antiguos que contengan tablas)" post_undo_action_window_mins: "Número de minutos durante los cuales los usuarios pueden deshacer sus acciones recientes en un post (me gusta, reportes, etc)." must_approve_users: "Los miembros administración deben aprobar todas las nuevas cuentas antes de que se les permita el acceso al sitio. AVISO: ¡habilitar esta opción en un sitio activo revocará el acceso a los usuarios que no sean moderadores o admin!" pending_users_reminder_delay: "Notificar a los moderadores si hay nuevos usuarios que hayan estado esperando aprbación durante más estas horas. Usa -1 para desactivar estas notificaciones." diff --git a/config/locales/server.fa_IR.yml b/config/locales/server.fa_IR.yml index 209ab34571..a1dc35576d 100644 --- a/config/locales/server.fa_IR.yml +++ b/config/locales/server.fa_IR.yml @@ -884,7 +884,6 @@ fa_IR: notify_mods_when_user_blocked: "اگر کاربر به‌طور خودکار مسدود شد، به تمام مدیران پیام بفرست." flag_sockpuppets: "اگر کاربری به موضوع با ای پی برابر با کاربری که نوشته را شروع کرده ٬ آنها را به عنوان هرزنامه پرچم گزاری کن." traditional_markdown_linebreaks: "در مدل‌های نشانه گزاری از خط جدید سنتی استفاده کن،‌ که برای linebreak نیاز به دو فضای انتهایی دارد ." - allow_html_tables: "اجازه ارسال جدول به صورت markdown با تگ های HTML. TABLE, THEAD, TD, TR, TH قابل استفاده هستند. (نیازمند ایجا دوباره در نوشته‌های قدیمی که شامل جدول هستند)" post_undo_action_window_mins: "تعداد دقایقی که کاربران اجازه دارند اقدامی را که در نوشته انجام داده اند باز گردانند. (پسند، پرچم گذاری،‌ چیزهای دیگر)." must_approve_users: "همکاران باید تمامی حساب‌های کاربری را قبل از اجازه دسترسی به سایت تایید کنند. اخطار: فعال‌سازی این گزینه ممکن است باعث جلوگیری از دسترسی کاربرانی که قبلا عضو شده‌اند نیز بشود!" pending_users_reminder_delay: "اگر کاربر‌ها بیشتر از این مقدار ساعت منتظر تایید بودند به مدیران اعلام کن. مقدار -1 برای غیرفعال‌سازی." diff --git a/config/locales/server.fi.yml b/config/locales/server.fi.yml index 9c699ac9ec..d2aad62daa 100644 --- a/config/locales/server.fi.yml +++ b/config/locales/server.fi.yml @@ -931,9 +931,7 @@ fi: notify_mods_when_user_blocked: "Jos käyttäjä estetään automaattisesti, lähetä viesti kaikille valvojille." flag_sockpuppets: "Jos uuden käyttäjän luomaan ketjuun vastaa toinen uusi käyttäjä samasta IP-osoitteesta, liputa molemmat viestit mahdolliseksi roskapostiksi." traditional_markdown_linebreaks: "Käytä perinteisiä rivinvaihtoja Markdownissa, joka vaatii kaksi perättäistä välilyöntiä rivin vaihtoon." - enable_experimental_markdown_it: "Ota käyttöön kokeellinen markdown.it Commonmark ohjelmistomoottori. VAROITUS: jotkut lisäosat voivat lakata toimimasta oikein" enable_markdown_typographer: "Käytetään tavanomaisia typografisia sääntöjä parantamaan tekstikappaleiden luettavuutta, (c), (tm) ym. korvataan symboleilla, kysymysmerkkien määrää vähennetään jne." - allow_html_tables: "Salli taulukoiden syöttäminen Markdowniin käyttäen HTML tageja. TABLE, THEAD, TD, TR, TH valkolistataan (edellyttää kaikkien taulukoita sisältävien vanhojen viestien uudelleen rakentamisen)" post_undo_action_window_mins: "Kuinka monta minuuttia käyttäjällä on aikaa perua viestiin kohdistuva toimi (tykkäys, liputus, etc)." must_approve_users: "Henkilökunnan täytyy hyväksyä kaikki uudet tilit, ennen uusien käyttäjien päästämistä sivustolle. VAROITUS: tämän asetuksen valitseminen poistaa pääsyn kaikilta jo olemassa olevilta henkilökuntaan kuulumattomilta käyttäjiltä." pending_users_reminder_delay: "Ilmoita valvojille, jos uusi käyttäjä on odottanut hyväksyntää kauemmin kuin näin monta tuntia. Aseta -1, jos haluat kytkeä ilmoitukset pois päältä." diff --git a/config/locales/server.fr.yml b/config/locales/server.fr.yml index 2b85e00688..cd44c0a458 100644 --- a/config/locales/server.fr.yml +++ b/config/locales/server.fr.yml @@ -931,7 +931,6 @@ fr: notify_mods_when_user_blocked: "Si un utilisateur est bloqué automatiquement, envoyer un message à tous les modérateurs." flag_sockpuppets: "Si un nouvel utilisateur répond à un sujet avec la même adresse IP que le nouvel utilisateur qui a commencé le sujet, alors leurs messages seront automatiquement marqués comme spam." traditional_markdown_linebreaks: "Utiliser le retour à la ligne traditionnel dans Markdown, qui nécessite deux espaces pour un saut de ligne." - allow_html_tables: "Autoriser la saisie des tableaux dans le Markdown en utilisant les tags HTML : TABLE, THEAD, TD, TR, TH sont autorisés (nécessite un rebake de tous les anciens messages contenant des tableaux)" post_undo_action_window_mins: "Nombre de minutes pendant lesquelles un utilisateur peut annuler une action sur un message (J'aime, signaler, etc.)" must_approve_users: "Les responsables doivent approuver les nouveaux utilisateurs afin qu'ils puissent accéder au site. ATTENTION : activer cette option sur un site en production suspendra l'accès des utilisateurs existants qui ne sont pas des responsables !" pending_users_reminder_delay: "Avertir les modérateurs si des nouveaux utilisateurs sont en attente d'approbation depuis x heures. Mettre -1 pour désactiver les notifications." diff --git a/config/locales/server.he.yml b/config/locales/server.he.yml index 5795be1d51..d809362bdc 100644 --- a/config/locales/server.he.yml +++ b/config/locales/server.he.yml @@ -921,7 +921,6 @@ he: notify_mods_when_user_blocked: "אם משתמש נחסם אוטומטית, שילחו הודעה לכל המנחים." flag_sockpuppets: "אם משתמשים חדשים מגיבים לנושא מכתובת IP זהה לזו של מי שהחל את הנושא, סמנו את הפוסטים של שניהם כספאם פוטנציאלי." traditional_markdown_linebreaks: "שימוש בשבירת שורות מסורתית בסימון, מה שדורש שני רווחים עוקבים למעבר שורה." - allow_html_tables: "אפשרו הכנסת טבלאות ב Markdown באמצעות תגיות HTML. התגיות TABLE, THEAD, TD, TR, TH יהיו ברשימה לבנה (מצריך אפייה מחדש של כל הפוסטים הישנים שכוללים טבלאות)" post_undo_action_window_mins: "מספר הדקות בהן מתאפשר למשתמשים לבטל פעולות אחרות בפוסט (לייק, סימון, וכו')." must_approve_users: "על הצוות לאשר את כל המשתמשים החדשים לפני שהם מקבלים גישה לאתר. אזהרה: בחירה זו עבור אתר קיים תשלול גישה ממשתמשים קיימים שאינם מנהלים." pending_users_reminder_delay: "הודיעו למנחים אם משתמשים חדשים ממתינים לאישור למעלה מכמות זו של שעות. קבעו ל -1 כדי לנטרל התראות." diff --git a/config/locales/server.it.yml b/config/locales/server.it.yml index 722c723d09..ad31c6d81b 100644 --- a/config/locales/server.it.yml +++ b/config/locales/server.it.yml @@ -934,7 +934,6 @@ it: notify_mods_when_user_blocked: "Se un utente è bloccato automaticamente, manda un messaggio ai moderatori." flag_sockpuppets: "Se un nuovo utente risponde ad un argomento dallo stesso indirizzo IP dell'utente che ha aperto l'argomento stesso, segnala entrambi i messaggi come potenziale spam." traditional_markdown_linebreaks: "Usa l'accapo tradizionale in Markdown, cioè due spazi a fine riga per andare a capo." - allow_html_tables: "Consenti di inserire tabelle in Markdown usando tag HTML. I tag TABLE, THEAD, TD, TR, TH saranno consentiti (richiede un full rebake di tutti i vecchi messaggi contenenti tabelle)" post_undo_action_window_mins: "Numero di minuti durante i quali gli utenti possono annullare le loro azioni recenti su un messaggio (segnalazioni, Mi piace, ecc.)." must_approve_users: "Lo staff deve approvare tutti i nuovi account utente prima che essi possano accedere al sito. ATTENZIONE: abilitare l'opzione per un sito live revocherà l'accesso per tutti gli utenti non-staff esistenti!" pending_users_reminder_delay: "Notifica i moderatori se nuovi utenti sono in attesa di approvazione per più di queste ore. Imposta a -1 per disabilitare le notifiche." diff --git a/config/locales/server.ko.yml b/config/locales/server.ko.yml index 30f6b772cd..2bffa0478e 100644 --- a/config/locales/server.ko.yml +++ b/config/locales/server.ko.yml @@ -876,9 +876,7 @@ ko: notify_mods_when_user_blocked: "만약 사용자가 자동 블락되면 중간 운영자에게 메시지 보내기" flag_sockpuppets: "어떤 신규 사용자(예:24이내 가입자)가 글타래를 생성하고 같은 IP주소의 또 다른 신규 사용자가 댓글을 쓰면 자동 스팸 신고" traditional_markdown_linebreaks: "Markdown에서 전통적인 linebreak를 사용, linebreak시 두개의 trailing space를 사용하는 것." - enable_experimental_markdown_it: "(실험) CommonMark를 지원하는 markdown.it 엔진을 사용합니다. 경고: 올바르게 작동하지 않는 플러그인이 있을 수 있습니다." enable_markdown_typographer: "문단의 가독성을 높이기 위해서 기본 타이포그라피 룰을 사용합니다. (c) (tm), 기타 기호를 교체하고 연달아 나오는 물음표의 갯수를 줄입니다." - allow_html_tables: "마크다운 문서에 HTML 테이블을 허용합니다. TABLE, THEAD, TD, TR, TH 태그를 사용할 수 있습니다.(테이블이 포함된 이전 게시물에 적용하려면 rebake 해야 합니다.)" post_undo_action_window_mins: "사용자가 어떤 글에 대해서 수행한 작업(신고 등)을 취소하는 것이 허용되는 시간(초)" must_approve_users: "스태프는 반드시 사이트 엑세스권한을 허용하기 전에 모든 신규가입계정을 승인해야 합니다. 경고: 이것을 활성화하면 기존 스태프 아닌 회원들의 엑세스권한이 회수됩니다." pending_users_reminder_delay: "새로운 사용자가 승인을 기다리는 시간이 여기에 지정된 시간횟수보다 더 길어길경우 운영자에게 알려줍니다. 알림을 해제하려면 -1로 설정하세요." diff --git a/config/locales/server.nl.yml b/config/locales/server.nl.yml index c28ee211c7..828a8c074b 100644 --- a/config/locales/server.nl.yml +++ b/config/locales/server.nl.yml @@ -895,7 +895,6 @@ nl: notify_mods_when_user_blocked: "Als een gebruiker automatisch geblokkeerd is, stuur dan een bericht naar alle moderatoren." flag_sockpuppets: "Als een nieuwe gebruiker antwoord op een topic vanaf hetzelfde ip-adres als de nieuwe gebruiker die het topic opende, markeer dan beide berichten als potentiële spam." traditional_markdown_linebreaks: "Gebruik traditionele regeleinden in Markdown, welke 2 spaties aan het einde van een regel nodig heeft voor een regeleinde." - allow_html_tables: "Sta toe dat tabellen in Markdown mogen worden ingevoerd met behulp van HTML-tags. TABLE, TD, TR, TH zullen aan de whitelist worden toegevoegd (vereist volledig herbouwen van alle oude berichten met tabellen)" post_undo_action_window_mins: "Het aantal minuten waarin gebruikers hun recente acties op een bericht nog terug kunnen draaien (liken, markeren, etc)." must_approve_users: "Stafleden moeten alle nieuwe gebruikersaccounts goedkeuren voordat ze de site mogen bezoeken. OPGELET: als dit wordt aangezet voor een actieve site wordt alle toegang voor bestaande niet stafleden ingetrokken." pending_users_reminder_delay: "Moderators informeren als nieuwe gebruikers al langer dan dit aantal uren op goedkeuring wachten. Stel dit in op -1 om meldingen uit te schakelen." diff --git a/config/locales/server.pl_PL.yml b/config/locales/server.pl_PL.yml index 67ddc8bdf6..bdf1636439 100644 --- a/config/locales/server.pl_PL.yml +++ b/config/locales/server.pl_PL.yml @@ -956,7 +956,6 @@ pl_PL: notify_mods_when_user_blocked: "If a user is automatically blocked, send a message to all moderators." flag_sockpuppets: "Jeśli nowy użytkownik odpowiada na dany temat z tego samego adresu IP co nowy użytkownik, który założył temat, oznacz ich posty jako potencjalny spam." traditional_markdown_linebreaks: "Używaj tradycyjnych znaków końca linii w Markdown, to znaczy dwóch spacji na końcu linii." - allow_html_tables: "Pozwalaj tabelom być zamieszczanym w Markdown przy użyciu tagów HTML. TABLE, THEAD, TD, TR, TH będą dozwolone (wymaga pełnego rebake na wszystkich starych postach zawierających tabele)." post_undo_action_window_mins: "Przez tyle minut użytkownicy mogą cofnąć swoje ostatnie działania przy danym poście (lajki, flagowanie, itd.)." must_approve_users: "Zespół musi zaakceptować wszystkie nowe konta zanim uzyskają dostęp do serwisu. UWAGA: włączenie tego dla już udostępnionej strony sprawi, że zostanie odebrany dostęp wszystkim istniejącym użytkownikom spoza zespołu." pending_users_reminder_delay: "Powiadomić moderatorów jeżeli nowi użytkownicy czekali na zatwierdzenie dłużej niż his mamy godzin. Ustaw -1 aby wyłączyć powiadomienia. " diff --git a/config/locales/server.pt.yml b/config/locales/server.pt.yml index a89dda5a2d..3f859154d6 100644 --- a/config/locales/server.pt.yml +++ b/config/locales/server.pt.yml @@ -802,7 +802,6 @@ pt: notify_mods_when_user_blocked: "Se um utilizador for bloqueado de forma automática, enviar uma mensagem a todos os moderadores." flag_sockpuppets: "Se um novo utilizador responde a um tópico a partir do mesmo endereço IP do novo utilizador que iniciou o tópico, sinalizar ambas as mensagens como potencial spam." traditional_markdown_linebreaks: "Utilize tradicionais quebras de linha no Markdown, que requer dois espaços no final para uma quebra de linha." - allow_html_tables: "Permitir inserção de tabelas em Markdown utilizando tags HTML. TABLE,THEAD, TD, TR,TH fazem parte da lista branca (requer que todas as mensagens antigas que contém tabelas sejam refeitas)" post_undo_action_window_mins: "Número de minutos durante o qual os utilizadores têm permissão para desfazer ações numa mensagem (gostos, sinalizações, etc)." must_approve_users: "O pessoal deve aprovar todas as novas contas de utilizador antes destas terem permissão para aceder ao sítio. AVISO: ativar isto para um sítio ativo irá revogar o acesso aos utilizadores existentes que não fazem parte do pessoal!" pending_users_reminder_delay: "Notificar moderadores se novos utilizadores estiverem à espera de aprovação por mais que esta quantidade de horas. Configurar com -1 para desativar notificações." diff --git a/config/locales/server.ro.yml b/config/locales/server.ro.yml index 8082e409bf..e4413b9253 100644 --- a/config/locales/server.ro.yml +++ b/config/locales/server.ro.yml @@ -878,7 +878,6 @@ ro: notify_mods_when_user_blocked: "Dacă un utilizator este blocat automat, trimite un mesaj tuturor moderatorilor." flag_sockpuppets: "Dacă un utilizator nou răspunde unui subiect de la același IP ca utilizatorul ce a pornit subiectul, marchează ambele postări ca potențial spam." traditional_markdown_linebreaks: "Folosește întreruperi de rând tradiționale în Markdown, ceea ce necesită două spații pentru un capăt de rând. " - allow_html_tables: "Permite introducerea de tabele în Markdown prin folosirea de etichete HTML. HEAD, TD, TR, TH vor fi autorizate (necesită un rebake pe toate postările vechi ce conțin tabele)" post_undo_action_window_mins: "Numărul de minute în care utilizatorii pot anula acțiunile recente asupra unei postări (aprecieri, marcări cu marcaje de avertizare, etc)." must_approve_users: "Membrii echipei trebuie să aprobe toate conturile noilor utilizatori înainte ca aceștia să poată accesa site-ul. ATENȚIE: activarea acestei opțiuni pentru un site în producție va revoca accesul tuturor utilizatorilor care nu sunt membri ai echipei!" pending_users_reminder_delay: "Notifică moderatorii dacă noii utilizatori sunt în așteptarea aprobării de mai mult de atâtea ore. Setează la -1 pentru a dezactiva notificările." diff --git a/config/locales/server.sk.yml b/config/locales/server.sk.yml index 71cd2e89a3..912d3d5da0 100644 --- a/config/locales/server.sk.yml +++ b/config/locales/server.sk.yml @@ -761,7 +761,6 @@ sk: notify_mods_when_user_blocked: "Ak je používateľ automaticky zablokovaný, pošli správu všetkým moderátorom." flag_sockpuppets: "Ak nový používateľ odpovedá na tému z rovnakej IP adresy, ako nový používateľ, ktorý danú tému vytvoril, označ oba ich príspevky ako potencionálny spam." traditional_markdown_linebreaks: "V Markdown použiť tradičné oddeľovače riadkov, čo vyžaduje dve koncové medzery ako oddeľovač riadku." - allow_html_tables: "V Markdown umožniť použitie tabuliek pomocou HTML značiek. TABLE, THEAD, TD, TR, TH budú umožnené (vyžaduje \"full rebake\" na všetkých starých príspevkoch ktoré obsahujú tabuľky)" post_undo_action_window_mins: "Počet minút počas ktorých môžu používatelia zrušiť poslednú akciu na príspevku (\"Páči sa\", označenie, atď..)." must_approve_users: "Obsluha musí povoliť účty všetkým novým používateľom skôr než im bude povolený prístup na stránku. UPOZORNENIE: zapnutie na živej stránke spôsobí zrušenie prístupu pre existujúcich používateľov, okrem obsluhy!" pending_users_reminder_delay: "Upozorni moderátora ak nový používateľ čaká na schválenie dlhšie ako tento počet hodín. Nastavte -1 pre vypnutie upozornenia." diff --git a/config/locales/server.sv.yml b/config/locales/server.sv.yml index 6ac8c593d3..e6242939a3 100644 --- a/config/locales/server.sv.yml +++ b/config/locales/server.sv.yml @@ -818,7 +818,6 @@ sv: notify_mods_when_user_blocked: "Om en användare blockeras automatiskt, skicka ett meddelande till alla moderatorer." flag_sockpuppets: "Flagga båda användarnas inlägg som potentiell skräppost om en ny användare svarar på ett ämne från samma IP-adress som den andra nya användaren som skapade ämnet." traditional_markdown_linebreaks: "Använd vanliga radmatningar i Markdown, vilka kräver 2 avslutande mellanslag för en radmatning." - allow_html_tables: "Tillåt tabeller att läggas in i Markdown genom användning av HTML-taggar. TABLE, THEAD, TD, TR, TH kommer att vitlistas (kräver full uppdatering/rebake av alla gamla inlägg som innehåller tabeller)" post_undo_action_window_mins: "Antal minuter som en användare tillåts att upphäva handlingar på ett inlägg som gjorts nyligen (gillning, flaggning osv)." must_approve_users: "Personal måste godkänna alla nya användarkonton innan de tillåts använda webbplatsen. VARNING: om det tillåts när webbplatsen är live så kommer det att upphäva tillgång för alla existerande användare som inte är personal!" pending_users_reminder_delay: "Notifiera moderatorer om nya användare har väntat på godkännande längre än så här många timmar. Ange -1 för att inaktivera notifikationer. " diff --git a/config/locales/server.tr_TR.yml b/config/locales/server.tr_TR.yml index 0430d47a47..1e9ec205cb 100644 --- a/config/locales/server.tr_TR.yml +++ b/config/locales/server.tr_TR.yml @@ -705,7 +705,6 @@ tr_TR: notify_mods_when_user_blocked: "Eğer bir kullanıcı otomatik olarak engellendiyse, tüm moderatörlere ileti yolla." flag_sockpuppets: "Eğer, yeni kullanıcı konuya, konuyu başlatan yeni kullanıcı ile aynı IP adresinden cevap yazarsa, her iki gönderiyi de potansiyel istenmeyen olarak bildir. " traditional_markdown_linebreaks: "Markdown'da, satır sonundan önce yazının sağında iki tane boşluk gerektiren, geleneksel satır sonu metodunu kullan." - allow_html_tables: "Tabloların HTML etiketleri kullanılarak Markdown ile oluşturulmasına izin verin. TABLE, THEAD, TD, TR, TH kabul edilir (tablo içeren tüm eski gönderilerin yenilenmesini gerektirir) " post_undo_action_window_mins: "Bir gönderide yapılan yeni eylemlerin (beğenme, bildirme vb) geri alınabileceği zaman, dakika olarak" must_approve_users: "Siteye erişimlerine izin verilmeden önce tüm yeni kullanıcı hesaplarının görevliler tarafından onaylanması gerekir. UYARI: yayındaki bir site için bunu etkinleştirmek görevli olmayan hesapların erişimini iptal edecek." pending_users_reminder_delay: "Belirtilen saatten daha uzun bir süredir onay bekleyen yeni kullanıcılar mevcutsa moderatörleri bilgilendir. Bilgilendirmeyi devre dışı bırakmak için -1 girin." diff --git a/config/locales/server.vi.yml b/config/locales/server.vi.yml index 62f13094af..346f3e5628 100644 --- a/config/locales/server.vi.yml +++ b/config/locales/server.vi.yml @@ -681,7 +681,6 @@ vi: notify_mods_when_user_blocked: "Nếu một thành viên được khóa tự động, gửi tin nhắn đến tất cả các điều hành viên." flag_sockpuppets: "Nếu thành viên mới trả lời chủ đề có cùng địa chỉ IP với thành viên mới tạo chủ đề, đánh dấu các bài viết của họ là spam tiềm năng." traditional_markdown_linebreaks: "Sử dụng ngắt dòng truyền thống trong Markdown, đòi hỏi hai khoảng trống kế tiếp cho một ngắt dòng." - allow_html_tables: "Cho phép nhập bảng trong Markdown sử dụng các thẻ HTML. TABLE, THEAD, TD, TR, TH sẽ được sử dụng (đòi hỏi thực hiện lại cho các bài viết cũ có chứa bảng)" post_undo_action_window_mins: "Số phút thành viên được phép làm lại các hành động gần đây với bài viết (like, đánh dấu...)." must_approve_users: "Quản trị viên phải duyệt tất cả các tài khoản thành viên mới trước khi họ có quyền truy cập website. LƯU Ý: bật tính năng này trên site đang hoạt động sẽ hủy bỏ quyền truy cập đối với các tài khoản thành viên hiện tại!" pending_users_reminder_delay: "Thông báo cho quản trị viên nếu thành viên mới đã chờ duyệt lâu hơn số giờ được thiết lập ở đây, đặt là -1 để tắt thông báo." diff --git a/config/locales/server.zh_CN.yml b/config/locales/server.zh_CN.yml index 297f1c3e33..b81aabde71 100644 --- a/config/locales/server.zh_CN.yml +++ b/config/locales/server.zh_CN.yml @@ -881,7 +881,6 @@ zh_CN: notify_mods_when_user_blocked: "如果一个用户被自动封禁了，发送一个私信给所有管理员。" flag_sockpuppets: "如果一个新用户开始了一个主题，并且同时另一个新用户以同一个 IP 在该主题回复，他们所有的帖子都将被自动标记为垃圾。" traditional_markdown_linebreaks: "在 Markdown 中使用传统换行符，即用两个尾随空格来换行" - allow_html_tables: "允许在输入 Markdown 文本时使用表格 HTML 标签。标签 TABLE、THEAD、TD、TR、TH 将被允许使用，即白名单这些标签（需要重置所有包含表格的老帖子的 HTML）" post_undo_action_window_mins: "允许用户在帖子上进行撤销操作（赞、标记等）所需等待的间隔分钟数" must_approve_users: "新用户在被允许访问站点前需要由管理人员批准。警告：在运行的站点中启用将解除所有非管理人员用户的访问权限！" pending_users_reminder_delay: "如果新用户等待批准时间超过此小时设置则通知版主。设置 -1 关闭通知。" diff --git a/config/locales/server.zh_TW.yml b/config/locales/server.zh_TW.yml index ebd13f4af5..16962647e7 100644 --- a/config/locales/server.zh_TW.yml +++ b/config/locales/server.zh_TW.yml @@ -832,7 +832,6 @@ zh_TW: notify_mods_when_user_blocked: "若有用戶被自動封鎖，將發送訊息給所有板主。" flag_sockpuppets: "如果一個新用戶開始了一個主題，並且同時另一個新用戶以同一個 IP 在該主題回復，他們所有的帖子都將被自動標記為垃圾。" traditional_markdown_linebreaks: "在 Markdown 中使用傳統的換行符號，即用兩個行末空格來換行" - allow_html_tables: "允許在輸入 Markdown 文本時使用表格 HTML 標籤。標籤 TABLE、THEAD、TD、TR、TH 將被允許使用，即白名單這些標籤（需要重置所有包含表格的老帖子的 HTML）" post_undo_action_window_mins: "允許用戶在帖子上進行撤銷操作（讚、標記等）所需等待的時間分隔（分鐘）" must_approve_users: "新用戶在被允許訪問站點前需要由管理人員批准。警告：在運行的站點中啟用將解除所有非管理人員用戶的訪問權限！" pending_users_reminder_delay: "如果新用戶等待批准時間超過此小時設置則通知版主。設置 -1 關閉通知。" diff --git a/config/site_settings.yml b/config/site_settings.yml index 9816e822ce..7d03976ad4 100644 --- a/config/site_settings.yml +++ b/config/site_settings.yml @@ -492,19 +492,12 @@ posting: delete_removed_posts_after: client: true default: 24 - enable_experimental_markdown_it: - client: true - default: false - shadowed_by_global: true traditional_markdown_linebreaks: client: true default: false enable_markdown_typographer: client: true - default: false - allow_html_tables: - client: true - default: false + default: true suppress_reply_directly_below: client: true default: true diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb index 2b9b3489b7..0fefc15b6e 100644 --- a/lib/pretty_text.rb +++ b/lib/pretty_text.rb @@ -80,11 +80,7 @@ module PrettyText ctx_load(ctx, "#{Rails.root}/app/assets/javascripts/discourse-loader.js") ctx_load(ctx, "vendor/assets/javascripts/lodash.js") ctx_load_manifest(ctx, "pretty-text-bundle.js") - - if SiteSetting.enable_experimental_markdown_it - ctx_load_manifest(ctx, "markdown-it-bundle.js") - end - + ctx_load_manifest(ctx, "markdown-it-bundle.js") root_path = "#{Rails.root}/app/assets/javascripts/" apply_es6_file(ctx, root_path, "discourse/lib/utilities") @@ -152,13 +148,6 @@ module PrettyText paths[:S3BaseUrl] = Discourse.store.absolute_base_url end - if SiteSetting.enable_experimental_markdown_it - # defer load markdown it - unless context.eval("window.markdownit") - ctx_load_manifest(context, "markdown-it-bundle.js") - end - end - custom_emoji = {} Emoji.custom.map { |e| custom_emoji[e.name] = e.url } diff --git a/lib/pretty_text/shims.js b/lib/pretty_text/shims.js index b3efe6cdf5..93a1ef592f 100644 --- a/lib/pretty_text/shims.js +++ b/lib/pretty_text/shims.js @@ -7,8 +7,6 @@ __utils = require('discourse/lib/utilities'); __emojiUnicodeReplacer = null; __setUnicode = function(replacements) { - require('pretty-text/engines/discourse-markdown/emoji').setUnicodeReplacements(replacements); - let unicodeRegexp = new RegExp(Object.keys(replacements).sort().reverse().join("|"), "g"); __emojiUnicodeReplacer = function(text) { diff --git a/script/import_scripts/lithium.rb b/script/import_scripts/lithium.rb index bd0a6f0055..54cd27e1d9 100644 --- a/script/import_scripts/lithium.rb +++ b/script/import_scripts/lithium.rb @@ -53,7 +53,6 @@ class ImportScripts::Lithium < ImportScripts::Base def execute - SiteSetting.allow_html_tables = true @max_start_id = Post.maximum(:id) import_categories diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb index ea6c749d5d..49bf5db18a 100644 --- a/spec/components/pretty_text_spec.rb +++ b/spec/components/pretty_text_spec.rb @@ -5,7 +5,7 @@ require 'html_normalize' describe PrettyText do before do - SiteSetting.enable_experimental_markdown_it = true + SiteSetting.enable_markdown_typographer = false end def n(html) diff --git a/test/javascripts/lib/pretty-text-test.js.es6 b/test/javascripts/lib/pretty-text-test.js.es6 index 558c105e3f..19ba8bc66c 100644 --- a/test/javascripts/lib/pretty-text-test.js.es6 +++ b/test/javascripts/lib/pretty-text-test.js.es6 @@ -42,9 +42,6 @@ QUnit.assert.cookedPara = function(input, expected, message) { }; QUnit.test("buildOptions", assert => { - assert.ok(buildOptions({ siteSettings: { allow_html_tables: true } }).features.table, 'tables enabled'); - assert.ok(!buildOptions({ siteSettings: { allow_html_tables: false } }).features.table, 'tables disabled'); - assert.ok(buildOptions({ siteSettings: { enable_emoji: true } }).features.emoji, 'emoji enabled'); assert.ok(!buildOptions({ siteSettings: { enable_emoji: false } }).features.emoji, 'emoji disabled'); }); diff --git a/test/javascripts/models/model-test.js.es6 b/test/javascripts/models/model-test.js.es6 index eb4d47b8d1..dba22bd390 100644 --- a/test/javascripts/models/model-test.js.es6 +++ b/test/javascripts/models/model-test.js.es6 @@ -18,4 +18,4 @@ QUnit.test("extractByKey: converts a list of hashes into a hash of instances of QUnit.test("extractByKey: returns an empty hash if there isn't anything to convert", assert => { assert.deepEqual(Model.extractByKey(), {}, "when called without parameters"); assert.deepEqual(Model.extractByKey([]), {}, "when called with an empty array"); -}); \ No newline at end of file +}); diff --git a/vendor/assets/javascripts/better_markdown.js b/vendor/assets/javascripts/better_markdown.js deleted file mode 100644 index 836b1c7e31..0000000000 --- a/vendor/assets/javascripts/better_markdown.js +++ /dev/null @@ -1,1518 +0,0 @@ -/* - This is a fork of markdown-js with a few changes to support discourse: - - * We have replaced the strong/em handlers because we prefer them only to work on word - boundaries. - - * [MOD]: non-url is fixed - - // Fix code within attrs - if (prev && (typeof prev[0] === "string") && prev[0].match(/<[^>]+$/)) { return; } - - // __RAW - - // if ( next_block.match(is_list_re) || (next_block.match(/^ /) && (!next_block.match(/^ *\>/))) ) { - -*/ - -// Released under MIT license -// Copyright (c) 2009-2010 Dominic Baggott -// Copyright (c) 2009-2010 Ash Berlin -// Copyright (c) 2011 Christoph Dorn (http://www.christophdorn.com) - -/*jshint browser:true, devel:true */ - -(function(expose) { - - var MarkdownHelpers = {}; - - // For Spidermonkey based engines - function mk_block_toSource() { - return "Markdown.mk_block( " + - uneval(this.toString()) + - ", " + - uneval(this.trailing) + - ", " + - uneval(this.lineNumber) + - " )"; - } - - // node - function mk_block_inspect() { - var util = require("util"); - return "Markdown.mk_block( " + - util.inspect(this.toString()) + - ", " + - util.inspect(this.trailing) + - ", " + - util.inspect(this.lineNumber) + - " )"; - - } - - MarkdownHelpers.mk_block = function(block, trail, line) { - // Be helpful for default case in tests. - if ( arguments.length === 1 ) - trail = "\n\n"; - - // We actually need a String object, not a string primitive - /* jshint -W053 */ - var s = new String(block); - s.trailing = trail; - // To make it clear its not just a string - s.inspect = mk_block_inspect; - s.toSource = mk_block_toSource; - - if ( line !== undefined ) - s.lineNumber = line; - - return s; - }; - - var isArray = MarkdownHelpers.isArray = Array.isArray || function(obj) { - return Object.prototype.toString.call(obj) === "[object Array]"; - }; - - // Don't mess with Array.prototype. Its not friendly - if ( Array.prototype.forEach ) { - MarkdownHelpers.forEach = function forEach( arr, cb, thisp ) { - return arr.forEach( cb, thisp ); - }; - } - else { - MarkdownHelpers.forEach = function forEach(arr, cb, thisp) { - for (var i = 0; i < arr.length; i++) - cb.call(thisp || arr, arr[i], i, arr); - }; - } - - MarkdownHelpers.isEmpty = function isEmpty( obj ) { - for ( var key in obj ) { - if ( hasOwnProperty.call( obj, key ) ) - return false; - } - return true; - }; - - MarkdownHelpers.extract_attr = function extract_attr( jsonml ) { - return isArray(jsonml) - && jsonml.length > 1 - && typeof jsonml[ 1 ] === "object" - && !( isArray(jsonml[ 1 ]) ) - ? jsonml[ 1 ] - : undefined; - }; - - /** - * class Markdown - * - * Markdown processing in Javascript done right. We have very particular views - * on what constitutes 'right' which include: - * - * - produces well-formed HTML (this means that em and strong nesting is - * important) - * - * - has an intermediate representation to allow processing of parsed data (We - * in fact have two, both as [JsonML]: a markdown tree and an HTML tree). - * - * - is easily extensible to add new dialects without having to rewrite the - * entire parsing mechanics - * - * - has a good test suite - * - * This implementation fulfills all of these (except that the test suite could - * do with expanding to automatically run all the fixtures from other Markdown - * implementations.) - * - * ##### Intermediate Representation - * - * *TODO* Talk about this :) Its JsonML, but document the node names we use. - * - * [JsonML]: http://jsonml.org/ "JSON Markup Language" - **/ - var Markdown = function(dialect) { - switch (typeof dialect) { - case "undefined": - this.dialect = Markdown.dialects.Gruber; - break; - case "object": - this.dialect = dialect; - break; - default: - if ( dialect in Markdown.dialects ) - this.dialect = Markdown.dialects[dialect]; - else - throw new Error("Unknown Markdown dialect '" + String(dialect) + "'"); - break; - } - this.em_state = []; - this.strong_state = []; - this.debug_indent = ""; - }; - - /** - * Markdown.dialects - * - * Namespace of built-in dialects. - **/ - Markdown.dialects = {}; - - // Imported functions - var mk_block = Markdown.mk_block = MarkdownHelpers.mk_block, - isArray = MarkdownHelpers.isArray; - - /** - * parse( markdown, [dialect] ) -> JsonML - * - markdown (String): markdown string to parse - * - dialect (String | Dialect): the dialect to use, defaults to gruber - * - * Parse `markdown` and return a markdown document as a Markdown.JsonML tree. - **/ - Markdown.parse = function( source, dialect ) { - // dialect will default if undefined - var md = new Markdown( dialect ); - return md.toTree( source ); - }; - - /** - * count_lines( str ) -> count - * - str (String): String whose lines we want to count - * - * Counts the number of linebreaks in `str` - **/ - function count_lines( str ) { - return str.split("\n").length - 1; - } - - // Internal - split source into rough blocks - Markdown.prototype.split_blocks = function splitBlocks( input ) { - input = input.replace(/(\r\n|\n|\r)/g, "\n"); - // [\s\S] matches _anything_ (newline or space) - // [^] is equivalent but doesn't work in IEs. - var re = /([\s\S]+?)($|\n#|\n(?:\s*\n|$)+)/g, - blocks = [], - m; - - var line_no = 1; - - if ( ( m = /^(\s*\n)/.exec(input) ) !== null ) { - // skip (but count) leading blank lines - line_no += count_lines( m[0] ); - re.lastIndex = m[0].length; - } - - while ( ( m = re.exec(input) ) !== null ) { - if (m[2] === "\n#") { - m[2] = "\n"; - re.lastIndex--; - } - blocks.push( mk_block( m[1], m[2], line_no ) ); - line_no += count_lines( m[0] ); - } - - return blocks; - }; - - /** - * Markdown#processBlock( block, next ) -> undefined | [ JsonML, ... ] - * - block (String): the block to process - * - next (Array): the following blocks - * - * Process `block` and return an array of JsonML nodes representing `block`. - * - * It does this by asking each block level function in the dialect to process - * the block until one can. Succesful handling is indicated by returning an - * array (with zero or more JsonML nodes), failure by a false value. - * - * Blocks handlers are responsible for calling [[Markdown#processInline]] - * themselves as appropriate. - * - * If the blocks were split incorrectly or adjacent blocks need collapsing you - * can adjust `next` in place using shift/splice etc. - * - * If any of this default behaviour is not right for the dialect, you can - * define a `__call__` method on the dialect that will get invoked to handle - * the block processing. - */ - Markdown.prototype.processBlock = function processBlock( block, next ) { - var cbs = this.dialect.block, - ord = cbs.__order__; - - if ( "__call__" in cbs ) - return cbs.__call__.call(this, block, next); - - for ( var i = 0; i < ord.length; i++ ) { - //D:this.debug( "Testing", ord[i] ); - var res = cbs[ ord[i] ].call( this, block, next ); - if ( res ) { - //D:this.debug(" matched"); - if ( !isArray(res) || ( res.length > 0 && !( isArray(res[0]) ) && ( typeof res[0] !== "string")) ) - this.debug(ord[i], "didn't return a proper array"); - //D:this.debug( "" ); - return res; - } - } - - // Uhoh! no match! Should we throw an error? - return []; - }; - - Markdown.prototype.processInline = function processInline( block ) { - return this.dialect.inline.__call__.call( this, String( block ) ); - }; - - /** - * Markdown#toTree( source ) -> JsonML - * - source (String): markdown source to parse - * - * Parse `source` into a JsonML tree representing the markdown document. - **/ - // custom_tree means set this.tree to `custom_tree` and restore old value on return - Markdown.prototype.toTree = function toTree( source, custom_root ) { - var blocks = source instanceof Array ? source : this.split_blocks( source ); - - // Make tree a member variable so its easier to mess with in extensions - var old_tree = this.tree; - try { - this.tree = custom_root || this.tree || [ "markdown" ]; - - blocks_loop: - while ( blocks.length ) { - var b = this.processBlock( blocks.shift(), blocks ); - - // Reference blocks and the like won't return any content - if ( !b.length ) - continue blocks_loop; - - this.tree.push.apply( this.tree, b ); - } - return this.tree; - } - finally { - if ( custom_root ) - this.tree = old_tree; - } - }; - - // Noop by default - Markdown.prototype.debug = function () { - var args = Array.prototype.slice.call( arguments); - args.unshift(this.debug_indent); - if ( typeof print !== "undefined" ) - print.apply( print, args ); - if ( typeof console !== "undefined" && typeof console.log !== "undefined" ) - console.log.apply( null, args ); - }; - - Markdown.prototype.loop_re_over_block = function( re, block, cb ) { - // Dont use /g regexps with this - var m, - b = block.valueOf(); - - while ( b.length && (m = re.exec(b) ) !== null ) { - b = b.substr( m[0].length ); - cb.call(this, m); - } - return b; - }; - - // Build default order from insertion order. - Markdown.buildBlockOrder = function(d) { - var ord = [[]]; - for ( var i in d ) { - if ( i === "__order__" || i === "__call__" ) - continue; - - var priority = d[i].priority || 0; - ord[priority] = ord[priority] || []; - ord[priority].push( i ); - } - - var flattend = []; - for (i=ord.length-1; i>=0; i--){ - if (ord[i]) { - for (var j=0; j String - * - jsonml (Array): JsonML array to render to XML - * - options (Object): options - * - * Converts the given JsonML into well-formed XML. - * - * The options currently understood are: - * - * - root (Boolean): wether or not the root node should be included in the - * output, or just its children. The default `false` is to not include the - * root itself. - */ - Markdown.renderJsonML = function( jsonml, options ) { - options = options || {}; - // include the root element in the rendered output? - options.root = options.root || false; - - var content = []; - - if ( options.root ) { - content.push( render_tree( jsonml ) ); - } - else { - jsonml.shift(); // get rid of the tag - if ( jsonml.length && typeof jsonml[ 0 ] === "object" && !( jsonml[ 0 ] instanceof Array ) ) - jsonml.shift(); // get rid of the attributes - - while ( jsonml.length ) - content.push( render_tree( jsonml.shift() ) ); - } - - return content.join( "\n\n" ); - }; - - /** - * toHTMLTree( markdown, [dialect] ) -> JsonML - * toHTMLTree( md_tree ) -> JsonML - * - markdown (String): markdown string to parse - * - dialect (String | Dialect): the dialect to use, defaults to gruber - * - md_tree (Markdown.JsonML): parsed markdown tree - * - * Turn markdown into HTML, represented as a JsonML tree. If a string is given - * to this function, it is first parsed into a markdown tree by calling - * [[parse]]. - **/ - Markdown.toHTMLTree = function toHTMLTree( input, dialect , options ) { - - // convert string input to an MD tree - if ( typeof input === "string" ) - input = this.parse( input, dialect ); - - // Now convert the MD tree to an HTML tree - - // remove references from the tree - var attrs = extract_attr( input ), - refs = {}; - - if ( attrs && attrs.references ) - refs = attrs.references; - - - var html = convert_tree_to_html( input, refs , options ); - merge_text_nodes( html ); - return html; - }; - - /** - * toHTML( markdown, [dialect] ) -> String - * toHTML( md_tree ) -> String - * - markdown (String): markdown string to parse - * - md_tree (Markdown.JsonML): parsed markdown tree - * - * Take markdown (either as a string or as a JsonML tree) and run it through - * [[toHTMLTree]] then turn it into a well-formated HTML fragment. - **/ - Markdown.toHTML = function toHTML( source , dialect , options ) { - var input = this.toHTMLTree( source , dialect , options ); - - return this.renderJsonML( input ); - }; - - function escapeHTML( text ) { - if (text && text.length > 0) { - return text.replace( /&/g, "&" ) - .replace( //g, ">" ) - .replace( /"/g, """ ) - .replace( /'/g, "'" ); - } else { - return ""; - } - } - - function render_tree( jsonml ) { - // basic case - if ( typeof jsonml === "string" ) - return jsonml; - - if ( jsonml[0] === "__RAW" ) { - return jsonml[1]; - } - - var tag = jsonml.shift(), - attributes = {}, - content = []; - - if ( jsonml.length && typeof jsonml[ 0 ] === "object" && !( jsonml[ 0 ] instanceof Array ) ) - attributes = jsonml.shift(); - - while ( jsonml.length ) - content.push( render_tree( jsonml.shift() ) ); - - var tag_attrs = ""; - if (typeof attributes.src !== 'undefined') { - tag_attrs += ' src="' + escapeHTML( attributes.src ) + '"'; - delete attributes.src; - } - - for ( var a in attributes ) { - var escaped = escapeHTML( attributes[ a ]); - if (escaped && escaped.length) { - tag_attrs += " " + a + '="' + escaped + '"'; - } - } - - // be careful about adding whitespace here for inline elements - if ( tag === "img" || tag === "br" || tag === "hr" ) - return "<"+ tag + tag_attrs + "/>"; - else - return "<"+ tag + tag_attrs + ">" + content.join( "" ) + ""; - } - - function convert_tree_to_html( tree, references, options ) { - var i; - options = options || {}; - - // shallow clone - var jsonml = tree.slice( 0 ); - - if ( typeof options.preprocessTreeNode === "function" ) - jsonml = options.preprocessTreeNode(jsonml, references); - - // Clone attributes if they exist - var attrs = extract_attr( jsonml ); - if ( attrs ) { - jsonml[ 1 ] = {}; - for ( i in attrs ) { - jsonml[ 1 ][ i ] = attrs[ i ]; - } - attrs = jsonml[ 1 ]; - } - - // basic case - if ( typeof jsonml === "string" ) - return jsonml; - - // convert this node - switch ( jsonml[ 0 ] ) { - case "header": - jsonml[ 0 ] = "h" + jsonml[ 1 ].level; - delete jsonml[ 1 ].level; - break; - case "bulletlist": - jsonml[ 0 ] = "ul"; - break; - case "numberlist": - jsonml[ 0 ] = "ol"; - break; - case "listitem": - jsonml[ 0 ] = "li"; - break; - case "para": - jsonml[ 0 ] = "p"; - break; - case "markdown": - jsonml[ 0 ] = "html"; - if ( attrs ) - delete attrs.references; - break; - case "code_block": - jsonml[ 0 ] = "pre"; - i = attrs ? 2 : 1; - var code = [ "code" ]; - code.push.apply( code, jsonml.splice( i, jsonml.length - i ) ); - jsonml[ i ] = code; - break; - case "inlinecode": - jsonml[ 0 ] = "code"; - break; - case "img": - jsonml[ 1 ].src = jsonml[ 1 ].href; - delete jsonml[ 1 ].href; - break; - case "linebreak": - jsonml[ 0 ] = "br"; - break; - case "link": - jsonml[ 0 ] = "a"; - break; - case "link_ref": - jsonml[ 0 ] = "a"; - - // grab this ref and clean up the attribute node - var ref = references[ attrs.ref ]; - - // if the reference exists, make the link - if ( ref ) { - delete attrs.ref; - - // add in the href and title, if present - attrs.href = ref.href; - if ( ref.title ) - attrs.title = ref.title; - - // get rid of the unneeded original text - delete attrs.original; - } - // the reference doesn't exist, so revert to plain text - else { - return attrs.original; - } - break; - case "img_ref": - jsonml[ 0 ] = "img"; - - // grab this ref and clean up the attribute node - var ref = references[ attrs.ref ]; - - // if the reference exists, make the link - if ( ref ) { - delete attrs.ref; - - // add in the href and title, if present - attrs.src = ref.href; - if ( ref.title ) - attrs.title = ref.title; - - // get rid of the unneeded original text - delete attrs.original; - } - // the reference doesn't exist, so revert to plain text - else { - return attrs.original; - } - break; - } - - // convert all the children - i = 1; - - // deal with the attribute node, if it exists - if ( attrs ) { - // if there are keys, skip over it - for ( var key in jsonml[ 1 ] ) { - i = 2; - break; - } - // if there aren't, remove it - if ( i === 1 ) - jsonml.splice( i, 1 ); - } - - for ( ; i < jsonml.length; ++i ) { - jsonml[ i ] = convert_tree_to_html( jsonml[ i ], references, options ); - } - - return jsonml; - } - - // merges adjacent text nodes into a single node - function merge_text_nodes( jsonml ) { - // skip the tag name and attribute hash - var i = extract_attr( jsonml ) ? 2 : 1; - - while ( i < jsonml.length ) { - // if it's a string check the next item too - if ( typeof jsonml[ i ] === "string" ) { - if ( i + 1 < jsonml.length && typeof jsonml[ i + 1 ] === "string" ) { - // merge the second string into the first and remove it - jsonml[ i ] += jsonml.splice( i + 1, 1 )[ 0 ]; - } - else { - ++i; - } - } - // if it's not a string recurse - else { - merge_text_nodes( jsonml[ i ] ); - ++i; - } - } - } - - var DialectHelpers = {}; - DialectHelpers.inline_until_char = function( text, want ) { - var consumed = 0, - nodes = [], - patterns = this.dialect.inline.__patterns__.replace('|_|', '|'); - - while ( true ) { - if ( text.charAt( consumed ) === want ) { - // Found the character we were looking for - consumed++; - return [ consumed, nodes ]; - } - - if ( consumed >= text.length ) { - // No closing char found. Abort. - return [consumed, null, nodes]; - } - - var res = this.dialect.inline.__oneElement__.call(this, text.substr( consumed ), patterns, [text.substr(0, consumed)]); - consumed += res[ 0 ]; - // Add any returned nodes. - nodes.push.apply( nodes, res.slice( 1 ) ); - } - }; - - // Helper function to make sub-classing a dialect easier - DialectHelpers.subclassDialect = function( d ) { - function Block() {} - Block.prototype = d.block; - function Inline() {} - Inline.prototype = d.inline; - - return { block: new Block(), inline: new Inline() }; - }; - - var forEach = MarkdownHelpers.forEach, - extract_attr = MarkdownHelpers.extract_attr, - mk_block = MarkdownHelpers.mk_block, - isEmpty = MarkdownHelpers.isEmpty, - inline_until_char = DialectHelpers.inline_until_char; - - // A robust regexp for matching URLs. Thakns: https://gist.github.com/dperini/729294 - var urlRegexp = /(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!(?:10|127)(?:\.\d{1,3}){3})(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]-*)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:\/[^\s]*)?/i.source; - - /** - * Gruber dialect - * - * The default dialect that follows the rules set out by John Gruber's - * markdown.pl as closely as possible. Well actually we follow the behaviour of - * that script which in some places is not exactly what the syntax web page - * says. - **/ - var Gruber = { - block: { - atxHeader: function atxHeader( block, next ) { - var m = block.match( /^(#{1,6})\s*(.*?)\s*#*\s*(?:\n|$)/ ); - - if ( !m ) - return undefined; - - var header = [ "header", { level: m[ 1 ].length } ]; - Array.prototype.push.apply(header, this.processInline(m[ 2 ])); - - if ( m[0].length < block.length ) - next.unshift( mk_block( block.substr( m[0].length ), block.trailing, block.lineNumber + 2 ) ); - - return [ header ]; - }, - - setextHeader: function setextHeader( block, next ) { - var m = block.match( /^(.*)\n([-=])\2\2+(?:\n|$)/ ); - - if ( !m ) - return undefined; - - var level = ( m[ 2 ] === "=" ) ? 1 : 2, - header = [ "header", { level : level } ].concat( this.processInline(m[ 1 ]) ); - - if ( m[0].length < block.length ) - next.unshift( mk_block( block.substr( m[0].length ), block.trailing, block.lineNumber + 2 ) ); - - return [ header ]; - }, - - code: function code( block, next ) { - // | Foo - // |bar - // should be a code block followed by a paragraph. Fun - // - // There might also be adjacent code block to merge. - - var ret = [], - re = /^(?: {0,3}\t| {4})(.*)\n?/; - - // 4 spaces + content - if ( !block.match( re ) ) - return undefined; - - block_search: - do { - // Now pull out the rest of the lines - var b = this.loop_re_over_block( - re, block.valueOf(), function( m ) { ret.push( m[1] ); } ); - - if ( b.length ) { - // Case alluded to in first comment. push it back on as a new block - next.unshift( mk_block(b, block.trailing) ); - break block_search; - } - else if ( next.length ) { - // Check the next block - it might be code too - if ( !next[0].match( re ) ) - break block_search; - - // Pull how how many blanks lines follow - minus two to account for .join - ret.push ( block.trailing.replace(/[^\n]/g, "").substring(2) ); - - block = next.shift(); - } - else { - break block_search; - } - } while ( true ); - - return [ [ "code_block", ret.join("\n") ] ]; - }, - - horizRule: function horizRule( block, next ) { - // this needs to find any hr in the block to handle abutting blocks - var m = block.match( /^(?:([\s\S]*?)\n)?[ \t]*([-_*])(?:[ \t]*\2){2,}[ \t]*(?:\n([\s\S]*))?$/ ); - - if ( !m ) - return undefined; - - var jsonml = [ [ "hr" ] ]; - - // if there's a leading abutting block, process it - if ( m[ 1 ] ) { - var contained = mk_block( m[ 1 ], "", block.lineNumber ); - jsonml.unshift.apply( jsonml, this.toTree( contained, [] ) ); - } - - // if there's a trailing abutting block, stick it into next - if ( m[ 3 ] ) - next.unshift( mk_block( m[ 3 ], block.trailing, block.lineNumber + 1 ) ); - - return jsonml; - }, - - // There are two types of lists. Tight and loose. Tight lists have no whitespace - // between the items (and result in text just in the
  1. ) and loose lists, - // which have an empty line between list items, resulting in (one or more) - // paragraphs inside the
  2. . - // - // There are all sorts weird edge cases about the original markdown.pl's - // handling of lists: - // - // * Nested lists are supposed to be indented by four chars per level. But - // if they aren't, you can get a nested list by indenting by less than - // four so long as the indent doesn't match an indent of an existing list - // item in the 'nest stack'. - // - // * The type of the list (bullet or number) is controlled just by the - // first item at the indent. Subsequent changes are ignored unless they - // are for nested lists - // - lists: (function( ) { - // Use a closure to hide a few variables. - var any_list = "[*+-]|\\d+\\.", - bullet_list = /[*+-]/, - // Capture leading indent as it matters for determining nested lists. - is_list_re = new RegExp( "^( {0,3})(" + any_list + ")[ \t]+" ), - indent_re = "(?: {0,3}\\t| {4})"; - - // TODO: Cache this regexp for certain depths. - // Create a regexp suitable for matching an li for a given stack depth - function regex_for_depth( depth ) { - return new RegExp( - // m[1] = indent, m[2] = list_type - "(?:^(" + indent_re + "{0," + depth + "} {0,3})(" + any_list + ")\\s+)|" + - // m[3] = cont - "(^" + indent_re + "{0," + (depth-1) + "}[ ]{0,4})" - ); - } - function expand_tab( input ) { - return input.replace( / {0,3}\t/g, " " ); - } - - // Add inline content `inline` to `li`. inline comes from processInline - // so is an array of content - function add(li, loose, inline, nl) { - if ( loose ) { - li.push( [ "para" ].concat(inline) ); - return; - } - // Hmmm, should this be any block level element or just paras? - var add_to = li[li.length -1] instanceof Array && li[li.length - 1][0] === "para" - ? li[li.length -1] - : li; - - // If there is already some content in this list, add the new line in - if ( nl && li.length > 1 ) - inline.unshift(nl); - - for ( var i = 0; i < inline.length; i++ ) { - var what = inline[i], - is_str = typeof what === "string"; - if ( is_str && add_to.length > 1 && typeof add_to[add_to.length-1] === "string" ) - add_to[ add_to.length-1 ] += what; - else - add_to.push( what ); - } - } - - // contained means have an indent greater than the current one. On - // *every* line in the block - function get_contained_blocks( depth, blocks ) { - - var re = new RegExp( "^(" + indent_re + "{" + depth + "}.*?\\n?)*$" ), - replace = new RegExp("^" + indent_re + "{" + depth + "}", "gm"), - ret = []; - - - while ( blocks.length > 0 ) { - // HACK: Fixes a v8 issue - test = blocks[0].replace(/^ {8,}/, ' '); - if ( re.exec( test ) ) { - var b = blocks.shift(), - // Now remove that indent - x = b.replace( replace, ""); - - ret.push( mk_block( x, b.trailing, b.lineNumber ) ); - } - else - break; - } - return ret; - } - - // passed to stack.forEach to turn list items up the stack into paras - function paragraphify(s, i, stack) { - var list = s.list; - var last_li = list[list.length-1]; - - if ( last_li[1] instanceof Array && last_li[1][0] === "para" ) - return; - - if ( i + 1 === stack.length ) { - // Last stack frame - // Keep the same array, but replace the contents - last_li.push( ["para"].concat( last_li.splice(1, last_li.length - 1) ) ); - } - else { - var sublist = last_li.pop(); - last_li.push( ["para"].concat( last_li.splice(1, last_li.length - 1) ), sublist ); - } - } - - // The matcher function - return function( block, next ) { - var m = block.match( is_list_re ); - if ( !m ) - return undefined; - - function make_list( m ) { - var list = bullet_list.exec( m[2] ) - ? ["bulletlist"] - : ["numberlist"]; - - stack.push( { list: list, indent: m[1] } ); - return list; - } - - var stack = [], // Stack of lists for nesting. - list = make_list( m ), - last_li, - loose = false, - ret = [ stack[0].list ], - i; - - // Loop to search over block looking for inner block elements and loose lists - loose_search: - while ( true ) { - // Split into lines preserving new lines at end of line - var lines = block.split( /(?=\n)/ ); - - // We have to grab all lines for a li and call processInline on them - // once as there are some inline things that can span lines. - var li_accumulate = "", nl = ""; - - // Loop over the lines in this block looking for tight lists. - tight_search: - for ( var line_no = 0; line_no < lines.length; line_no++ ) { - nl = ""; - var l = lines[line_no].replace(/^\n/, function(n) { nl = n; return ""; }); - - // TODO: really should cache this - var line_re = regex_for_depth( stack.length ); - - m = l.match( line_re ); - //print( "line:", uneval(l), "\nline match:", uneval(m) ); - - // We have a list item - if ( m[1] !== undefined ) { - // Process the previous list item, if any - if ( li_accumulate.length ) { - add( last_li, loose, this.processInline( li_accumulate ), nl ); - // Loose mode will have been dealt with. Reset it - loose = false; - li_accumulate = ""; - } - - m[1] = expand_tab( m[1] ); - var wanted_depth = Math.floor(m[1].length/4)+1; - //print( "want:", wanted_depth, "stack:", stack.length); - if ( wanted_depth > stack.length ) { - // Deep enough for a nested list outright - //print ( "new nested list" ); - list = make_list( m ); - last_li.push( list ); - last_li = list[1] = [ "listitem" ]; - } - else { - // We aren't deep enough to be strictly a new level. This is - // where Md.pl goes nuts. If the indent matches a level in the - // stack, put it there, else put it one deeper then the - // wanted_depth deserves. - var found = false; - for ( i = 0; i < stack.length; i++ ) { - if ( stack[ i ].indent !== m[1] ) - continue; - - list = stack[ i ].list; - stack.splice( i+1, stack.length - (i+1) ); - found = true; - break; - } - - if (!found) { - //print("not found. l:", uneval(l)); - wanted_depth++; - if ( wanted_depth <= stack.length ) { - stack.splice(wanted_depth, stack.length - wanted_depth); - //print("Desired depth now", wanted_depth, "stack:", stack.length); - list = stack[wanted_depth-1].list; - //print("list:", uneval(list) ); - } - else { - //print ("made new stack for messy indent"); - list = make_list(m); - last_li.push(list); - } - } - - //print( uneval(list), "last", list === stack[stack.length-1].list ); - last_li = [ "listitem" ]; - list.push(last_li); - } // end depth of shenegains - nl = ""; - } - - // Add content - if ( l.length > m[0].length ) - li_accumulate += nl + l.substr( m[0].length ); - } // tight_search - - if ( li_accumulate.length ) { - - var contents = this.processBlock(li_accumulate, []), - firstBlock = contents[0]; - - if (firstBlock) { - firstBlock.shift(); - contents.splice.apply(contents, [0, 1].concat(firstBlock)); - add( last_li, loose, contents, nl ); - - // Let's not creating a trailing \n after content in the li - if(last_li[last_li.length-1] === "\n") { - last_li.pop(); - } - - // Loose mode will have been dealt with. Reset it - loose = false; - li_accumulate = ""; - } - } - - // Look at the next block - we might have a loose list. Or an extra - // paragraph for the current li - var contained = get_contained_blocks( stack.length, next ); - - // Deal with code blocks or properly nested lists - if ( contained.length > 0 ) { - // Make sure all listitems up the stack are paragraphs - forEach( stack, paragraphify, this); - - last_li.push.apply( last_li, this.toTree( contained, [] ) ); - } - - var next_block = next[0] && next[0].valueOf() || ""; - - if ( next_block.match(is_list_re) ) { - block = next.shift(); - - // Check for an HR following a list: features/lists/hr_abutting - var hr = this.dialect.block.horizRule.call( this, block, next ); - - if ( hr ) { - ret.push.apply(ret, hr); - break; - } - - // Add paragraphs if the indentation level stays the same - if (stack[stack.length-1].indent === block.match(/^\s*/)[0]) { - forEach( stack, paragraphify, this); - } - - loose = true; - continue loose_search; - } - break; - } // loose_search - - return ret; - }; - })(), - - blockquote: function blockquote( block, next ) { - - // Handle quotes that have spaces before them - var m = /(^|\n) +(\>[\s\S]*)/.exec(block); - if (m && m[2] && m[2].length) { - var blockContents = block.replace(/(^|\n) +\>/, "$1>"); - next.unshift(blockContents); - return []; - } - - if ( !block.match( /^>/m ) ) - return undefined; - - var jsonml = []; - - // separate out the leading abutting block, if any. I.e. in this case: - // - // a - // > b - // - if ( block[ 0 ] !== ">" ) { - var lines = block.split( /\n/ ), - prev = [], - line_no = block.lineNumber; - - // keep shifting lines until you find a crotchet - while ( lines.length && lines[ 0 ][ 0 ] !== ">" ) { - prev.push( lines.shift() ); - line_no++; - } - - var abutting = mk_block( prev.join( "\n" ), "\n", block.lineNumber ); - jsonml.push.apply( jsonml, this.processBlock( abutting, [] ) ); - // reassemble new block of just block quotes! - block = mk_block( lines.join( "\n" ), block.trailing, line_no ); - } - - // if the next block is also a blockquote merge it in - while ( next.length && next[ 0 ][ 0 ] === ">" ) { - var b = next.shift(); - block = mk_block( block + block.trailing + b, b.trailing, block.lineNumber ); - } - - // Strip off the leading "> " and re-process as a block. - var input = block.replace( /^> ?/gm, "" ), - old_tree = this.tree, - processedBlock = this.toTree( input, [ "blockquote" ] ), - attr = extract_attr( processedBlock ); - - // If any link references were found get rid of them - if ( attr && attr.references ) { - delete attr.references; - // And then remove the attribute object if it's empty - if ( isEmpty( attr ) ) - processedBlock.splice( 1, 1 ); - } - - jsonml.push( processedBlock ); - return jsonml; - }, - - referenceDefn: function referenceDefn( block, next) { - var re = /^\s*\[([^\[\]]+)\]:\s*(\S+)(?:\s+(?:(['"])(.*)\3|\((.*?)\)))?\n?/; - // interesting matches are [ , ref_id, url, , title, title ] - - if ( !block.match(re) ) - return undefined; - - var attrs = create_attrs.call( this ); - - var b = this.loop_re_over_block(re, block, function( m ) { - create_reference(attrs, m); - } ); - - if ( b.length ) - next.unshift( mk_block( b, block.trailing ) ); - - return []; - }, - - para: function para( block ) { - // everything's a para! - return [ ["para"].concat( this.processInline( block ) ) ]; - } - }, - - inline: { - - __oneElement__: function oneElement( text, patterns_or_re, previous_nodes ) { - - // PERF NOTE: rewritten to avoid greedy match regex \([\s\S]*?)(...)\ - // greedy match performs horribly with large inline blocks, it can be so - // slow it will crash chrome - patterns_or_re = patterns_or_re || this.dialect.inline.__patterns__; - - var search_re = new RegExp(patterns_or_re.source || patterns_or_re); - var pos = text.search(search_re); - - if (pos === -1) { - return [ text.length, text ]; - } else if (pos !== 0) { - // Some un-interesting text matched. Return that first - return [pos, text.substring(0,pos)]; - } - - var match_re = new RegExp( "^(" + (patterns_or_re.source || patterns_or_re) + ")" ); - var m = match_re.exec( text ); - var res; - if ( m[1] in this.dialect.inline ) { - res = this.dialect.inline[ m[1] ].call( - this, - text.substr( m.index ), m, previous_nodes || [] ); - - // If no inline code executed, fallback - if (!res) { - var fn = this.dialect.inline[m[1][0]]; - if (fn) { - res = fn.call( - this, - text.substr( m.index ), m, previous_nodes || [] ); - } - } - } - // Default for now to make dev easier. just slurp special and output it. - res = res || [ m[1].length, m[1] ]; - return res; - }, - - __call__: function inline( text, patterns ) { - - var out = [], - res; - - function add(x) { - //D:self.debug(" adding output", uneval(x)); - if ( typeof x === "string" && typeof out[out.length-1] === "string" ) - out[ out.length-1 ] += x; - else - out.push(x); - } - - while ( text.length > 0 ) { - res = this.dialect.inline.__oneElement__.call(this, text, patterns, out ); - text = text.substr( res.shift() ); - forEach(res, add ); - } - - return out; - }, - - // These characters are interesting elsewhere, so have rules for them so that - // chunks of plain text blocks don't include them - "]": function () {}, - "}": function () {}, - - __escape__ : /^\\[\\`\*_{}<>\[\]()#\+.!\-]/, - - "\\": function escaped( text ) { - // [ length of input processed, node/children to add... ] - // Only esacape: \ ` * _ { } [ ] ( ) # * + - . ! - if ( this.dialect.inline.__escape__.exec( text ) ) - return [ 2, text.charAt( 1 ) ]; - else - // Not an esacpe - return [ 1, "\\" ]; - }, - - "![": function image( text ) { - - // Without this guard V8 crashes hard on the RegExp - if (text.indexOf('(') >= 0 && text.indexOf(')') === -1) { return; } - - // Unlike images, alt text is plain text only. no other elements are - // allowed in there - - // ![Alt text](/path/to/img.jpg "Optional title") - // 1 2 3 4 <--- captures - // - // First attempt to use a strong URL regexp to catch things like parentheses. If it misses, use the - // old one. - var origMatcher = /^!\[(.*?)\][ \t]*\([ \t]*([^")]*?)(?:[ \t]+(["'])(.*?)\3)?[ \t]*\)/; - m = text.match(new RegExp("^!\\[(.*?)][ \\t]*\\((" + urlRegexp + ")\\)([ \\t])*([\"'].*[\"'])?")) || - text.match(origMatcher); - - if (m && m[2].indexOf(")]") !== -1) { m = text.match(origMatcher); } - - if ( m ) { - if ( m[2] && m[2][0] === "<" && m[2][m[2].length-1] === ">" ) - m[2] = m[2].substring( 1, m[2].length - 1 ); - - m[2] = this.dialect.inline.__call__.call( this, m[2], /\\/ )[0]; - - var attrs = { alt: m[1], href: m[2] || "" }; - if ( m[4] !== undefined) - attrs.title = m[4]; - - return [ m[0].length, [ "img", attrs ] ]; - } - - // ![Alt text][id] - m = text.match( /^!\[(.*?)\][ \t]*\[(.*?)\]/ ); - - if ( m ) { - // We can't check if the reference is known here as it likely wont be - // found till after. Check it in md tree->hmtl tree conversion - return [ m[0].length, [ "img_ref", { alt: m[1], ref: m[2].toLowerCase(), original: m[0] } ] ]; - } - - // Just consume the '![' - return [ 2, "![" ]; - }, - - "[": function link( text ) { - - var open = 1; - for (var i=0; i 3) { return [1, "["]; } - } - - var orig = String(text); - // Inline content is possible inside `link text` - var res = inline_until_char.call( this, text.substr(1), "]" ); - - // No closing ']' found. Just consume the [ - if ( !res[1] ) { - return [ res[0] + 1, text.charAt(0) ].concat(res[2]); - } - - if ( res[0] == 1 ) { return [ 2, "[]" ]; } // empty link found. - - var consumed = 1 + res[ 0 ], - children = res[ 1 ], - link, - attrs; - - // At this point the first [...] has been parsed. See what follows to find - // out which kind of link we are (reference or direct url) - text = text.substr( consumed ); - - // [link text](/path/to/img.jpg "Optional title") - // 1 2 3 <--- captures - // This will capture up to the last paren in the block. We then pull - // back based on if there a matching ones in the url - // ([here](/url/(test)) - // The parens have to be balanced - var m = text.match( /^\s*\([ \t]*([^"'\s]*)(?:[ \t]+(["'])(.*?)\2)?[ \t]*\)/ ); - if ( m ) { - var url = m[1].replace(/\s+$/, ''); - consumed += m[0].length; - - if ( url && url[0] === "<" && url[url.length-1] === ">" ) - url = url.substring( 1, url.length - 1 ); - - // If there is a title we don't have to worry about parens in the url - if ( !m[3] ) { - var open_parens = 1; // One open that isn't in the capture - for ( var len = 0; len < url.length; len++ ) { - switch ( url[len] ) { - case "(": - open_parens++; - break; - case ")": - if ( --open_parens === 0) { - consumed -= url.length - len; - url = url.substring(0, len); - } - break; - } - } - } - - // Process escapes only - url = this.dialect.inline.__call__.call( this, url, /\\/ )[0]; - - attrs = { href: url || "" }; - if ( m[3] !== undefined) - attrs.title = m[3]; - - link = [ "link", attrs ].concat( children ); - return [ consumed, link ]; - } - - if (text.indexOf('(') === 0 && text.indexOf(')') !== -1) { - m = text.match(new RegExp("^\\((" + urlRegexp + ")\\)")); - if (m && m[1]) { - consumed += m[0].length; - link = ["link", {href: m[1]}].concat(children); - return [consumed, link]; - } - } - - // [Alt text][id] - // [Alt text] [id] - m = text.match( /^\s*\[(.*?)\]/ ); - if ( m ) { - - consumed += m[ 0 ].length; - - // [links][] uses links as its reference - attrs = { ref: ( m[ 1 ] || String(children) ).toLowerCase(), original: orig.substr( 0, consumed ) }; - - if (children && children.length > 0) { - link = [ "link_ref", attrs ].concat( children ); - - // We can't check if the reference is known here as it likely wont be - // found till after. Check it in md tree->hmtl tree conversion. - // Store the original so that conversion can revert if the ref isn't found. - return [ consumed, link ]; - } - } - - // Another check for references - m = orig.match(/^\s*\[(.*?)\]:\s*(\S+)(?:\s+(?:(['"])(.*?)\3|\((.*?)\)))?\n?/); - if (m && - (/^(?:(?:https?|ftp):\/\/)(?:\S+(?::\S*)?@)?(?:(?!10(?:\.\d{1,3}){3})(?!127(?:\.\d{1,3}){3})(?!169\.254(?:\.\d{1,3}){2})(?!192\.168(?:\.\d{1,3}){2})(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))|(?:(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)(?:\.(?:[a-z\u00a1-\uffff0-9]+-?)*[a-z\u00a1-\uffff0-9]+)*(?:\.(?:[a-z\u00a1-\uffff]{2,})))(?::\d{2,5})?(?:\/[^\s]*)?$/i.test(m[2]) || - /(\/[\w~,;\-\./?%&+#=]*)/.test(m[2]))) { - attrs = create_attrs.call(this); - create_reference(attrs, m); - - return [ m[0].length ]; - } - - // [id] - // Only if id is plain (no formatting.) - if ( children.length === 1 && typeof children[0] === "string" ) { - - var normalized = children[0].toLowerCase().replace(/\s+/, ' '); - attrs = { ref: normalized, original: orig.substr( 0, consumed ) }; - link = [ "link_ref", attrs, children[0] ]; - return [ consumed, link ]; - } - - // Just consume the "[" - return [ 1, "[" ]; - }, - - "<": function autoLink( text ) { - var m; - - if ( ( m = text.match( /^<(?:((https?|ftp|mailto):[^>]+)|(.*?@.*?\.[a-zA-Z]+))>/ ) ) !== null ) { - if ( m[3] ) - return [ m[0].length, [ "link", { href: "mailto:" + m[3] }, m[3] ] ]; - else if ( m[2] === "mailto" ) - return [ m[0].length, [ "link", { href: m[1] }, m[1].substr("mailto:".length ) ] ]; - else - return [ m[0].length, [ "link", { href: m[1] }, m[1] ] ]; - } - - return [ 1, "<" ]; - }, - - "`": function inlineCode( text, match, prev ) { - - // If we're in a tag, don't do it. - if (prev && (typeof prev[0] === "string") && prev[0].match(/<[^>]+$/)) { return; } - - // Inline code block. as many backticks as you like to start it - // Always skip over the opening ticks. - var m = text.match( /(`+)(([\s\S]*?)\1)/ ); - - if ( m && m[2] ) - return [ m[1].length + m[2].length, [ "inlinecode", m[3] ] ]; - else { - // TODO: No matching end code found - warn! - return [ 1, "`" ]; - } - }, - - " \n": function lineBreak() { - return [ 3, [ "linebreak" ] ]; - } - - } - }; - - // A helper function to create attributes - function create_attrs() { - if ( !extract_attr( this.tree ) ) { - this.tree.splice( 1, 0, {} ); - } - - var attrs = extract_attr( this.tree ); - - // make a references hash if it doesn't exist - if ( attrs.references === undefined ) { - attrs.references = {}; - } - - return attrs; - } - - // Create references for attributes - function create_reference(attrs, m) { - if ( m[2] && m[2][0] === "<" && m[2][m[2].length-1] === ">" ) - m[2] = m[2].substring( 1, m[2].length - 1 ); - - var ref = attrs.references[ m[1].toLowerCase() ] = { - href: m[2] - }; - - if ( m[4] !== undefined ) - ref.title = m[4]; - else if ( m[5] !== undefined ) - ref.title = m[5]; - } - - Markdown.dialects.Gruber = Gruber; - Markdown.buildBlockOrder ( Markdown.dialects.Gruber.block ); - Markdown.buildInlinePatterns( Markdown.dialects.Gruber.inline ); - -// Include all our dependencies and return the resulting library. - - expose.Markdown = Markdown; - expose.parse = Markdown.parse; - expose.toHTML = Markdown.toHTML; - expose.toHTMLTree = Markdown.toHTMLTree; - expose.renderJsonML = Markdown.renderJsonML; - expose.DialectHelpers = DialectHelpers; - -})(function() { - window.BetterMarkdown = {}; - return window.BetterMarkdown; -}()); From f1b38ba4fb778c08a45dd001300fc651c92718ba Mon Sep 17 00:00:00 2001 From: Sam Date: Wed, 12 Jul 2017 16:59:58 -0400 Subject: [PATCH 048/382] Integrate new engine, correct old specs corrects edge cases with - full quotes - [url] with nested tags - engine overrides - onebox applying to non http srcs --- .../engines/discourse-markdown-it.js.es6 | 3 +- .../engines/discourse-markdown.js.es6 | 597 ------------------ .../discourse-markdown/bbcode-inline.js.es6 | 47 +- .../engines/discourse-markdown/onebox.js.es6 | 8 +- .../engines/discourse-markdown/quotes.js.es6 | 2 +- .../pretty-text/pretty-text.js.es6 | 32 +- lib/pretty_text.rb | 10 +- spec/components/pretty_text_spec.rb | 30 +- test/javascripts/lib/pretty-text-test.js.es6 | 382 ++++++----- test/javascripts/lib/sanitizer-test.js.es6 | 10 +- 10 files changed, 341 insertions(+), 780 deletions(-) delete mode 100644 app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6 diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 index 5ff68ef795..781a186d84 100644 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 +++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown-it.js.es6 @@ -227,10 +227,11 @@ export function setup(opts, siteSettings, state) { opts.markdownIt = true; opts.setup = true; - if (!opts.discourse.sanitizer) { + if (!opts.discourse.sanitizer || !opts.sanitizer) { const whiteLister = new WhiteLister(opts.discourse); opts.sanitizer = opts.discourse.sanitizer = (!!opts.discourse.sanitize) ? a=>sanitize(a, whiteLister) : a=>a; } + } export function cook(raw, opts) { diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6 deleted file mode 100644 index 3303cc34b9..0000000000 --- a/app/assets/javascripts/pretty-text/engines/discourse-markdown.js.es6 +++ /dev/null @@ -1,597 +0,0 @@ -import guid from 'pretty-text/guid'; -import { default as WhiteLister, whiteListFeature } from 'pretty-text/white-lister'; -import { escape } from 'pretty-text/sanitizer'; - -var parser = window.BetterMarkdown, - MD = parser.Markdown, - DialectHelpers = parser.DialectHelpers, - hoisted; - -let currentOpts; - -const emitters = []; -const preProcessors = []; -const parseNodes = []; - -function findEndPos(text, start, stop, args, offset) { - let endPos, nextStart; - do { - endPos = text.indexOf(stop, offset); - if (endPos === -1) { return -1; } - nextStart = text.indexOf(start, offset); - offset = endPos + stop.length; - } while (nextStart !== -1 && nextStart < endPos); - return endPos; -} - -class DialectHelper { - constructor() { - this._dialect = MD.dialects.Discourse = DialectHelpers.subclassDialect(MD.dialects.Gruber); - this._setup = false; - } - - escape(str) { - return escape(str); - } - - getOptions() { - return currentOpts; - } - - registerInlineFeature(featureName, start, fn) { - this._dialect.inline[start] = function() { - if (!currentOpts.features[featureName]) { return; } - return fn.apply(this, arguments); - }; - } - - addPreProcessorFeature(featureName, fn) { - preProcessors.push(raw => { - if (!currentOpts.features[featureName]) { return raw; } - return fn(raw, hoister); - }); - } - - /** - The simplest kind of replacement possible. Replace a stirng token with JsonML. - - For example to replace all occurrances of :) with a smile image: - - ```javascript - helper.inlineReplace(':)', text => ['img', {src: '/images/smile.png'}]); - ``` - **/ - inlineReplaceFeature(featureName, token, emitter) { - this.registerInline(token, (text, match, prev) => { - if (!currentOpts.features[featureName]) { return; } - return [token.length, emitter.call(this, token, match, prev)]; - }); - } - - /** - After the parser has been executed, change the contents of a HTML tag. - - Let's say you want to replace the contents of all code tags to prepend - "EVIL TROUT HACKED YOUR CODE!": - - ```javascript - helper.postProcessTag('code', contents => `EVIL TROUT HACKED YOUR CODE!\n\n${contents}`); - ``` - **/ - postProcessTagFeature(featureName, tag, emitter) { - this.onParseNode(event => { - if (!currentOpts.features[featureName]) { return; } - const node = event.node; - if (node[0] === tag) { - node[node.length-1] = emitter(node[node.length-1]); - } - }); - } - - /** - Matches inline using a regular expression. The emitter function is passed - the matches from the regular expression. - - For example, this auto links URLs: - - ```javascript - helper.inlineRegexp({ - matcher: /((?:https?:(?:\/{1,3}|[a-z0-9%])|www\d{0,3}[.])(?:[^\s()<>]+|\([^\s()<>]+\))+(?:\([^\s()<>]+\)|[^`!()\[\]{};:'".,<>?«»“”‘’\s]))/gm, - spaceBoundary: true, - start: 'http', - - emitter(matches) { - const url = matches[1]; - return ['a', {href: url}, url]; - } - }); - ``` - **/ - inlineRegexpFeature(featureName, args) { - this.registerInline(args.start, function(text, match, prev) { - if (!currentOpts.features[featureName]) { return; } - if (invalidBoundary(args, prev)) { return; } - - args.matcher.lastIndex = 0; - const m = args.matcher.exec(text); - if (m) { - const result = args.emitter.call(this, m); - if (result) { - return [m[0].length, result]; - } - } - }); - } - - /** - Handles inline replacements surrounded by tokens. - - For example, to handle markdown style bold. Note we use `concat` on the array because - the contents are JsonML too since we didn't pass `rawContents` as true. This supports - recursive markup. - - ```javascript - helper.inlineBetween({ - between: '**', - wordBoundary: true. - emitter(contents) { - return ['strong'].concat(contents); - } - }); - ``` - **/ - inlineBetweenFeature(featureName, args) { - const start = args.start || args.between; - const stop = args.stop || args.between; - const startLength = start.length; - - this.registerInline(start, function(text, match, prev) { - if (!currentOpts.features[featureName]) { return; } - if (invalidBoundary(args, prev)) { return; } - - const endPos = findEndPos(text, start, stop, args, startLength); - if (endPos === -1) { return; } - var between = text.slice(startLength, endPos); - - // If rawcontents is set, don't process inline - if (!args.rawContents) { - between = this.processInline(between); - } - - var contents = args.emitter.call(this, between); - if (contents) { - return [endPos+stop.length, contents]; - } - }); - } - - /** - Replaces a block of text between a start and stop. As opposed to inline, these - might span multiple lines. - - Here's an example that takes the content between `[code]` ... `[/code]` and - puts them inside a `pre` tag: - - ```javascript - helper.replaceBlock({ - start: /(\[code\])([\s\S]*)/igm, - stop: '[/code]', - rawContents: true, - - emitter(blockContents) { - return ['p', ['pre'].concat(blockContents)]; - } - }); - ``` - **/ - replaceBlockFeature(featureName, args) { - function blockFunc(block, next) { - if (!currentOpts.features[featureName]) { return; } - - const linebreaks = currentOpts.traditionalMarkdownLinebreaks; - if (linebreaks && args.skipIfTradtionalLinebreaks) { return; } - - args.start.lastIndex = 0; - const result = []; - const match = (args.start).exec(block); - if (!match) { return; } - - const lastChance = () => !next.some(blk => blk.match(args.stop)); - - // shave off start tag and leading text, if any. - const pos = args.start.lastIndex - match[0].length; - const leading = block.slice(0, pos); - const trailing = match[2] ? match[2].replace(/^\n*/, "") : ""; - - // The other leading block should be processed first! eg a code block wrapped around a code block. - if (args.withoutLeading && args.withoutLeading.test(leading)) { - return; - } - - // just give up if there's no stop tag in this or any next block - args.stop.lastIndex = block.length - trailing.length; - if (!args.stop.exec(block) && lastChance()) { return; } - if (leading.length > 0) { - var parsedLeading = this.processBlock(MD.mk_block(leading), []); - if (parsedLeading && parsedLeading[0]) { - result.push(parsedLeading[0]); - } - } - if (trailing.length > 0) { - next.unshift(MD.mk_block(trailing, block.trailing, - block.lineNumber + countLines(leading) + (match[2] ? match[2].length : 0) - trailing.length)); - } - - // go through the available blocks to find the matching stop tag. - const contentBlocks = []; - let nesting = 0; - let actualEndPos = -1; - let currentBlock; - - blockloop: - while (currentBlock = next.shift()) { - - // collect all the start and stop tags in the current block - args.start.lastIndex = 0; - const startPos = []; - let m; - while (m = (args.start).exec(currentBlock)) { - startPos.push(args.start.lastIndex - m[0].length); - args.start.lastIndex = args.start.lastIndex - (m[2] ? m[2].length : 0); - } - args.stop.lastIndex = 0; - const endPos = []; - while (m = (args.stop).exec(currentBlock)) { - endPos.push(args.stop.lastIndex - m[0].length); - } - - // go through the available end tags: - let ep = 0; - let sp = 0; - while (ep < endPos.length) { - if (sp < startPos.length && startPos[sp] < endPos[ep]) { - // there's an end tag, but there's also another start tag first. we need to go deeper. - sp++; nesting++; - } else if (nesting > 0) { - // found an end tag, but we must go up a level first. - ep++; nesting--; - } else { - // found an end tag and we're at the top: done! -- or: start tag and end tag are - // identical, (i.e. startPos[sp] == endPos[ep]), so we don't do nesting at all. - actualEndPos = endPos[ep]; - break blockloop; - } - } - - if (lastChance()) { - // when lastChance() becomes true the first time, currentBlock contains the last - // end tag available in the input blocks but it's not on the right nesting level - // or we would have terminated the loop already. the only thing we can do is to - // treat the last available end tag as tho it were matched with our start tag - // and let the emitter figure out how to render the garbage inside. - actualEndPos = endPos[endPos.length - 1]; - break; - } - - // any left-over start tags still increase the nesting level - nesting += startPos.length - sp; - contentBlocks.push(currentBlock); - } - - const stopLen = currentBlock.match(args.stop)[0].length; - const before = currentBlock.slice(0, actualEndPos).replace(/\n*$/, ""); - const after = currentBlock.slice(actualEndPos + stopLen).replace(/^\n*/, ""); - if (before.length > 0) contentBlocks.push(MD.mk_block(before, "", currentBlock.lineNumber)); - if (after.length > 0) next.unshift(MD.mk_block(after, currentBlock.trailing, currentBlock.lineNumber + countLines(before))); - - const emitterResult = args.emitter.call(this, contentBlocks, match); - if (emitterResult) { result.push(emitterResult); } - return result; - }; - - if (args.priority) { - blockFunc.priority = args.priority; - } - - this.registerBlock(args.start.toString(), blockFunc); - } - - /** - After the parser has been executed, post process any text nodes in the HTML document. - This is useful if you want to apply a transformation to the text. - - If you are generating HTML from the text, it is preferable to use the replacer - functions and do it in the parsing part of the pipeline. This function is best for - simple transformations or transformations that have to happen after all earlier - processing is done. - - For example, to convert all text to upper case: - - ```javascript - helper.postProcessText(function (text) { - return text.toUpperCase(); - }); - ``` - **/ - postProcessTextFeature(featureName, fn) { - emitters.push(function () { - if (!currentOpts.features[featureName]) { return; } - return fn.apply(this, arguments); - }); - } - - onParseNodeFeature(featureName, fn) { - parseNodes.push(function () { - if (!currentOpts.features[featureName]) { return; } - return fn.apply(this, arguments); - }); - } - - registerBlockFeature(featureName, name, fn) { - const blockFunc = function() { - if (!currentOpts.features[featureName]) { return; } - return fn.apply(this, arguments); - }; - - blockFunc.priority = fn.priority; - this._dialect.block[name] = blockFunc; - } - - applyFeature(featureName, module) { - helper.registerInline = (code, fn) => helper.registerInlineFeature(featureName, code, fn); - helper.replaceBlock = args => helper.replaceBlockFeature(featureName, args); - helper.addPreProcessor = fn => helper.addPreProcessorFeature(featureName, fn); - helper.inlineReplace = (token, emitter) => helper.inlineReplaceFeature(featureName, token, emitter); - helper.postProcessTag = (token, emitter) => helper.postProcessTagFeature(featureName, token, emitter); - helper.inlineRegexp = args => helper.inlineRegexpFeature(featureName, args); - helper.inlineBetween = args => helper.inlineBetweenFeature(featureName, args); - helper.postProcessText = fn => helper.postProcessTextFeature(featureName, fn); - helper.onParseNode = fn => helper.onParseNodeFeature(featureName, fn); - helper.registerBlock = (name, fn) => helper.registerBlockFeature(featureName, name, fn); - - module.setup(this); - } - - setup() { - if (this._setup) { return; } - this._setup = true; - - Object.keys(require._eak_seen).forEach(entry => { - if (entry.indexOf('discourse-markdown') !== -1) { - const module = requirejs(entry); - if (module && module.setup) { - const featureName = entry.split('/').reverse()[0]; - helper.whiteList = info => whiteListFeature(featureName, info); - - this.applyFeature(featureName, module); - helper.whiteList = undefined; - } - } - }); - - MD.buildBlockOrder(this._dialect.block); - var index = this._dialect.block.__order__.indexOf("code"); - if (index > -1) { - this._dialect.block.__order__.splice(index, 1); - this._dialect.block.__order__.unshift("code"); - } - MD.buildInlinePatterns(this._dialect.inline); - } -}; - -const helper = new DialectHelper(); - -export function cook(raw, opts) { - currentOpts = opts; - - hoisted = {}; - - if (!currentOpts.enableExperimentalMarkdownIt) { - raw = hoistCodeBlocksAndSpans(raw); - preProcessors.forEach(p => raw = p(raw)); - } - - const whiteLister = new WhiteLister(opts); - - let result; - - if (currentOpts.enableExperimentalMarkdownIt) { - result = opts.sanitizer( - requirejs('pretty-text/engines/markdown-it/instance').default(opts).render(raw), - whiteLister - ); - } else { - const tree = parser.toHTMLTree(raw, 'Discourse'); - result = opts.sanitizer(parser.renderJsonML(parseTree(tree, opts)), whiteLister); - } - - // If we hoisted out anything, put it back - const keys = Object.keys(hoisted); - if (keys.length) { - let found = true; - - const unhoist = function(key) { - result = result.replace(new RegExp(key, "g"), function() { - found = true; - return hoisted[key]; - }); - }; - - while (found) { - found = false; - keys.forEach(unhoist); - } - } - - return result.trim(); -} - -export function setup() { - helper.setup(); -} - -function processTextNodes(node, event, emitter) { - if (node.length < 2) { return; } - - if (node[0] === '__RAW') { - const hash = guid(); - hoisted[hash] = node[1]; - node[1] = hash; - return; - } - - for (var j=1; j fn(event)); - - for (var j=0; j$/.exec(n[1])) { - // Remove paragraphs around comment-only nodes. - tree[i] = n[1]; - } else { - parseTree(n, options, path, insideCounts); - } - - insideCounts[tagName] = insideCounts[tagName] - 1; - } - - // If raw nodes are in paragraphs, pull them up - if (tree.length === 2 && tree[0] === 'p' && tree[1] instanceof Array && tree[1][0] === "__RAW") { - var text = tree[1][1]; - tree[0] = "__RAW"; - tree[1] = text; - } - - path.pop(); - } - return tree; -} - -// Returns true if there's an invalid word boundary for a match. -function invalidBoundary(args, prev) { - if (!(args.wordBoundary || args.spaceBoundary || args.spaceOrTagBoundary)) { return false; } - - var last = prev[prev.length - 1]; - if (typeof last !== "string") { return false; } - - if (args.wordBoundary && (!last.match(/\W$/))) { return true; } - if (args.spaceBoundary && (!last.match(/\s$/))) { return true; } - if (args.spaceOrTagBoundary && (!last.match(/(\s|\>|\()$/))) { return true; } -} - -function countLines(str) { - let index = -1, count = 0; - while ((index = str.indexOf("\n", index + 1)) !== -1) { count++; } - return count; -} - -function hoister(t, target, replacement) { - const regexp = new RegExp(target.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&'), "g"); - if (t.match(regexp)) { - const hash = guid(); - t = t.replace(regexp, hash); - hoisted[hash] = replacement; - } - return t; -} - -function outdent(t) { - return t.replace(/^([ ]{4}|\t)/gm, ""); -} - -function removeEmptyLines(t) { - return t.replace(/^\n+/, "").replace(/\s+$/, ""); -} - -function hideBackslashEscapedCharacters(t) { - return t.replace(/\\\\/g, "\u1E800").replace(/\\`/g, "\u1E8001"); -} - -function showBackslashEscapedCharacters(t) { - return t.replace(/\u1E8001/g, "\\`").replace(/\u1E800/g, "\\\\"); -} - -function hoistCodeBlocksAndSpans(text) { - // replace all "\`" with a single character - text = hideBackslashEscapedCharacters(text); - - // /!\ the order is important /!\ - - // fenced code blocks (AKA GitHub code blocks) - text = text.replace(/(^\n*|\n)```([a-z0-9\-]*)\n([\s\S]*?)\n```/g, function(_, before, language, content) { - const hash = guid(); - hoisted[hash] = escape(showBackslashEscapedCharacters(removeEmptyLines(content))); - return before + "```" + language + "\n" + hash + "\n```"; - }); - - // markdown code blocks - text = text.replace(/(^\n*|\n\n)((?:(?:[ ]{4}|\t).*\n*)+)/g, function(match, before, content, index) { - // make sure we aren't in a list - var previousLine = text.slice(0, index).trim().match(/.*$/); - if (previousLine && previousLine[0].length) { - previousLine = previousLine[0].trim(); - if (/^(?:\*|\+|-|\d+\.)\s+/.test(previousLine)) { - return match; - } - } - // we can safely hoist the code block - const hash = guid(); - hoisted[hash] = escape(outdent(showBackslashEscapedCharacters(removeEmptyLines(content)))); - return before + " " + hash + "\n"; - }); - - // 
    ...
    code blocks - text = text.replace(/(\s|^)
    ([\s\S]*?)<\/pre>/ig, function(_, before, content) {
-    const hash = guid();
-    hoisted[hash] = escape(showBackslashEscapedCharacters(removeEmptyLines(content)));
-    return before + "
    " + hash + "
    ";
-  });
-
-  // code spans (double & single `)
-  ["``", "`"].forEach(function(delimiter) {
-    var regexp = new RegExp("(^|[^`])" + delimiter + "([^`\\n]+?)" + delimiter + "([^`]|$)", "g");
-    text = text.replace(regexp, function(_, before, content, after) {
-      const hash = guid();
-      hoisted[hash] = escape(showBackslashEscapedCharacters(content.trim()));
-      return before + delimiter + hash + delimiter + after;
-    });
-  });
-
-  // replace back all weird character with "\`"
-  return showBackslashEscapedCharacters(text);
-}
diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6
index 71ce50462a..a2dfa57143 100644
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/bbcode-inline.js.es6
@@ -57,6 +57,7 @@ function tokanizeBBCode(state, silent, ruler) {
 
     let token = state.push('text', '' , 0);
     token.content = state.src.slice(pos, pos+tagInfo.length);
+    token.meta = 'bbcode';
 
     state.delimiters.push({
       bbInfo: tagInfo,
@@ -105,10 +106,15 @@ function processBBCode(state, silent) {
     let tag, className;
 
     if (typeof tagInfo.rule.wrap === 'function') {
-      if (!tagInfo.rule.wrap(token, tagInfo)) {
-        return false;
+      let content = "";
+      for (let j = startDelim.token+1; j < endDelim.token; j++) {
+        let inner = state.tokens[j];
+        if (inner.type === 'text' && inner.meta !== 'bbcode') {
+          content += inner.content;
+        }
       }
-      tag = token.tag;
+      tagInfo.rule.wrap(token, state.tokens[endDelim.token], tagInfo, content);
+      continue;
     } else {
       let split = tagInfo.rule.wrap.split('.');
       tag = split[0];
@@ -160,19 +166,35 @@ export function setup(helper) {
       }
     });
 
+    const simpleUrlRegex = /^http[s]?:\/\//;
     ruler.push('url', {
       tag: 'url',
-      replace: function(state, tagInfo, content) {
-        let token;
+      wrap: function(startToken, endToken, tagInfo, content) {
 
-        token = state.push('link_open', 'a', 1);
-        token.attrs = [['href', content], ['data-bbcode', 'true']];
+        const url = (tagInfo.attrs['_default'] || content).trim();
 
-        token = state.push('text', '', 0);
-        token.content = content;
+        if (simpleUrlRegex.test(url)) {
+          startToken.type = 'link_open';
+          startToken.tag = 'a';
+          startToken.attrs = [['href', url], ['data-bbcode', 'true']];
+          startToken.content = '';
+          startToken.nesting = 1;
 
-        token = state.push('link_close', 'a', -1);
-        return true;
+          endToken.type = 'link_close';
+          endToken.tag = 'a';
+          endToken.content = '';
+          endToken.nesting = -1;
+        } else {
+          // just strip the bbcode tag
+          endToken.content = '';
+          startToken.content = '';
+
+          // edge case, we don't want this detected as a onebox if auto linked
+          // this ensures it is not stripped
+          startToken.type = 'html_inline';
+        }
+
+        return false;
       }
     });
 
@@ -180,9 +202,10 @@ export function setup(helper) {
       tag: 'email',
       replace: function(state, tagInfo, content) {
         let token;
+        let email = tagInfo.attrs['_default'] || content;
 
         token = state.push('link_open', 'a', 1);
-        token.attrs = [['href', 'mailto:' + content], ['data-bbcode', 'true']];
+        token.attrs = [['href', 'mailto:' + email], ['data-bbcode', 'true']];
 
         token = state.push('text', '', 0);
         token.content = content;
diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6
index 5f04b72b1d..788cdaed14 100644
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/onebox.js.es6
@@ -22,6 +22,7 @@ function applyOnebox(state, silent) {
           if (j === 0 && token.leading_space) {
             continue;
           } else if (j > 0) {
+
             let prevSibling = token.children[j-1];
 
             if (prevSibling.tag !== 'br' || prevSibling.leading_space) {
@@ -45,8 +46,12 @@ function applyOnebox(state, silent) {
             continue;
           }
 
-          // we already know text matches cause it is an auto link
+          // edge case ... what if this is not http or protocoless?
+          if (!/^http|^\/\//i.test(attrs[0][1])) {
+            continue;
+          }
 
+          // we already know text matches cause it is an auto link
           if (!close || close.type !== "link_close") {
             continue;
           }
@@ -71,6 +76,7 @@ function applyOnebox(state, silent) {
           } else {
             // decorate...
             attrs.push(["class", "onebox"]);
+            attrs.push(["target", "_blank"]);
           }
         }
       }
diff --git a/app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6 b/app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6
index 05afd6bc9e..ba892ef706 100644
--- a/app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6
+++ b/app/assets/javascripts/pretty-text/engines/discourse-markdown/quotes.js.es6
@@ -26,7 +26,7 @@ const rule = {
           continue;
         }
 
-        if (split[i].indexOf(/full:\s*true/) === 0) {
+        if (/full:\s*true/.test(split[i])) {
           full = true;
           continue;
         }
diff --git a/app/assets/javascripts/pretty-text/pretty-text.js.es6 b/app/assets/javascripts/pretty-text/pretty-text.js.es6
index bfd6eaf904..32064527ae 100644
--- a/app/assets/javascripts/pretty-text/pretty-text.js.es6
+++ b/app/assets/javascripts/pretty-text/pretty-text.js.es6
@@ -1,12 +1,9 @@
 import { cook as cookIt, setup as setupIt } from 'pretty-text/engines/discourse-markdown-it';
-import { sanitize } from 'pretty-text/sanitizer';
-import WhiteLister from 'pretty-text/white-lister';
 
-const _registerFns = [];
-const identity = value => value;
-
-export function registerOption(fn) {
-  _registerFns.push(fn);
+export function registerOption() {
+  if (window.console) {
+    window.console.log("registerOption is deprecated");
+  }
 }
 
 export function buildOptions(state) {
@@ -24,7 +21,7 @@ export function buildOptions(state) {
     emojiUnicodeReplacer
   } = state;
 
-  const features = {
+  let features = {
     'bold-italics': true,
     'auto-link': true,
     'mentions': true,
@@ -36,6 +33,10 @@ export function buildOptions(state) {
     'newline': !siteSettings.traditional_markdown_linebreaks
   };
 
+  if (state.features) {
+    features = _.merge(features, state.features);
+  }
+
   const options = {
     sanitize: true,
     getURL,
@@ -54,6 +55,8 @@ export function buildOptions(state) {
     markdownIt: true
   };
 
+  // note, this will mutate options due to the way the API is designed
+  // may need a refactor
   setupIt(options, siteSettings, state);
 
   return options;
@@ -61,9 +64,14 @@ export function buildOptions(state) {
 
 export default class {
   constructor(opts) {
-    this.opts = opts || {};
-    this.opts.features = this.opts.features || {};
-    this.opts.sanitizer = (!!this.opts.sanitize) ? (this.opts.sanitizer || sanitize) : identity;
+    if (!opts) {
+      opts = buildOptions({ siteSettings: {}});
+    }
+    this.opts = opts;
+  }
+
+  disableSanitizer() {
+    this.opts.sanitizer = this.opts.discourse.sanitizer = ident => ident;
   }
 
   cook(raw) {
@@ -75,6 +83,6 @@ export default class {
   }
 
   sanitize(html) {
-    return this.opts.sanitizer(html, new WhiteLister(this.opts));
+    return this.opts.sanitizer(html).trim();
   }
 };
diff --git a/lib/pretty_text.rb b/lib/pretty_text.rb
index 0fefc15b6e..fd1e73c68a 100644
--- a/lib/pretty_text.rb
+++ b/lib/pretty_text.rb
@@ -175,12 +175,14 @@ module PrettyText
 
       buffer << "__textOptions = __buildOptions(__optInput);\n"
 
-      # Be careful disabling sanitization. We allow for custom emails
-      if opts[:sanitize] == false
-        buffer << ('__textOptions.sanitize = false;')
-      end
 
       buffer << ("__pt = new __PrettyText(__textOptions);")
+
+      # Be careful disabling sanitization. We allow for custom emails
+      if opts[:sanitize] == false
+        buffer << ('__pt.disableSanitizer();')
+      end
+
       opts = context.eval(buffer)
 
       DiscourseEvent.trigger(:markdown_context, context)
diff --git a/spec/components/pretty_text_spec.rb b/spec/components/pretty_text_spec.rb
index 49bf5db18a..b3aa2468c9 100644
--- a/spec/components/pretty_text_spec.rb
+++ b/spec/components/pretty_text_spec.rb
@@ -61,7 +61,7 @@ describe PrettyText do
           [/quote]
         MD
         html = <<~HTML
-