From 788719d27143ec2a26feba1a84c413d958df13ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9gis=20Hanol?= Date: Fri, 4 Jan 2019 15:30:17 +0100 Subject: [PATCH] DEV: speed up posts base imports --- lib/post_creator.rb | 22 +++++------- lib/upload_creator.rb | 14 ++------ script/import_scripts/base.rb | 64 +++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 46 deletions(-) diff --git a/lib/post_creator.rb b/lib/post_creator.rb index c5c09d04d1..173a30cf20 100644 --- a/lib/post_creator.rb +++ b/lib/post_creator.rb @@ -174,15 +174,14 @@ class PostCreator update_user_counts create_embedded_topic link_post_uploads - ensure_in_allowed_users if guardian.is_staff? unarchive_message - @post.advance_draft_sequence + @post.advance_draft_sequence unless @opts[:import_mode] @post.save_reply_relationships end end - if @post && errors.blank? + if @post && errors.blank? && !@opts[:import_mode] # update counters etc. @post.topic.reload @@ -194,12 +193,10 @@ class PostCreator trigger_after_events unless opts[:skip_events] - auto_close unless @opts[:import_mode] + auto_close end - if @post || @spam - handle_spam unless @opts[:import_mode] - end + handle_spam if !opts[:import_mode] && (@post || @spam) @post end @@ -428,6 +425,8 @@ class PostCreator end def update_topic_auto_close + return if @opts[:import_mode] + if @topic.closed? @topic.delete_topic_timer(TopicTimer.types[:close]) else @@ -510,9 +509,7 @@ class PostCreator end def publish - return if @opts[:import_mode] - return unless @post.post_number > 1 - + return if @opts[:import_mode] || @post.post_number == 1 @post.publish_change_to_clients! :created end @@ -522,7 +519,7 @@ class PostCreator end def track_topic - return if @opts[:auto_track] == false + return if @opts[:import_mode] || @opts[:auto_track] == false unless @user.user_option.disable_jump_reply? TopicUser.change(@post.user_id, @@ -540,8 +537,7 @@ class PostCreator if @user.staged TopicUser.auto_notification_for_staging(@user.id, @topic.id, TopicUser.notification_reasons[:auto_watch]) - else - return if @topic.private_message? + elsif !@topic.private_message? notification_level = @user.user_option.notification_level_when_replying || NotificationLevels.topic_levels[:tracking] TopicUser.auto_notification(@user.id, @topic.id, TopicUser.notification_reasons[:created_post], notification_level) end diff --git a/lib/upload_creator.rb b/lib/upload_creator.rb index 2d7cc9166c..a1386feb2e 100644 --- a/lib/upload_creator.rb +++ b/lib/upload_creator.rb @@ -80,8 +80,8 @@ class UploadCreator end fixed_original_filename = nil - if is_image + if is_image current_extension = File.extname(@filename).downcase.sub("jpeg", "jpg") expected_extension = ".#{image_type}".downcase.sub("jpeg", "jpg") @@ -89,11 +89,7 @@ class UploadCreator # otherwise validation will fail and we can not save # TODO decide if we only run the validation on the extension if current_extension != expected_extension - basename = File.basename(@filename, current_extension) - - if basename.length == 0 - basename = "image" - end + basename = File.basename(@filename, current_extension).presence || "image" fixed_original_filename = "#{basename}#{expected_extension}" end end @@ -173,10 +169,7 @@ class UploadCreator MIN_CONVERT_TO_JPEG_SAVING_RATIO = 0.70 def convert_to_jpeg! - - if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED - return - end + return if filesize < MIN_CONVERT_TO_JPEG_BYTES_SAVED jpeg_tempfile = Tempfile.new(["image", ".jpg"]) @@ -290,7 +283,6 @@ class UploadCreator def crop! max_pixel_ratio = Discourse::PIXEL_RATIOS.max - filename_with_correct_ext = "image.#{@image_info.type}" case @opts[:type] diff --git a/script/import_scripts/base.rb b/script/import_scripts/base.rb index 3ce3ef0452..30dfb7cb08 100644 --- a/script/import_scripts/base.rb +++ b/script/import_scripts/base.rb @@ -54,6 +54,8 @@ class ImportScripts::Base update_last_posted_at update_last_seen_at update_user_stats + update_topic_users + update_post_timings update_feature_topic_users update_category_featured_topics update_topic_count_replies @@ -313,7 +315,7 @@ class ImportScripts::Base unless opts[:email][EmailValidator.email_regex] opts[:email] = fake_email - puts "Invalid email #{original_email} for #{opts[:username]}. Using: #{opts[:email]}" + puts "Invalid email '#{original_email}' for '#{opts[:username]}'. Using '#{opts[:email]}'" end opts[:name] = original_username if original_name.blank? && opts[:username] != original_username @@ -326,7 +328,7 @@ class ImportScripts::Base u = User.new(opts) (opts[:custom_fields] || {}).each { |k, v| u.custom_fields[k] = v } u.custom_fields["import_id"] = import_id - u.custom_fields["import_username"] = opts[:username] if original_username.present? + u.custom_fields["import_username"] = original_username if original_username.present? && original_username != opts[:username] u.custom_fields["import_avatar_url"] = avatar_url if avatar_url.present? u.custom_fields["import_pass"] = opts[:password] if opts[:password].present? u.custom_fields["import_email"] = original_email if original_email != opts[:email] @@ -506,21 +508,19 @@ class ImportScripts::Base import_id = params.delete(:id).to_s if post_id_from_imported_post_id(import_id) - skipped += 1 # already imported this post + skipped += 1 else begin new_post = create_post(params, import_id) if new_post.is_a?(Post) add_post(import_id, new_post) add_topic(new_post) - created_post(new_post) - created += 1 else skipped += 1 puts "Error creating post #{import_id}. Skipping." - puts new_post.inspect + p new_post end rescue Discourse::InvalidAccess => e skipped += 1 @@ -632,7 +632,7 @@ class ImportScripts::Base def update_topic_status puts "", "Updating topic status" - DB.exec(<<~SQL) + DB.exec <<~SQL UPDATE topics AS t SET closed = TRUE WHERE EXISTS( @@ -642,7 +642,7 @@ class ImportScripts::Base ) SQL - DB.exec(<<~SQL) + DB.exec <<~SQL UPDATE topics AS t SET archived = TRUE WHERE EXISTS( @@ -652,7 +652,7 @@ class ImportScripts::Base ) SQL - DB.exec(<<~SQL) + DB.exec <<~SQL DELETE FROM topic_custom_fields WHERE name IN ('import_closed', 'import_archived') SQL @@ -660,13 +660,16 @@ class ImportScripts::Base def update_bumped_at puts "", "Updating bumped_at on topics" - DB.exec("update topics t set bumped_at = COALESCE((select max(created_at) from posts where topic_id = t.id and post_type = #{Post.types[:regular]}), bumped_at)") + DB.exec <<~SQL + UPDATE topics t + SET bumped_at = COALESCE((SELECT MAX(created_at) FROM posts WHERE topic_id = t.id AND post_type = #{Post.types[:regular]}), bumped_at) + SQL end def update_last_posted_at puts "", "Updating last posted at on users" - sql = <<-SQL + DB.exec <<~SQL WITH lpa AS ( SELECT user_id, MAX(posts.created_at) AS last_posted_at FROM posts @@ -679,8 +682,6 @@ class ImportScripts::Base WHERE u1.id = users.id AND users.last_posted_at <> lpa.last_posted_at SQL - - DB.exec(sql) end def update_user_stats @@ -699,7 +700,7 @@ class ImportScripts::Base puts "", "Updating first_post_created_at..." - sql = <<-SQL + DB.exec <<~SQL WITH sub AS ( SELECT user_id, MIN(posts.created_at) AS first_post_created_at FROM posts @@ -713,11 +714,9 @@ class ImportScripts::Base AND user_stats.first_post_created_at <> sub.first_post_created_at SQL - DB.exec(sql) - puts "", "Updating user post_count..." - sql = <<-SQL + DB.exec <<~SQL WITH sub AS ( SELECT user_id, COUNT(*) AS post_count FROM posts @@ -731,11 +730,9 @@ class ImportScripts::Base AND user_stats.post_count <> sub.post_count SQL - DB.exec(sql) - puts "", "Updating user topic_count..." - sql = <<-SQL + DB.exec <<~SQL WITH sub AS ( SELECT user_id, COUNT(*) AS topic_count FROM topics @@ -748,8 +745,6 @@ class ImportScripts::Base WHERE u1.user_id = user_stats.user_id AND user_stats.topic_count <> sub.topic_count SQL - - DB.exec(sql) end # scripts that are able to import last_seen_at from the source data should override this method @@ -760,6 +755,31 @@ class ImportScripts::Base DB.exec("UPDATE users SET last_seen_at = last_posted_at WHERE last_posted_at IS NOT NULL") end + def update_topic_users + puts "", "Updating topic users" + + DB.exec <<~SQL + INSERT INTO topic_users (user_id, topic_id, posted, last_read_post_number, highest_seen_post_number, first_visited_at, last_visited_at, total_msecs_viewed) + SELECT user_id, topic_id, 't' , MAX(post_number), MAX(post_number), MIN(created_at), MAX(created_at), COUNT(id) * 5000 + FROM posts + WHERE user_id > 0 + GROUP BY user_id, topic_id + ON CONFLICT DO NOTHING + SQL + end + + def update_post_timings + puts "", "Updating post timings" + + DB.exec <<~SQL + INSERT INTO post_timings (topic_id, post_number, user_id, msecs) + SELECT topic_id, post_number, user_id, 5000 + FROM posts + WHERE user_id > 0 + ON CONFLICT DO NOTHING + SQL + end + def update_feature_topic_users puts "", "Updating featured topic users"