diff --git a/Gemfile.checksum b/Gemfile.checksum index 07512b4ede8..3cce76c0e85 100644 --- a/Gemfile.checksum +++ b/Gemfile.checksum @@ -319,7 +319,7 @@ {"name":"health_check","version":"3.1.0","platform":"ruby","checksum":"10146508237dc54ed7e24c292d8ba7fb8f9590cf26c66e325b947438c4103b57"}, {"name":"heapy","version":"0.2.0","platform":"ruby","checksum":"74141e845d61ffc7c1e8bf8b127c8cf94544ec7a1181aec613288682543585ea"}, {"name":"html-pipeline","version":"2.14.3","platform":"ruby","checksum":"8a1d4d7128b2141913387cac0f8ba898bb6812557001acc0c2b46910f59413a0"}, -{"name":"html2text","version":"0.2.0","platform":"ruby","checksum":"31c2f0be9ab7aa4fc780b07d5f84882ebc22a9024c29a45f4f5adfe42e92ad4f"}, +{"name":"html2text","version":"0.4.0","platform":"ruby","checksum":"b1becfa0b9150739633f7dc6d8637a49d7e38c3223bcb3afa3cebf59960afdc5"}, {"name":"htmlbeautifier","version":"1.4.2","platform":"ruby","checksum":"9de0c98480fe80d795ed5734a11f183563cd969686f25a04609c0f5a446fa5f8"}, {"name":"htmlentities","version":"4.3.4","platform":"ruby","checksum":"125a73c6c9f2d1b62100b7c3c401e3624441b663762afa7fe428476435a673da"}, {"name":"http","version":"5.1.1","platform":"ruby","checksum":"fcaec14a4f82de6d2f9cb978c07326814c6c2b42b8974f6ec166ff19c645ebaf"}, diff --git a/Gemfile.lock b/Gemfile.lock index 9cfca04950c..18fe5c7d25d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1009,8 +1009,8 @@ GEM html-pipeline (2.14.3) activesupport (>= 2) nokogiri (>= 1.4) - html2text (0.2.0) - nokogiri (~> 1.6) + html2text (0.4.0) + nokogiri (>= 1.0, < 2.0) htmlbeautifier (1.4.2) htmlentities (4.3.4) http (5.1.1) diff --git a/Gemfile.next.checksum b/Gemfile.next.checksum index a47d597d85e..62bc7035f93 100644 --- a/Gemfile.next.checksum +++ b/Gemfile.next.checksum @@ -319,7 +319,7 @@ {"name":"health_check","version":"3.1.0","platform":"ruby","checksum":"10146508237dc54ed7e24c292d8ba7fb8f9590cf26c66e325b947438c4103b57"}, {"name":"heapy","version":"0.2.0","platform":"ruby","checksum":"74141e845d61ffc7c1e8bf8b127c8cf94544ec7a1181aec613288682543585ea"}, {"name":"html-pipeline","version":"2.14.3","platform":"ruby","checksum":"8a1d4d7128b2141913387cac0f8ba898bb6812557001acc0c2b46910f59413a0"}, -{"name":"html2text","version":"0.2.0","platform":"ruby","checksum":"31c2f0be9ab7aa4fc780b07d5f84882ebc22a9024c29a45f4f5adfe42e92ad4f"}, +{"name":"html2text","version":"0.4.0","platform":"ruby","checksum":"b1becfa0b9150739633f7dc6d8637a49d7e38c3223bcb3afa3cebf59960afdc5"}, {"name":"htmlbeautifier","version":"1.4.2","platform":"ruby","checksum":"9de0c98480fe80d795ed5734a11f183563cd969686f25a04609c0f5a446fa5f8"}, {"name":"htmlentities","version":"4.3.4","platform":"ruby","checksum":"125a73c6c9f2d1b62100b7c3c401e3624441b663762afa7fe428476435a673da"}, {"name":"http","version":"5.1.1","platform":"ruby","checksum":"fcaec14a4f82de6d2f9cb978c07326814c6c2b42b8974f6ec166ff19c645ebaf"}, diff --git a/Gemfile.next.lock b/Gemfile.next.lock index ea75ca100df..7ae9dc13f0b 100644 --- a/Gemfile.next.lock +++ b/Gemfile.next.lock @@ -1003,8 +1003,8 @@ GEM html-pipeline (2.14.3) activesupport (>= 2) nokogiri (>= 1.4) - html2text (0.2.0) - nokogiri (~> 1.6) + html2text (0.4.0) + nokogiri (>= 1.0, < 2.0) htmlbeautifier (1.4.2) htmlentities (4.3.4) http (5.1.1) diff --git a/lib/gitlab/database/repair_index.rb b/lib/gitlab/database/repair_index.rb index ef7067bab7b..2a1e865eefa 100644 --- a/lib/gitlab/database/repair_index.rb +++ b/lib/gitlab/database/repair_index.rb @@ -15,6 +15,7 @@ module Gitlab FIND_DUPLICATE_SETS_SQL = <<~SQL SELECT ARRAY_AGG(id ORDER BY id ASC) as ids FROM %{table_name} + WHERE %{not_null_conditions} GROUP BY %{column_list} HAVING COUNT(*) > 1 SQL @@ -323,10 +324,15 @@ module Gitlab def find_duplicate_sets(table_name, columns) logger.info("Checking for duplicates in '#{table_name}' for columns: #{columns.join(',')}...") + not_null_conditions = columns.map do |col| + "#{connection.quote_column_name(col)} IS NOT NULL" + end.join(' AND ') + sql = format( FIND_DUPLICATE_SETS_SQL, table_name: connection.quote_table_name(table_name), - column_list: columns.map { |col| connection.quote_column_name(col) }.join(', ') + column_list: columns.map { |col| connection.quote_column_name(col) }.join(', '), + not_null_conditions: not_null_conditions ) execute_local(sql, read_only: true) do diff --git a/spec/lib/gitlab/database/repair_index_spec.rb b/spec/lib/gitlab/database/repair_index_spec.rb index 105ae0bd8c5..d3d462522fc 100644 --- a/spec/lib/gitlab/database/repair_index_spec.rb +++ b/spec/lib/gitlab/database/repair_index_spec.rb @@ -74,8 +74,8 @@ RSpec.describe Gitlab::Database::RepairIndex, feature_category: :database do connection.execute(<<~SQL) CREATE TABLE #{test_table} ( id serial PRIMARY KEY, - name varchar(255) NOT NULL, - email varchar(255) NOT NULL + name varchar(255) NULL, + email varchar(255) NULL ); SQL @@ -184,7 +184,9 @@ RSpec.describe Gitlab::Database::RepairIndex, feature_category: :database do INSERT INTO #{test_table} (name, email) VALUES ('test_user', 'test@example.com'), -- ID 1 ('test_user', 'test@example.com'), -- ID 2 (duplicate) - ('other_user', 'other@example.com'); -- ID 3 + ('test_user', NULL), -- ID 3, email NULL, should be preserved + (NULL, 'other@example.com'), -- ID 4, name NULL, should be preserved + ('other_user', 'other@example.com'); -- ID 5 SQL # Create standard references (no entity column) @@ -217,7 +219,7 @@ RSpec.describe Gitlab::Database::RepairIndex, feature_category: :database do it 'handles all reference types correctly' do # before: 3 users, various references user_count_before = connection.select_value("SELECT COUNT(*) FROM #{test_table}") - expect(user_count_before).to eq(3) + expect(user_count_before).to eq(5) # unique index doesn't exist yet index_exists_before = connection.select_value(<<~SQL).present? @@ -230,9 +232,16 @@ RSpec.describe Gitlab::Database::RepairIndex, feature_category: :database do repairer.run - # after: 2 users (duplicate removed) + # after: 4 users (only true duplicate ID 2 removed) + # ID 3 with NULL value preserved user_count_after = connection.select_value("SELECT COUNT(*) FROM #{test_table}") - expect(user_count_after).to eq(2) + expect(user_count_after).to eq(4) + + # Verify NULL values are preserved + null_records = connection.select_value( + "SELECT COUNT(*) FROM #{test_table} WHERE email IS NULL or name is NULL" + ) + expect(null_records).to eq(2) # standard reference updated to good ID standard_ref = connection.select_value( diff --git a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb index 59c488739dc..49693636fce 100644 --- a/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb +++ b/spec/lib/gitlab/email/html_to_markdown_parser_spec.rb @@ -17,8 +17,11 @@ RSpec.describe Gitlab::Email::HtmlToMarkdownParser, feature_category: :service_d expect(subject).to eq( <<~BODY.chomp Hello, World! + This is some e-mail content. Even though it has whitespace and newlines, the e-mail converter will handle it correctly. + *Even* mismatched tags. + A div Another div A div @@ -26,7 +29,9 @@ RSpec.describe Gitlab::Email::HtmlToMarkdownParser, feature_category: :service_d Another line Yet another line + [A link](http://foo.com) +
One diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb index 05c8559e30f..1766b040646 100644 --- a/spec/lib/gitlab/email/reply_parser_spec.rb +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -70,6 +70,7 @@ RSpec.describe Gitlab::Email::ReplyParser, feature_category: :team_planning do Company Contact Country Alfreds Futterkiste Maria Anders Germany Centro comercial Moctezuma Francisco Chang Mexico + Words can be like X-rays, if you use them properly—they’ll go through anything. You read and you’re pierced. BODY )