Add latest changes from gitlab-org/gitlab@master

app/models/ci/build.rb

@@ -221,6 +221,7 @@ module Ci
     end
 
     after_commit :track_ci_secrets_management_id_tokens_usage, on: :create, if: :id_tokens?
+    after_commit :track_ci_build_created_event, on: :create
 
     class << self
       # This is needed for url_for to work,
@@ -1244,6 +1245,16 @@ module Ci
       )
     end
 
+    def track_ci_build_created_event
+      return unless Feature.enabled?(:track_ci_build_created_internal_event, project, type: :gitlab_com_derisk)
+
+      if user
+        Gitlab::InternalEvents.track_event('create_ci_build', project: project, user: user)
+      else
+        Gitlab::InternalEvents.track_event('create_ci_build', project: project)
+      end
+    end
+
     def partition_id_prefix_in_16_bit_encode
       "#{partition_id.to_s(16)}_"
     end

app/models/namespace_setting.rb

@@ -7,6 +7,7 @@ class NamespaceSetting < ApplicationRecord
   include IgnorableColumns
 
   ignore_column :project_import_level, remove_with: '16.10', remove_after: '2024-02-22'
+  ignore_column :third_party_ai_features_enabled, remove_with: '16.10', remove_after: '2024-02-22'
 
   cascading_attr :delayed_project_removal
   cascading_attr :toggle_security_policy_custom_ci

config/events/create_ci_build.yml

@@ -0,0 +1,20 @@
+---
+description: Ci Build created
+category: InternalEventTracking
+action: create_ci_build
+identifiers:
+- project
+- namespace
+- user
+product_section: ci
+product_stage: verify
+product_group: pipeline_execution
+milestone: '16.9'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141421
+distributions:
+- ce
+- ee
+tiers:
+- free
+- premium
+- ultimate

config/feature_flags/gitlab_com_derisk/track_ci_build_created_internal_event.yml

@@ -0,0 +1,9 @@
+---
+name: track_ci_build_created_internal_event
+feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/429063
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141421
+rollout_issue_url: https://gitlab.com/gitlab-com/gl-infra/production/-/issues/17409
+milestone: '16.9'
+group: group::pipeline execution
+type: gitlab_com_derisk
+default_enabled: false

config/metrics/counts_28d/count_distinct_user_id_from_create_ci_build_monthly.yml

@@ -0,0 +1,27 @@
+---
+key_path: redis_hll_counters.count_distinct_user_id_from_create_ci_build_monthly
+description: Monthly count of unique users triggering jobs
+product_section: ci
+product_stage: verify
+product_group: pipeline_execution
+performance_indicator_type: []
+value_type: number
+status: active
+milestone: '16.9'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141421
+time_frame: 28d
+data_source: internal_events
+data_category: optional
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - create_ci_build
+events:
+- name: create_ci_build
+  unique: user.id

config/metrics/counts_28d/count_total_create_ci_build_monthly.yml

@@ -0,0 +1,26 @@
+---
+key_path: counts.count_total_create_ci_build_monthly
+description: Monthly count of ci builds created across all projects and project types
+product_section: ci
+product_stage: verify
+product_group: pipeline_execution
+performance_indicator_type: []
+value_type: number
+status: active
+milestone: '16.9'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141421
+time_frame: 28d
+data_source: internal_events
+data_category: optional
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - create_ci_build
+events:
+- name: create_ci_build

config/metrics/counts_all/count_total_create_ci_build.yml

@@ -0,0 +1,26 @@
+---
+key_path: counts.count_total_create_ci_build
+description: Total count of ci builds across all project and project types
+product_section: ci
+product_stage: verify
+product_group: pipeline_execution
+performance_indicator_type: []
+value_type: number
+status: active
+milestone: '16.9'
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/141421
+time_frame: all
+data_source: internal_events
+data_category: optional
+distribution:
+- ce
+- ee
+tier:
+- free
+- premium
+- ultimate
+options:
+  events:
+  - create_ci_build
+events:
+- name: create_ci_build

db/post_migrate/20240123040846_swap_columns_for_system_note_metadata_id.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class SwapColumnsForSystemNoteMetadataId < Gitlab::Database::Migration[2.2]
+  include Gitlab::Database::MigrationHelpers::ConvertToBigint
+
+  disable_ddl_transaction!
+
+  milestone '16.9'
+
+  TABLE_NAME = 'system_note_metadata'
+  PRIMARY_KEY_CONSTRAINT_NAME = 'system_note_metadata_pkey'
+  PRIMARY_KEY_INDEX_NAME = 'index_system_note_metadata_pkey'
+  NEW_PRIMARY_KEY_INDEX_NAME = 'index_system_note_metadata_pkey_on_id_convert_to_bigint'
+
+  # For the FK from 'resource_link_events' table referencing 'system_note_metadata'.
+  FK_SOURCE_TABLE_NAME = :resource_link_events
+  FK_COLUMN_NAME = :system_note_metadata_id
+  FK_NAME = 'fk_2a039c40f4'
+  TEMP_FK_NAME = 'fk_system_note_metadata_id_convert_to_bigint'
+
+  def up
+    swap
+  end
+
+  def down
+    swap
+
+    add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: NEW_PRIMARY_KEY_INDEX_NAME
+
+    add_concurrent_foreign_key FK_SOURCE_TABLE_NAME, TABLE_NAME,
+      name: TEMP_FK_NAME,
+      column: FK_COLUMN_NAME,
+      target_column: :id_convert_to_bigint,
+      on_delete: :cascade,
+      validate: false
+  end
+
+  def swap
+    # This will replace the existing system_note_metadata_pkey index for the primary key
+    add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: NEW_PRIMARY_KEY_INDEX_NAME
+
+    # This will replace the existing fk_2a039c40f4
+    add_concurrent_foreign_key FK_SOURCE_TABLE_NAME, TABLE_NAME,
+      name: TEMP_FK_NAME,
+      column: FK_COLUMN_NAME,
+      target_column: :id_convert_to_bigint,
+      on_delete: :cascade
+
+    with_lock_retries(raise_on_exhaustion: true) do
+      execute "LOCK TABLE #{TABLE_NAME}, #{FK_SOURCE_TABLE_NAME} IN ACCESS EXCLUSIVE MODE"
+
+      # Swap columns
+      temp_column_name = quote_column_name(:id_tmp)
+      id_column_name = quote_column_name(:id)
+      id_convert_to_bigint_name = quote_column_name(:id_convert_to_bigint)
+      execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_column_name} TO #{temp_column_name}"
+      execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{id_convert_to_bigint_name} TO #{id_column_name}"
+      execute "ALTER TABLE #{TABLE_NAME} RENAME COLUMN #{temp_column_name} TO #{id_convert_to_bigint_name}"
+
+      # Reset trigger
+      function_name = Gitlab::Database::UnidirectionalCopyTrigger
+        .on_table(TABLE_NAME, connection: connection)
+        .name(:id, :id_convert_to_bigint)
+      execute "ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL"
+
+      # Swap defaults
+      seq_name = "system_note_metadata_id_seq"
+      execute "ALTER SEQUENCE #{seq_name} OWNED BY #{TABLE_NAME}.id"
+      change_column_default TABLE_NAME, :id, -> { "nextval('#{seq_name}'::regclass)" }
+      change_column_default TABLE_NAME, :id_convert_to_bigint, 0
+
+      # Swap pkey constraint
+      # This will drop fk_2a039c40f4 ("resource_link_events" REFERENCES system_note_metadata(id) ON DELETE CASCADE)
+      execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT #{PRIMARY_KEY_CONSTRAINT_NAME} CASCADE"
+      rename_index TABLE_NAME, NEW_PRIMARY_KEY_INDEX_NAME, PRIMARY_KEY_INDEX_NAME
+      execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT #{PRIMARY_KEY_CONSTRAINT_NAME} PRIMARY KEY USING INDEX #{PRIMARY_KEY_INDEX_NAME}" # rubocop:disable Layout/LineLength -- for readability
+
+      # Rename the new FK
+      rename_constraint FK_SOURCE_TABLE_NAME, TEMP_FK_NAME, FK_NAME
+    end
+  end
+end

db/schema_migrations/20240123040846

@@ -0,0 +1 @@
+c579e5722d58bb422a82f52706e37e1e89ee18c90772ce1da20ae94f670618ab

db/structure.sql

@@ -24497,14 +24497,14 @@ CREATE SEQUENCE system_access_microsoft_graph_access_tokens_id_seq
 ALTER SEQUENCE system_access_microsoft_graph_access_tokens_id_seq OWNED BY system_access_microsoft_graph_access_tokens.id;
 
 CREATE TABLE system_note_metadata (
-    id integer NOT NULL,
+    id_convert_to_bigint integer DEFAULT 0 NOT NULL,
     commit_count integer,
     action character varying,
     created_at timestamp without time zone NOT NULL,
     updated_at timestamp without time zone NOT NULL,
     description_version_id bigint,
     note_id bigint NOT NULL,
-    id_convert_to_bigint bigint DEFAULT 0 NOT NULL
+    id bigint NOT NULL
 );
 
 CREATE SEQUENCE system_note_metadata_id_seq
@@ -35566,8 +35566,6 @@ CREATE UNIQUE INDEX index_system_note_metadata_on_description_version_id ON syst
 
 CREATE UNIQUE INDEX index_system_note_metadata_on_note_id ON system_note_metadata USING btree (note_id);
 
-CREATE UNIQUE INDEX index_system_note_metadata_pkey_on_id_convert_to_bigint ON system_note_metadata USING btree (id_convert_to_bigint);
-
 CREATE INDEX index_taggings_on_tag_id ON taggings USING btree (tag_id);
 
 CREATE INDEX index_taggings_on_taggable_id_and_taggable_type_and_context ON taggings USING btree (taggable_id, taggable_type, context);
@@ -41166,9 +41164,6 @@ ALTER TABLE ONLY integrations
 ALTER TABLE ONLY merge_requests
     ADD CONSTRAINT fk_source_project FOREIGN KEY (source_project_id) REFERENCES projects(id) ON DELETE SET NULL;
 
-ALTER TABLE ONLY resource_link_events
-    ADD CONSTRAINT fk_system_note_metadata_id_convert_to_bigint FOREIGN KEY (system_note_metadata_id) REFERENCES system_note_metadata(id_convert_to_bigint) ON DELETE CASCADE NOT VALID;
-
 ALTER TABLE ONLY timelogs
     ADD CONSTRAINT fk_timelogs_issues_issue_id FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
 

doc/development/fe_guide/graphql.md

@@ -1301,7 +1301,7 @@ If we need to test a query error, we need to mock a rejected value as request ha
 ```javascript
 it('renders error if query fails', async () => {
   const wrapper = createComponent({
-    designListHandler: jest.fn.mockRejectedValue('Houston, we have a problem!')
+    designListHandler: jest.fn().mockRejectedValue('Houston, we have a problem!')
   });
 
   await waitForPromises()

doc/install/requirements.md

@@ -262,7 +262,7 @@ Redis stores all user sessions and the background task queue.
 
 The requirements for Redis are as follows:
 
-- Redis 6.x or 7.x is required in GitLab 16.0 and later.
+- Redis 6.x or 7.x is required in GitLab 16.0 and later. In GitLab 16.8 and later, Redis 6.2 or later is required.
 - Redis Cluster mode is not supported. Redis Standalone must be used.
 - Storage requirements for Redis are minimal, about 25 kB per user on average.
 

doc/update/versions/gitlab_16_changes.md

@@ -100,6 +100,11 @@ Specific information applies to Linux package installations:
   | 16.6                    |  All                    | None     |
   | 16.7                    |  All                    | None     |
 
+## 16.8.0
+
+- Redis 6.2 or later is now required by Sidekiq. For installations that use an external Redis service, you should upgrade your Redis service to 6.2 - 7.x
+before upgrading to GitLab 16.8.
+
 ## 16.6.0
 
 - Old [CI Environment destroy jobs may be spawned](https://gitlab.com/gitlab-org/gitlab/-/issues/433264#) after upgrading to GitLab 16.6.

lib/api/api.rb

@@ -151,7 +151,7 @@ module API
     end
 
     rescue_from Grape::Exceptions::Base do |e|
-      error! e.message, e.status, e.headers
+      error!(e.message, e.status, e.headers || {})
     end
 
     rescue_from MovedPermanentlyError do |e|

qa/qa/page/element.rb

@@ -23,10 +23,7 @@ module QA
       end
 
       def selector_css
-        [
-          %([data-testid="#{name}"]#{additional_selectors}),
-          %([data-qa-selector="#{name}"]#{additional_selectors})
-        ].join(',')
+        %([data-testid="#{name}"]#{additional_selectors})
       end
 
       def matches?(line)

qa/qa/page/project/web_ide/vscode.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# VSCode WebIDE is built off an iFrame application therefore we are unable to use `testids`
+# VSCode WebIDE is built off an iFrame application therefore we are unable to use `data-testids`
 module QA
   module Page
     module Project

qa/spec/page/element_spec.rb

@@ -4,7 +4,7 @@ RSpec.describe QA::Page::Element do
   describe '#selector_css' do
     it 'transforms element name into QA-specific clickable css selector' do
       expect(described_class.new(:sign_in_button).selector_css)
-        .to eq('[data-testid="sign_in_button"],[data-qa-selector="sign_in_button"]')
+        .to eq('[data-testid="sign_in_button"]')
     end
   end
 
@@ -94,11 +94,11 @@ RSpec.describe QA::Page::Element do
     end
   end
 
-  describe 'data-qa selectors' do
+  describe 'data-testid selectors' do
     subject { described_class.new(:my_element) }
 
-    it 'properly translates to a data-qa-selector' do
-      expect(subject.selector_css).to include(%q([data-qa-selector="my_element"]))
+    it 'does not translate to a deprecated qa selector' do
+      expect(subject.selector_css).not_to include(%q([data-qa-selector="my_element"]))
     end
 
     it 'properly translates to a data-testid' do
@@ -112,8 +112,6 @@ RSpec.describe QA::Page::Element do
       it 'matches on additional data-qa properties translating snake_case to kebab-case' do
         expect(element.selector_css)
           .to include('[data-testid="my_element"][data-qa-index="3"][data-qa-another-match="something"]')
-        expect(element.selector_css)
-          .to include('[data-qa-selector="my_element"][data-qa-index="3"][data-qa-another-match="something"]')
       end
 
       it 'doesnt conflict with element requirement' do

spec/migrations/swap_columns_for_system_note_metadata_id_spec.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe SwapColumnsForSystemNoteMetadataId, feature_category: :team_planning do
+  describe '#up' do
+    let(:pk_name) { "system_note_metadata_pkey" }
+    let(:fk_name) { "fk_2a039c40f4" } # FK on 'resource_link_events'
+
+    before do
+      # A we call `schema_migrate_down!` before each example, and for this migration
+      # `#down` is same as `#up`, we need to ensure we start from the expected state.
+      connection = described_class.new.connection
+      connection.execute('ALTER TABLE system_note_metadata ALTER COLUMN id TYPE integer')
+      connection.execute('ALTER TABLE system_note_metadata ALTER COLUMN id_convert_to_bigint TYPE bigint')
+    end
+
+    it 'swaps the integer and bigint columns with correct constraint names' do
+      table = table(:system_note_metadata)
+
+      disable_migrations_output do
+        reversible_migration do |migration|
+          migration.before -> {
+            table.reset_column_information
+
+            primary_key_naming_check = query_constraint_by_name(:system_note_metadata, pk_name).first
+            foreign_key_naming_check = query_constraint_by_name(:resource_link_events, fk_name).first
+            expect(primary_key_naming_check).to match("bool" => true)
+            expect(foreign_key_naming_check).to match("bool" => true)
+            expect(table.columns.find { |c| c.name == 'id' }.sql_type).to eq('integer')
+            expect(table.columns.find { |c| c.name == 'id_convert_to_bigint' }.sql_type).to eq('bigint')
+          }
+
+          migration.after -> {
+            table.reset_column_information
+
+            primary_key_naming_check = query_constraint_by_name(:system_note_metadata, pk_name).first
+            foreign_key_naming_check = query_constraint_by_name(:resource_link_events, fk_name).first
+            expect(primary_key_naming_check).to match("bool" => true)
+            expect(foreign_key_naming_check).to match("bool" => true)
+            expect(table.columns.find { |c| c.name == 'id' }.sql_type).to eq('bigint')
+            expect(table.columns.find { |c| c.name == 'id_convert_to_bigint' }.sql_type).to eq('integer')
+          }
+        end
+      end
+    end
+
+    def query_constraint_by_name(table_name, conname)
+      described_class.new.connection.execute <<~SQL
+        SELECT true FROM pg_constraint c JOIN pg_class t ON t.oid = c.conrelid
+        WHERE t.relname = \'#{table_name}\' AND c.conname = \'#{conname}\';
+      SQL
+    end
+  end
+end

spec/models/ci/build_spec.rb

@@ -89,6 +89,47 @@ RSpec.describe Ci::Build, feature_category: :continuous_integration, factory_def
         create(:ci_build, pipeline: pipeline)
       end
     end
+
+    context 'when running after_commit callbacks' do
+      context 'without user present' do
+        it 'tracks creation event' do
+          build = FactoryBot.build(:ci_build)
+
+          expect(Gitlab::InternalEvents).to receive(:track_event).with(
+            'create_ci_build',
+            project: build.project
+          )
+
+          build.save!
+        end
+      end
+
+      context 'with user present' do
+        it 'tracks creation event' do
+          build = FactoryBot.build(:ci_build, user: create(:user))
+
+          expect(Gitlab::InternalEvents).to receive(:track_event).with(
+            'create_ci_build',
+            project: build.project,
+            user: build.user
+          )
+
+          build.save!
+        end
+      end
+
+      context 'with FF track_ci_build_created_internal_event disabled' do
+        before do
+          stub_feature_flags(track_ci_build_created_internal_event: false)
+        end
+
+        it 'does not track creation event' do
+          expect(Gitlab::InternalEvents).not_to receive(:track_event)
+
+          create(:ci_build)
+        end
+      end
+    end
   end
 
   describe 'status' do
@@ -5615,6 +5656,10 @@ RSpec.describe Ci::Build, feature_category: :continuous_integration, factory_def
       context 'on create' do
         let(:ci_build) { FactoryBot.build(:ci_build, user: user, id_tokens: { 'ID_TOKEN_1' => { aud: 'developers' } }) }
 
+        before do
+          allow(Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:track_event).and_call_original
+        end
+
         it 'tracks RedisHLL event with user_id' do
           expect(::Gitlab::UsageDataCounters::HLLRedisCounter).to receive(:track_event)
             .with('i_ci_secrets_management_id_tokens_build_created', values: user.id)
@@ -5664,6 +5709,7 @@ RSpec.describe Ci::Build, feature_category: :continuous_integration, factory_def
       context 'on create' do
         it 'does not track RedisHLL event' do
           expect(Gitlab::UsageDataCounters::HLLRedisCounter).not_to receive(:track_event)
+            .with('i_ci_secrets_management_id_tokens_build_created', values: user.id)
 
           ci_build.save!
         end

spec/requests/api/api_spec.rb

@@ -381,4 +381,14 @@ RSpec.describe API::API, feature_category: :system_access do
       )
     end
   end
+
+  describe 'Grape::Exceptions::Base handler' do
+    it 'returns 400 on JSON parse errors' do
+      post api('/projects'),
+        params: '{"test":"random_\$escaped/symbols\;here"}',
+        headers: { 'content-type' => 'application/json' }
+
+      expect(response).to have_gitlab_http_status(:bad_request)
+    end
+  end
 end