Add latest changes from gitlab-org/gitlab@master

db/docs/batched_background_migrations/resync_has_vulnerabilities.yml

@@ -0,0 +1,8 @@
+---
+migration_job_name: ResyncHasVulnerabilities
+description: 'Fixes incorrect project_settings.has_vulnerabilities values'
+feature_category: vulnerability_management
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/171029
+milestone: '17.6'
+queued_migration_version: 20241029170331
+finalized_by:

db/post_migrate/20241029170331_queue_resync_has_vulnerabilities.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class QueueResyncHasVulnerabilities < Gitlab::Database::Migration[2.2]
+  milestone '17.6'
+
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  MIGRATION = "ResyncHasVulnerabilities"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 5000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :project_settings,
+      :project_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(MIGRATION, :project_settings, :project_id, [])
+  end
+end

db/schema_migrations/20241029170331

@@ -0,0 +1 @@
+7d30f02f075d8acdb5a4c4f7291ed6baadba6c32598e98b3ae6141c4d30dd5f8

doc/ci/docker/using_docker_build.md

@@ -189,6 +189,46 @@ To use Docker-in-Docker with TLS enabled:
        - docker run my-docker-image /script/to/run/tests
    ```
 
+##### Use a Unix socket on a shared volume between Docker-in-Docker and build container
+
+Directories defined in `volumes = ["/certs/client", "/cache"]` in the
+[Docker-in-Docker with TLS enabled in the Docker executor](#docker-in-docker-with-tls-enabled-in-the-docker-executor)
+approach are [persistent between builds](https://docs.gitlab.com/runner/executors/docker.html#persistent-storage).
+If multiple CI/CD jobs using a Docker executor runner have Docker-in-Docker services enabled, then each job
+writes to the directory path. This approach might result in a conflict.
+
+To address this conflict, use a Unix socket on a volume shared between the Docker-in-Docker service and the build container.
+This approach improves performance and establishes a secure connection between the service and client.
+
+The following is a sample `config.toml` with temporary volume shared between build and service containers:
+
+```toml
+[[runners]]
+  url = "https://gitlab.com/"
+  token = TOKEN
+  executor = "docker"
+  [runners.docker]
+    image = "docker:24.0.5"
+    privileged = true
+    volumes = ["/runner/services/docker"] # Temporary volume shared between build and service containers.
+```
+
+The Docker-in-Docker service creates a `docker.sock`. The Docker client connects to `docker.sock` through a Docker Unix socket volume.
+
+```yaml
+job:
+  variables:
+    # This variable is shared by both the DinD service and Docker client.
+    # For the service, it will instruct DinD to create `docker.sock` here.
+    # For the client, it tells the Docker client which Docker Unix socket to connect to.
+    DOCKER_HOST: "unix:///runner/services/docker/docker.sock"
+  services:
+    - docker:24.0.5-dind
+  image: docker:24.0.5
+  script:
+    - docker version
+```
+
 ##### Docker-in-Docker with TLS disabled in the Docker executor
 
 Sometimes there are legitimate reasons to disable TLS.

doc/development/secure_coding_guidelines.md

@@ -100,6 +100,12 @@ Some example of well implemented access controls and tests:
 
 **NB:** any input from development team is welcome, for example, about RuboCop rules.
 
+## CI/CD development
+
+When developing features that interact with or trigger pipelines, it's essential to consider the broader implications these actions have on the system's security and operational integrity.
+
+The [CI/CD development guidelines](../development/cicd/index.md) are essential reading material. No SAST or RuboCop rules enforce these guidelines.
+
 ## Regular Expressions guidelines
 
 ### Anchors / Multi line

lib/gitlab/background_migration/resync_has_vulnerabilities.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class ResyncHasVulnerabilities < BatchedMigrationJob
+      operation_name :rsync_has_vulnerabilities
+      feature_category :vulnerability_management
+
+      class Vulnerability < Database::SecApplicationRecord
+        self.table_name = 'vulnerabilities'
+      end
+
+      class ProjectSetting < ApplicationRecord
+        self.table_name = 'project_settings'
+      end
+
+      def perform
+        each_sub_batch do |sub_batch|
+          project_ids = sub_batch.pluck(:project_id)
+          bulk_update!(update_values_sql(project_ids))
+        end
+      end
+
+      private
+
+      def update_values_sql(project_ids)
+        project_ids_values = Arel::Nodes::ValuesList.new(project_ids.zip).to_sql
+
+        results = Vulnerability
+          .select('project_ids.id AS project_id', exists_select)
+          .from("(#{project_ids_values}) AS project_ids (id)")
+
+        update_values = results.map { |result| [result.project_id, result.has_vulnerabilities] }
+        Arel::Nodes::ValuesList.new(update_values).to_sql
+      end
+
+      def exists_select
+        <<~SQL
+          EXISTS (
+            SELECT
+                1
+            FROM
+                vulnerabilities
+            WHERE
+                vulnerabilities.project_id = project_ids.id AND
+                vulnerabilities.present_on_default_branch IS TRUE
+          ) AS has_vulnerabilities
+        SQL
+      end
+
+      def bulk_update!(update_values)
+        sql = <<~SQL
+          UPDATE
+            project_settings
+          SET
+            has_vulnerabilities = update_values.has_vulnerabilities,
+            updated_at = NOW()
+          FROM
+            (#{update_values}) AS update_values (project_id, has_vulnerabilities)
+          WHERE
+            project_settings.project_id = update_values.project_id;
+        SQL
+
+        ProjectSetting.connection.execute(sql)
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/resync_has_vulnerabilities_spec.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::ResyncHasVulnerabilities, feature_category: :vulnerability_management do
+  let(:project_settings) { table(:project_settings) }
+  let(:findings) { table(:vulnerability_occurrences) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let!(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  let(:true_without_vulnerabilities) do
+    create_project_setting(
+      'true-without-vulnerabilities',
+      has_vulnerabilities_setting: true,
+      actually_has_vulnerabilities: false
+    )
+  end
+
+  let(:true_with_vulnerabilities) do
+    create_project_setting(
+      'true-with-vulnerabilities',
+      has_vulnerabilities_setting: true,
+      actually_has_vulnerabilities: true
+    )
+  end
+
+  let(:false_with_vulnerabilities) do
+    create_project_setting(
+      'false-with-vulnerabilities',
+      has_vulnerabilities_setting: false,
+      actually_has_vulnerabilities: true
+    )
+  end
+
+  let(:false_without_vulnerabilities) do
+    create_project_setting(
+      'false-without-vulnerabilities',
+      has_vulnerabilities_setting: false,
+      actually_has_vulnerabilities: false
+    )
+  end
+
+  let(:has_vulnerabilities_on_non_default_branch) do
+    create_project_setting(
+      'non-default-branch',
+      has_vulnerabilities_setting: false,
+      actually_has_vulnerabilities: true,
+      present_on_default_branch: false
+    )
+  end
+
+  let(:args) do
+    {
+      start_id: project_settings.minimum(:project_id),
+      end_id: project_settings.maximum(:project_id),
+      batch_table: :project_settings,
+      batch_column: :project_id,
+      sub_batch_size: 100,
+      pause_ms: 0,
+      connection: ApplicationRecord.connection
+    }
+  end
+
+  subject(:perform_migration) { described_class.new(**args).perform }
+
+  def create_project_setting(
+    name,
+    has_vulnerabilities_setting:,
+    actually_has_vulnerabilities:,
+    present_on_default_branch: true
+  )
+    group_namespace = namespaces.create!(name: name, path: name)
+    project_namespace = namespaces.create!(name: name, path: name)
+    project = projects.create!(
+      name: name,
+      path: name,
+      namespace_id: group_namespace.id,
+      project_namespace_id: project_namespace.id
+    )
+    create_vulnerability(project, present_on_default_branch) if actually_has_vulnerabilities
+    project_settings.create!(project_id: project.id, has_vulnerabilities: has_vulnerabilities_setting)
+  end
+
+  def create_vulnerability(project, present_on_default_branch)
+    common_args = {
+      project_id: project.id,
+      severity: 1,
+      report_type: 1
+    }
+
+    scanner = scanners.create!(project_id: project.id, external_id: 'id', name: 'name')
+
+    primary_identifier = identifiers.create!(
+      project_id: project.id,
+      external_id: "CVE-2018-1234",
+      external_type: "CVE",
+      name: "CVE-2018-1234",
+      fingerprint: SecureRandom.hex(20)
+    )
+
+    finding = findings.create!(
+      scanner_id: scanner.id,
+      primary_identifier_id: primary_identifier.id,
+      project_fingerprint: SecureRandom.hex(20),
+      location_fingerprint: SecureRandom.hex(20),
+      uuid: SecureRandom.uuid,
+      name: 'name',
+      raw_metadata: '{}',
+      metadata_version: 'test:1.0',
+      **common_args
+    )
+
+    vulnerabilities.create!(
+      author_id: user.id,
+      finding_id: finding.id,
+      title: 'title',
+      present_on_default_branch: present_on_default_branch,
+      state: 1,
+      **common_args
+    )
+  end
+
+  it 'fixes only the incorrect records' do
+    expect { perform_migration }.to change { true_without_vulnerabilities.reload.has_vulnerabilities }
+      .from(true).to(false).and change { false_with_vulnerabilities.reload.has_vulnerabilities }.from(false).to(true)
+      .and not_change { true_with_vulnerabilities.reload.has_vulnerabilities }
+      .and not_change { false_without_vulnerabilities.reload.has_vulnerabilities }
+      .and not_change { has_vulnerabilities_on_non_default_branch.reload.has_vulnerabilities }
+  end
+end

spec/migrations/20241029170331_queue_resync_has_vulnerabilities_spec.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueResyncHasVulnerabilities, migration: :gitlab_main, feature_category: :vulnerability_management do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          gitlab_schema: :gitlab_main,
+          table_name: :project_settings,
+          column_name: :project_id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE
+        )
+      }
+    end
+  end
+end