diff --git a/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml b/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml index ad37f7ced96..08f5d240d3c 100644 --- a/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml +++ b/.gitlab/ci/includes/gitlab-com/danger-review.gitlab-ci.yml @@ -1,6 +1,6 @@ include: - project: gitlab-org/quality/pipeline-common - ref: 8.5.0 + ref: 8.5.1 file: - /ci/danger-review.yml diff --git a/AI_GATEWAY_VERSION b/AI_GATEWAY_VERSION new file mode 100644 index 00000000000..ada8921c1f0 --- /dev/null +++ b/AI_GATEWAY_VERSION @@ -0,0 +1 @@ +1dbddb919823f69cb58bf36343c84c52ec4a3418 diff --git a/app/assets/images/bot_avatars/security-bot.png b/app/assets/images/bot_avatars/security-bot.png new file mode 100644 index 00000000000..0709f62f07b Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot.png differ diff --git a/app/assets/images/bot_avatars/security-bot_120.png b/app/assets/images/bot_avatars/security-bot_120.png new file mode 100644 index 00000000000..28a8fd37bef Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_120.png differ diff --git a/app/assets/images/bot_avatars/security-bot_16.png b/app/assets/images/bot_avatars/security-bot_16.png new file mode 100644 index 00000000000..b1f41ad7e3c Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_16.png differ diff --git a/app/assets/images/bot_avatars/security-bot_160.png b/app/assets/images/bot_avatars/security-bot_160.png new file mode 100644 index 00000000000..3a4c460c94d Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_160.png differ diff --git a/app/assets/images/bot_avatars/security-bot_20.png b/app/assets/images/bot_avatars/security-bot_20.png new file mode 100644 index 00000000000..f762342d379 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_20.png differ diff --git a/app/assets/images/bot_avatars/security-bot_23.png b/app/assets/images/bot_avatars/security-bot_23.png new file mode 100644 index 00000000000..dc298418573 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_23.png differ diff --git a/app/assets/images/bot_avatars/security-bot_24.png b/app/assets/images/bot_avatars/security-bot_24.png new file mode 100644 index 00000000000..f9de38cc630 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_24.png differ diff --git a/app/assets/images/bot_avatars/security-bot_26.png b/app/assets/images/bot_avatars/security-bot_26.png new file mode 100644 index 00000000000..20a39a06f93 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_26.png differ diff --git a/app/assets/images/bot_avatars/security-bot_32.png b/app/assets/images/bot_avatars/security-bot_32.png new file mode 100644 index 00000000000..9ffd9032e3f Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_32.png differ diff --git a/app/assets/images/bot_avatars/security-bot_36.png b/app/assets/images/bot_avatars/security-bot_36.png new file mode 100644 index 00000000000..177a3bf4f4e Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_36.png differ diff --git a/app/assets/images/bot_avatars/security-bot_38.png b/app/assets/images/bot_avatars/security-bot_38.png new file mode 100644 index 00000000000..2d18b75e785 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_38.png differ diff --git a/app/assets/images/bot_avatars/security-bot_40.png b/app/assets/images/bot_avatars/security-bot_40.png new file mode 100644 index 00000000000..5e2a99c7d48 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_40.png differ diff --git a/app/assets/images/bot_avatars/security-bot_48.png b/app/assets/images/bot_avatars/security-bot_48.png new file mode 100644 index 00000000000..cc5fb7053b0 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_48.png differ diff --git a/app/assets/images/bot_avatars/security-bot_60.png b/app/assets/images/bot_avatars/security-bot_60.png new file mode 100644 index 00000000000..4f76b3e3ae7 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_60.png differ diff --git a/app/assets/images/bot_avatars/security-bot_64.png b/app/assets/images/bot_avatars/security-bot_64.png new file mode 100644 index 00000000000..dd0ef99b968 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_64.png differ diff --git a/app/assets/images/bot_avatars/security-bot_90.png b/app/assets/images/bot_avatars/security-bot_90.png new file mode 100644 index 00000000000..4c3de4ec29f Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_90.png differ diff --git a/app/assets/images/bot_avatars/security-bot_96.png b/app/assets/images/bot_avatars/security-bot_96.png new file mode 100644 index 00000000000..ff26c6cc311 Binary files /dev/null and b/app/assets/images/bot_avatars/security-bot_96.png differ diff --git a/app/assets/javascripts/graphql_shared/possible_types.json b/app/assets/javascripts/graphql_shared/possible_types.json index e0024378422..7067241285e 100644 --- a/app/assets/javascripts/graphql_shared/possible_types.json +++ b/app/assets/javascripts/graphql_shared/possible_types.json @@ -56,7 +56,8 @@ "InstanceExternalAuditEventDestination" ], "AuditEventStreamingDestinationInterface": [ - "GroupAuditEventStreamingDestination" + "GroupAuditEventStreamingDestination", + "InstanceAuditEventStreamingDestination" ], "GoogleCloudArtifactRegistryArtifact": [ "GoogleCloudArtifactRegistryDockerImage" diff --git a/app/assets/javascripts/group_settings/allow_runner_registration_token_toggle.js b/app/assets/javascripts/group_settings/allow_runner_registration_token_toggle.js new file mode 100644 index 00000000000..b947d2f39be --- /dev/null +++ b/app/assets/javascripts/group_settings/allow_runner_registration_token_toggle.js @@ -0,0 +1,30 @@ +import { initToggle } from '~/toggles'; + +/** + * Uses a toggle element in combination with a hidden field + * to force submit a form that updates the settings. + * + * Unlike other settings in this page, updating this setting uses + * a full page refresh (submit). This is done to avoid adding + * `allow_runner_registration_token` to any API as this setting + * is discouraged. + */ +export const initAllowRunnerRegistrationTokenToggle = () => { + const el = document.querySelector('.js-allow-runner-registration-token-toggle'); + const input = document.querySelector('.js-allow-runner-registration-token-input'); + + if (el && input) { + const toggle = initToggle(el); + + toggle.$on('change', (isEnabled) => { + input.value = isEnabled; + + toggle.isLoading = true; + + toggle.$el.closest('form').requestSubmit(); + }); + return toggle; + } + + return null; +}; diff --git a/app/assets/javascripts/issues/dashboard/components/issues_dashboard_app.vue b/app/assets/javascripts/issues/dashboard/components/issues_dashboard_app.vue index 5bce755105b..e8af3a2a7f3 100644 --- a/app/assets/javascripts/issues/dashboard/components/issues_dashboard_app.vue +++ b/app/assets/javascripts/issues/dashboard/components/issues_dashboard_app.vue @@ -529,7 +529,6 @@ export default { diff --git a/app/assets/javascripts/pages/groups/settings/ci_cd/show/index.js b/app/assets/javascripts/pages/groups/settings/ci_cd/show/index.js index 184958bd189..f3dae99711a 100644 --- a/app/assets/javascripts/pages/groups/settings/ci_cd/show/index.js +++ b/app/assets/javascripts/pages/groups/settings/ci_cd/show/index.js @@ -1,4 +1,6 @@ import initStaleRunnerCleanupSetting from 'ee_else_ce/group_settings/stale_runner_cleanup'; +import { initAllowRunnerRegistrationTokenToggle } from '~/group_settings/allow_runner_registration_token_toggle'; + import initVariableList from '~/ci/ci_variable_list'; import initSharedRunnersForm from '~/group_settings/mount_shared_runners'; import initSettingsPanels from '~/settings_panels'; @@ -7,7 +9,7 @@ import initDeployTokens from '~/deploy_tokens'; // Initialize expandable settings panels initSettingsPanels(); initDeployTokens(); - +initAllowRunnerRegistrationTokenToggle(); initSharedRunnersForm(); initStaleRunnerCleanupSetting(); initVariableList(); diff --git a/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue b/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue index 7b919952db8..975a0947648 100644 --- a/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue +++ b/app/assets/javascripts/projects/settings/branch_rules/components/view/index.vue @@ -351,7 +351,7 @@ export default { @submitted="$apollo.queries.project.refetch()" > diff --git a/app/controllers/groups/settings/ci_cd_controller.rb b/app/controllers/groups/settings/ci_cd_controller.rb index 0340fc15f91..715555fdd8c 100644 --- a/app/controllers/groups/settings/ci_cd_controller.rb +++ b/app/controllers/groups/settings/ci_cd_controller.rb @@ -79,7 +79,7 @@ module Groups end def update_group_params - params.require(:group).permit(:max_artifacts_size) + params.require(:group).permit(:max_artifacts_size, :allow_runner_registration_token) end # Overridden in EE diff --git a/app/helpers/issues_helper.rb b/app/helpers/issues_helper.rb index 9c6554d5cde..8be11276ad5 100644 --- a/app/helpers/issues_helper.rb +++ b/app/helpers/issues_helper.rb @@ -207,7 +207,7 @@ module IssuesHelper dashboard_labels_path: dashboard_labels_path(format: :json, include_ancestor_groups: true), dashboard_milestones_path: dashboard_milestones_path(format: :json), empty_state_with_filter_svg_path: image_path('illustrations/empty-state/empty-issues-md.svg'), - empty_state_without_filter_svg_path: image_path('illustrations/issue-dashboard_results-without-filter.svg'), + empty_state_without_filter_svg_path: image_path('illustrations/empty-state/empty-search-md.svg'), has_issue_date_filter_feature: Feature.enabled?(:issue_date_filter, current_user).to_s, initial_sort: current_user&.user_preference&.issues_sort, is_public_visibility_restricted: diff --git a/app/models/concerns/avatarable.rb b/app/models/concerns/avatarable.rb index 142308db6c4..6cd84a415fd 100644 --- a/app/models/concerns/avatarable.rb +++ b/app/models/concerns/avatarable.rb @@ -61,6 +61,10 @@ module Avatarable return uncached_avatar_path(only_path: only_path, size: size) end + if self.try(:static_avatar_path?) + return self.static_avatar_path(size) + end + # Cache this avatar path only within the request because avatars in # object storage may be generated with time-limited, signed URLs. key = "#{self.class.name}:#{self.id}:#{only_path}:#{size}" diff --git a/app/views/groups/runners/_settings.html.haml b/app/views/groups/runners/_settings.html.haml index 24b2469c501..6269ca2bbbd 100644 --- a/app/views/groups/runners/_settings.html.haml +++ b/app/views/groups/runners/_settings.html.haml @@ -3,3 +3,13 @@ - if @group.licensed_feature_available?(:stale_runner_cleanup_for_namespace) .gl-mb-5 #stale-runner-cleanup-form{ data: { group_full_path: @group.full_path, stale_timeout_secs: ::Ci::Runner::STALE_TIMEOUT.to_i } } + +- if @group.root? && Gitlab::CurrentSettings.allow_runner_registration_token + -# Only available to top-level groups when the registration token is available in the instance + .gl-mb-5 + = gitlab_ui_form_for @group, url: group_settings_ci_cd_path(@group, anchor: 'runners-settings') do |f| + = f.hidden_field :allow_runner_registration_token, class: 'js-allow-runner-registration-token-input', value: @group.allow_runner_registration_token? + = render Pajamas::ToggleComponent.new(classes: 'js-allow-runner-registration-token-toggle', + label: s_("GroupSettings|Allow members of projects and groups to create runners with runner registration tokens"), + is_checked: @group.allow_runner_registration_token?) do + = s_("GroupSettings|When disabled, members will not be able to register runners using runner registration tokens. They can instead use runner authentication tokens as a more secure runner registration method.") diff --git a/db/docs/batched_background_migrations/fix_corrupted_scanner_ids_of_vulnerability_reads.yml b/db/docs/batched_background_migrations/fix_corrupted_scanner_ids_of_vulnerability_reads.yml new file mode 100644 index 00000000000..5fa1e3b1179 --- /dev/null +++ b/db/docs/batched_background_migrations/fix_corrupted_scanner_ids_of_vulnerability_reads.yml @@ -0,0 +1,19 @@ +--- +migration_job_name: FixCorruptedScannerIdsOfVulnerabilityReads +description: >- + There was a bug in production where the scanner_id of a + vulnerability finding would be updated, yet the scanner_id of the + related vulnerability_read would remain unchanged. This is an issue + as the vulnerability_read should always be in sync with the + vulnerability finding. + + The bug has been fixed in production. This + migration finds any vulnerability_reads with a scanner_id mismatch + with the vulnerability_occurrence. It updates the vulnerability_read + to have the value from the vulnerability_occurrence +feature_category: vulnerability_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/148807 +milestone: '16.11' +queued_migration_version: 20240409023046 +# Replace with the approximate date you think it's best to ensure the completion of this BBM. +finalize_after: '2024-05-01' diff --git a/db/docs/zoekt_nodes.yml b/db/docs/zoekt_nodes.yml index 2c0740d8b60..0a77bb78542 100644 --- a/db/docs/zoekt_nodes.yml +++ b/db/docs/zoekt_nodes.yml @@ -7,4 +7,5 @@ feature_categories: description: Describes a Zoekt server that will be used for indexing and search for some configured namespaces introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134901 milestone: '16.6' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_cell +exempt_from_sharding: true diff --git a/db/post_migrate/20240409023046_queue_fix_corrupted_scanner_ids_of_vulnerability_reads.rb b/db/post_migrate/20240409023046_queue_fix_corrupted_scanner_ids_of_vulnerability_reads.rb new file mode 100644 index 00000000000..be24a259e39 --- /dev/null +++ b/db/post_migrate/20240409023046_queue_fix_corrupted_scanner_ids_of_vulnerability_reads.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +class QueueFixCorruptedScannerIdsOfVulnerabilityReads < Gitlab::Database::Migration[2.2] + milestone '16.11' + + MIGRATION = "FixCorruptedScannerIdsOfVulnerabilityReads" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + def up + queue_batched_background_migration( + MIGRATION, + :vulnerability_reads, + :id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :vulnerability_reads, :id, []) + end +end diff --git a/db/schema_migrations/20240409023046 b/db/schema_migrations/20240409023046 new file mode 100644 index 00000000000..4923e0c756d --- /dev/null +++ b/db/schema_migrations/20240409023046 @@ -0,0 +1 @@ +b652ab8839ccdf19995d8f26c776a8946294367f21dd50279fd8894768f9c47a \ No newline at end of file diff --git a/doc/administration/audit_event_types.md b/doc/administration/audit_event_types.md index 83cda27a0b9..8bc33d680ae 100644 --- a/doc/administration/audit_event_types.md +++ b/doc/administration/audit_event_types.md @@ -60,6 +60,7 @@ Audit event types belong to the following product categories. | [`create_event_streaming_destination`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74632) | Event triggered when an external audit event destination is created| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/344664) | Group | | [`create_group_audit_event_streaming_destination`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/147888) | Event triggered when an external audit event destination for a top-level group is created.| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [16.11](https://gitlab.com/gitlab-org/gitlab/-/issues/436610) | Group | | [`create_http_namespace_filter`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136047) | Event triggered when a namespace filter for an external audit event destination for a top-level group is created.| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [16.6](https://gitlab.com/gitlab-org/gitlab/-/issues/424176) | Group | +| [`create_instance_audit_event_streaming_destination`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/148383) | Event triggered when an external audit event destination for a GitLab instance is created.| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [16.11](https://gitlab.com/gitlab-org/gitlab/-/issues/436615) | Instance | | [`create_instance_event_streaming_destination`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123882) | Event triggered when an instance level external audit event destination is created| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [16.2](https://gitlab.com/gitlab-org/gitlab/-/issues/404730) | Instance | | [`delete_http_namespace_filter`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136302) | Event triggered when a namespace filter for an external audit event destination for a top-level group is deleted.| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [16.7](https://gitlab.com/gitlab-org/gitlab/-/issues/424177) | Group | | [`destroy_event_streaming_destination`](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/74632) | Event triggered when an external audit event destination is deleted| **{check-circle}** Yes | **{check-circle}** Yes | GitLab [14.6](https://gitlab.com/gitlab-org/gitlab/-/issues/344664) | Group | diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index a0fbc616cb5..f8f58e443f2 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -4950,6 +4950,32 @@ Input type: `HttpIntegrationUpdateInput` | `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | | `integration` | [`AlertManagementHttpIntegration`](#alertmanagementhttpintegration) | HTTP integration. | +### `Mutation.instanceAuditEventStreamingDestinationsCreate` + +DETAILS: +**Introduced** in GitLab 16.11. +**Status**: Experiment. + +Input type: `InstanceAuditEventStreamingDestinationsCreateInput` + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `category` | [`String!`](#string) | Destination category. | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `config` | [`JSON!`](#json) | Destination config. | +| `name` | [`String`](#string) | Destination name. | +| `secretToken` | [`String!`](#string) | Secret token. | + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `clientMutationId` | [`String`](#string) | A unique identifier for the client performing the mutation. | +| `errors` | [`[String!]!`](#string) | Errors encountered during execution of the mutation. | +| `externalAuditEventDestination` | [`InstanceAuditEventStreamingDestination`](#instanceauditeventstreamingdestination) | Destination created. | + ### `Mutation.instanceExternalAuditEventDestinationCreate` Input type: `InstanceExternalAuditEventDestinationCreateInput` @@ -21984,6 +22010,19 @@ Stores instance level Amazon S3 configurations for audit event streaming. | `id` | [`ID!`](#id) | ID of the configuration. | | `name` | [`String!`](#string) | Name of the external destination to send audit events to. | +### `InstanceAuditEventStreamingDestination` + +Represents an external destination to stream instance level audit events. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `category` | [`String!`](#string) | Category of the external destination to send audit events to. | +| `config` | [`JSON!`](#json) | Config of the external destination. | +| `id` | [`ID!`](#id) | ID of the destination. | +| `name` | [`String!`](#string) | Name of the external destination to send audit events to. | + ### `InstanceExternalAuditEventDestination` Represents an external resource to send instance audit events to. @@ -34908,6 +34947,7 @@ Implementations: Implementations: - [`GroupAuditEventStreamingDestination`](#groupauditeventstreamingdestination) +- [`InstanceAuditEventStreamingDestination`](#instanceauditeventstreamingdestination) ##### Fields diff --git a/doc/architecture/blueprints/ai_gateway/index.md b/doc/architecture/blueprints/ai_gateway/index.md index d970094d269..c883a674694 100644 --- a/doc/architecture/blueprints/ai_gateway/index.md +++ b/doc/architecture/blueprints/ai_gateway/index.md @@ -558,3 +558,60 @@ Alternative solutions were discussed in ## Decisions - [ADR-001: Allow direct connections](decisions/001_direct_connections.md) + +## Future work + +AI Gateway aim is to become the primary method for the monolith to **access** machine learning models across all usages of GitLab and create a consistent user journey when developing AI-backed features. To do so, these goal is split down into three categories: + +- Centralized Access Through AI Gateway +- Self Managed AI Gateway +- Unit Primitives + +### Centralized Access Through AI Gateway + +The AI Gateway, a standalone service, is the sole access point for all communication between GitLab installations and third-party AI models. It is designed to centralize and manage access to all GitLab features, whether they are in-app functionalities or code suggestions, irrespective of their deployment methods. + +This strategy significantly simplifies enterprise management and abstracts machine learning away from the monolith. With future expansions including telemetry, embeddings API, and multi-region/customer-specific deployments, our goal is to provide a scalable, comprehensive AI solution for all GitLab users, regardless of their installation type. + +[Model registry](../../../user/project/ml/model_registry/index.md) is a feature that allows users to use GitLab to manage the machine learning models. While not solely focused on large language models, and currently more targeted at smaller model applications, which could be deployed in various ways: as a standalone library, a service, a pod, a cloud deployment, and so forth. For these user-deployed models, the ability to auto-configure an API that's accessible through the AI Gateway could be a significant feature. + +- [AI Gateway as the Sole Access Point for Monolith to Access Models](https://gitlab.com/groups/gitlab-org/-/epics/13024) + +### Unit Primitives + +Unit Primitives are a fundamental part of our strategy for managing access to AI features through the AI Gateway. They represent the smallest unit of functionality that can be accessed and managed through the Gateway. This approach provides a more granular control over the functionalities exposed through the AI Gateway and simplifies the management of AI features. It also paves the way for future work on supporting user-deployed models and locally hosted models. From a business perspective, unit primitives are the smallest pieces that may be shuffled across various tiers or packaging models, providing flexibility and adaptability in our offerings. + +In the initial iteration, we will support two primitives: Code Suggestions and Chat. The latter will encompass all Chat features in one primitive. + +In the next iteration, we plan to decompose the Chat primitive into multiple primitives based on top-level tools. This work is dependent on the completion of the task to move classification into the AI Gateway. + +The introduction of Unit Primitives will simplify the management of AI features and provide a more granular control over the functionalities exposed through the AI Gateway. This will also pave the way for future work on supporting user-deployed models and locally hosted models. + +For more details, refer to the [Initial Set of Unit Primitives](https://gitlab.com/gitlab-org/gitlab/-/issues/444934) issue. + +- [Unit Primitives for Accessing CC Features](https://gitlab.com/groups/gitlab-org/-/epics/12556) + +### Self Managed AI Gateway + +Self-managed instances can either use GitLab-hosted AI Gateway or have their own AI Gateway if they want to use self-deployed models, with Runway likely being the deployment method. This means part of our work will be to ensure that the AI Gateway can be deployed in a self-managed environment. This work will go hand-in-hand with the work to support locally hosted models (local inference) in support of GitLab AI features. + +- [Self Managed AI Gateway](https://gitlab.com/groups/gitlab-org/-/epics/13162) + +## Other components in the AI stack + +While AI Gateway centralizes _access_ to AI features and models, it interacts with other components to help users achieve their goals: + +- AI Agents: create and manage agents and prompts +- Model registry: manage and deployment machine learning models + +### Model registry + +[Model registry](../../../user/project/ml/model_registry/index.md) is a feature that allows users to use GitLab to manage the machine learning models. While not solely focused on large language models, and currently more targeted at smaller model applications, which could be deployed in various ways: as a standalone library, a service, a pod, a cloud deployment, and so forth. For these user-deployed models, the ability to auto-configure an API that's accessible through the AI Gateway could be a significant feature. + +### AI Agents + +[AI Agents](https://gitlab.com/groups/gitlab-org/-/epics/12330) is a feature that allows users to implement and manage their own chats and AI features, managing prompts, models and tools. Development is currently in its early stages. Once mature, we intend to move GitLab feature to agents, but there are blockers that currently prevent us from doing so: + +- [Lack of prompt templating](https://gitlab.com/gitlab-org/gitlab/-/issues/441081). +- Implement replication of user-defined prompts into ai-gateway. +- Implement replication of GitLab-defined prompts into self-managed installations (e.g., organization-level agents where we prepopulate with a few agents). diff --git a/doc/development/documentation/feature_flags.md b/doc/development/documentation/feature_flags.md index 54696a0db47..5035e6f09bd 100644 --- a/doc/development/documentation/feature_flags.md +++ b/doc/development/documentation/feature_flags.md @@ -17,7 +17,7 @@ When the state of a feature flag changes, the developer who made the change Every feature introduced to the codebase, even if it's behind a disabled flag, must be documented. For more information, see -[the discussion that led to this decision](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47917#note_459984428). [Experiment or Beta](../../policy/experiment-beta-support.md) features are usually behind a feature flag and must also be documented. For more information, see [Document Experiment or Beta features](experiment_beta.md). +[the discussion that led to this decision](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47917#note_459984428). [Experiment or Beta](../../policy/experiment-beta-support.md) features are usually behind a flag and must also be documented. For more information, see [Document Experiment or Beta features](experiment_beta.md). When the feature is [implemented in multiple merge requests](../feature_flags/index.md#feature-flags-in-gitlab-development), discuss the plan with your technical writer. @@ -43,7 +43,7 @@ When you document feature flags, you must: ## Add history text -When the state of a flag changes (for example, disabled by default to enabled by default), add the change to the +When the state of a flag changes (for example, from disabled by default to enabled by default), add the change to the [history](versions.md#add-a-history-item). Possible history entries are: @@ -132,7 +132,7 @@ To make it available, an administrator can [enable the feature flag](../administ On GitLab.com and GitLab Dedicated, this feature is not available. ``` -When the feature is enabled by default on self-managed GitLab: +When the feature is enabled by default on self-managed and GitLab Dedicated: ```markdown > - [Introduced](issue-link) in GitLab 13.7 [with a flag](../../administration/feature_flags.md) named `forti_token_cloud`. Disabled by default. @@ -188,7 +188,7 @@ Combine entries if they happened in the same release: > - [Enabled on self-managed, GitLab.com, and GitLab Dedicated](issue-link) in GitLab 14.3. ``` -Delete `Enabled` entries when the feature is enabled by default for all offerings and the flag is removed: +Delete `Enabled on GitLab.com` entries only when the feature is enabled by default for all offerings and the flag is removed: - Before: @@ -203,5 +203,6 @@ Delete `Enabled` entries when the feature is enabled by default for all offering ```markdown > - [Introduced](issue-link) in GitLab 15.6 [with a flag](../../administration/feature_flags.md) named `ci_hooks_pre_get_sources_script`. Disabled by default. + > - [Enabled on self-managed and GitLab Dedicated](issue-link) in GitLab 15.7. > - [Generally available](issue-link) in GitLab 15.9. Feature flag `ci_hooks_pre_get_sources_script` removed. ``` diff --git a/doc/development/fe_guide/view_component.md b/doc/development/fe_guide/view_component.md index c795882bbc1..53c389cef8c 100644 --- a/doc/development/fe_guide/view_component.md +++ b/doc/development/fe_guide/view_component.md @@ -194,7 +194,7 @@ The `Pajamas::ToggleComponent` follows the [Pajamas Toggle](https://design.gitla = render Pajamas::ToggleComponent.new(classes: 'js-force-push-toggle', label: s_("ProtectedBranch|Toggle allowed to force push"), is_checked: protected_branch.allow_force_push, - label_position: :hidden) + label_position: :hidden) do Leverage this block to render a rich help text. To render a plain text help text, prefer the `help` parameter. ``` diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 06fb09f702f..357427903e5 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -128,7 +128,7 @@ The following languages and dependency managers are supported: Y - Java and Kotlin (not Android)1 + Java and Kotlin1 8 LTS, 11 LTS, @@ -237,7 +237,7 @@ The following languages and dependency managers are supported:
  • - Support for Kotlin projects for Android is tracked in issue 336866. + Android is supported when DS_EXPERIMENTAL_GRADLE_BUILTIN_PARSER is set to true.

  • diff --git a/doc/user/gitlab_com/index.md b/doc/user/gitlab_com/index.md index 59a2e68d730..2b691402b57 100644 --- a/doc/user/gitlab_com/index.md +++ b/doc/user/gitlab_com/index.md @@ -124,6 +124,8 @@ Groups are permanently deleted after a seven-day delay. If you are on the Free tier, your groups are immediately deleted, and you will not be able to restore them. +You can [view and restore groups marked for deletion](../../user/group/index.md#restore-a-group). + ## Delayed project deletion DETAILS: @@ -136,6 +138,8 @@ Projects are permanently deleted after a seven-day delay. If you are on the Free tier, your projects are immediately deleted, and you will not be able to restore them. +You can [view and restore projects marked for deletion](../../user/project/working_with_projects.md#restore-a-project). + ## Inactive project deletion [Inactive project deletion](../../administration/inactive_project_deletion.md) is disabled on GitLab.com. diff --git a/gems/gitlab-housekeeper/bin/gitlab-housekeeper b/gems/gitlab-housekeeper/bin/gitlab-housekeeper index e81508b859d..fb169d33761 100755 --- a/gems/gitlab-housekeeper/bin/gitlab-housekeeper +++ b/gems/gitlab-housekeeper/bin/gitlab-housekeeper @@ -8,6 +8,10 @@ options = {} OptionParser.new do |opts| opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase' + opts.on('-b=BRANCH', '--target-branch=BRANCH', String, 'Target branch to use. Defaults to master.') do |branch| + options[:target_branch] = branch + end + opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m| options[:max_mrs] = m end diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb index bd3624b5620..e1ba91db146 100644 --- a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb +++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb @@ -13,10 +13,11 @@ require 'digest' module Gitlab module Housekeeper class Runner - def initialize(max_mrs: 1, dry_run: false, keeps: nil, filter_identifiers: []) + def initialize(max_mrs: 1, dry_run: false, keeps: nil, filter_identifiers: [], target_branch: 'master') @max_mrs = max_mrs @dry_run = dry_run @logger = Logger.new($stdout) + @target_branch = target_branch require_keeps @keeps = if keeps @@ -95,7 +96,7 @@ module Gitlab end def git - @git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger) + @git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger, branch_from: @target_branch) end def require_keeps @@ -127,7 +128,7 @@ module Gitlab end puts '=> Diff:' - puts Shell.execute('git', '--no-pager', 'diff', '--color=always', 'master', branch_name, '--', + puts Shell.execute('git', '--no-pager', 'diff', '--color=always', @target_branch, branch_name, '--', *change.changed_files) puts end @@ -136,7 +137,7 @@ module Gitlab non_housekeeper_changes = gitlab_client.non_housekeeper_changes( source_project_id: housekeeper_fork_project_id, source_branch: branch_name, - target_branch: 'master', + target_branch: @target_branch, target_project_id: housekeeper_target_project_id ) @@ -146,7 +147,7 @@ module Gitlab change: change, source_project_id: housekeeper_fork_project_id, source_branch: branch_name, - target_branch: 'master', + target_branch: @target_branch, target_project_id: housekeeper_target_project_id, update_title: !non_housekeeper_changes.include?(:title), update_description: !non_housekeeper_changes.include?(:description), @@ -159,7 +160,7 @@ module Gitlab gitlab_client.get_existing_merge_request( source_project_id: housekeeper_fork_project_id, source_branch: branch_name, - target_branch: 'master', + target_branch: @target_branch, target_project_id: housekeeper_target_project_id ) end diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb index d0b7521879a..d223ea7c654 100644 --- a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb +++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb @@ -135,6 +135,36 @@ RSpec.describe ::Gitlab::Housekeeper::Runner do expect(change2.keep_class).to eq(fake_keep) end + context 'when given target_branch' do + it 'branches from that target branch' do + # Branches get created + expect(::Gitlab::Housekeeper::Git).to receive(:new) + .with(logger: anything, branch_from: 'the-target-branch') + .and_return(git) + + # Branches get shown and pushed + expect(::Gitlab::Housekeeper::Shell).to receive(:execute) + .with('git', '--no-pager', 'diff', '--color=always', 'the-target-branch', + 'the-identifier-for-the-first-change', '--', 'change1.txt', 'change2.txt') + + # Merge requests get created + expect(gitlab_client).to receive(:create_or_update_merge_request) + .with( + change: change1, + source_project_id: '123', + source_branch: 'the-identifier-for-the-first-change', + target_branch: 'the-target-branch', + target_project_id: '456', + update_title: true, + update_description: true, + update_labels: true, + update_reviewers: true + ).twice.and_return({ 'web_url' => 'https://example.com' }) + + described_class.new(max_mrs: 1, keeps: [fake_keep], target_branch: 'the-target-branch').run + end + end + context 'when given filter_identifiers' do it 'skips a change that does not match the filter_identifiers' do # Branches get created diff --git a/lib/gitlab/background_migration/fix_corrupted_scanner_ids_of_vulnerability_reads.rb b/lib/gitlab/background_migration/fix_corrupted_scanner_ids_of_vulnerability_reads.rb new file mode 100644 index 00000000000..ad18e5bf182 --- /dev/null +++ b/lib/gitlab/background_migration/fix_corrupted_scanner_ids_of_vulnerability_reads.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This batched background migration is EE-only, see + # ee/lib/ee/gitlab/background_migration/fix_corrupted_scanner_ids_of_vulnerability_reads.rb + # for the actual migration code. + # + # This batched background migration will update any + # vulnerability_read that has a scanner_id that differs from the + # related vulnerability_occurrence.scanner_id. The + # vulnerability_occurrence.scanner_id is the correct value, so the + # mismatched vulnerability_read will be updated to the + # vulnerability_occurrence.scanner_id value + class FixCorruptedScannerIdsOfVulnerabilityReads < BatchedMigrationJob + feature_category :vulnerability_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::FixCorruptedScannerIdsOfVulnerabilityReads.prepend_mod diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 407ce45b2fd..ba2bd605d69 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -24722,6 +24722,9 @@ msgstr "" msgid "GroupSettings|After the instance reaches the user cap, any user who is added or requests access must be approved by an administrator. Leave empty for an unlimited user cap. If you change the user cap to unlimited, you must re-enable %{project_sharing_docs_link_start}project sharing%{link_end} and %{group_sharing_docs_link_start}group sharing%{link_end}. Increasing the user cap does not automatically approve pending users." msgstr "" +msgid "GroupSettings|Allow members of projects and groups to create runners with runner registration tokens" +msgstr "" + msgid "GroupSettings|An experiment is a feature that is in the process of being developed. It is not production-ready. We encourage users to try experimental features and provide feedback. %{link_start}Learn more%{link_end}." msgstr "" @@ -24950,6 +24953,9 @@ msgstr "" msgid "GroupSettings|What is Insights?" msgstr "" +msgid "GroupSettings|When disabled, members will not be able to register runners using runner registration tokens. They can instead use runner authentication tokens as a more secure runner registration method." +msgstr "" + msgid "GroupSettings|When you enable any of these features, you accept the %{link_start}GitLab Testing Agreement%{link_end}." msgstr "" diff --git a/package.json b/package.json index 1995c3419ce..24db0c10bf5 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,7 @@ "@gitlab/favicon-overlay": "2.0.0", "@gitlab/fonts": "^1.3.0", "@gitlab/svgs": "3.95.0", - "@gitlab/ui": "78.10.0", + "@gitlab/ui": "78.10.1", "@gitlab/visual-review-tools": "1.7.3", "@gitlab/web-ide": "^0.0.1-dev-20240226152102", "@mattiasbuelens/web-streams-adapter": "^0.1.0", diff --git a/qa/Gemfile b/qa/Gemfile index 3753333a580..6d9145f98e6 100644 --- a/qa/Gemfile +++ b/qa/Gemfile @@ -9,7 +9,7 @@ gem 'activesupport', '~> 7.0.8.1' # This should stay in sync with the root's Gem gem 'allure-rspec', '~> 2.24.2' gem 'capybara', '~> 3.40.0' gem 'capybara-screenshot', '~> 1.0.26' -gem 'rake', '~> 13', '>= 13.2.0' +gem 'rake', '~> 13', '>= 13.2.1' gem 'rspec', '~> 3.13' gem 'selenium-webdriver', '= 4.19.0' gem 'airborne', '~> 0.3.7', require: false # airborne is messing with rspec sandboxed mode so not requiring by default diff --git a/qa/Gemfile.lock b/qa/Gemfile.lock index 883443f49e8..1f315e7c79e 100644 --- a/qa/Gemfile.lock +++ b/qa/Gemfile.lock @@ -251,7 +251,7 @@ GEM rails-html-sanitizer (1.5.0) loofah (~> 2.19, >= 2.19.1) rainbow (3.1.1) - rake (13.2.0) + rake (13.2.1) regexp_parser (2.1.1) representable (3.2.0) declarative (< 0.1.0) @@ -371,7 +371,7 @@ DEPENDENCIES parallel_tests (~> 4.4) pry-byebug (~> 3.10.1) rainbow (~> 3.1.1) - rake (~> 13, >= 13.2.0) + rake (~> 13, >= 13.2.1) rest-client (~> 2.1.0) rotp (~> 6.3.0) rspec (~> 3.13) diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb index 6e26a1c64dd..7c1d560d4cd 100644 --- a/qa/qa/runtime/env.rb +++ b/qa/qa/runtime/env.rb @@ -86,6 +86,10 @@ module QA ENV['CI_PROJECT_NAME'] end + def ci_project_path + ENV['CI_PROJECT_PATH'] + end + def schedule_type ENV['SCHEDULE_TYPE'] end diff --git a/qa/qa/service/docker_run/saml_idp.rb b/qa/qa/service/docker_run/saml_idp.rb index c5bcd043a9c..a51ad5e7d69 100644 --- a/qa/qa/service/docker_run/saml_idp.rb +++ b/qa/qa/service/docker_run/saml_idp.rb @@ -97,7 +97,7 @@ module QA config = ERB.new(read_fixture('saml', "#{AUTHSOURCES_FILENAME}.erb")).result(binding) ::File.write(config_path, config) - config_path.gsub("/home/gitlab/qa", "/builds/gitlab-org/gitlab/qa") + config_path.gsub("/home/gitlab/qa", "/builds/#{File.join(Runtime::Env.ci_project_path, 'qa')}") end end end diff --git a/scripts/lint-doc.sh b/scripts/lint-doc.sh index 53ebd087943..349731a0644 100755 --- a/scripts/lint-doc.sh +++ b/scripts/lint-doc.sh @@ -235,7 +235,7 @@ fi # shellcheck disable=2059 printf "${COLOR_GREEN}INFO: Looking for Vale to lint prose, either installed locally or available in documentation linting image...${COLOR_RESET}\n" -run_locally_or_in_container 'vale' "--minAlertLevel error --output=doc/.vale/vale.tmpl --glob=${MD_DOC_PATH}" 'doc' +run_locally_or_in_container 'vale' "--minAlertLevel error --output=doc/.vale/vale.tmpl" "${MD_DOC_PATH}" if [ "$ERRORCODE" -ne 0 ] then diff --git a/spec/frontend/group_settings/allow_runner_registration_token_toggle_spec.js b/spec/frontend/group_settings/allow_runner_registration_token_toggle_spec.js new file mode 100644 index 00000000000..19678fc5741 --- /dev/null +++ b/spec/frontend/group_settings/allow_runner_registration_token_toggle_spec.js @@ -0,0 +1,79 @@ +import { initAllowRunnerRegistrationTokenToggle } from '~/group_settings/allow_runner_registration_token_toggle'; + +import { setHTMLFixture, resetHTMLFixture } from 'helpers/fixtures'; + +describe('initAllowRunnerRegistrationTokenToggle', () => { + let form; + let requestSubmitMock; + + const setFormFixture = ({ + action = '/settings', + hiddenInputValue = 'false', + toggleDisabled = 'false', + toggleIsChecked = 'false', + toggleLabel = 'Toggle Label', + } = {}) => { + setHTMLFixture(` +
    + + +
    + `); + + initAllowRunnerRegistrationTokenToggle(); + + form = document.querySelector('form'); + requestSubmitMock = jest.spyOn(form, 'requestSubmit').mockImplementation(() => {}); + }; + + const findInput = () => form.querySelector('[name="group[allow_runner_registration_token]"]'); + const findToggle = () => form.querySelector('[data-testid="toggle-wrapper"] button'); + + afterEach(() => { + resetHTMLFixture(); + }); + + it('renders a toggle and hidden input', () => { + setFormFixture(); + + expect(form.textContent).toContain('Toggle Label'); + + expect(findToggle()).toBeDefined(); + expect(findInput()).toBeDefined(); + }); + + describe('when setting is enabled', () => { + beforeEach(() => { + setFormFixture({ hiddenInputValue: 'true', toggleIsChecked: 'true' }); + }); + + it('shows an "on" toggle', () => { + expect(findInput().value).toBe('true'); + expect(findToggle().getAttribute('aria-checked')).toBe('true'); + }); + + it('when clicked, toggles the setting', () => { + findToggle().click(); + + expect(findInput().value).toBe('false'); + expect(requestSubmitMock).toHaveBeenCalledTimes(1); + }); + }); + describe('when setting is disabled', () => { + beforeEach(() => { + setFormFixture({ hiddenInputValue: 'false', toggleIsChecked: 'false' }); + }); + + it('shows an "off toggle"', () => { + expect(findInput().value).toBe('false'); + expect(findToggle().getAttribute('aria-checked')).toBe('false'); + }); + + it('when clicked, toggles the setting', () => { + findToggle().click(); + + expect(findInput().value).toBe('true'); + expect(requestSubmitMock).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index ee03e7c3299..73b9bc125f2 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -217,6 +217,7 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :cell do def tables_missing_sharding_key(starting_from_milestone:) ::Gitlab::Database::Dictionary.entries.filter_map do |entry| entry.table_name if entry.sharding_key.blank? && + !entry.exempt_from_sharding? && entry.milestone_greater_than_or_equal_to?(starting_from_milestone) && ::Gitlab::Database::GitlabSchema.cell_local?(entry.gitlab_schema) end diff --git a/spec/migrations/20240409023046_queue_fix_corrupted_scanner_ids_of_vulnerability_reads_spec.rb b/spec/migrations/20240409023046_queue_fix_corrupted_scanner_ids_of_vulnerability_reads_spec.rb new file mode 100644 index 00000000000..c5e365e7079 --- /dev/null +++ b/spec/migrations/20240409023046_queue_fix_corrupted_scanner_ids_of_vulnerability_reads_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueFixCorruptedScannerIdsOfVulnerabilityReads, feature_category: :vulnerability_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :vulnerability_reads, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE + ) + } + end + end +end diff --git a/yarn.lock b/yarn.lock index f9b49631962..cf1e1d4e5e4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1326,10 +1326,10 @@ resolved "https://registry.yarnpkg.com/@gitlab/svgs/-/svgs-3.95.0.tgz#80b128efbfb7ad4a46d94ac97f0bc326433ed256" integrity sha512-UQ91tQNRtsVnEoR/sd6Lk0dNZKLAIeTyadwg0BhW7i6KL7V9XN3UQkvoo4K1KIDwBSnOSm1WPEy4Xvm4Vx/60Q== -"@gitlab/ui@78.10.0": - version "78.10.0" - resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-78.10.0.tgz#6859da363a880b98d6bfebd593a9ada6d46a2e83" - integrity sha512-m3zIkbhTW69KLnrCQ4hvy/pcPcJ3YUEOpP9HMw3zLfKlYEZXy2y9RVbBGsa5lRfMJPmW+IuPvafEVCWqjUtXEA== +"@gitlab/ui@78.10.1": + version "78.10.1" + resolved "https://registry.yarnpkg.com/@gitlab/ui/-/ui-78.10.1.tgz#7f5dead96ad0ba26ad2e2fd50f4acf74b46999f6" + integrity sha512-H6tJ4UTUKn/LY1g7tRwKoV31fmDyr/MMW22r8dRDpwrhq2s4bBaT2SU8QKyQ9pvCUQvaOU67ogooBc8fkuOPLA== dependencies: "@floating-ui/dom" "1.4.3" bootstrap-vue "2.23.1"