From 38df3b61726ad699a5603702af2a0fb6b2df9d20 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Fri, 9 Aug 2024 21:09:19 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .rubocop_todo/rails/strong_params.yml | 1 - .../javascripts/ide/init_gitlab_web_ide.js | 40 --- app/assets/javascripts/ide/remote/index.js | 42 --- .../pages/web_ide/remote_ide/index.js | 3 - .../web_ide/remote_ide_controller.rb | 53 ---- app/helpers/ide_helper.rb | 2 - app/views/web_ide/remote_ide/index.html.haml | 5 - .../increase_diff_file_performance.yml | 2 +- config/routes.rb | 6 - doc/api/graphql/reference/index.md | 50 ++++ doc/ci/pipelines/merge_trains.md | 36 ++- .../secret_push_protection/index.md | 21 +- doc/user/gitlab_duo/turn_on_off.md | 9 +- doc/user/gitlab_duo/use_cases.md | 2 +- doc/user/markdown.md | 6 +- .../code_suggestions/supported_extensions.md | 71 +++-- doc/user/report_abuse.md | 6 + gems/gitlab-secret_detection/README.md | 8 +- .../lib/gitlab/secret_detection/scan.rb | 198 +++++++------ .../lib/gitlab/secret_detection/status.rb | 2 +- .../lib/gitlab/secret_detection/scan_spec.rb | 269 +++++++++++------- locale/gitlab.pot | 9 - package.json | 2 +- ...ab+web-ide+0.0.1-dev-20240731185426.patch} | 124 +++++++- qa/gdk/Dockerfile.gdk | 2 +- qa/qa/page/project/web_ide/vscode.rb | 2 +- spec/frontend/ide/init_gitlab_web_ide_spec.js | 62 ---- spec/frontend/ide/remote/index_spec.js | 92 ------ spec/helpers/ide_helper_spec.rb | 3 +- .../web_ide/remote_ide_controller_spec.rb | 134 --------- spec/routing/web_ide_routing_spec.rb | 22 -- yarn.lock | 39 +-- 32 files changed, 560 insertions(+), 763 deletions(-) delete mode 100644 app/assets/javascripts/ide/remote/index.js delete mode 100644 app/assets/javascripts/pages/web_ide/remote_ide/index.js delete mode 100644 app/controllers/web_ide/remote_ide_controller.rb delete mode 100644 app/views/web_ide/remote_ide/index.html.haml rename patches/{@gitlab+web-ide+0.0.1-dev-20240613133550.patch => @gitlab+web-ide+0.0.1-dev-20240731185426.patch} (98%) delete mode 100644 spec/frontend/ide/remote/index_spec.js delete mode 100644 spec/requests/web_ide/remote_ide_controller_spec.rb delete mode 100644 spec/routing/web_ide_routing_spec.rb diff --git a/.rubocop_todo/rails/strong_params.yml b/.rubocop_todo/rails/strong_params.yml index 1eb545023bb..8b4b0ece4bd 100644 --- a/.rubocop_todo/rails/strong_params.yml +++ b/.rubocop_todo/rails/strong_params.yml @@ -245,7 +245,6 @@ Rails/StrongParams: - 'app/controllers/users/terms_controller.rb' - 'app/controllers/users/unsubscribes_controller.rb' - 'app/controllers/users_controller.rb' - - 'app/controllers/web_ide/remote_ide_controller.rb' - 'ee/app/controllers/admin/ai/feature_settings_controller.rb' - 'ee/app/controllers/admin/ai/self_hosted_models_controller.rb' - 'ee/app/controllers/admin/elasticsearch_controller.rb' diff --git a/app/assets/javascripts/ide/init_gitlab_web_ide.js b/app/assets/javascripts/ide/init_gitlab_web_ide.js index 11929fb4ef5..1ade6433d63 100644 --- a/app/assets/javascripts/ide/init_gitlab_web_ide.js +++ b/app/assets/javascripts/ide/init_gitlab_web_ide.js @@ -1,9 +1,5 @@ import { start } from '@gitlab/web-ide'; -import { __ } from '~/locale'; -import { cleanLeadingSeparator } from '~/lib/utils/url_utility'; import { convertObjectPropsToCamelCase } from '~/lib/utils/common_utils'; -import { confirmAction } from '~/lib/utils/confirm_via_gl_modal/confirm_action'; -import { createAndSubmitForm } from '~/lib/utils/create_and_submit_form'; import csrf from '~/lib/utils/csrf'; import Tracking from '~/tracking'; import { @@ -15,20 +11,6 @@ import { import { GITLAB_WEB_IDE_FEEDBACK_ISSUE } from './constants'; import { renderWebIdeError } from './render_web_ide_error'; -const buildRemoteIdeURL = (ideRemotePath, remoteHost, remotePathArg) => { - const remotePath = cleanLeadingSeparator(remotePathArg); - - const replacers = { - ':remote_host': encodeURIComponent(remoteHost), - ':remote_path': encodeURIComponent(remotePath).replaceAll('%2F', '/'), - }; - - // why: Use the function callback of "replace" so we replace both keys at once - return ideRemotePath.replace(/(:remote_host|:remote_path)/g, (key) => { - return replacers[key]; - }); -}; - const getMRTargetProject = () => { const url = new URL(window.location.href); @@ -49,7 +31,6 @@ export const initGitlabWebIDE = async (el) => { cspNonce: nonce, branchName: ref, projectPath, - ideRemotePath, filePath, mergeRequest: mrId, forkInfo: forkInfoJSON, @@ -106,27 +87,6 @@ export const initGitlabWebIDE = async (el) => { handleTracking, // See https://gitlab.com/gitlab-org/gitlab-web-ide/-/blob/main/packages/web-ide-types/src/config.ts#L86 telemetryEnabled: Tracking.enabled(), - async handleStartRemote({ remoteHost, remotePath, connectionToken }) { - const confirmed = await confirmAction( - __('Are you sure you want to leave the Web IDE? All unsaved changes will be lost.'), - { - primaryBtnText: __('Start remote connection'), - cancelBtnText: __('Continue editing'), - }, - ); - - if (!confirmed) { - return; - } - - createAndSubmitForm({ - url: buildRemoteIdeURL(ideRemotePath, remoteHost, remotePath), - data: { - connection_token: connectionToken, - return_url: window.location.href, - }, - }); - }, }); } catch (error) { renderWebIdeError({ error, signOutPath }); diff --git a/app/assets/javascripts/ide/remote/index.js b/app/assets/javascripts/ide/remote/index.js deleted file mode 100644 index 6966786ca4e..00000000000 --- a/app/assets/javascripts/ide/remote/index.js +++ /dev/null @@ -1,42 +0,0 @@ -import { startRemote } from '@gitlab/web-ide'; -import { getBaseConfig, setupRootElement } from '~/ide/lib/gitlab_web_ide'; -import { isSameOriginUrl, joinPaths } from '~/lib/utils/url_utility'; -import { handleTracking } from '~/ide/lib/gitlab_web_ide/handle_tracking_event'; - -/** - * @param {Element} rootEl - */ -export const mountRemoteIDE = async (el) => { - const { - remoteHost: remoteAuthority, - remotePath: hostPath, - cspNonce, - connectionToken, - returnUrl, - } = el.dataset; - - const rootEl = setupRootElement(el); - - const visitReturnUrl = () => { - // security: Only change `href` if of the same origin as current page - if (returnUrl && isSameOriginUrl(returnUrl)) { - window.location.href = returnUrl; - } else { - window.location.reload(); - } - }; - - startRemote(rootEl, { - ...getBaseConfig(), - nonce: cspNonce, - connectionToken, - // remoteAuthority must start with "/" - remoteAuthority: joinPaths('/', remoteAuthority), - // hostPath must start with "/" - hostPath: joinPaths('/', hostPath), - // TODO Handle error better - handleError: visitReturnUrl, - handleClose: visitReturnUrl, - handleTracking, - }); -}; diff --git a/app/assets/javascripts/pages/web_ide/remote_ide/index.js b/app/assets/javascripts/pages/web_ide/remote_ide/index.js deleted file mode 100644 index 463798e85b9..00000000000 --- a/app/assets/javascripts/pages/web_ide/remote_ide/index.js +++ /dev/null @@ -1,3 +0,0 @@ -import { mountRemoteIDE } from '~/ide/remote'; - -mountRemoteIDE(document.getElementById('ide')); diff --git a/app/controllers/web_ide/remote_ide_controller.rb b/app/controllers/web_ide/remote_ide_controller.rb deleted file mode 100644 index 8392e7a190c..00000000000 --- a/app/controllers/web_ide/remote_ide_controller.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -require 'uri' - -module WebIde - class RemoteIdeController < ApplicationController - include WebIdeCSP - - rescue_from URI::InvalidComponentError, with: :render_404 - - before_action :allow_remote_ide_content_security_policy - - feature_category :remote_development - - urgency :low - - def index - return render_404 unless Feature.enabled?(:vscode_web_ide, current_user) - - render layout: 'fullscreen', locals: { data: root_element_data } - end - - private - - def allow_remote_ide_content_security_policy - return if request.content_security_policy.directives.blank? - - default_src = Array(request.content_security_policy.directives['default-src'] || []) - - request.content_security_policy.directives['connect-src'] ||= default_src - request.content_security_policy.directives['connect-src'].concat(connect_src_urls) - end - - def connect_src_urls - # It's okay if "port" is null - host, port = params.require(:remote_host).split(':') - - # This could throw URI::InvalidComponentError. We go ahead and let it throw - # and let the controller recover with a bad_request response - %w[ws wss http https].map { |scheme| URI::Generic.build(scheme: scheme, host: host, port: port).to_s } - end - - def root_element_data - { - connection_token: params.fetch(:connection_token, ''), - remote_host: params.require(:remote_host), - remote_path: params.fetch(:remote_path, ''), - return_url: params.fetch(:return_url, ''), - csp_nonce: content_security_policy_nonce - } - end - end -end diff --git a/app/helpers/ide_helper.rb b/app/helpers/ide_helper.rb index e36dc3c75c6..e19a8b09f79 100644 --- a/app/helpers/ide_helper.rb +++ b/app/helpers/ide_helper.rb @@ -82,8 +82,6 @@ module IdeHelper { 'project-path' => project&.path_with_namespace, 'csp-nonce' => content_security_policy_nonce, - # We will replace these placeholders in the FE - 'ide-remote-path' => ide_remote_path(remote_host: ':remote_host', remote_path: ':remote_path'), 'editor-font' => new_ide_fonts.to_json, 'extensions-gallery-settings' => extensions_gallery_settings }.merge(new_ide_code_suggestions_data).merge(new_ide_oauth_data) diff --git a/app/views/web_ide/remote_ide/index.html.haml b/app/views/web_ide/remote_ide/index.html.haml deleted file mode 100644 index f007794d056..00000000000 --- a/app/views/web_ide/remote_ide/index.html.haml +++ /dev/null @@ -1,5 +0,0 @@ -- data = local_assigns.fetch(:data) - -- page_title _('Web IDE') - -= render partial: 'shared/ide_root', locals: { data: data, loading_text: _('Connecting to the remote environment...') } diff --git a/config/feature_flags/development/increase_diff_file_performance.yml b/config/feature_flags/development/increase_diff_file_performance.yml index e78bce2b5d6..37fc6d5770b 100644 --- a/config/feature_flags/development/increase_diff_file_performance.yml +++ b/config/feature_flags/development/increase_diff_file_performance.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432600 milestone: '16.9' type: development group: group::code review -default_enabled: false +default_enabled: true diff --git a/config/routes.rb b/config/routes.rb index fb3427696f4..44903738510 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -159,12 +159,6 @@ InitializerConnections.raise_if_new_database_connection do get '/', to: 'ide#index' end - # Remote host can contain "." characters so it needs a constraint - post 'remote/:remote_host(/*remote_path)', - as: :remote, - to: 'web_ide/remote_ide#index', - constraints: { remote_host: %r{[^/?]+} } - post '/reset_oauth_application_settings' => 'admin/applications#reset_web_ide_oauth_application_settings' end diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index b05d066acf6..8ce3026624b 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -13062,6 +13062,29 @@ The edge type for [`ForecastDatapoint`](#forecastdatapoint). | `cursor` | [`String!`](#string) | A cursor for use in pagination. | | `node` | [`ForecastDatapoint`](#forecastdatapoint) | The item at the end of the edge. | +#### `GitlabSubscriptionHistoryConnection` + +The connection type for [`GitlabSubscriptionHistory`](#gitlabsubscriptionhistory). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[GitlabSubscriptionHistoryEdge]`](#gitlabsubscriptionhistoryedge) | A list of edges. | +| `nodes` | [`[GitlabSubscriptionHistory]`](#gitlabsubscriptionhistory) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `GitlabSubscriptionHistoryEdge` + +The edge type for [`GitlabSubscriptionHistory`](#gitlabsubscriptionhistory). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`GitlabSubscriptionHistory`](#gitlabsubscriptionhistory) | The item at the end of the edge. | + #### `GoogleCloudArtifactRegistryArtifactConnection` The connection type for [`GoogleCloudArtifactRegistryArtifact`](#googlecloudartifactregistryartifact). @@ -22278,6 +22301,22 @@ four standard [pagination arguments](#pagination-arguments): | `replicationState` | [`ReplicationStateEnum`](#replicationstateenum) | Filters registries by their replication state. | | `verificationState` | [`VerificationStateEnum`](#verificationstateenum) | Filters registries by their verification state. | +### `GitlabSubscriptionHistory` + +Describes the subscription history of a given namespace. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `changeType` | [`SubscriptionHistoryChangeType`](#subscriptionhistorychangetype) | Indicates what type of change in the subscription has happened. | +| `createdAt` | [`Time`](#time) | Timestamp of the subscription history entry creation. | +| `endDate` | [`Time`](#time) | Subscription end date. | +| `maxSeatsUsed` | [`Int`](#int) | Maximum seats used in subscription. | +| `seats` | [`Int`](#int) | Seats purchased in subscription. | +| `seatsInUse` | [`Int`](#int) | Seats being used in subscription. | +| `startDate` | [`Time`](#time) | Subscription start date. | + ### `GoogleCloudArtifactRegistryDockerImage` Represents a docker artifact of Google Artifact Registry. @@ -22467,6 +22506,7 @@ GPG signature for a signed commit. | `stats` | [`GroupStats`](#groupstats) | Group statistics. | | `storageSizeLimit` | [`Float`](#float) | The storage limit (in bytes) included with the root namespace plan. This limit only applies to namespaces under namespace limit enforcement. | | `subgroupCreationLevel` | [`String`](#string) | Permission level required to create subgroups within the group. | +| `subscriptionHistory` **{warning-solid}** | [`GitlabSubscriptionHistoryConnection`](#gitlabsubscriptionhistoryconnection) | **Introduced** in GitLab 17.3. **Status**: Experiment. Find subscription history records. | | `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in GitLab 15.3. **Status**: Experiment. Timelog categories for the namespace. | | `totalRepositorySize` | [`Float`](#float) | Total repository size of all projects in the root namespace in bytes. | | `totalRepositorySizeExcess` | [`Float`](#float) | Total excess repository size of all projects in the root namespace in bytes. This only applies to namespaces under Project limit enforcement. | @@ -27089,6 +27129,7 @@ Product analytics events for a specific month and year. | `securityPolicyProject` | [`Project`](#project) | Security policy project assigned to the namespace. | | `sharedRunnersSetting` | [`SharedRunnersSetting`](#sharedrunnerssetting) | Shared runners availability for the namespace and its descendants. | | `storageSizeLimit` | [`Float`](#float) | The storage limit (in bytes) included with the root namespace plan. This limit only applies to namespaces under namespace limit enforcement. | +| `subscriptionHistory` **{warning-solid}** | [`GitlabSubscriptionHistoryConnection`](#gitlabsubscriptionhistoryconnection) | **Introduced** in GitLab 17.3. **Status**: Experiment. Find subscription history records. | | `timelogCategories` **{warning-solid}** | [`TimeTrackingTimelogCategoryConnection`](#timetrackingtimelogcategoryconnection) | **Introduced** in GitLab 15.3. **Status**: Experiment. Timelog categories for the namespace. | | `totalRepositorySize` | [`Float`](#float) | Total repository size of all projects in the root namespace in bytes. | | `totalRepositorySizeExcess` | [`Float`](#float) | Total excess repository size of all projects in the root namespace in bytes. This only applies to namespaces under Project limit enforcement. | @@ -37146,6 +37187,15 @@ Values for sorting the mapping of users on source instance to users on destinati | `STATUS_ASC` | Status of the mapping by ascending order. | | `STATUS_DESC` | Status of the mapping by descending order. | +### `SubscriptionHistoryChangeType` + +Types of change for a subscription history record. + +| Value | Description | +| ----- | ----------- | +| `GITLAB_SUBSCRIPTION_DESTROYED` | This was the previous state before the subscription was destroyed. | +| `GITLAB_SUBSCRIPTION_UPDATED` | This was the previous state before the subscription was updated. | + ### `TestCaseStatus` | Value | Description | diff --git a/doc/ci/pipelines/merge_trains.md b/doc/ci/pipelines/merge_trains.md index e0e2fb8aeab..e96f4815faf 100644 --- a/doc/ci/pipelines/merge_trains.md +++ b/doc/ci/pipelines/merge_trains.md @@ -118,16 +118,38 @@ Prerequisites: To start a merge train: -1. Visit a merge request. +1. Go to a merge request. 1. Select: - When no pipeline is running, **Merge**. - When a pipeline is running, [**Set to auto-merge**](../../user/project/merge_requests/auto_merge.md). The merge request's merge train status displays under the pipeline widget with a -message similar to `A new merge train has started and this merge request is the first of the queue.` +message similar to `A new merge train has started and this merge request is the first of the queue. View merge train details.` +You can select the link to view the merge train. Other merge requests can now be added to the train. +## View a merge train + +> - Merge train visualization [introduced](https://gitlab.com/groups/gitlab-org/-/epics/13705) in GitLab 17.3. + +You can view the merge train to gain better insight into the order and status of merge requests in the queue. +The merge train details page shows active merge requests in the queue and merged merge requests that were part of the train. + +To access the merge train details from the list of merge requests: + +1. On the left sidebar, select **Search or go to** and find your project. +1. Select **Code > Merge requests**. +1. Above the list of merge requests, select **Merge trains**. +1. Optional. Filter the merge trains by target branch. + +You also access this view by selecting **View merge train details** from: + +- The pipeline widget and system notes on a merge request added to a merge train. +- The pipeline details page for a merge train pipeline. + +You can also remove (**{close}**) a merge request from the merge train details view. + ## Add a merge request to a merge train > - Auto-merge for merge trains [introduced](https://gitlab.com/groups/gitlab-org/-/epics/10874) in GitLab 17.2 [with a flag](../../administration/feature_flags.md) named `merge_when_checks_pass_merge_train`. Disabled by default. @@ -154,14 +176,18 @@ waiting to join the merge train. ## Remove a merge request from a merge train -To remove a merge request from a merge train, select **Cancel auto-merge**. -You can add the merge request to a merge train again later. - When you remove a merge request from a merge train: - All pipelines for merge requests queued after the removed merge request restart. - Redundant pipelines [are cancelled](#automatic-pipeline-cancellation). +You can add the merge request to a merge train again later. + +To remove a merge request from a merge train: + +- From a merge request, select **Cancel auto-merge**. +- From the [merge train details](#view-a-merge-train), next to the merge request, select **{close}**. + ## Skip the merge train and merge immediately If you have a high-priority merge request, like a critical patch that must diff --git a/doc/user/application_security/secret_detection/secret_push_protection/index.md b/doc/user/application_security/secret_detection/secret_push_protection/index.md index dd0d4700326..59e621a5699 100644 --- a/doc/user/application_security/secret_detection/secret_push_protection/index.md +++ b/doc/user/application_security/secret_detection/secret_push_protection/index.md @@ -93,13 +93,13 @@ To enable secret push protection in a project: ## Coverage -Secret push protection checks the content of each commit when it is pushed to GitLab. +Secret push protection checks the content of each commit diff when it is pushed to GitLab. However, the following exclusions apply. Secret push protection does not check a file in a commit when: - The file is a binary file. -- The file is larger than 1 MiB. +- The diff patch for the file is larger than 1 MiB. - The file was renamed, deleted, or moved without changes to the content. - The content of the file is identical to the content of another file in the source code. - The file is contained in the initial push that created the repository. @@ -186,20 +186,3 @@ To skip secret push protection when using any Git client: For example, you are using the GitLab Web IDE and have several commits that are blocked from being pushed because one of them contains a secret. To skip secret push protection, edit the latest commit message and add `[skip secret push protection]`, then push the commits. - -## Troubleshooting - -When working with secret push protection, you may encounter the following situations. - -### Push blocked unexpectedly - -Secret Push Protection scans all contents of modified files. This can cause a push to be -unexpectedly blocked if a modified file contains a secret, even if the secret is not part of the diff. - -To push a change to a file that contains a secret, you need to [skip secret push protection](#skip-secret-push-protection). - -[Issue 469161](https://gitlab.com/gitlab-org/gitlab/-/issues/469161) proposes to change the scanning logic to scan only diffs. - -### File was not scanned - -Some files are excluded from scanning. For a list of exclusions, see [coverage](#coverage). diff --git a/doc/user/gitlab_duo/turn_on_off.md b/doc/user/gitlab_duo/turn_on_off.md index b821569eabf..66bf2ec9ac0 100644 --- a/doc/user/gitlab_duo/turn_on_off.md +++ b/doc/user/gitlab_duo/turn_on_off.md @@ -67,16 +67,13 @@ DETAILS: **Tier:** Premium, Ultimate **Status:** Beta -> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/161997) in GitLab 17.3 [with a flag](../../administration/feature_flags.md) named `cloud_connector_status`. Enabled by default. - -FLAG: -The availability of this feature is controlled by a feature flag. -For more information, see the history. -This feature is available for testing, but not ready for production use. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/161997) in GitLab 17.3. Run a health check to test if your instance meets the requirements to use GitLab Duo. When the health check completes, it displays a pass or fail result and the types of issues. +This is a [beta](../../policy/experiment-beta-support.md) feature. + Prerequisites: - You must be an administrator. diff --git a/doc/user/gitlab_duo/use_cases.md b/doc/user/gitlab_duo/use_cases.md index db74902c93f..b771c56f1b9 100644 --- a/doc/user/gitlab_duo/use_cases.md +++ b/doc/user/gitlab_duo/use_cases.md @@ -453,7 +453,7 @@ fun main() { ### Get Started with PowerShell NOTE: -PowerShell support is [experimental](../project/repository/code_suggestions/supported_extensions.md#add-additional-languages-for-code-suggestions). +PowerShell support is [experimental](../project/repository/code_suggestions/supported_extensions.md#add-support-for-more-languages). 1. Use Duo Chat to ask how to get started with a PowerShell script that prints the file size of the current directory. diff --git a/doc/user/markdown.md b/doc/user/markdown.md index 942f9568a0b..0131185316c 100644 --- a/doc/user/markdown.md +++ b/doc/user/markdown.md @@ -1587,14 +1587,16 @@ When rendered, the escaped characters look like this: Exceptions: -A backslash doesn't always escape the following character. The backslash appears as regular text in the following cases: +A backslash doesn't always escape the character that follows it. The backslash appears as regular text in the following cases: - When the backslash appears before a non-reserved character, such as `A`, `3`, or a space. - When the backslash appears inside of these Markdown elements: - Code blocks - Code spans - Auto-links - - Inline HTML + - Inline HTML, such as `` + +In these instances you might need to use the equivalent HTML entity, such as `]` for `]`. ## Footnotes diff --git a/doc/user/project/repository/code_suggestions/supported_extensions.md b/doc/user/project/repository/code_suggestions/supported_extensions.md index ac786912568..2a7ad61c67a 100644 --- a/doc/user/project/repository/code_suggestions/supported_extensions.md +++ b/doc/user/project/repository/code_suggestions/supported_extensions.md @@ -2,7 +2,7 @@ stage: Create group: Code Creation info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://handbook.gitlab.com/handbook/product/ux/technical-writing/#assignments -description: "Code Suggestions supports multiple editors and languages." +description: "Code suggestions supports multiple editors and languages." --- # Supported extensions and languages @@ -11,12 +11,12 @@ DETAILS: **Tier:** Premium with GitLab Duo Pro or Ultimate with [GitLab Duo Pro or Enterprise](../../../../subscriptions/subscription-add-ons.md) **Offering:** GitLab.com, Self-managed, GitLab Dedicated -Code Suggestions is available in the following editor extensions and +Code suggestions are available in the following editor extensions and for the following languages. ## Supported editor extensions -To use Code Suggestions, use one of these editor extensions: +To use code suggestions, use one of these editor extensions: | IDE | Extension | |----------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------| @@ -26,17 +26,17 @@ To use Code Suggestions, use one of these editor extensions: | JetBrains IDEs | [GitLab Duo Plugin for JetBrains](https://plugins.jetbrains.com/plugin/22325-gitlab-duo) | | Neovim | [`gitlab.vim` plugin](https://gitlab.com/gitlab-org/editor-extensions/gitlab.vim) | -A [GitLab Language Server](https://gitlab.com/gitlab-org/editor-extensions/gitlab-lsp) is used in VS Code, Visual Studio, and Neovim. The Language Server supports faster iteration across more platforms. You can also configure it to support Code Suggestions in IDEs where GitLab doesn't provide official support. +A [GitLab Language Server](https://gitlab.com/gitlab-org/editor-extensions/gitlab-lsp) is used in VS Code, Visual Studio, and Neovim. The Language Server supports faster iteration across more platforms. You can also configure it to support code suggestions in IDEs where GitLab doesn't provide official support. You can express interest in other IDE extension support [in this issue](https://gitlab.com/gitlab-org/editor-extensions/meta/-/issues/78). ## Supported languages -Code Suggestions is aware of common popular programming languages, concepts, and +Code suggestions are aware of common popular programming languages, concepts, and infrastructure-as-code interfaces, like Kubernetes Resource Model (KRM), Google Cloud CLI, and Terraform. -Code Suggestions supports these languages: +Code suggestions support these languages: | Language | Web IDE | VS Code | JetBrains IDEs | Visual Studio 2022 for Windows | Neovim | |-------------------------------|----------------------------|---------------------------------------------------------------------------------------------|-----------------------|--------------------------------|--------------------------------------------------------------------------------------------------------| @@ -68,31 +68,33 @@ NOTE: Some languages are not supported in all JetBrains IDEs, or might require additional plugin support. Refer to the JetBrains documentation for specifics on your IDE. -For languages not listed in the table, Code Suggestions might not function as expected. +Locally, you can add [more languages](#add-support-for-more-languages). For languages not listed in the table, +code suggestions might not function as expected. -## View multiple code suggestions +## Manage languages for code suggestions -> - [Introduced](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/1325) in GitLab 17.1. +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/blob/main/CHANGELOG.md#4210-2024-07-16) in GitLab Workflow for VS Code 4.21.0 -For a code completion suggestion in VS Code, multiple suggestion options -might be available. To view all available suggestions: +You can customize your coding experience in VS Code by enabling or disabling code suggestions for specific supported languages. +You can do this by editing your `settings.json` file directly, or from the VS Code user interface: -1. Hover over the code completion suggestion. -1. Scroll through the alternatives. Either: - - Use keyboard shortcuts: - - On a Mac, press Option + ] to view the - next suggestion, and Option + [ to view the previous - suggestions. - - On Windows, press Alt + ] to view the - next suggestion, and Alt + [ to view the previous - suggestions. - - On the dialog that's displayed, select the right or left arrow to see next or previous options. -1. Press Tab to apply the suggestion you prefer. +1. In VS Code, open the extension settings for **GitLab Workflow**: + 1. On the top bar, go to **Code > Settings > Extensions**. + 1. Search for **GitLab Workflow** in the list, and select **Manage** (**{settings}**). + 1. Select **Extension Settings**. +1. In your **User** settings, find the section titled **AI Assisted Code Suggestions: Enabled Supported Languages**. +1. You will see a list of all supported languages with checkboxes next to each language. +1. To enable code suggestions for a language, ensure its checkbox is checked. +1. To disable code suggestions for a language, uncheck its checkbox. +1. Your changes are automatically saved and will take effect immediately. -## Add additional languages for Code Suggestions +When you disable code suggestions for a language, the Duo icon changes to show that suggestions are disabled +for this language. On hover, it shows **Code Suggestions are disabled for this language**. -If your desired language isn't [enabled by default](#supported-languages) for Code Suggestions, -you can add support for it locally. +### Add support for more languages + +If your desired language doesn't have code suggestions available by default, +you can add support for your language locally. Prerequisites: @@ -113,3 +115,22 @@ To do this: 1. In your **User** settings, find **GitLab › Ai Assisted Code Suggestions: Additional Languages** and select **Add Item**. 1. In **Item**, add the language identifier, and select **OK**. + +## View multiple code suggestions + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab-vscode-extension/-/issues/1325) in GitLab 17.1. + +For a code completion suggestion in VS Code, multiple suggestion options +might be available. To view all available suggestions: + +1. Hover over the code completion suggestion. +1. Scroll through the alternatives. Either: + - Use keyboard shortcuts: + - On a Mac, press Option + ] to view the + next suggestion, and Option + [ to view the previous + suggestions. + - On Windows, press Alt + ] to view the + next suggestion, and Alt + [ to view the previous + suggestions. + - On the dialog that's displayed, select the right or left arrow to see next or previous options. +1. Press Tab to apply the suggestion you prefer. diff --git a/doc/user/report_abuse.md b/doc/user/report_abuse.md index 808c8cc0570..df0de351571 100644 --- a/doc/user/report_abuse.md +++ b/doc/user/report_abuse.md @@ -69,6 +69,8 @@ A URL to the reported user's comment is pre-filled in the abuse report's ## Report abuse from a task +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/461848) in GitLab 17.3. + 1. On the task, in the upper-right corner, select **More actions** (**{ellipsis_v}**). 1. Select **Report abuse**. 1. Select a reason for reporting the user. @@ -77,6 +79,8 @@ A URL to the reported user's comment is pre-filled in the abuse report's ## Report abuse from an objective +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/461848) in GitLab 17.3. + 1. On the objective, in the upper-right corner, select **More actions** (**{ellipsis_v}**). 1. Select **Report abuse**. 1. Select a reason for reporting the user. @@ -85,6 +89,8 @@ A URL to the reported user's comment is pre-filled in the abuse report's ## Report abuse from a key result +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/461848) in GitLab 17.3. + 1. On the key result, in the upper-right corner, select **More actions** (**{ellipsis_v}**). 1. Select **Report abuse**. 1. Select a reason for reporting the user. diff --git a/gems/gitlab-secret_detection/README.md b/gems/gitlab-secret_detection/README.md index 9fc770a8bf5..ede38a5a01a 100644 --- a/gems/gitlab-secret_detection/README.md +++ b/gems/gitlab-secret_detection/README.md @@ -1,6 +1,6 @@ # Gitlab::SecretDetection -The gitlab-secret_detection gem performs keyword and regex matching on git blobs that may include secrets. The gem accepts one or more git blobs, matches them against a defined ruleset of regular expressions, and returns scan results. +The gitlab-secret_detection gem performs keyword and regex matching on git diffs that may include secrets. The gem accepts one or more git diffs, matches them against a defined ruleset of regular expressions, and returns scan results. ##### Scan parameters @@ -10,10 +10,10 @@ accepts the following parameters: | Parameter | Type | Required | Default | Description | |----------------|---------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `blobs` | Array | Yes | NA | Array of blobs with each blob to have `id` and `data` properties. `id` represents the uniqueness of the blob in the given array and `data` is the content of the blob to scan. | +| `diffs` | Array | Yes | NA | Array of diffs. Each diff has attributes: left_blob_id, right_blob_id, patch, status, binary, and over_patch_bytes_limit. | | `timeout` | Number | No | [`60s`](https://gitlab.com/gitlab-org/gitlab/-/blob/5dfcf7431bfff25519c05a7e66c0cbb8d7b362be/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb#L22) | The maximum duration allowed for the scan to run on a commit request comprising multiple blobs. If the specified timeout elapses, the scan is automatically terminated. The timeout duration is specified in seconds but can also accept floating-point values to denote smaller units. For instance, use `0.5` to represent `500ms`. | -| `blob_timeout` | Number | No | [`5s`](https://gitlab.com/gitlab-org/gitlab/-/blob/5dfcf7431bfff25519c05a7e66c0cbb8d7b362be/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb#L24) | The maximum duration allowed for the scan to run on an individual blob. Upon expiration of the specified timeout, the scan is interrupted for the current blob and advances to the next blob in the request. The timeout duration is specified in seconds but can also accept floating-point values to denote smaller units. For instance, use `0.5` to represent `500ms`. | -| `subprocess` | Boolean | No | [`true`](https://gitlab.com/gitlab-org/gitlab/-/blob/5dfcf7431bfff25519c05a7e66c0cbb8d7b362be/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb#L34) | Runs the scan operation within a subprocess rather than the main process. This design aims to mitigate memory overconsumption issues that may arise from scanning multiple large blobs within a single subprocess. Check [here](https://docs.gitlab.com/ee/architecture/blueprints/secret_detection/decisions/002_run_scan_within_subprocess.html) for more details. | +| `diff_timeout` | Number | No | [`5s`](https://gitlab.com/gitlab-org/gitlab/-/blob/5dfcf7431bfff25519c05a7e66c0cbb8d7b362be/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb#L24) | The maximum duration allowed for the scan to run on an individual diff. Upon expiration of the specified timeout, the scan is interrupted for the current diff and advances to the next diff in the request. The timeout duration is specified in seconds but can also accept floating-point values to denote smaller units. For instance, use `0.5` to represent `500ms`. | +| `subprocess` | Boolean | No | [`true`](https://gitlab.com/gitlab-org/gitlab/-/blob/5dfcf7431bfff25519c05a7e66c0cbb8d7b362be/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb#L34) | Runs the scan operation within a subprocess rather than the main process. This design aims to mitigate memory overconsumption issues that may arise from scanning multiple large diffs within a single subprocess. Check [here](https://docs.gitlab.com/ee/architecture/blueprints/secret_detection/decisions/002_run_scan_within_subprocess.html) for more details. | ##### Scan Constraints diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb index 3918d584ccd..0b58bc4213e 100644 --- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb +++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb @@ -20,14 +20,14 @@ module Gitlab # default time limit(in seconds) for running the scan operation per invocation DEFAULT_SCAN_TIMEOUT_SECS = 60 - # default time limit(in seconds) for running the scan operation on a single blob - DEFAULT_BLOB_TIMEOUT_SECS = 5 + # default time limit(in seconds) for running the scan operation on a single diff + DEFAULT_DIFF_TIMEOUT_SECS = 5 # file path where the secrets ruleset file is located RULESET_FILE_PATH = File.expand_path('../../gitleaks.toml', __dir__) # Max no of child processes to spawn per request # ref: https://gitlab.com/gitlab-org/gitlab/-/issues/430160 MAX_PROCS_PER_REQUEST = 5 - # Minimum cumulative size of the blobs required to spawn and + # Minimum cumulative size of the diffs required to spawn and # run the scan within a new subprocess. MIN_CHUNK_SIZE_PER_PROC_BYTES = 2_097_152 # 2MiB # Whether to run scan in subprocesses or not. Default is true. @@ -46,23 +46,24 @@ module Gitlab @pattern_matcher = build_pattern_matcher(rules) end - # Runs Secret Detection scan on the list of given blobs. Both the total scan duration and - # the duration for each blob is time bound via +timeout+ and +blob_timeout+ respectively. + # Runs Secret Detection scan on the list of given diffs. Both the total scan duration and + # the duration for each diff is time bound via +timeout+ and +diff_timeout+ respectively. # - # +blobs+:: Array of blobs with each blob to have `id` and `data` properties. + # +diffs+:: Array of diffs between diff pairs. Each diff has attributes: left_blob_id, right_blob_id, + # patch, status, binary, and over_patch_bytes_limit. # +timeout+:: No of seconds(accepts floating point for smaller time values) to limit the total scan duration - # +blob_timeout+:: No of seconds(accepts floating point for smaller time values) to limit - # the scan duration on each blob + # +diff_timeout+:: No of seconds(accepts floating point for smaller time values) to limit + # the scan duration on each diff # +subprocess+:: If passed true, the scan is performed within subprocess instead of main process. - # To avoid over-consuming memory by running scan on multiple large blobs within a single subprocess, - # it instead groups the blobs into smaller array where each array contains blobs with cumulative size of + # To avoid over-consuming memory by running scan on multiple large diffs within a single subprocess, + # it instead groups the diffs into smaller array where each array contains diffs with cumulative size of # +MIN_CHUNK_SIZE_PER_PROC_BYTES+ bytes and each group runs in a separate sub-process. Default value # is true. # # NOTE: # Running the scan in fork mode primarily focuses on reducing the memory consumption of the scan by - # offloading regex operations on large blobs to sub-processes. However, it does not assure the improvement - # in the overall latency of the scan, specifically in the case of smaller blob sizes, where the overhead of + # offloading regex operations on large diffs to sub-processes. However, it does not assure the improvement + # in the overall latency of the scan, specifically in the case of smaller diff sizes, where the overhead of # forking a new process adds to the overall latency of the scan instead. More reference on Subprocess-based # execution is found here: https://gitlab.com/gitlab-org/gitlab/-/issues/430160. # @@ -73,23 +74,25 @@ module Gitlab # } # def secrets_scan( - blobs, + diffs, timeout: DEFAULT_SCAN_TIMEOUT_SECS, - blob_timeout: DEFAULT_BLOB_TIMEOUT_SECS, + diff_timeout: DEFAULT_DIFF_TIMEOUT_SECS, subprocess: RUN_IN_SUBPROCESS ) - return SecretDetection::Response.new(SecretDetection::Status::INPUT_ERROR) unless validate_scan_input(blobs) + + return SecretDetection::Response.new(SecretDetection::Status::INPUT_ERROR) unless validate_scan_input(diffs) Timeout.timeout(timeout) do - matched_blobs = filter_by_keywords(blobs) + matched_diffs = filter_by_keywords(diffs) - next SecretDetection::Response.new(SecretDetection::Status::NOT_FOUND) if matched_blobs.empty? + next SecretDetection::Response.new(SecretDetection::Status::NOT_FOUND) if matched_diffs.empty? - secrets = if subprocess - run_scan_within_subprocess(matched_blobs, blob_timeout) - else - run_scan(matched_blobs, blob_timeout) - end + secrets = + if subprocess + run_scan_within_subprocess(matched_diffs, diff_timeout) + else + run_scan(matched_diffs, diff_timeout) + end scan_status = overall_scan_status(secrets) @@ -143,104 +146,137 @@ module Gitlab secrets_keywords.flatten.compact.to_set end - # returns only those blobs that contain at least one of the keywords + # returns only those diffs that contain at least one of the keywords # from the keywords list - def filter_by_keywords(blobs) - matched_blobs = [] + def filter_by_keywords(diffs) + matched_diffs = [] - blobs.each do |blob| - matched_blobs << blob if keywords.any? { |keyword| blob.data.include?(keyword) } + diffs.each do |diff| + matched_diffs << diff if keywords.any? { |keyword| diff.patch.include?(keyword) } end - matched_blobs.freeze + matched_diffs.freeze end - def run_scan(blobs, blob_timeout) - found_secrets = blobs.flat_map do |blob| - Timeout.timeout(blob_timeout) do - find_secrets(blob) + def run_scan(diffs, diff_timeout) + found_secrets = diffs.flat_map do |diff| + Timeout.timeout(diff_timeout) do + find_secrets(diff) end rescue Timeout::Error => e - logger.error "Secret Detection scan timed out on the blob(id:#{blob.id}): #{e}" - SecretDetection::Finding.new(blob.id, - SecretDetection::Status::BLOB_TIMEOUT) + logger.error "Secret Detection scan timed out on the diff(id:#{diff.right_blob_id}): #{e}" + SecretDetection::Finding.new(diff.right_blob_id, + SecretDetection::Status::DIFF_TIMEOUT) end found_secrets.freeze end - def run_scan_within_subprocess(blobs, blob_timeout) - blob_sizes = blobs.map(&:size) - grouped_blob_indicies = group_by_chunk_size(blob_sizes) + def run_scan_within_subprocess(diffs, diff_timeout) + diff_sizes = diffs.map { |diff| diff.patch.length } + grouped_diff_indicies = group_by_chunk_size(diff_sizes) - grouped_blobs = grouped_blob_indicies.map { |idx_arr| idx_arr.map { |i| blobs[i] } } + grouped_diffs = grouped_diff_indicies.map { |idx_arr| idx_arr.map { |i| diffs[i] } } found_secrets = Parallel.flat_map( - grouped_blobs, + grouped_diffs, in_processes: MAX_PROCS_PER_REQUEST, isolation: true # do not reuse sub-processes - ) do |grouped_blob| - grouped_blob.flat_map do |blob| - Timeout.timeout(blob_timeout) do - find_secrets(blob) + ) do |grouped_diff| + grouped_diff.flat_map do |diff| + Timeout.timeout(diff_timeout) do + find_secrets(diff) end rescue Timeout::Error => e - logger.error "Secret Detection scan timed out on the blob(id:#{blob.id}): #{e}" - SecretDetection::Finding.new(blob.id, - SecretDetection::Status::BLOB_TIMEOUT) + logger.error "Secret Detection scan timed out on the diff(id:#{diff.right_blob_id}): #{e}" + SecretDetection::Finding.new(diff.right_blob_id, + SecretDetection::Status::DIFF_TIMEOUT) end end found_secrets.freeze end - # finds secrets in the given blob with a timeout circuit breaker - def find_secrets(blob) + # finds secrets in the given diff with a timeout circuit breaker + def find_secrets(diff) + line_number_offset = 0 secrets = [] - blob.data.each_line.with_index do |line, index| - patterns = pattern_matcher.match(line, exception: false) + lines = diff.patch.split("\n") - next unless patterns.any? + # The following section parses the diff patch. + # + # If the line starts with @@, it is the hunk header, used to calculate the line number. + # If the line starts with +, it is newly added in this diff, and we + # scan the line for newly added secrets. Also increment line number. + # If the line starts with -, it is removed in this diff, do not increment line number. + # If the line starts with \\, it is the no newline marker, do not increment line number. + # If the line starts with a space character, it is a context line, just increment the line number. + # + # A context line that starts with an important character would still be treated + # like a context line, as shown below: + # @@ -1,5 +1,5 @@ + # context line + # -removed line + # +added line + # @@this context line has a @@ but starts with a space so isnt a header + # +this context line has a + but starts with a space so isnt an addition + # -this context line has a - but starts with a space so isnt a removal + lines.each do |line| + # Parse hunk header for start line + if line.start_with?("@@") + hunk_info = line.match(/@@ -\d+(,\d+)? \+(\d+)(,\d+)? @@/) + start_line = hunk_info[2].to_i + line_number_offset = start_line - 1 + # Line added in this commit + elsif line.start_with?('+') + line_number_offset += 1 + # Remove leading + + line_content = line[1..] - line_number = index + 1 - patterns.each do |pattern| - type = rules[pattern]["id"] - description = rules[pattern]["description"] + patterns = pattern_matcher.match(line_content, exception: false) + next unless patterns.any? - secrets << SecretDetection::Finding.new( - blob.id, - SecretDetection::Status::FOUND, - line_number, - type, - description - ) + patterns.each do |pattern| + type = rules[pattern]["id"] + description = rules[pattern]["description"] + + secrets << SecretDetection::Finding.new( + diff.right_blob_id, + SecretDetection::Status::FOUND, + line_number_offset, + type, + description + ) + end + # Line not added in this commit, just increment line number + elsif line.start_with?(' ') + line_number_offset += 1 + # Line removed in this commit or no newline marker, do not increment line number + elsif line.start_with?('-', '\\') + # No increment end end secrets rescue StandardError => e - logger.error "Secret Detection scan failed on the blob(id:#{blob.id}): #{e}" + logger.error "Secret Detection scan failed on the diff(id:#{diff.right_blob_id}): #{e}" - SecretDetection::Finding.new(blob.id, SecretDetection::Status::SCAN_ERROR) + SecretDetection::Finding.new(diff.right_blob_id, SecretDetection::Status::SCAN_ERROR) end - def validate_scan_input(blobs) - return false if blobs.nil? || !blobs.instance_of?(Array) + def validate_scan_input(diffs) + return false if diffs.nil? || !diffs.instance_of?(Array) - blobs.all? do |blob| - next false unless blob.respond_to?(:id) || blob.respond_to?(:data) - - blob.data.freeze # freeze blobs to avoid additional object allocations on strings - end + diffs.each { |diff| diff.patch.freeze } end def overall_scan_status(found_secrets) return SecretDetection::Status::NOT_FOUND if found_secrets.empty? - timed_out_blobs = found_secrets.count { |el| el.status == SecretDetection::Status::BLOB_TIMEOUT } + timed_out_diffs = found_secrets.count { |el| el.status == SecretDetection::Status::DIFF_TIMEOUT } - case timed_out_blobs + case timed_out_diffs when 0 SecretDetection::Status::FOUND when found_secrets.length @@ -250,15 +286,15 @@ module Gitlab end end - # This method accepts an array of blob sizes(in bytes) and groups them into an array + # This method accepts an array of diff sizes(in bytes) and groups them into an array # of arrays structure where each element is the group of indicies of the input - # array whose cumulative blob sizes has at least +MIN_CHUNK_SIZE_PER_PROC_BYTES+ - def group_by_chunk_size(blob_size_arr) + # array whose cumulative diff sizes has at least +MIN_CHUNK_SIZE_PER_PROC_BYTES+ + def group_by_chunk_size(diff_size_arr) cumulative_size = 0 chunk_indexes = [] chunk_idx_start = 0 - blob_size_arr.each_with_index do |size, index| + diff_size_arr.each_with_index do |size, index| cumulative_size += size next unless cumulative_size >= MIN_CHUNK_SIZE_PER_PROC_BYTES @@ -268,11 +304,11 @@ module Gitlab cumulative_size = 0 end - if cumulative_size.positive? && (chunk_idx_start < blob_size_arr.length) - chunk_indexes << if chunk_idx_start == blob_size_arr.length - 1 + if cumulative_size.positive? && (chunk_idx_start < diff_size_arr.length) + chunk_indexes << if chunk_idx_start == diff_size_arr.length - 1 [chunk_idx_start] else - (chunk_idx_start..blob_size_arr.length - 1).to_a + (chunk_idx_start..diff_size_arr.length - 1).to_a end end diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb index 200294fc2e7..85d60b86009 100644 --- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb +++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb @@ -8,7 +8,7 @@ module Gitlab FOUND = 1 # When scan operation completes with one or more findings FOUND_WITH_ERRORS = 2 # When scan operation completes with one or more findings along with some errors SCAN_TIMEOUT = 3 # When the scan operation runs beyond given time out - BLOB_TIMEOUT = 4 # When the scan operation on a blob runs beyond given time out + DIFF_TIMEOUT = 4 # When the scan operation on a diff runs beyond given time out SCAN_ERROR = 5 # When the scan operation fails due to regex error INPUT_ERROR = 6 # When the scan operation fails due to invalid input end diff --git a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb index 71899cd77a1..8cba55865d1 100644 --- a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb +++ b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb @@ -5,10 +5,13 @@ require 'spec_helper' RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detection do subject(:scan) { described_class.new } - def new_blob(id:, data:) - Struct.new(:id, :data).new(id, data) + let(:diff_blob) do + Struct.new(:left_blob_id, :right_blob_id, :patch, :status, :binary, :over_patch_bytes_limit, keyword_init: true) end + let(:sha1_blank_sha) { ('0' * 40).freeze } + let(:sample_blob_id) { 'fe29d93da4843da433e62711ace82db601eb4f8f' } + let(:ruleset) do { "title" => "gitleaks config", @@ -64,47 +67,108 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio allow(scan).to receive(:parse_ruleset).and_return(ruleset) end - context 'when the blob does not contain a secret' do - let(:blobs) do + context 'when the diff does not contain a secret' do + let(:diffs) do [ - new_blob(id: 1234, data: "no secrets") + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1 @@\n+BASE_URL=https://foo.bar\n\\ No newline at end of file\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ) ] end it "does not match" do expected_response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::NOT_FOUND) - expect(scan.secrets_scan(blobs)).to eq(expected_response) + expect(scan.secrets_scan(diffs)).to eq(expected_response) end - it "attempts to keyword match returning no blobs for further scan" do + it "attempts to keyword match returning no diffs for further scan" do expect(scan).to receive(:filter_by_keywords) - .with(blobs) + .with(diffs) .and_return([]) - scan.secrets_scan(blobs) + scan.secrets_scan(diffs) end it "does not attempt to regex match" do expect(scan).not_to receive(:match_rules_bulk) - scan.secrets_scan(blobs) + scan.secrets_scan(diffs) end end - context "when multiple blobs contains secrets" do - let(:blobs) do + context "when multiple diffs contains secrets" do + let(:diffs) do [ - new_blob(id: 111, data: "glpat-12312312312312312312"), # gitleaks:allow - new_blob(id: 222, data: "\n\nglptt-1231231231231231231212312312312312312312"), # gitleaks:allow - new_blob(id: 333, data: "data with no secret"), - new_blob(id: 444, - data: "GR134894112312312312312312312\nglft-12312312312312312312"), # gitleaks:allow - new_blob(id: 555, data: "data with no secret"), - new_blob(id: 666, data: "data with no secret"), - new_blob(id: 777, data: "\nglptt-1231231231231231231212312312312312312312"), # gitleaks:allow - new_blob(id: 888, - data: "glpat-12312312312312312312;GR134894112312312312312312312") # gitleaks:allow + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1 @@\n+glpat-12312312312312312312\n", # gitleaks:allow + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,3 @@\n+\n+\n+glptt-1231231231231231231212312312312312312312\n", # gitleaks:allow + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1 @@\n+data with no secret\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,2 @@\n+GR134894112312312312312312312\n+glft-12312312312312312312\n", # gitleaks:allow + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1 @@\n+data with no secret\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1 @@\n+data with no secret\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1 @@\n+glptt-1231231231231231231212312312312312312312\n", # gitleaks:allow + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,2 @@\n+glpat-12312312312312312312\n+GR134894112312312312312312312\n", # gitleaks:allow + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ) ] end @@ -113,51 +177,51 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio Gitlab::SecretDetection::Status::FOUND, [ Gitlab::SecretDetection::Finding.new( - blobs[0].id, + diffs[0].right_blob_id, Gitlab::SecretDetection::Status::FOUND, 1, ruleset['rules'][0]['id'], ruleset['rules'][0]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[1].id, + diffs[1].right_blob_id, Gitlab::SecretDetection::Status::FOUND, 3, ruleset['rules'][1]['id'], ruleset['rules'][1]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[3].id, + diffs[3].right_blob_id, Gitlab::SecretDetection::Status::FOUND, 1, ruleset['rules'][2]['id'], ruleset['rules'][2]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[3].id, + diffs[3].right_blob_id, Gitlab::SecretDetection::Status::FOUND, 2, ruleset['rules'][3]['id'], ruleset['rules'][3]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[6].id, + diffs[6].right_blob_id, Gitlab::SecretDetection::Status::FOUND, - 2, + 1, ruleset['rules'][1]['id'], ruleset['rules'][1]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[7].id, + diffs[7].right_blob_id, Gitlab::SecretDetection::Status::FOUND, 1, ruleset['rules'][0]['id'], ruleset['rules'][0]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[7].id, + diffs[7].right_blob_id, Gitlab::SecretDetection::Status::FOUND, - 1, + 2, ruleset['rules'][2]['id'], ruleset['rules'][2]['description'] ) @@ -165,90 +229,93 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio ) end - it "attempts to keyword match returning only filtered blobs for further scan" do - expected = blobs.filter { |b| b.data != "data with no secret" } + it "attempts to keyword match returning only filtered diffs for further scan" do + expected = diffs.reject { |d| d.patch.include?("data with no secret") } expect(scan).to receive(:filter_by_keywords) - .with(blobs) + .with(diffs) .and_return(expected) - scan.secrets_scan(blobs) + scan.secrets_scan(diffs) end it "matches multiple rules when running in main process" do - expect(scan.secrets_scan(blobs, subprocess: false)).to eq(expected_response) - end - - context "in subprocess" do - let(:dummy_lines) do - 10_000 - end - - let(:large_blobs) do - dummy_data = "\nrandom data" * dummy_lines - [ - new_blob(id: 111, data: "glpat-12312312312312312312#{dummy_data}"), # gitleaks:allow - new_blob(id: 222, data: "\n\nglptt-1231231231231231231212312312312312312312#{dummy_data}"), # gitleaks:allow - new_blob(id: 333, data: "data with no secret#{dummy_data}"), - new_blob(id: 444, - data: "GR134894112312312312312312312\nglft-12312312312312312312#{dummy_data}"), # gitleaks:allow - new_blob(id: 555, data: "data with no secret#{dummy_data}"), - new_blob(id: 666, data: "data with no secret#{dummy_data}"), - new_blob(id: 777, data: "#{dummy_data}\nglptt-1231231231231231231212312312312312312312") # gitleaks:allow - ] - end - - it "matches multiple rules" do - expect(scan.secrets_scan(blobs, subprocess: true)).to eq(expected_response) - end - - it "allocates less memory than when running in main process" do - forked_stats = Benchmark::Malloc.new.run { scan.secrets_scan(large_blobs, subprocess: true) } - non_forked_stats = Benchmark::Malloc.new.run { scan.secrets_scan(large_blobs, subprocess: false) } - - max_processes = Gitlab::SecretDetection::Scan::MAX_PROCS_PER_REQUEST - - forked_memory = forked_stats.allocated.total_memory - non_forked_memory = non_forked_stats.allocated.total_memory - forked_obj_allocs = forked_stats.allocated.total_objects - non_forked_obj_allocs = non_forked_stats.allocated.total_objects - - expect(non_forked_memory).to be >= forked_memory * max_processes - expect(non_forked_obj_allocs).to be >= forked_obj_allocs * max_processes - end + expect(scan.secrets_scan(diffs, subprocess: false)).to eq(expected_response) end end context "when configured with time out" do - let(:each_blob_timeout_secs) { 0.000_001 } # 1 micro-sec to intentionally timeout large blob + let(:each_diff_timeout_secs) { 0.000_001 } # 1 micro-sec to intentionally timeout large diff let(:large_data) do - ("large data with a secret glpat-12312312312312312312\n" * 10_000_000).freeze # gitleaks:allow + ("\n+large data with a secret glpat-12312312312312312312" * 10_000_000).freeze # gitleaks:allow end - let(:blobs) do + let(:diffs) do [ - new_blob(id: 111, data: "GR134894112312312312312312312"), # gitleaks:allow - new_blob(id: 333, data: "data with no secret"), - new_blob(id: 333, data: large_data) + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,2 @@\n+GR134894112312312312312312312\n", # gitleaks:allow + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,2 @@\n+data with no secret\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,10000001 @@\n#{large_data}\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ) ] end - let(:all_large_blobs) do + let(:all_large_diffs) do [ - new_blob(id: 111, data: large_data), - new_blob(id: 222, data: large_data), - new_blob(id: 333, data: large_data) + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,10000001 @@\n#{large_data}\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,10000001 @@\n#{large_data}\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ), + diff_blob.new( + left_blob_id: sha1_blank_sha, + right_blob_id: sample_blob_id, + patch: "@@ -0,0 +1,10000001 @@\n#{large_data}\n", + status: :STATUS_END_OF_PATCH, + binary: false, + over_patch_bytes_limit: false + ) ] end it "whole secret detection scan operation times out" do - scan_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob + scan_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large diff expected_response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::SCAN_TIMEOUT) begin - response = scan.secrets_scan(blobs, timeout: scan_timeout_secs) + response = scan.secrets_scan(diffs, timeout: scan_timeout_secs) expect(response).to eq(expected_response) rescue ArgumentError # When RSpec's main process terminates and attempts to clean up child processes upon completion, it terminates @@ -264,50 +331,50 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio end end - it "one of the blobs times out while others continue to get scanned" do + it "one of the diffs times out while others continue to get scanned" do expected_response = Gitlab::SecretDetection::Response.new( Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS, [ Gitlab::SecretDetection::Finding.new( - blobs[0].id, + diffs[0].right_blob_id, Gitlab::SecretDetection::Status::FOUND, 1, ruleset['rules'][2]['id'], ruleset['rules'][2]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[2].id, - Gitlab::SecretDetection::Status::BLOB_TIMEOUT + diffs[2].right_blob_id, + Gitlab::SecretDetection::Status::DIFF_TIMEOUT ) ] ) - expect(scan.secrets_scan(blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response) + expect(scan.secrets_scan(diffs, diff_timeout: each_diff_timeout_secs)).to eq(expected_response) end - it "all the blobs time out" do - # scan status changes to SCAN_TIMEOUT when *all* the blobs time out + it "all the diffs time out" do + # scan status changes to SCAN_TIMEOUT when *all* the diffs time out expected_scan_status = Gitlab::SecretDetection::Status::SCAN_TIMEOUT expected_response = Gitlab::SecretDetection::Response.new( expected_scan_status, [ Gitlab::SecretDetection::Finding.new( - all_large_blobs[0].id, - Gitlab::SecretDetection::Status::BLOB_TIMEOUT + all_large_diffs[0].right_blob_id, + Gitlab::SecretDetection::Status::DIFF_TIMEOUT ), Gitlab::SecretDetection::Finding.new( - all_large_blobs[1].id, - Gitlab::SecretDetection::Status::BLOB_TIMEOUT + all_large_diffs[1].right_blob_id, + Gitlab::SecretDetection::Status::DIFF_TIMEOUT ), Gitlab::SecretDetection::Finding.new( - all_large_blobs[2].id, - Gitlab::SecretDetection::Status::BLOB_TIMEOUT + all_large_diffs[2].right_blob_id, + Gitlab::SecretDetection::Status::DIFF_TIMEOUT ) ] ) - expect(scan.secrets_scan(all_large_blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response) + expect(scan.secrets_scan(all_large_diffs, diff_timeout: each_diff_timeout_secs)).to eq(expected_response) end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index b5fd0e5f54c..01696aecf35 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -7155,9 +7155,6 @@ msgid_plural "Are you sure you want to import %d repositories?" msgstr[0] "" msgstr[1] "" -msgid "Are you sure you want to leave the Web IDE? All unsaved changes will be lost." -msgstr "" - msgid "Are you sure you want to lock %{path}?" msgstr "" @@ -14337,9 +14334,6 @@ msgstr "" msgid "Connecting to terminal sync service" msgstr "" -msgid "Connecting to the remote environment..." -msgstr "" - msgid "Connecting..." msgstr "" @@ -51527,9 +51521,6 @@ msgstr "" msgid "Start merge train..." msgstr "" -msgid "Start remote connection" -msgstr "" - msgid "Start search" msgstr "" diff --git a/package.json b/package.json index c146efa9a57..0a2816120d4 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "@gitlab/fonts": "^1.3.0", "@gitlab/svgs": "3.112.0", "@gitlab/ui": "88.0.0", - "@gitlab/web-ide": "^0.0.1-dev-20240613133550", + "@gitlab/web-ide": "^0.0.1-dev-20240731185426", "@mattiasbuelens/web-streams-adapter": "^0.1.0", "@rails/actioncable": "7.0.8-4", "@rails/ujs": "7.0.8-4", diff --git a/patches/@gitlab+web-ide+0.0.1-dev-20240613133550.patch b/patches/@gitlab+web-ide+0.0.1-dev-20240731185426.patch similarity index 98% rename from patches/@gitlab+web-ide+0.0.1-dev-20240613133550.patch rename to patches/@gitlab+web-ide+0.0.1-dev-20240731185426.patch index 0a9e1141f95..815bfb36eeb 100644 --- a/patches/@gitlab+web-ide+0.0.1-dev-20240613133550.patch +++ b/patches/@gitlab+web-ide+0.0.1-dev-20240731185426.patch @@ -301,10 +301,10 @@ index 960d118..0000000 - diff --git a/node_modules/@gitlab/web-ide/dist/public/vscode/out/vs/workbench/contrib/webview/browser/pre/index-no-csp.html b/node_modules/@gitlab/web-ide/dist/public/vscode/out/vs/workbench/contrib/webview/browser/pre/index-no-csp.html deleted file mode 100644 -index 70c8500..0000000 +index 6eb5b28..0000000 --- a/node_modules/@gitlab/web-ide/dist/public/vscode/out/vs/workbench/contrib/webview/browser/pre/index-no-csp.html +++ /dev/null -@@ -1,1256 +0,0 @@ +@@ -1,1309 +0,0 @@ - - - @@ -319,6 +319,32 @@ index 70c8500..0000000 - - - +- +- +- - +- -