diff --git a/app/controllers/users_controller.rb b/app/controllers/users_controller.rb index d52bc07af38..2053ba26764 100644 --- a/app/controllers/users_controller.rb +++ b/app/controllers/users_controller.rb @@ -28,7 +28,7 @@ class UsersController < ApplicationController before_action only: [:exists] do check_rate_limit!(:username_exists, scope: request.ip) end - before_action only: [:show] do + before_action only: [:show, :activity, :groups, :projects, :contributed, :starred, :snippets, :followers, :following] do push_frontend_feature_flag(:profile_tabs_vue, current_user) end diff --git a/db/migrate/20241007071632_add_ml_models_project_id_foreign_key_as_cascade_delete.rb b/db/migrate/20241007071632_add_ml_models_project_id_foreign_key_as_cascade_delete.rb new file mode 100644 index 00000000000..7a0cdb9dfd5 --- /dev/null +++ b/db/migrate/20241007071632_add_ml_models_project_id_foreign_key_as_cascade_delete.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddMlModelsProjectIdForeignKeyAsCascadeDelete < Gitlab::Database::Migration[2.2] + milestone '17.5' + disable_ddl_transaction! + + FK_NAME = 'fk_51e87f7c50_new' + + def up + add_concurrent_foreign_key :ml_models, :projects, name: FK_NAME, column: :project_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key_if_exists(:ml_models, :projects, name: FK_NAME) + end + end +end diff --git a/db/migrate/20241007071637_remove_ml_models_project_id_plain_foreign_key.rb b/db/migrate/20241007071637_remove_ml_models_project_id_plain_foreign_key.rb new file mode 100644 index 00000000000..bec32ebd2e4 --- /dev/null +++ b/db/migrate/20241007071637_remove_ml_models_project_id_plain_foreign_key.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class RemoveMlModelsProjectIdPlainForeignKey < Gitlab::Database::Migration[2.2] + milestone '17.5' + disable_ddl_transaction! + + FK_NAME = 'fk_rails_51e87f7c50' + + def up + with_lock_retries do + remove_foreign_key_if_exists(:ml_models, :projects, name: FK_NAME) + end + end + + def down + add_concurrent_foreign_key :ml_models, :projects, name: FK_NAME, column: :project_id, on_delete: nil + end +end diff --git a/db/schema_migrations/20241007071632 b/db/schema_migrations/20241007071632 new file mode 100644 index 00000000000..0273bdea84a --- /dev/null +++ b/db/schema_migrations/20241007071632 @@ -0,0 +1 @@ +6c238462ec40a00e6cfe211c4b87fbf2abfc2903225dc99ee57dfce659a782f1 \ No newline at end of file diff --git a/db/schema_migrations/20241007071637 b/db/schema_migrations/20241007071637 new file mode 100644 index 00000000000..85e3609a885 --- /dev/null +++ b/db/schema_migrations/20241007071637 @@ -0,0 +1 @@ +011735ac60dcb4fd62fb8b8757b7f455138ddc99447a5c0dc55bd3c439c28009 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index ffda14de1a3..b24c9c2dda4 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -34347,6 +34347,9 @@ ALTER TABLE ONLY approval_group_rules_protected_branches ALTER TABLE ONLY deploy_tokens ADD CONSTRAINT fk_51bf7bfb69 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE ONLY ml_models + ADD CONSTRAINT fk_51e87f7c50_new FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY path_locks ADD CONSTRAINT fk_5265c98f24 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -36051,9 +36054,6 @@ ALTER TABLE ONLY ml_candidate_metadata ALTER TABLE zoekt_tasks ADD CONSTRAINT fk_rails_51af186590 FOREIGN KEY (zoekt_node_id) REFERENCES zoekt_nodes(id) ON DELETE CASCADE; -ALTER TABLE ONLY ml_models - ADD CONSTRAINT fk_rails_51e87f7c50 FOREIGN KEY (project_id) REFERENCES projects(id); - ALTER TABLE ONLY merge_request_merge_schedules ADD CONSTRAINT fk_rails_5294434bc3 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 7868c3bca86..00e9fb935c0 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -1077,7 +1077,9 @@ four standard [pagination arguments](#pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `addOnPurchaseIds` | [`[GitlabSubscriptionsAddOnPurchaseID!]!`](#gitlabsubscriptionsaddonpurchaseid) | Global IDs of the add on purchases to find assignments for. | | `addOnType` | [`GitlabSubscriptionsAddOnType!`](#gitlabsubscriptionsaddontype) | Type of add on to filter the eligible users by. | +| `filterByAssignedSeat` | [`String`](#string) | Filter users list by assigned seat. | | `search` | [`String`](#string) | Search the user list. | | `sort` | [`GitlabSubscriptionsUserSort`](#gitlabsubscriptionsusersort) | Sort the user list. | @@ -23420,7 +23422,9 @@ four standard [pagination arguments](#pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `addOnPurchaseIds` | [`[GitlabSubscriptionsAddOnPurchaseID!]!`](#gitlabsubscriptionsaddonpurchaseid) | Global IDs of the add on purchases to find assignments for. | | `addOnType` | [`GitlabSubscriptionsAddOnType!`](#gitlabsubscriptionsaddontype) | Type of add on to filter the eligible users by. | +| `filterByAssignedSeat` | [`String`](#string) | Filter users list by assigned seat. | | `search` | [`String`](#string) | Search the user list. | | `sort` | [`GitlabSubscriptionsUserSort`](#gitlabsubscriptionsusersort) | Sort the user list. | @@ -28190,7 +28194,9 @@ four standard [pagination arguments](#pagination-arguments): | Name | Type | Description | | ---- | ---- | ----------- | +| `addOnPurchaseIds` | [`[GitlabSubscriptionsAddOnPurchaseID!]!`](#gitlabsubscriptionsaddonpurchaseid) | Global IDs of the add on purchases to find assignments for. | | `addOnType` | [`GitlabSubscriptionsAddOnType!`](#gitlabsubscriptionsaddontype) | Type of add on to filter the eligible users by. | +| `filterByAssignedSeat` | [`String`](#string) | Filter users list by assigned seat. | | `search` | [`String`](#string) | Search the user list. | | `sort` | [`GitlabSubscriptionsUserSort`](#gitlabsubscriptionsusersort) | Sort the user list. | diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb index fa329c8365a..b8fa1e08fc8 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb @@ -12,6 +12,7 @@ module Gitlab # GitLab Backup CLI module Cli autoload :BackupExecutor, 'gitlab/backup/cli/backup_executor' + autoload :BaseExecutor, 'gitlab/backup/cli/base_executor' autoload :Commands, 'gitlab/backup/cli/commands' autoload :Context, 'gitlab/backup/cli/context' autoload :Dependencies, 'gitlab/backup/cli/dependencies' diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/backup_executor.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/backup_executor.rb index b834916c34e..8df1a3edc89 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/backup_executor.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/backup_executor.rb @@ -10,20 +10,26 @@ module Gitlab # # It also allows for multiple backups to happen in parallel # without one overwriting data from another - class BackupExecutor - attr_reader :context, :metadata, :workdir, :archive_directory, :backup_bucket, :wait_for_completion, - :registry_bucket, :service_account_file + class BackupExecutor < BaseExecutor + attr_reader :context, :metadata, :workdir, :archive_directory - # @param [Context::SourceContext, Context::OmnibusContext] context - def initialize(context:, backup_options: {}) + # @param [Gitlab::Backup::Cli::SourceContext, Context::OmnibusContext] context + def initialize( + context:, + backup_bucket: nil, + wait_for_completion: nil, + registry_bucket: nil, + service_account_file: nil) @context = context @metadata = build_metadata @workdir = create_temporary_workdir! @archive_directory = context.backup_basedir.join(metadata.backup_id) - @backup_bucket = backup_options["backup_bucket"] - @registry_bucket = backup_options["registry_bucket"] - @wait_for_completion = backup_options["wait_for_completion"] - @service_account_file = backup_options["service_account_file"] + super( + backup_bucket: backup_bucket, + wait_for_completion: wait_for_completion, + registry_bucket: registry_bucket, + service_account_file: service_account_file + ) end def execute @@ -58,19 +64,18 @@ module Gitlab Gitlab::Backup::Cli::Output.info("Executing Backup of #{task.human_name}...") duration = measure_duration do - tasks << { name: task.human_name, result: task.backup!(workdir, metadata.backup_id) } + task.backup!(workdir, metadata.backup_id) + tasks << task end - next unless task.object_storage? + next if task.asynchronous? Gitlab::Backup::Cli::Output.success("Finished Backup of #{task.human_name}! (#{duration.in_seconds}s)") end if wait_for_completion tasks.each do |task| - next unless task[:result].respond_to?(:wait_until_done!) - - wait_for_task(task[:result]) + wait_for_task(task) end else Gitlab::Backup::Cli::Output.info('Backup tasks completed! Not waiting for object storage tasks to complete') @@ -107,13 +112,15 @@ module Gitlab end def wait_for_task(task) - Gitlab::Backup::Cli::Output.info("Waiting for Backup of #{task.name} to finish...") + return unless task.asynchronous? + + Gitlab::Backup::Cli::Output.info("Waiting for Backup of #{task.human_name} to finish...") r = task.wait_until_done! if r.error? - Gitlab::Backup::Cli::Output.error("Backup of #{task.name} failed!") + Gitlab::Backup::Cli::Output.error("Backup of #{task.human_name} failed!") else - Gitlab::Backup::Cli::Output.success("Finished Backup of #{task.name}!") + Gitlab::Backup::Cli::Output.success("Finished Backup of #{task.human_name}!") end end end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/base_executor.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/base_executor.rb new file mode 100644 index 00000000000..b3160a9b3da --- /dev/null +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/base_executor.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Backup + module Cli + class BaseExecutor + attr_reader :backup_bucket, :wait_for_completion, :registry_bucket, :service_account_file + + def initialize(backup_bucket:, wait_for_completion:, registry_bucket:, service_account_file:) + @backup_bucket = backup_bucket + @registry_bucket = registry_bucket + @wait_for_completion = wait_for_completion + @service_account_file = service_account_file + end + end + end + end +end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands.rb index e0ad6132040..9302d83c491 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands.rb @@ -6,6 +6,7 @@ module Gitlab module Commands autoload :BackupSubcommand, 'gitlab/backup/cli/commands/backup_subcommand' autoload :Command, 'gitlab/backup/cli/commands/command' + autoload :ObjectStorageCommand, 'gitlab/backup/cli/commands/object_storage_command' autoload :RestoreSubcommand, 'gitlab/backup/cli/commands/restore_subcommand' end end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/backup_subcommand.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/backup_subcommand.rb index 010a94ad11c..0f1db9cf20f 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/backup_subcommand.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/backup_subcommand.rb @@ -4,28 +4,9 @@ module Gitlab module Backup module Cli module Commands - class BackupSubcommand < Command + class BackupSubcommand < ObjectStorageCommand package_name 'Backup' - EXECUTOR_OPTIONS = %w[backup_bucket wait_for_completion registry_bucket service_account_file].freeze - - class_option :backup_bucket, - desc: "When backing up object storage, this is the bucket to backup to", - required: false - - class_option :wait_for_completion, - desc: "Wait for object storage backups to complete", - type: :boolean, - default: true - - class_option :registry_bucket, - desc: "When backing up registry from object storage, this is the source bucket", - required: false - - class_option :service_account_file, - desc: "JSON file containing the Google service account credentials", - default: "/etc/gitlab/backup-account-credentials.json" - desc 'all', 'Creates a backup including repositories, database and local files' def all Gitlab::Backup::Cli.update_process_title!('backup all') @@ -37,7 +18,11 @@ module Gitlab Gitlab::Backup::Cli::Output.success("Environment loaded. (#{duration.in_seconds}s)") backup_executor = Gitlab::Backup::Cli::BackupExecutor.new( - context: build_context, backup_options: executor_options + context: build_context, + backup_bucket: options["backup_bucket"], + wait_for_completion: options["wait_for_completion"], + registry_bucket: options["registry_bucket"], + service_account_file: options["service_account_file"] ) backup_id = backup_executor.metadata.backup_id @@ -67,10 +52,6 @@ module Gitlab ActiveSupport::Duration.build(Time.now - start) end - - def executor_options - options.select { |key, _| EXECUTOR_OPTIONS.include?(key) } - end end end end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/object_storage_command.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/object_storage_command.rb new file mode 100644 index 00000000000..ff5727141a8 --- /dev/null +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/object_storage_command.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Backup + module Cli + module Commands + class ObjectStorageCommand < Command + class_option :backup_bucket, + desc: "When backing up object storage, this is the bucket to backup to", + required: false, + type: :string + + class_option :wait_for_completion, + desc: "Wait for object storage backups to complete", + type: :boolean, + default: true + + class_option :registry_bucket, + desc: "When backing up registry from object storage, this is the source bucket", + required: false, + type: :string + + class_option :service_account_file, + desc: "JSON file containing the Google service account credentials", + default: "/etc/gitlab/backup-account-credentials.json", + type: :string + end + end + end + end +end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/restore_subcommand.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/restore_subcommand.rb index d8f56922687..22df2ea6123 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/restore_subcommand.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/restore_subcommand.rb @@ -4,7 +4,7 @@ module Gitlab module Backup module Cli module Commands - class RestoreSubcommand < Command + class RestoreSubcommand < ObjectStorageCommand package_name 'Restore' desc 'all BACKUP_ID', 'Restores a backup including repositories, database and local files' @@ -20,7 +20,11 @@ module Gitlab restore_executor = Gitlab::Backup::Cli::RestoreExecutor.new( context: build_context, - backup_id: backup_id + backup_id: backup_id, + backup_bucket: options["backup_bucket"], + wait_for_completion: options["wait_for_completion"], + registry_bucket: options["registry_bucket"], + service_account_file: options["service_account_file"] ) duration = measure_duration do diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/restore_executor.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/restore_executor.rb index b697b044c7e..25a76382d5e 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/restore_executor.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/restore_executor.rb @@ -8,19 +8,31 @@ module Gitlab # A Restore Executor handles the creation and deletion of # temporary environment necessary for a restoration to happen # - class RestoreExecutor + class RestoreExecutor < BaseExecutor attr_reader :context, :backup_id, :workdir, :archive_directory # @param [Context::SourceContext|Context::OmnibusContext] context # @param [String] backup_id - def initialize(context:, backup_id:) + def initialize( + context:, + backup_id: nil, + backup_bucket: nil, + wait_for_completion: nil, + registry_bucket: nil, + service_account_file: nil + ) @context = context @backup_id = backup_id @workdir = create_temporary_workdir! @archive_directory = context.backup_basedir.join(backup_id) @metadata = nil - @backup_options = nil + super( + backup_bucket: backup_bucket, + wait_for_completion: wait_for_completion, + registry_bucket: registry_bucket, + service_account_file: service_account_file + ) end def execute @@ -47,15 +59,28 @@ module Gitlab def execute_all_tasks # TODO: when we migrate targets to the new codebase, recreate options to have only what we need here # https://gitlab.com/gitlab-org/gitlab/-/issues/454906 + tasks = [] Gitlab::Backup::Cli::Tasks.build_each(context: context, options: backup_options) do |task| Gitlab::Backup::Cli::Output.info("Executing restoration of #{task.human_name}...") duration = measure_duration do - task.restore!(archive_directory) + tasks << { name: task.human_name, result: task.restore!(archive_directory, backup_id) } end + next if task.object_storage? + Gitlab::Backup::Cli::Output.success("Finished restoration of #{task.human_name}! (#{duration.in_seconds}s)") end + + if wait_for_completion + tasks.each do |task| + next unless task[:result].respond_to?(:wait_until_done) + + wait_for_task(task[:result]) + end + else + Gitlab::Backup::Cli::Output.info("Restore tasks complete! Not waiting for object storage tasks to complete") + end end def read_metadata! @@ -64,7 +89,10 @@ module Gitlab def build_backup_options! ::Backup::Options.new( - backup_id: backup_id + backup_id: backup_id, + remote_directory: backup_bucket, + container_registry_bucket: registry_bucket, + service_account_file: service_account_file ) end @@ -83,6 +111,17 @@ module Gitlab ActiveSupport::Duration.build(Time.now - start) end + + def wait_for_task(task) + Gitlab::Backup::Cli::Output.info("Waiting for Restore of #{task.name} to finish...") + + r = task.wait_until_done! + if r.error? + Gitlab::Backup::Cli::Output.error("Restore of #{task.name} failed!") + else + Gitlab::Backup::Cli::Output.success("Finished Restore of #{task.name}!") + end + end end end end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/object_storage/google.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/object_storage/google.rb index 655b049c2d2..79001062857 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/object_storage/google.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/object_storage/google.rb @@ -8,10 +8,11 @@ module Gitlab module Targets class ObjectStorage class Google < Target - attr_accessor :object_type, :backup_bucket, :client, :config + OperationNotFoundError = Class.new(StandardError) + + attr_accessor :object_type, :backup_bucket, :client, :config, :results def initialize(object_type, options, config) - check_env @object_type = object_type @backup_bucket = options.remote_directory @config = config @@ -19,65 +20,107 @@ module Gitlab end def dump(_, backup_id) - response = find_or_create_job(backup_id) + response = find_or_create_job(backup_id, "backup") run_request = { - project_id: job_spec(backup_id)[:project_id], + project_id: backup_job_spec(backup_id)[:project_id], job_name: response.name } - client.run_transfer_job run_request + @results = client.run_transfer_job run_request end - def job_name - "transferJobs/#{object_type}-backup" + def restore(_, backup_id) + response = find_or_create_job(backup_id, "restore") + run_request = { + project_id: restore_job_spec(backup_id)[:project_id], + job_name: response.name + } + @results = client.run_transfer_job run_request end - def job_spec(backup_id) + def job_name(operation) + "transferJobs/#{object_type}-#{operation}" + end + + def backup_job_spec(backup_id) + job_spec( + config.object_store.remote_directory, + backup_bucket, + operation: "backup", + destination_path: backup_path(backup_id) + ) + end + + def restore_job_spec(backup_id) + job_spec( + backup_bucket, + config.object_store.remote_directory, + operation: "restore", + source_path: backup_path(backup_id) + ) + end + + def backup_path(backup_id) + "backups/#{backup_id}/#{object_type}/" + end + + def find_job_spec(backup_id, operation) + case operation + when "backup" + backup_job_spec(backup_id) + when "restore" + restore_job_spec(backup_id) + else + raise StandardError "Operation #{operation} not found" + end + end + + def job_spec(source, destination, operation:, source_path: nil, destination_path: nil) { project_id: config.object_store.connection.google_project, - name: job_name, + name: job_name(operation), transfer_spec: { gcs_data_source: { - bucket_name: config.object_store.remote_directory + bucket_name: source, + path: source_path }, gcs_data_sink: { - bucket_name: backup_bucket, + bucket_name: destination, # NOTE: The trailing '/' is required - path: "backups/#{backup_id}/#{object_type}/" + path: destination_path } }, status: :ENABLED } end - private - - def check_env - # We expect service account credentials to be passed via env variables. If they are not, attempt - # to use the local service account credentials and warn. - return unless ENV.key?("GOOGLE_CLOUD_CREDENTIALS") || ENV.key?("GOOGLE_APPLICATION_CREDENTIALS") - - log.warning("No credentials provided.") - log.warning("If we're in GCP, we will attempt to use the machine service account.") - log.warning("This is not recommended.") + def asynchronous? + true end - def find_or_create_job(backup_id) + def wait_until_done! + @results.wait_until_done! + end + + private + + def find_or_create_job(backup_id, operation) begin + name = job_name(operation) response = client.get_transfer_job( - job_name: job_name, project_id: config.object_store.connection.google_project + job_name: name, project_id: config.object_store.connection.google_project ) log.info("Existing job for #{object_type} found, using") - job_update = job_spec(backup_id) + job_update = find_job_spec(backup_id, operation) job_update.delete(:project_id) client.update_transfer_job( - job_name: job_name, + job_name: name, project_id: config.object_store.connection.google_project, transfer_job: job_update ) rescue ::Google::Cloud::NotFoundError log.info("Existing job for #{object_type} not found, creating one") - response = client.create_transfer_job transfer_job: job_spec(backup_id) + response = client.create_transfer_job transfer_job: find_job_spec(backup_id, operation) end response end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/target.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/target.rb index 835e364dc3f..d7757026423 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/target.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/target.rb @@ -16,6 +16,10 @@ module Gitlab @options = options end + def asynchronous? + false + end + # dump task backup to `path` # # @param [String] path fully qualified backup task destination diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb index 4d306bbb0c3..14ed04f567c 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb @@ -13,8 +13,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']) end def storage_path = context.ci_job_artifacts_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb index 64a68f6a5cc..f819d3fd3ab 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb @@ -13,8 +13,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']) end def storage_path = context.ci_secure_files_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb index 1c5d39adf2b..46c96d4c8ec 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb @@ -13,8 +13,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options)) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options) end def storage_path = context.ci_lfs_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb index 282d4bb8932..f3bca473d97 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb @@ -13,8 +13,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']) end def storage_path = context.packages_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb index 2f6f444a914..f2c12e1d709 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb @@ -17,10 +17,8 @@ module Gitlab private - def target - check_object_storage( - ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: [LEGACY_PAGES_TMP_PATH]) - ) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: [LEGACY_PAGES_TMP_PATH]) end def storage_path = context.pages_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb index 962a8dab098..80f97fbb66a 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb @@ -30,8 +30,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options)) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options) end def storage_path = context.registry_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb index e26ab4df32e..1d1fde51fcd 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb @@ -6,6 +6,7 @@ module Gitlab module Tasks class Task attr_reader :options, :context + attr_writer :target # Identifier used as parameter in the CLI to skip from executing def self.id @@ -30,10 +31,10 @@ module Gitlab target.dump(backup_output, backup_id) end - def restore!(archive_directory) + def restore!(archive_directory, backup_id) archived_data_location = Pathname(archive_directory).join(destination_path) - target.restore(archived_data_location, nil) + target.restore(archived_data_location, backup_id) end # Key string that identifies the task @@ -83,20 +84,32 @@ module Gitlab true end + def asynchronous? + target.asynchronous? || false + end + + def wait_until_done! + target.wait_until_done! + end + + def target + return @target unless @target.nil? + + @target = if object_storage? + ::Gitlab::Backup::Cli::Targets::ObjectStorage.find_task(id, options, config) + else + local + end + + @target + end + private # The target factory method - def target + def local raise NotImplementedError end - - def check_object_storage(file_target) - if object_storage? - ::Gitlab::Backup::Cli::Targets::ObjectStorage.find_task(id, options, config) - else - file_target - end - end end end end diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb index fc9b0b4a7bf..be821ca5966 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb @@ -13,8 +13,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']) end def storage_path = context.terraform_state_path diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb index 53eecdc7871..2846528073d 100644 --- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb +++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb @@ -13,8 +13,8 @@ module Gitlab private - def target - check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])) + def local + ::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']) end def storage_path = context.upload_path diff --git a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/commands/backup_subcommand_spec.rb b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/commands/backup_subcommand_spec.rb deleted file mode 100644 index 2b50095d3a2..00000000000 --- a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/commands/backup_subcommand_spec.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe Gitlab::Backup::Cli::Commands::BackupSubcommand do - describe "#executor_options" do - it "returns the expected hash" do - expect(described_class.new.send(:executor_options).keys).to eq( - %w[wait_for_completion service_account_file] - ) - end - end -end diff --git a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/targets/object_storage/google_spec.rb b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/targets/object_storage/google_spec.rb index 87394210a61..1ef0090d93a 100644 --- a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/targets/object_storage/google_spec.rb +++ b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/targets/object_storage/google_spec.rb @@ -2,24 +2,31 @@ RSpec.describe Gitlab::Backup::Cli::Targets::ObjectStorage::Google do let(:gitlab_config) { class_double("GitlabSettings::Settings") } - let(:supported_object_store) do + + let(:supported_config) { instance_double("GitlabSettings::Options", object_store: supported_object_store) } + let(:supported_provider) do instance_double( - "GitlabSettings::Options", - enabled: true, - connection: instance_double("GitlabSettings::Options", provider: "Google") + "GitlabSettings::Options", provider: "Google", google_application_default: true, google_project: "fake_project" + ) + end + + let(:supported_object_store) do + instance_double( + "GitlabSettings::Options", enabled: true, connection: supported_provider, remote_directory: "fake_source_bucket" ) end - let(:supported_config) { instance_double("GitlabSettings::Options", object_store: supported_object_store) } let(:client) { instance_double("::Google::Cloud::StorageTransfer::V1::StorageTransferService::Client") } - let(:existing_transfer_job) { build(:google_cloud_storage_transfer_job) } - let(:new_transfer_job_spec) do + let(:backup_transfer_job) { build(:google_cloud_storage_transfer_job) } + let(:restore_transfer_job) { build(:google_cloud_storage_transfer_job) } + let(:new_backup_transfer_job_spec) do { name: "transferJobs/fake_object-backup", project_id: "fake_project", transfer_spec: { gcs_data_source: { - bucket_name: "fake_source_bucket" + bucket_name: "fake_source_bucket", + path: nil }, gcs_data_sink: { bucket_name: "fake_backup_bucket", @@ -30,41 +37,42 @@ RSpec.describe Gitlab::Backup::Cli::Targets::ObjectStorage::Google do } end - let(:backup_options) { instance_double("Gitlab::Backup::Options", remote_directory: 'fake_backup_bucket') } + let(:new_restore_transfer_job_spec) do + { + name: "transferJobs/fake_object-restore", + project_id: "fake_project", + transfer_spec: { + gcs_data_source: { + bucket_name: "fake_backup_bucket", + path: "backups/12345/fake_object/" + }, + gcs_data_sink: { + bucket_name: "fake_source_bucket", + path: nil + } + }, + status: :ENABLED + } + end + + let(:backup_options) { instance_double("::Backup::Options", remote_directory: 'fake_backup_bucket') } before do allow(Gitlab).to receive(:config).and_return(gitlab_config) allow(::Google::Cloud::StorageTransfer).to receive(:storage_transfer_service).and_return(client) + allow(gitlab_config).to receive(:[]).with('fake_object').and_return(supported_config) end subject(:object_storage) { described_class.new("fake_object", backup_options, supported_config) } describe "#dump" do - let(:supported_provider) do - instance_double( - "GitlabSettings::Options", provider: "Google", google_application_default: true, google_project: "fake_project" - ) - end - - let(:supported_object_store) do - instance_double( - "GitlabSettings::Options", enabled: true, connection: supported_provider, remote_directory: "fake_source_bucket" - ) - end - - let(:supported_config) { instance_double("GitlabSettings::Options", object_store: supported_object_store) } - - before do - allow(gitlab_config).to receive(:[]).with('fake_object').and_return(supported_config) - end - context "when job exists" do before do - allow(client).to receive(:get_transfer_job).and_return(existing_transfer_job) + allow(client).to receive(:get_transfer_job).and_return(backup_transfer_job) end it "reuses existing job" do - updated_spec = new_transfer_job_spec + updated_spec = new_backup_transfer_job_spec expect(client).to receive(:update_transfer_job).with( job_name: updated_spec[:name], project_id: updated_spec.delete(:project_id), @@ -85,9 +93,43 @@ RSpec.describe Gitlab::Backup::Cli::Targets::ObjectStorage::Google do it "creates a new job" do expect(client).to receive(:create_transfer_job) - .with(transfer_job: new_transfer_job_spec).and_return(existing_transfer_job) + .with(transfer_job: new_backup_transfer_job_spec).and_return(backup_transfer_job) object_storage.dump(nil, 12345) end end end + + describe "#restore" do + context "when job exists" do + before do + allow(client).to receive(:get_transfer_job).and_return(restore_transfer_job) + end + + it "reuses existing job" do + updated_spec = new_restore_transfer_job_spec + expect(client).to receive(:update_transfer_job).with( + job_name: updated_spec[:name], + project_id: updated_spec.delete(:project_id), + transfer_job: updated_spec + ) + expect(client).to receive(:run_transfer_job).with({ job_name: "fake_transfer_job", project_id: "fake_project" }) + object_storage.restore(nil, 12345) + end + end + + context "when job does not exist" do + before do + allow(client).to receive(:get_transfer_job).with( + job_name: "transferJobs/fake_object-restore", project_id: "fake_project" + ).and_raise(::Google::Cloud::NotFoundError) + allow(client).to receive(:run_transfer_job) + end + + it "creates a new job" do + expect(client).to receive(:create_transfer_job) + .with(transfer_job: new_restore_transfer_job_spec).and_return(restore_transfer_job) + object_storage.restore(nil, 12345) + end + end + end end diff --git a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb index 7b008cea5f0..2148df8cafd 100644 --- a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb +++ b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true RSpec.describe Gitlab::Backup::Cli::Tasks::Task do - let(:options) { nil } + let(:options) { instance_double("::Backup::Option", backup_id: "abc123") } let(:context) { build_fake_context } let(:tmpdir) { Pathname.new(Dir.mktmpdir('task', temp_path)) } let(:metadata) { build(:backup_metadata) } @@ -59,7 +59,7 @@ RSpec.describe Gitlab::Backup::Cli::Tasks::Task do expect(task).to receive(:destination_path).and_return(tmpdir.join('test_task')) expect(task).to receive_message_chain(:target, :restore) - task.restore!(archive_directory) + task.restore!(archive_directory, options.backup_id) end end end diff --git a/gems/gitlab-backup-cli/spec/thor/gitlab_backup_cli_restore_spec.rb b/gems/gitlab-backup-cli/spec/thor/gitlab_backup_cli_restore_spec.rb index 8366b331831..b3861af064d 100644 --- a/gems/gitlab-backup-cli/spec/thor/gitlab_backup_cli_restore_spec.rb +++ b/gems/gitlab-backup-cli/spec/thor/gitlab_backup_cli_restore_spec.rb @@ -13,6 +13,14 @@ RSpec.describe 'gitlab-backup-cli restore subcommand', type: :thor do gitlab-backup-cli restore all BACKUP_ID # Restores a backup including repositories, database and local files gitlab-backup-cli restore help [COMMAND] # Describe subcommands or one specific subcommand + Options: + [--backup-bucket=BACKUP_BUCKET] # When backing up object storage, this is the bucket to backup to + [--wait-for-completion], [--no-wait-for-completion], [--skip-wait-for-completion] # Wait for object storage backups to complete + # Default: true + [--registry-bucket=REGISTRY_BUCKET] # When backing up registry from object storage, this is the source bucket + [--service-account-file=SERVICE_ACCOUNT_FILE] # JSON file containing the Google service account credentials + # Default: /etc/gitlab/backup-account-credentials.json + COMMAND end diff --git a/lib/backup/targets/files.rb b/lib/backup/targets/files.rb index f5f6560bf17..5613240f02b 100644 --- a/lib/backup/targets/files.rb +++ b/lib/backup/targets/files.rb @@ -172,6 +172,10 @@ module Backup raise FileBackupError.new(storage_realpath, backup_tarball) end + def asynchronous? + false + end + private def storage_realpath diff --git a/lib/backup/targets/repositories.rb b/lib/backup/targets/repositories.rb index cfa615f4b3e..9fc87ffc693 100644 --- a/lib/backup/targets/repositories.rb +++ b/lib/backup/targets/repositories.rb @@ -48,6 +48,10 @@ module Backup restore_object_pools end + def asynchronous? + false + end + private attr_reader :strategy, :storages, :paths, :skip_paths, :logger diff --git a/lib/sidebars/user_profile/panel.rb b/lib/sidebars/user_profile/panel.rb index b15538c698b..7137ab7e194 100644 --- a/lib/sidebars/user_profile/panel.rb +++ b/lib/sidebars/user_profile/panel.rb @@ -25,8 +25,17 @@ module Sidebars private + def add_legacy_menu? + # When `profile_tabs_vue` feature flag is enabled, legacy profile pages + # will be replaced by routes in `app/assets/javascripts/profile/components/app.vue` + Feature.disabled?(:profile_tabs_vue, context.current_user) + end + def add_menus add_menu(Sidebars::UserProfile::Menus::OverviewMenu.new(context)) + + return unless add_legacy_menu? + add_menu(Sidebars::UserProfile::Menus::ActivityMenu.new(context)) add_menu(Sidebars::UserProfile::Menus::GroupsMenu.new(context)) add_menu(Sidebars::UserProfile::Menus::ContributedProjectsMenu.new(context)) diff --git a/spec/features/users/show_spec.rb b/spec/features/users/show_spec.rb index 5253034d8ca..272cf8c304a 100644 --- a/spec/features/users/show_spec.rb +++ b/spec/features/users/show_spec.rb @@ -158,10 +158,8 @@ RSpec.describe 'User page', feature_category: :user_profile do end end - it_behaves_like 'follower links with count badges' - context 'with profile_tabs_vue feature flag disabled' do - before_all do + before do stub_feature_flags(profile_tabs_vue: false) end diff --git a/spec/lib/backup/targets/files_spec.rb b/spec/lib/backup/targets/files_spec.rb index 0ea0f4aba55..36d4076b17d 100644 --- a/spec/lib/backup/targets/files_spec.rb +++ b/spec/lib/backup/targets/files_spec.rb @@ -392,4 +392,14 @@ RSpec.describe Backup::Targets::Files, feature_category: :backup_restore do ).to be_falsey end end + + context 'with unified backup' do + subject(:files) do + described_class.new(progress, '/fake/path', options: backup_options) + end + + it 'is not asynchronous by default' do + expect(files.asynchronous?).to be_falsey + end + end end diff --git a/spec/lib/sidebars/user_profile/panel_spec.rb b/spec/lib/sidebars/user_profile/panel_spec.rb index 97fe13397a9..e93ca544e9b 100644 --- a/spec/lib/sidebars/user_profile/panel_spec.rb +++ b/spec/lib/sidebars/user_profile/panel_spec.rb @@ -3,6 +3,17 @@ require 'spec_helper' RSpec.describe Sidebars::UserProfile::Panel, feature_category: :navigation do + legacy_menu_classes = [ + Sidebars::UserProfile::Menus::ActivityMenu, + Sidebars::UserProfile::Menus::GroupsMenu, + Sidebars::UserProfile::Menus::ContributedProjectsMenu, + Sidebars::UserProfile::Menus::PersonalProjectsMenu, + Sidebars::UserProfile::Menus::StarredProjectsMenu, + Sidebars::UserProfile::Menus::SnippetsMenu, + Sidebars::UserProfile::Menus::FollowersMenu, + Sidebars::UserProfile::Menus::FollowingMenu + ] + let(:current_user) { build_stubbed(:user) } let(:user) { build_stubbed(:user) } @@ -13,11 +24,29 @@ RSpec.describe Sidebars::UserProfile::Panel, feature_category: :navigation do it_behaves_like 'a panel with uniquely identifiable menu items' it_behaves_like 'a panel instantiable by the anonymous user' - it 'implements #aria_label' do - expect(subject.aria_label).to eq(s_('UserProfile|User profile navigation')) + describe '#aria_label' do + specify { expect(subject.aria_label).to eq(s_('UserProfile|User profile navigation')) } end - it 'implements #super_sidebar_context_header' do - expect(subject.super_sidebar_context_header).to eq(_('Profile')) + describe '#super_sidebar_context_header' do + specify { expect(subject.super_sidebar_context_header).to eq(_('Profile')) } + end + + it 'does not add legacy menu items' do + menu_classes = subject.renderable_menus.map(&:class) + + expect(menu_classes).not_to include(*legacy_menu_classes) + end + + context 'when profile_tabs_vue feature is disabled' do + before do + stub_feature_flags(profile_tabs_vue: false) + end + + it 'add legacy menu items' do + menu_classes = subject.renderable_menus.map(&:class) + + expect(menu_classes).to include(*legacy_menu_classes) + end end end