VladPolskiy/gitlab-foss
1
0
Fork 0
mirror of https://gitlab.com/gitlab-org/gitlab-foss.git synced 2025-07-21 23:43:41 +00:00
Code Issues Packages Projects Releases 1 Wiki Activity

Add latest changes from gitlab-org/gitlab@master

Browse Source
This commit is contained in:
GitLab Bot
2024-10-08 06:18:18 +00:00
parent b0d99982d5
commit ea9bc357ed
36 changed files with 436 additions and 159 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
app/controllers/users_controller.rb
View File

@ -28,7 +28,7 @@ class UsersController < ApplicationController
before_action only: [:exists] do
check_rate_limit!(:username_exists, scope: request.ip)
end
before_action only: [:show] do
before_action only: [:show, :activity, :groups, :projects, :contributed, :starred, :snippets, :followers, :following] do
push_frontend_feature_flag(:profile_tabs_vue, current_user)
end

18
db/migrate/20241007071632_add_ml_models_project_id_foreign_key_as_cascade_delete.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class AddMlModelsProjectIdForeignKeyAsCascadeDelete < Gitlab::Database::Migration[2.2]
milestone '17.5'
disable_ddl_transaction!
FK_NAME = 'fk_51e87f7c50_new'
def up
add_concurrent_foreign_key :ml_models, :projects, name: FK_NAME, column: :project_id, on_delete: :cascade
end
def down
with_lock_retries do
remove_foreign_key_if_exists(:ml_models, :projects, name: FK_NAME)
end
end
end

18
db/migrate/20241007071637_remove_ml_models_project_id_plain_foreign_key.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
class RemoveMlModelsProjectIdPlainForeignKey < Gitlab::Database::Migration[2.2]
milestone '17.5'
disable_ddl_transaction!
FK_NAME = 'fk_rails_51e87f7c50'
def up
with_lock_retries do
remove_foreign_key_if_exists(:ml_models, :projects, name: FK_NAME)
end
end
def down
add_concurrent_foreign_key :ml_models, :projects, name: FK_NAME, column: :project_id, on_delete: nil
end
end

1
db/schema_migrations/20241007071632 Normal file
View File

@ -0,0 +1 @@
6c238462ec40a00e6cfe211c4b87fbf2abfc2903225dc99ee57dfce659a782f1

1
db/schema_migrations/20241007071637 Normal file
View File

@ -0,0 +1 @@
011735ac60dcb4fd62fb8b8757b7f455138ddc99447a5c0dc55bd3c439c28009

6
db/structure.sql
View File

@ -34347,6 +34347,9 @@ ALTER TABLE ONLY approval_group_rules_protected_branches
ALTER TABLE ONLY deploy_tokens
ADD CONSTRAINT fk_51bf7bfb69 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
ALTER TABLE ONLY ml_models
ADD CONSTRAINT fk_51e87f7c50_new FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
ALTER TABLE ONLY path_locks
ADD CONSTRAINT fk_5265c98f24 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
@ -36051,9 +36054,6 @@ ALTER TABLE ONLY ml_candidate_metadata
ALTER TABLE zoekt_tasks
ADD CONSTRAINT fk_rails_51af186590 FOREIGN KEY (zoekt_node_id) REFERENCES zoekt_nodes(id) ON DELETE CASCADE;
ALTER TABLE ONLY ml_models
ADD CONSTRAINT fk_rails_51e87f7c50 FOREIGN KEY (project_id) REFERENCES projects(id);
ALTER TABLE ONLY merge_request_merge_schedules
ADD CONSTRAINT fk_rails_5294434bc3 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE;

6
doc/api/graphql/reference/index.md
View File

@ -1077,7 +1077,9 @@ four standard [pagination arguments](#pagination-arguments):
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="queryselfmanagedaddoneligibleusersaddonpurchaseids"></a>`addOnPurchaseIds` | [`[GitlabSubscriptionsAddOnPurchaseID!]!`](#gitlabsubscriptionsaddonpurchaseid) | Global IDs of the add on purchases to find assignments for. |
| <a id="queryselfmanagedaddoneligibleusersaddontype"></a>`addOnType` | [`GitlabSubscriptionsAddOnType!`](#gitlabsubscriptionsaddontype) | Type of add on to filter the eligible users by. |
| <a id="queryselfmanagedaddoneligibleusersfilterbyassignedseat"></a>`filterByAssignedSeat` | [`String`](#string) | Filter users list by assigned seat. |
| <a id="queryselfmanagedaddoneligibleuserssearch"></a>`search` | [`String`](#string) | Search the user list. |
| <a id="queryselfmanagedaddoneligibleuserssort"></a>`sort` | [`GitlabSubscriptionsUserSort`](#gitlabsubscriptionsusersort) | Sort the user list. |
@ -23420,7 +23422,9 @@ four standard [pagination arguments](#pagination-arguments):
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="groupaddoneligibleusersaddonpurchaseids"></a>`addOnPurchaseIds` | [`[GitlabSubscriptionsAddOnPurchaseID!]!`](#gitlabsubscriptionsaddonpurchaseid) | Global IDs of the add on purchases to find assignments for. |
| <a id="groupaddoneligibleusersaddontype"></a>`addOnType` | [`GitlabSubscriptionsAddOnType!`](#gitlabsubscriptionsaddontype) | Type of add on to filter the eligible users by. |
| <a id="groupaddoneligibleusersfilterbyassignedseat"></a>`filterByAssignedSeat` | [`String`](#string) | Filter users list by assigned seat. |
| <a id="groupaddoneligibleuserssearch"></a>`search` | [`String`](#string) | Search the user list. |
| <a id="groupaddoneligibleuserssort"></a>`sort` | [`GitlabSubscriptionsUserSort`](#gitlabsubscriptionsusersort) | Sort the user list. |
@ -28190,7 +28194,9 @@ four standard [pagination arguments](#pagination-arguments):
| Name | Type | Description |
| ---- | ---- | ----------- |
| <a id="namespaceaddoneligibleusersaddonpurchaseids"></a>`addOnPurchaseIds` | [`[GitlabSubscriptionsAddOnPurchaseID!]!`](#gitlabsubscriptionsaddonpurchaseid) | Global IDs of the add on purchases to find assignments for. |
| <a id="namespaceaddoneligibleusersaddontype"></a>`addOnType` | [`GitlabSubscriptionsAddOnType!`](#gitlabsubscriptionsaddontype) | Type of add on to filter the eligible users by. |
| <a id="namespaceaddoneligibleusersfilterbyassignedseat"></a>`filterByAssignedSeat` | [`String`](#string) | Filter users list by assigned seat. |
| <a id="namespaceaddoneligibleuserssearch"></a>`search` | [`String`](#string) | Search the user list. |
| <a id="namespaceaddoneligibleuserssort"></a>`sort` | [`GitlabSubscriptionsUserSort`](#gitlabsubscriptionsusersort) | Sort the user list. |

1
gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
View File

@ -12,6 +12,7 @@ module Gitlab
# GitLab Backup CLI
module Cli
autoload :BackupExecutor, 'gitlab/backup/cli/backup_executor'
autoload :BaseExecutor, 'gitlab/backup/cli/base_executor'
autoload :Commands, 'gitlab/backup/cli/commands'
autoload :Context, 'gitlab/backup/cli/context'
autoload :Dependencies, 'gitlab/backup/cli/dependencies'

41
gems/gitlab-backup-cli/lib/gitlab/backup/cli/backup_executor.rb
View File

@ -10,20 +10,26 @@ module Gitlab
#
# It also allows for multiple backups to happen in parallel
# without one overwriting data from another
class BackupExecutor
attr_reader :context, :metadata, :workdir, :archive_directory, :backup_bucket, :wait_for_completion,
:registry_bucket, :service_account_file
class BackupExecutor < BaseExecutor
attr_reader :context, :metadata, :workdir, :archive_directory
# @param [Context::SourceContext, Context::OmnibusContext] context
def initialize(context:, backup_options: {})
# @param [Gitlab::Backup::Cli::SourceContext, Context::OmnibusContext] context
def initialize(
context:,
backup_bucket: nil,
wait_for_completion: nil,
registry_bucket: nil,
service_account_file: nil)
@context = context
@metadata = build_metadata
@workdir = create_temporary_workdir!
@archive_directory = context.backup_basedir.join(metadata.backup_id)
@backup_bucket = backup_options["backup_bucket"]
@registry_bucket = backup_options["registry_bucket"]
@wait_for_completion = backup_options["wait_for_completion"]
@service_account_file = backup_options["service_account_file"]
super(
backup_bucket: backup_bucket,
wait_for_completion: wait_for_completion,
registry_bucket: registry_bucket,
service_account_file: service_account_file
)
end
def execute
@ -58,19 +64,18 @@ module Gitlab
Gitlab::Backup::Cli::Output.info("Executing Backup of #{task.human_name}...")
duration = measure_duration do
tasks << { name: task.human_name, result: task.backup!(workdir, metadata.backup_id) }
task.backup!(workdir, metadata.backup_id)
tasks << task
end
next unless task.object_storage?
next if task.asynchronous?
Gitlab::Backup::Cli::Output.success("Finished Backup of #{task.human_name}! (#{duration.in_seconds}s)")
end
if wait_for_completion
tasks.each do |task|
next unless task[:result].respond_to?(:wait_until_done!)
wait_for_task(task[:result])
wait_for_task(task)
end
else
Gitlab::Backup::Cli::Output.info('Backup tasks completed! Not waiting for object storage tasks to complete')
@ -107,13 +112,15 @@ module Gitlab
end
def wait_for_task(task)
Gitlab::Backup::Cli::Output.info("Waiting for Backup of #{task.name} to finish...")
return unless task.asynchronous?
Gitlab::Backup::Cli::Output.info("Waiting for Backup of #{task.human_name} to finish...")
r = task.wait_until_done!
if r.error?
Gitlab::Backup::Cli::Output.error("Backup of #{task.name} failed!")
Gitlab::Backup::Cli::Output.error("Backup of #{task.human_name} failed!")
else
Gitlab::Backup::Cli::Output.success("Finished Backup of #{task.name}!")
Gitlab::Backup::Cli::Output.success("Finished Backup of #{task.human_name}!")
end
end
end

18
gems/gitlab-backup-cli/lib/gitlab/backup/cli/base_executor.rb Normal file
View File

@ -0,0 +1,18 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
class BaseExecutor
attr_reader :backup_bucket, :wait_for_completion, :registry_bucket, :service_account_file
def initialize(backup_bucket:, wait_for_completion:, registry_bucket:, service_account_file:)
@backup_bucket = backup_bucket
@registry_bucket = registry_bucket
@wait_for_completion = wait_for_completion
@service_account_file = service_account_file
end
end
end
end
end

1
gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands.rb
View File

@ -6,6 +6,7 @@ module Gitlab
module Commands
autoload :BackupSubcommand, 'gitlab/backup/cli/commands/backup_subcommand'
autoload :Command, 'gitlab/backup/cli/commands/command'
autoload :ObjectStorageCommand, 'gitlab/backup/cli/commands/object_storage_command'
autoload :RestoreSubcommand, 'gitlab/backup/cli/commands/restore_subcommand'
end
end

31
gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/backup_subcommand.rb
View File

@ -4,28 +4,9 @@ module Gitlab
module Backup
module Cli
module Commands
class BackupSubcommand < Command
class BackupSubcommand < ObjectStorageCommand
package_name 'Backup'
EXECUTOR_OPTIONS = %w[backup_bucket wait_for_completion registry_bucket service_account_file].freeze
class_option :backup_bucket,
desc: "When backing up object storage, this is the bucket to backup to",
required: false
class_option :wait_for_completion,
desc: "Wait for object storage backups to complete",
type: :boolean,
default: true
class_option :registry_bucket,
desc: "When backing up registry from object storage, this is the source bucket",
required: false
class_option :service_account_file,
desc: "JSON file containing the Google service account credentials",
default: "/etc/gitlab/backup-account-credentials.json"
desc 'all', 'Creates a backup including repositories, database and local files'
def all
Gitlab::Backup::Cli.update_process_title!('backup all')
@ -37,7 +18,11 @@ module Gitlab
Gitlab::Backup::Cli::Output.success("Environment loaded. (#{duration.in_seconds}s)")
backup_executor = Gitlab::Backup::Cli::BackupExecutor.new(
context: build_context, backup_options: executor_options
context: build_context,
backup_bucket: options["backup_bucket"],
wait_for_completion: options["wait_for_completion"],
registry_bucket: options["registry_bucket"],
service_account_file: options["service_account_file"]
)
backup_id = backup_executor.metadata.backup_id
@ -67,10 +52,6 @@ module Gitlab
ActiveSupport::Duration.build(Time.now - start)
end
def executor_options
options.select { |key, _| EXECUTOR_OPTIONS.include?(key) }
end
end
end
end

31
gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/object_storage_command.rb Normal file
View File

@ -0,0 +1,31 @@
# frozen_string_literal: true
module Gitlab
module Backup
module Cli
module Commands
class ObjectStorageCommand < Command
class_option :backup_bucket,
desc: "When backing up object storage, this is the bucket to backup to",
required: false,
type: :string
class_option :wait_for_completion,
desc: "Wait for object storage backups to complete",
type: :boolean,
default: true
class_option :registry_bucket,
desc: "When backing up registry from object storage, this is the source bucket",
required: false,
type: :string
class_option :service_account_file,
desc: "JSON file containing the Google service account credentials",
default: "/etc/gitlab/backup-account-credentials.json",
type: :string
end
end
end
end
end

8
gems/gitlab-backup-cli/lib/gitlab/backup/cli/commands/restore_subcommand.rb
View File

@ -4,7 +4,7 @@ module Gitlab
module Backup
module Cli
module Commands
class RestoreSubcommand < Command
class RestoreSubcommand < ObjectStorageCommand
package_name 'Restore'
desc 'all BACKUP_ID', 'Restores a backup including repositories, database and local files'
@ -20,7 +20,11 @@ module Gitlab
restore_executor =
Gitlab::Backup::Cli::RestoreExecutor.new(
context: build_context,
backup_id: backup_id
backup_id: backup_id,
backup_bucket: options["backup_bucket"],
wait_for_completion: options["wait_for_completion"],
registry_bucket: options["registry_bucket"],
service_account_file: options["service_account_file"]
)
duration = measure_duration do

49
gems/gitlab-backup-cli/lib/gitlab/backup/cli/restore_executor.rb
View File

@ -8,19 +8,31 @@ module Gitlab
# A Restore Executor handles the creation and deletion of
# temporary environment necessary for a restoration to happen
#
class RestoreExecutor
class RestoreExecutor < BaseExecutor
attr_reader :context, :backup_id, :workdir, :archive_directory
# @param [Context::SourceContext|Context::OmnibusContext] context
# @param [String] backup_id
def initialize(context:, backup_id:)
def initialize(
context:,
backup_id: nil,
backup_bucket: nil,
wait_for_completion: nil,
registry_bucket: nil,
service_account_file: nil
)
@context = context
@backup_id = backup_id
@workdir = create_temporary_workdir!
@archive_directory = context.backup_basedir.join(backup_id)
@metadata = nil
@backup_options = nil
super(
backup_bucket: backup_bucket,
wait_for_completion: wait_for_completion,
registry_bucket: registry_bucket,
service_account_file: service_account_file
)
end
def execute
@ -47,15 +59,28 @@ module Gitlab
def execute_all_tasks
# TODO: when we migrate targets to the new codebase, recreate options to have only what we need here
# https://gitlab.com/gitlab-org/gitlab/-/issues/454906
tasks = []
Gitlab::Backup::Cli::Tasks.build_each(context: context, options: backup_options) do |task|
Gitlab::Backup::Cli::Output.info("Executing restoration of #{task.human_name}...")
duration = measure_duration do
task.restore!(archive_directory)
tasks << { name: task.human_name, result: task.restore!(archive_directory, backup_id) }
end
next if task.object_storage?
Gitlab::Backup::Cli::Output.success("Finished restoration of #{task.human_name}! (#{duration.in_seconds}s)")
end
if wait_for_completion
tasks.each do |task|
next unless task[:result].respond_to?(:wait_until_done)
wait_for_task(task[:result])
end
else
Gitlab::Backup::Cli::Output.info("Restore tasks complete! Not waiting for object storage tasks to complete")
end
end
def read_metadata!
@ -64,7 +89,10 @@ module Gitlab
def build_backup_options!
::Backup::Options.new(
backup_id: backup_id
backup_id: backup_id,
remote_directory: backup_bucket,
container_registry_bucket: registry_bucket,
service_account_file: service_account_file
)
end
@ -83,6 +111,17 @@ module Gitlab
ActiveSupport::Duration.build(Time.now - start)
end
def wait_for_task(task)
Gitlab::Backup::Cli::Output.info("Waiting for Restore of #{task.name} to finish...")
r = task.wait_until_done!
if r.error?
Gitlab::Backup::Cli::Output.error("Restore of #{task.name} failed!")
else
Gitlab::Backup::Cli::Output.success("Finished Restore of #{task.name}!")
end
end
end
end
end

97
gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/object_storage/google.rb
View File

@ -8,10 +8,11 @@ module Gitlab
module Targets
class ObjectStorage
class Google < Target
attr_accessor :object_type, :backup_bucket, :client, :config
OperationNotFoundError = Class.new(StandardError)
attr_accessor :object_type, :backup_bucket, :client, :config, :results
def initialize(object_type, options, config)
check_env
@object_type = object_type
@backup_bucket = options.remote_directory
@config = config
@ -19,65 +20,107 @@ module Gitlab
end
def dump(_, backup_id)
response = find_or_create_job(backup_id)
response = find_or_create_job(backup_id, "backup")
run_request = {
project_id: job_spec(backup_id)[:project_id],
project_id: backup_job_spec(backup_id)[:project_id],
job_name: response.name
}
client.run_transfer_job run_request
@results = client.run_transfer_job run_request
end
def job_name
"transferJobs/#{object_type}-backup"
def restore(_, backup_id)
response = find_or_create_job(backup_id, "restore")
run_request = {
project_id: restore_job_spec(backup_id)[:project_id],
job_name: response.name
}
@results = client.run_transfer_job run_request
end
def job_spec(backup_id)
def job_name(operation)
"transferJobs/#{object_type}-#{operation}"
end
def backup_job_spec(backup_id)
job_spec(
config.object_store.remote_directory,
backup_bucket,
operation: "backup",
destination_path: backup_path(backup_id)
)
end
def restore_job_spec(backup_id)
job_spec(
backup_bucket,
config.object_store.remote_directory,
operation: "restore",
source_path: backup_path(backup_id)
)
end
def backup_path(backup_id)
"backups/#{backup_id}/#{object_type}/"
end
def find_job_spec(backup_id, operation)
case operation
when "backup"
backup_job_spec(backup_id)
when "restore"
restore_job_spec(backup_id)
else
raise StandardError "Operation #{operation} not found"
end
end
def job_spec(source, destination, operation:, source_path: nil, destination_path: nil)
{
project_id: config.object_store.connection.google_project,
name: job_name,
name: job_name(operation),
transfer_spec: {
gcs_data_source: {
bucket_name: config.object_store.remote_directory
bucket_name: source,
path: source_path
},
gcs_data_sink: {
bucket_name: backup_bucket,
bucket_name: destination,
# NOTE: The trailing '/' is required
path: "backups/#{backup_id}/#{object_type}/"
path: destination_path
}
},
status: :ENABLED
}
end
private
def check_env
# We expect service account credentials to be passed via env variables. If they are not, attempt
# to use the local service account credentials and warn.
return unless ENV.key?("GOOGLE_CLOUD_CREDENTIALS") || ENV.key?("GOOGLE_APPLICATION_CREDENTIALS")
log.warning("No credentials provided.")
log.warning("If we're in GCP, we will attempt to use the machine service account.")
log.warning("This is not recommended.")
def asynchronous?
true
end
def find_or_create_job(backup_id)
def wait_until_done!
@results.wait_until_done!
end
private
def find_or_create_job(backup_id, operation)
begin
name = job_name(operation)
response = client.get_transfer_job(
job_name: job_name, project_id: config.object_store.connection.google_project
job_name: name, project_id: config.object_store.connection.google_project
)
log.info("Existing job for #{object_type} found, using")
job_update = job_spec(backup_id)
job_update = find_job_spec(backup_id, operation)
job_update.delete(:project_id)
client.update_transfer_job(
job_name: job_name,
job_name: name,
project_id: config.object_store.connection.google_project,
transfer_job: job_update
)
rescue ::Google::Cloud::NotFoundError
log.info("Existing job for #{object_type} not found, creating one")
response = client.create_transfer_job transfer_job: job_spec(backup_id)
response = client.create_transfer_job transfer_job: find_job_spec(backup_id, operation)
end
response
end

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/target.rb
View File

@ -16,6 +16,10 @@ module Gitlab
@options = options
end
def asynchronous?
false
end
# dump task backup to `path`
#
# @param [String] path fully qualified backup task destination

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/artifacts.rb
View File

@ -13,8 +13,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
end
def storage_path = context.ci_job_artifacts_path

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/ci_secure_files.rb
View File

@ -13,8 +13,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
end
def storage_path = context.ci_secure_files_path

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/lfs.rb
View File

@ -13,8 +13,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options)
end
def storage_path = context.ci_lfs_path

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/packages.rb
View File

@ -13,8 +13,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
end
def storage_path = context.packages_path

6
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/pages.rb
View File

@ -17,10 +17,8 @@ module Gitlab
private
def target
check_object_storage(
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: [LEGACY_PAGES_TMP_PATH])
)
def local
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: [LEGACY_PAGES_TMP_PATH])
end
def storage_path = context.pages_path

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/registry.rb
View File

@ -30,8 +30,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options)
end
def storage_path = context.registry_path

35
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/task.rb
View File

@ -6,6 +6,7 @@ module Gitlab
module Tasks
class Task
attr_reader :options, :context
attr_writer :target
# Identifier used as parameter in the CLI to skip from executing
def self.id
@ -30,10 +31,10 @@ module Gitlab
target.dump(backup_output, backup_id)
end
def restore!(archive_directory)
def restore!(archive_directory, backup_id)
archived_data_location = Pathname(archive_directory).join(destination_path)
target.restore(archived_data_location, nil)
target.restore(archived_data_location, backup_id)
end
# Key string that identifies the task
@ -83,20 +84,32 @@ module Gitlab
true
end
def asynchronous?
target.asynchronous? || false
end
def wait_until_done!
target.wait_until_done!
end
def target
return @target unless @target.nil?
@target = if object_storage?
::Gitlab::Backup::Cli::Targets::ObjectStorage.find_task(id, options, config)
else
local
end
@target
end
private
# The target factory method
def target
def local
raise NotImplementedError
end
def check_object_storage(file_target)
if object_storage?
::Gitlab::Backup::Cli::Targets::ObjectStorage.find_task(id, options, config)
else
file_target
end
end
end
end
end

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/terraform_state.rb
View File

@ -13,8 +13,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
end
def storage_path = context.terraform_state_path

4
gems/gitlab-backup-cli/lib/gitlab/backup/cli/tasks/uploads.rb
View File

@ -13,8 +13,8 @@ module Gitlab
private
def target
check_object_storage(::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp']))
def local
::Backup::Targets::Files.new(nil, storage_path, options: options, excludes: ['tmp'])
end
def storage_path = context.upload_path

11
gems/gitlab-backup-cli/spec/gitlab/backup/cli/commands/backup_subcommand_spec.rb
View File

@ -1,11 +0,0 @@
# frozen_string_literal: true
RSpec.describe Gitlab::Backup::Cli::Commands::BackupSubcommand do
describe "#executor_options" do
it "returns the expected hash" do
expect(described_class.new.send(:executor_options).keys).to eq(
%w[wait_for_completion service_account_file]
)
end
end
end

102
gems/gitlab-backup-cli/spec/gitlab/backup/cli/targets/object_storage/google_spec.rb
View File

@ -2,24 +2,31 @@
RSpec.describe Gitlab::Backup::Cli::Targets::ObjectStorage::Google do
let(:gitlab_config) { class_double("GitlabSettings::Settings") }
let(:supported_object_store) do
let(:supported_config) { instance_double("GitlabSettings::Options", object_store: supported_object_store) }
let(:supported_provider) do
instance_double(
"GitlabSettings::Options",
enabled: true,
connection: instance_double("GitlabSettings::Options", provider: "Google")
"GitlabSettings::Options", provider: "Google", google_application_default: true, google_project: "fake_project"
)
end
let(:supported_object_store) do
instance_double(
"GitlabSettings::Options", enabled: true, connection: supported_provider, remote_directory: "fake_source_bucket"
)
end
let(:supported_config) { instance_double("GitlabSettings::Options", object_store: supported_object_store) }
let(:client) { instance_double("::Google::Cloud::StorageTransfer::V1::StorageTransferService::Client") }
let(:existing_transfer_job) { build(:google_cloud_storage_transfer_job) }
let(:new_transfer_job_spec) do
let(:backup_transfer_job) { build(:google_cloud_storage_transfer_job) }
let(:restore_transfer_job) { build(:google_cloud_storage_transfer_job) }
let(:new_backup_transfer_job_spec) do
{
name: "transferJobs/fake_object-backup",
project_id: "fake_project",
transfer_spec: {
gcs_data_source: {
bucket_name: "fake_source_bucket"
bucket_name: "fake_source_bucket",
path: nil
},
gcs_data_sink: {
bucket_name: "fake_backup_bucket",
@ -30,41 +37,42 @@ RSpec.describe Gitlab::Backup::Cli::Targets::ObjectStorage::Google do
}
end
let(:backup_options) { instance_double("Gitlab::Backup::Options", remote_directory: 'fake_backup_bucket') }
let(:new_restore_transfer_job_spec) do
{
name: "transferJobs/fake_object-restore",
project_id: "fake_project",
transfer_spec: {
gcs_data_source: {
bucket_name: "fake_backup_bucket",
path: "backups/12345/fake_object/"
},
gcs_data_sink: {
bucket_name: "fake_source_bucket",
path: nil
}
},
status: :ENABLED
}
end
let(:backup_options) { instance_double("::Backup::Options", remote_directory: 'fake_backup_bucket') }
before do
allow(Gitlab).to receive(:config).and_return(gitlab_config)
allow(::Google::Cloud::StorageTransfer).to receive(:storage_transfer_service).and_return(client)
allow(gitlab_config).to receive(:[]).with('fake_object').and_return(supported_config)
end
subject(:object_storage) { described_class.new("fake_object", backup_options, supported_config) }
describe "#dump" do
let(:supported_provider) do
instance_double(
"GitlabSettings::Options", provider: "Google", google_application_default: true, google_project: "fake_project"
)
end
let(:supported_object_store) do
instance_double(
"GitlabSettings::Options", enabled: true, connection: supported_provider, remote_directory: "fake_source_bucket"
)
end
let(:supported_config) { instance_double("GitlabSettings::Options", object_store: supported_object_store) }
before do
allow(gitlab_config).to receive(:[]).with('fake_object').and_return(supported_config)
end
context "when job exists" do
before do
allow(client).to receive(:get_transfer_job).and_return(existing_transfer_job)
allow(client).to receive(:get_transfer_job).and_return(backup_transfer_job)
end
it "reuses existing job" do
updated_spec = new_transfer_job_spec
updated_spec = new_backup_transfer_job_spec
expect(client).to receive(:update_transfer_job).with(
job_name: updated_spec[:name],
project_id: updated_spec.delete(:project_id),
@ -85,9 +93,43 @@ RSpec.describe Gitlab::Backup::Cli::Targets::ObjectStorage::Google do
it "creates a new job" do
expect(client).to receive(:create_transfer_job)
.with(transfer_job: new_transfer_job_spec).and_return(existing_transfer_job)
.with(transfer_job: new_backup_transfer_job_spec).and_return(backup_transfer_job)
object_storage.dump(nil, 12345)
end
end
end
describe "#restore" do
context "when job exists" do
before do
allow(client).to receive(:get_transfer_job).and_return(restore_transfer_job)
end
it "reuses existing job" do
updated_spec = new_restore_transfer_job_spec
expect(client).to receive(:update_transfer_job).with(
job_name: updated_spec[:name],
project_id: updated_spec.delete(:project_id),
transfer_job: updated_spec
)
expect(client).to receive(:run_transfer_job).with({ job_name: "fake_transfer_job", project_id: "fake_project" })
object_storage.restore(nil, 12345)
end
end
context "when job does not exist" do
before do
allow(client).to receive(:get_transfer_job).with(
job_name: "transferJobs/fake_object-restore", project_id: "fake_project"
).and_raise(::Google::Cloud::NotFoundError)
allow(client).to receive(:run_transfer_job)
end
it "creates a new job" do
expect(client).to receive(:create_transfer_job)
.with(transfer_job: new_restore_transfer_job_spec).and_return(restore_transfer_job)
object_storage.restore(nil, 12345)
end
end
end
end

4
gems/gitlab-backup-cli/spec/gitlab/backup/cli/tasks/task_spec.rb
View File

@ -1,7 +1,7 @@
# frozen_string_literal: true
RSpec.describe Gitlab::Backup::Cli::Tasks::Task do
let(:options) { nil }
let(:options) { instance_double("::Backup::Option", backup_id: "abc123") }
let(:context) { build_fake_context }
let(:tmpdir) { Pathname.new(Dir.mktmpdir('task', temp_path)) }
let(:metadata) { build(:backup_metadata) }
@ -59,7 +59,7 @@ RSpec.describe Gitlab::Backup::Cli::Tasks::Task do
expect(task).to receive(:destination_path).and_return(tmpdir.join('test_task'))
expect(task).to receive_message_chain(:target, :restore)
task.restore!(archive_directory)
task.restore!(archive_directory, options.backup_id)
end
end
end

8
gems/gitlab-backup-cli/spec/thor/gitlab_backup_cli_restore_spec.rb
View File

@ -13,6 +13,14 @@ RSpec.describe 'gitlab-backup-cli restore subcommand', type: :thor do
gitlab-backup-cli restore all BACKUP_ID # Restores a backup including repositories, database and local files
gitlab-backup-cli restore help [COMMAND] # Describe subcommands or one specific subcommand
Options:
[--backup-bucket=BACKUP_BUCKET] # When backing up object storage, this is the bucket to backup to
[--wait-for-completion], [--no-wait-for-completion], [--skip-wait-for-completion] # Wait for object storage backups to complete
# Default: true
[--registry-bucket=REGISTRY_BUCKET] # When backing up registry from object storage, this is the source bucket
[--service-account-file=SERVICE_ACCOUNT_FILE] # JSON file containing the Google service account credentials
# Default: /etc/gitlab/backup-account-credentials.json
COMMAND
end

4
lib/backup/targets/files.rb
View File

@ -172,6 +172,10 @@ module Backup
raise FileBackupError.new(storage_realpath, backup_tarball)
end
def asynchronous?
false
end
private
def storage_realpath

4
lib/backup/targets/repositories.rb
View File

@ -48,6 +48,10 @@ module Backup
restore_object_pools
end
def asynchronous?
false
end
private
attr_reader :strategy, :storages, :paths, :skip_paths, :logger

9
lib/sidebars/user_profile/panel.rb
View File

@ -25,8 +25,17 @@ module Sidebars
private
def add_legacy_menu?
# When `profile_tabs_vue` feature flag is enabled, legacy profile pages
# will be replaced by routes in `app/assets/javascripts/profile/components/app.vue`
Feature.disabled?(:profile_tabs_vue, context.current_user)
end
def add_menus
add_menu(Sidebars::UserProfile::Menus::OverviewMenu.new(context))
return unless add_legacy_menu?
add_menu(Sidebars::UserProfile::Menus::ActivityMenu.new(context))
add_menu(Sidebars::UserProfile::Menus::GroupsMenu.new(context))
add_menu(Sidebars::UserProfile::Menus::ContributedProjectsMenu.new(context))

4
spec/features/users/show_spec.rb
View File

@ -158,10 +158,8 @@ RSpec.describe 'User page', feature_category: :user_profile do
end
end
it_behaves_like 'follower links with count badges'
context 'with profile_tabs_vue feature flag disabled' do
before_all do
before do
stub_feature_flags(profile_tabs_vue: false)
end

10
spec/lib/backup/targets/files_spec.rb
View File

@ -392,4 +392,14 @@ RSpec.describe Backup::Targets::Files, feature_category: :backup_restore do
).to be_falsey
end
end
context 'with unified backup' do
subject(:files) do
described_class.new(progress, '/fake/path', options: backup_options)
end
it 'is not asynchronous by default' do
expect(files.asynchronous?).to be_falsey
end
end
end

37
spec/lib/sidebars/user_profile/panel_spec.rb
View File

@ -3,6 +3,17 @@
require 'spec_helper'
RSpec.describe Sidebars::UserProfile::Panel, feature_category: :navigation do
legacy_menu_classes = [
Sidebars::UserProfile::Menus::ActivityMenu,
Sidebars::UserProfile::Menus::GroupsMenu,
Sidebars::UserProfile::Menus::ContributedProjectsMenu,
Sidebars::UserProfile::Menus::PersonalProjectsMenu,
Sidebars::UserProfile::Menus::StarredProjectsMenu,
Sidebars::UserProfile::Menus::SnippetsMenu,
Sidebars::UserProfile::Menus::FollowersMenu,
Sidebars::UserProfile::Menus::FollowingMenu
]
let(:current_user) { build_stubbed(:user) }
let(:user) { build_stubbed(:user) }
@ -13,11 +24,29 @@ RSpec.describe Sidebars::UserProfile::Panel, feature_category: :navigation do
it_behaves_like 'a panel with uniquely identifiable menu items'
it_behaves_like 'a panel instantiable by the anonymous user'
it 'implements #aria_label' do
expect(subject.aria_label).to eq(s_('UserProfile|User profile navigation'))
describe '#aria_label' do
specify { expect(subject.aria_label).to eq(s_('UserProfile|User profile navigation')) }
end
it 'implements #super_sidebar_context_header' do
expect(subject.super_sidebar_context_header).to eq(_('Profile'))
describe '#super_sidebar_context_header' do
specify { expect(subject.super_sidebar_context_header).to eq(_('Profile')) }
end
it 'does not add legacy menu items' do
menu_classes = subject.renderable_menus.map(&:class)
expect(menu_classes).not_to include(*legacy_menu_classes)
end
context 'when profile_tabs_vue feature is disabled' do
before do
stub_feature_flags(profile_tabs_vue: false)
end
it 'add legacy menu items' do
menu_classes = subject.renderable_menus.map(&:class)
expect(menu_classes).to include(*legacy_menu_classes)
end
end
end