Add latest changes from gitlab-org/gitlab@master

gems/gitlab-backup-cli/.gitlab-ci.yml

@@ -17,13 +17,18 @@ rspec:
     - name: postgres:${POSTGRES_VERSION}
       command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
   before_script:
+    - apt update && apt install -y postgresql-client
+    - psql -h postgres -U $POSTGRES_USER -c 'create database gitlabhq_test;'
+    - psql -h postgres -U $POSTGRES_USER -c 'create database gitlabhq_ci_test;'
     - cp gems/gitlab-backup-cli/spec/fixtures/config/database.yml config/
     - "sed -i \"s/username: postgres$/username: $POSTGRES_USER/g\" config/database.yml"
     - "sed -i \"s/password:\\s*$/password: $POSTGRES_PASSWORD/g\" config/database.yml"
     - "sed -i \"s/host: localhost$/host: postgres/g\" config/database.yml"
-    - apt update && apt install -y postgresql-client
-    - psql -h postgres -U $POSTGRES_USER -c 'create database gitlabhq_test;'
-    - psql -h postgres -U $POSTGRES_USER -c 'create database gitlabhq_ci_test;'
+    - |
+      cd gems/gitlab-backup-cli/spec/fixtures/gitlab_fake &&
+      [ -n "$BUNDLE_GEMFILE" ] && mv Gemfile ${BUNDLE_GEMFILE} && mv Gemfile.lock ${BUNDLE_GEMFILE}.lock
+    - bundle install --retry=3
+    - cd -
     - !reference [.default, before_script]
   script:
     - RAILS_ENV=test bundle exec rspec

gems/gitlab-backup-cli/.rubocop.yml

@@ -11,3 +11,6 @@ Rails/Exit:
 RSpec/MultipleMemoizedHelpers:
   Max: 25
   AllowSubject: true
+
+Rails/RakeEnvironment:
+  Enabled: false

gems/gitlab-backup-cli/Rakefile

@@ -10,3 +10,7 @@ require "rubocop/rake_task"
 RuboCop::RakeTask.new
 
 task default: %i[spec rubocop]
+
+task :version do |_|
+  puts Gitlab::Backup::Cli::VERSION
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/errors.rb

@@ -5,6 +5,7 @@ module Gitlab
     module Cli
       module Errors
         autoload :DatabaseBackupError, 'gitlab/backup/cli/errors/database_backup_error'
+        autoload :DatabaseCleanupError, 'gitlab/backup/cli/errors/database_cleanup_error'
         autoload :DatabaseConfigMissingError, 'gitlab/backup/cli/errors/database_config_missing_error'
         autoload :DatabaseMissingConnectionError, 'gitlab/backup/cli/errors/database_missing_connection_error'
         autoload :FileBackupError, 'gitlab/backup/cli/errors/file_backup_error'

gems/gitlab-backup-cli/lib/gitlab/backup/cli/errors/database_cleanup_error.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Errors
+        class DatabaseCleanupError < StandardError
+          attr_reader :task, :path, :error
+
+          def initialize(task:, path:, error:)
+            @task = task
+            @path = path
+            @error = error
+
+            super(build_message)
+          end
+
+          private
+
+          def build_message
+            "Failed to cleanup GitLab databases \n" \
+              "Running the following rake task: '#{task}' (from: #{path}) failed:\n" \
+              "#{error}"
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/database.rb

@@ -18,6 +18,10 @@ module Gitlab
           ].freeze
           IGNORED_ERRORS_REGEXP = Regexp.union(IGNORED_ERRORS).freeze
 
+          # Rake task used to drop all tables from GitLab databases
+          # This task is executed before restoring data
+          DROP_TABLES_TASK = "gitlab:db:drop_tables"
+
           attr_reader :errors
 
           def initialize(context)
@@ -66,6 +70,10 @@ module Gitlab
           def restore(source)
             databases = Gitlab::Backup::Cli::Services::Postgres.new(context)
 
+            # Drop all tables Load the schema to ensure we don't have any newer tables
+            # hanging out from a failed upgrade
+            drop_tables!
+
             databases.each do |db|
               database_name = db.configuration.name
               pg_database_name = db.configuration.database
@@ -89,10 +97,6 @@ module Gitlab
                 next
               end
 
-              # Drop all tables Load the schema to ensure we don't have any newer tables
-              # hanging out from a failed upgrade
-              drop_tables(db)
-
               Gitlab::Backup::Cli::Output.info "Restoring PostgreSQL database #{pg_database_name} ... "
 
               status = restore_tables(database: db, filepath: db_file_name)
@@ -151,18 +155,22 @@ module Gitlab
             Gitlab::Backup::Cli::Output.print_tag(status ? :success : :failure)
           end
 
-          def drop_tables(database)
-            pg_database_name = database.configuration.database
-            Gitlab::Backup::Cli::Output.print_info "Cleaning the '#{pg_database_name}' database ... "
+          def drop_tables!
+            Gitlab::Backup::Cli::Output.print_info "Cleaning existing databases ... "
 
-            if Rake::Task.task_defined? "gitlab:db:drop_tables:#{database.configuration.name}"
-              Rake::Task["gitlab:db:drop_tables:#{database.configuration.name}"].invoke
-            else
-              # In single database (single or two connections)
-              Rake::Task["gitlab:db:drop_tables"].invoke
+            gitlab_path = context.gitlab_basepath
+
+            # Drop existing tables from configured databases before restoring from a backup
+            rake = Utils::Rake.new(DROP_TABLES_TASK, chdir: gitlab_path).execute
+
+            unless rake.success?
+              Gitlab::Backup::Cli::Output.print_tag(:failure)
+
+              raise Errors::DatabaseCleanupError.new(task: DROP_TABLES_TASK, path: gitlab_path, error: rake.stderr)
             end
 
             Gitlab::Backup::Cli::Output.print_tag(:success)
+            Gitlab::Backup::Cli::Output.info(rake.output) unless rake.output.empty?
           end
 
           def restore_tables(database:, filepath:)

gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils.rb

@@ -6,6 +6,7 @@ module Gitlab
       module Utils
         autoload :Compression, 'gitlab/backup/cli/utils/compression'
         autoload :PgDump, 'gitlab/backup/cli/utils/pg_dump'
+        autoload :Rake, 'gitlab/backup/cli/utils/rake'
         autoload :Tar, 'gitlab/backup/cli/utils/tar'
       end
     end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils/rake.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Backup
+    module Cli
+      module Utils
+        class Rake
+          # @return [Array<String>] a list of tasks to be executed
+          attr_reader :tasks
+
+          # @return [String|Pathname] a path where rake tasks are run from
+          attr_reader :chdir
+
+          # @param [Array<String>] *tasks a list of tasks to be executed
+          # @param [String|Pathname] chdir a path where rake tasks are run from
+          def initialize(*tasks, chdir: Gitlab::Backup::Cli.root)
+            @tasks = tasks
+            @chdir = chdir
+          end
+
+          # @return [self]
+          def execute
+            Bundler.with_original_env do
+              @result = Shell::Command.new(*rake_command, chdir: chdir).capture
+            end
+
+            self
+          end
+
+          # Return whether the execution was a success or not
+          #
+          # @return [Boolean] whether the execution was a success
+          def success?
+            @result&.status&.success? || false
+          end
+
+          # Return the captured rake output
+          #
+          # @return [String] stdout content
+          def output
+            @result&.stdout || ''
+          end
+
+          # Return the captured error content
+          #
+          # @return [String] stdout content
+          def stderr
+            @result&.stderr || ''
+          end
+
+          # Return the captured execution duration
+          #
+          # @return [Float] execution duration
+          def duration
+            @result&.duration || 0.0
+          end
+
+          private
+
+          # Return a list of commands necessary to execute `rake`
+          #
+          # @return [Array<String (frozen)>] array of commands to be used by Shellout
+          def rake_command
+            %w[bundle exec rake] + tasks
+          end
+        end
+      end
+    end
+  end
+end

gems/gitlab-backup-cli/spec/fixtures/gitlab_fake/Gemfile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+gem 'rake', '~> 13.0'

gems/gitlab-backup-cli/spec/fixtures/gitlab_fake/Gemfile.lock

@@ -0,0 +1,18 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    rake (13.2.1)
+
+PLATFORMS
+  aarch64-linux
+  arm64-darwin
+  ruby
+  x86-linux
+  x86_64-darwin
+  x86_64-linux
+
+DEPENDENCIES
+  rake (~> 13.0)
+
+BUNDLED WITH
+   2.5.22

gems/gitlab-backup-cli/spec/fixtures/gitlab_fake/Rakefile

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+  namespace :db do
+    task :drop_tables do |_|
+      exit 0
+    end
+  end
+end
+
+task :current_pwd do |_|
+  puts Dir.getwd
+end

gems/gitlab-backup-cli/spec/gitlab/backup/cli/errors/database_cleanup_error_spec.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+RSpec.describe Gitlab::Backup::Cli::Errors::DatabaseCleanupError do
+  let(:task) { 'gitlab:task' }
+  let(:path) { fixtures_path }
+  let(:error) { 'error message from task execution' }
+
+  subject(:database_error) { described_class.new(task: task, path: path, error: error) }
+
+  describe '#initialize' do
+    it 'sets task, path and error attributes' do
+      expect(database_error.path).to eq(path)
+      expect(database_error.task).to eq(task)
+      expect(database_error.error).to eq(error)
+    end
+  end
+end

gems/gitlab-backup-cli/spec/gitlab/backup/cli/services/database_spec.rb

@@ -1,16 +1,20 @@
 # frozen_string_literal: true
 
 RSpec.describe Gitlab::Backup::Cli::Services::Database do
-  let(:database_yml) { YAML.load_file(fixtures_path.join('config/database.yml'), aliases: true) }
+  let(:context) { build_test_context }
+  let(:connection) { database.send(:connection) }
   let(:mocked_configuration) do
+    database_yml = YAML.load_file(fixtures_path.join('config/database.yml'), aliases: true)
     ActiveRecord::DatabaseConfigurations.new(database_yml).configs_for(env_name: 'test', include_hidden: false).first
   end
 
   let(:test_configuration) do
-    Gitlab::Backup::Cli::Services::Postgres.new(build_test_context).send(:database_configurations).first
+    Gitlab::Backup::Cli::Services::Postgres.new(context).send(:database_configurations).first
   end
 
-  let(:connection) { database.send(:connection) }
+  after do
+    context.cleanup!
+  end
 
   context 'with mocked configuration' do
     subject(:database) { described_class.new(mocked_configuration) }

gems/gitlab-backup-cli/spec/gitlab/backup/cli/services/postgres_spec.rb

@@ -5,6 +5,10 @@ RSpec.describe Gitlab::Backup::Cli::Services::Postgres do
 
   subject(:postgres) { described_class.new(context) }
 
+  after do
+    context.cleanup!
+  end
+
   describe '#entries' do
     context 'with missing database configuration' do
       it 'raises an error' do

gems/gitlab-backup-cli/spec/gitlab/backup/cli/targets/database_spec.rb

@@ -7,6 +7,10 @@ RSpec.describe Gitlab::Backup::Cli::Targets::Database do
   let(:database) { described_class.new(context) }
   let(:pipeline_success) { instance_double(Gitlab::Backup::Cli::Shell::Pipeline::Result, success?: true) }
 
+  after do
+    context.cleanup!
+  end
+
   describe '#dump', :silence_output do
     let(:destination) { Pathname(Dir.mktmpdir('database-target', temp_path)) }
 
@@ -17,7 +21,7 @@ RSpec.describe Gitlab::Backup::Cli::Targets::Database do
     it 'creates the destination directory' do
       mock_database_dump!
 
-      expect(FileUtils).to receive(:mkdir_p).with(destination)
+      expect(destination).to be_directory
 
       database.dump(destination)
     end
@@ -99,18 +103,16 @@ RSpec.describe Gitlab::Backup::Cli::Targets::Database do
           pipeline_success
         )
 
-        mock_databases_collection('main') do |db|
+        mock_databases_collection('main') do |_|
           FileUtils.touch(source.join('database.sql.gz'))
-
-          expect(database).to receive(:drop_tables).with(db)
         end
 
+        expect(database).to receive(:drop_tables!)
+
         database.restore(source)
       end
 
       it 'restores the database' do
-        allow(database).to receive(:drop_tables)
-
         mock_databases_collection('main') do |db|
           filepath = source.join('database.sql.gz')
           FileUtils.touch(filepath)

gems/gitlab-backup-cli/spec/gitlab/backup/cli/utils/rake_spec.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+RSpec.describe Gitlab::Backup::Cli::Utils::Rake do
+  subject(:rake) { described_class.new('version') }
+
+  describe '#execute' do
+    it 'clears out bundler environment' do
+      expect(Bundler).to receive(:with_original_env).and_yield
+
+      rake.execute
+    end
+
+    it 'runs rake using bundle exec' do
+      expect_next_instance_of(Gitlab::Backup::Cli::Shell::Command) do |shell|
+        expect(shell.cmd_args).to start_with(%w[bundle exec rake])
+      end
+
+      rake.execute
+    end
+
+    it 'runs rake command with the defined tasks' do
+      expect_next_instance_of(Gitlab::Backup::Cli::Shell::Command) do |shell|
+        expect(shell.cmd_args).to end_with(%w[version])
+      end
+
+      rake.execute
+
+      expect(rake.success?).to eq(true)
+    end
+
+    context 'when chdir is set' do
+      let(:tmpdir) { Dir.mktmpdir }
+
+      after do
+        FileUtils.rm_rf(tmpdir)
+      end
+
+      subject(:rake) { described_class.new('current_pwd', chdir: tmpdir) }
+
+      it 'runs rake in the provided chdir directory' do
+        expect_next_instance_of(Gitlab::Backup::Cli::Shell::Command) do |shell|
+          expect(shell.chdir).to eq(tmpdir)
+        end
+
+        FileUtils.cp_r(fixtures_path.join('gitlab_fake').glob('*'), tmpdir)
+
+        rake.execute
+
+        expect(rake.success?).to eq(true)
+        expect(rake.output).to match(/#{tmpdir}/)
+      end
+    end
+  end
+
+  describe '#success?' do
+    subject(:rake) { described_class.new('--version') } # valid command that has no side-effect
+
+    context 'with a successful rake execution' do
+      it 'returns true' do
+        rake.execute
+
+        expect(rake.success?).to be_truthy
+      end
+    end
+
+    context 'with a failed rake execution', :hide_output do
+      subject(:invalid_rake) { described_class.new('--invalid') } # valid command that has no side-effect
+
+      it 'returns false when a previous execution failed' do
+        invalid_rake.execute
+
+        expect(invalid_rake.duration).to be > 0.0
+        expect(invalid_rake.success?).to be_falsey
+      end
+    end
+
+    it 'returns false when no execution was done before' do
+      expect(rake.success?).to be_falsey
+    end
+  end
+
+  describe '#output' do
+    it 'returns the output from running a rake task' do
+      rake.execute
+
+      expect(rake.output).to match(Gitlab::Backup::Cli::VERSION)
+    end
+
+    it 'returns an empty string when the task has not been run' do
+      expect(rake.output).to eq('')
+    end
+  end
+
+  describe '#stderr' do
+    subject(:invalid_rake) { described_class.new('--invalid') } # valid command that has no side-effect
+
+    it 'returns the content from stderr when available' do
+      invalid_rake.execute
+
+      expect(invalid_rake.stderr).to match('invalid option: --invalid')
+    end
+
+    it 'returns an empty string when the task has not been run' do
+      expect(invalid_rake.stderr).to eq('')
+    end
+  end
+
+  describe '#duration' do
+    it 'returns a duration time' do
+      rake.execute
+
+      expect(rake.duration).to be > 0.0
+    end
+
+    it 'returns 0.0 when the task has not been run' do
+      expect(rake.duration).to eq(0.0)
+    end
+  end
+end

gems/gitlab-backup-cli/spec/support/helpers.rb

@@ -34,7 +34,16 @@ module GitlabBackupHelpers
   end
 
   def build_test_context
-    TestContext.new
+    TestContext.new.tap do |context|
+      # config/database.yml
+      db = context.gitlab_original_basepath.join('config/database.yml')
+      test_db = context.gitlab_basepath.join('config/database.yml')
+      FileUtils.mkdir_p(File.dirname(test_db))
+      FileUtils.copy(db, test_db)
+
+      # Mocked Rakefile and Gemfile
+      FileUtils.cp_r(fixtures_path.join('gitlab_fake').glob('*'), context.gitlab_basepath)
+    end
   end
 end
 

gems/gitlab-backup-cli/spec/support/test_context.rb

@@ -2,11 +2,22 @@
 
 class TestContext < Gitlab::Backup::Cli::Context::SourceContext
   def gitlab_basepath
-    test_helpers.spec_path.join('../../..')
+    @gitlab_basepath ||= Pathname(Dir.mktmpdir('gitlab', test_helpers.temp_path))
   end
 
   def backup_basedir
-    test_helpers.temp_path.join('backups')
+    gitlab_basepath.join('backups')
+  end
+
+  def gitlab_original_basepath
+    test_helpers.spec_path.join('../../..')
+  end
+
+  # Deletes the temporary folders
+  def cleanup!
+    dir_permissions = (File.stat(gitlab_basepath).mode & 0o777).to_s(8) # retrieve permissions in octal format)
+
+    FileUtils.rm_rf(gitlab_basepath) if dir_permissions == "700" # ensure it's a temporary dir before deleting
   end
 
   private