Add latest changes from gitlab-org/gitlab@master

gems/gitlab-backup-cli/lib/gitlab/backup/cli/shell/command.rb

@@ -7,7 +7,7 @@ module Gitlab
         # Abstraction to control shell command execution
         # It provides an easier API to common usages
         class Command
-          attr_reader :cmd_args, :env
+          attr_reader :env
 
           # Result data structure from running a command
           #
@@ -29,8 +29,24 @@ module Gitlab
           # @param [Array<String>] cmd_args
           # @param [Hash<String,String>] env
           def initialize(*cmd_args, env: {})
-            @cmd_args = cmd_args
-            @env = env
+            @cmd_args = cmd_args.freeze
+            @env = env.freeze
+          end
+
+          # List of command arguments
+          #
+          # @param [Boolean] with_env whether to include env hash in the returned list
+          # @return [Array<Hash|String>]
+          def cmd_args(with_env: false)
+            if with_env && env.any?
+              # When providing cmd_args to `Open3.pipeline`, the env needs to be the first element of the array.
+              #
+              # While `Open3.capture3` accepts an empty hash as a valid parameter, it doesn't work with
+              # `Open3.pipeline`, so we modify the returned array only when the env hash is not empty.
+              @cmd_args.dup.prepend(env)
+            else
+              @cmd_args.dup
+            end
           end
 
           # Execute a process and return its output and status
@@ -61,7 +77,7 @@ module Gitlab
             options[:in] = input if input # redirect stdin
             options[:out] = output if output # redirect stdout
 
-            status_list = Open3.pipeline(cmd_args, **options)
+            status_list = Open3.pipeline(cmd_args(with_env: true), **options)
             duration = Time.now - start
 
             err_write.close # close the pipe before reading

gems/gitlab-backup-cli/lib/gitlab/backup/cli/shell/pipeline.rb

@@ -18,9 +18,10 @@ module Gitlab
             end
           end
 
+          # List of Shell::Commands that are part of the pipeline
           attr_reader :shell_commands
 
-          # @param [Array<Shell::Command>] shell_commands
+          # @param [Array<Shell::Command>] shell_commands list of commands
           def initialize(*shell_commands)
             @shell_commands = shell_commands
           end
@@ -58,8 +59,13 @@ module Gitlab
 
           private
 
+          # Returns an array of arrays that contains the expanded command args with their env hashes when available
+          #
+          # The output is intended to be used directly by Open3.pipeline
+          #
+          # @return [Array<Array<Hash,String>>]
           def build_command_list
-            @shell_commands.map(&:cmd_args)
+            @shell_commands.map { |command| command.cmd_args(with_env: true) }
           end
         end
       end

gems/gitlab-backup-cli/lib/gitlab/backup/cli/targets/database.rb

@@ -5,7 +5,7 @@ module Gitlab
     module Cli
       module Targets
         class Database < Target
-          attr_reader :force, :errors
+          attr_reader :errors
 
           IGNORED_ERRORS = [
             # Ignore warnings
@@ -19,11 +19,6 @@ module Gitlab
 
           def initialize
             @errors = []
-
-            # This flag will be removed as part of https://gitlab.com/gitlab-org/gitlab/-/issues/494209
-            # This option will be reintroduced as part of
-            # https://gitlab.com/gitlab-org/gitlab/-/issues/498453
-            @force = false
           end
 
           def dump(destination_dir)
@@ -58,7 +53,7 @@ module Gitlab
 
               raise DatabaseBackupError.new(active_record_config, dump_file_name) unless success
 
-              report_success(success)
+              report_finish_status(success)
             end
           ensure
             if multiple_databases?
@@ -77,8 +72,6 @@ module Gitlab
           end
 
           def restore(destination_dir)
-            @errors = []
-
             base_models_for_backup.each do |database_name, _|
               backup_connection = ::Backup::DatabaseConnection.new(database_name)
 
@@ -89,7 +82,7 @@ module Gitlab
 
               unless File.exist?(db_file_name)
                 if main_database?(database_name)
-                  raise(::Backup::Error, "Source database file does not exist #{db_file_name}")
+                  raise(DatabaseBackupError, "Source database file does not exist #{db_file_name}")
                 end
 
                 Gitlab::Backup::Cli::Output.warning(
@@ -99,48 +92,33 @@ module Gitlab
                 return false
               end
 
-              unless force
-                Gitlab::Backup::Cli::Output.warning(
-                  'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'
-                )
-
-                sleep(5)
-              end
+              Gitlab::Backup::Cli::Output.warning("Removing all tables from #{database_name}.")
 
               # Drop all tables Load the schema to ensure we don't have any newer tables
               # hanging out from a failed upgrade
               drop_tables(database_name)
 
-              tracked_errors = []
               pg_env = backup_connection.database_configuration.pg_env_variables
-              success = with_transient_pg_env(pg_env) do
-                decompress_rd, decompress_wr = IO.pipe
 
-                decompress_pid = spawn(decompression_cmd, out: decompress_wr, in: db_file_name)
-                decompress_wr.close
+              pipeline = Gitlab::Backup::Cli::Shell::Pipeline.new(
+                Utils::Compression.decompression_command,
+                pg_restore_cmd(database, pg_env)
+              )
 
-                status, tracked_errors =
-                  case config[:adapter]
-                  when "postgresql"
-                    Gitlab::Backup::Cli::Output.print_info "Restoring PostgreSQL database #{database} ... "
-                    execute_and_track_errors(pg_restore_cmd(database), decompress_rd)
-                  end
-                decompress_rd.close
+              Gitlab::Backup::Cli::Output.print_info "Restoring PostgreSQL database #{database} ... "
 
-                Process.waitpid(decompress_pid)
-                $CHILD_STATUS.success? && status.success?
-              end
+              pipeline_status = pipeline.run!(input: db_file_name)
+              tracked_errors = pipeline_status.stderr
 
               unless tracked_errors.empty?
                 Gitlab::Backup::Cli::Output.error "------ BEGIN ERRORS -----"
                 Gitlab::Backup::Cli::Output.print(tracked_errors.join, stderr: true)
                 Gitlab::Backup::Cli::Output.error "------ END ERRORS -------"
-
-                @errors += tracked_errors
               end
 
-              report_success(success)
-              raise ::Backup::Error, 'Restore failed' unless success
+              report_finish_status(pipeline_status.success?)
+
+              raise DatabaseBackupError, 'Restore failed' unless pipeline_status.success?
             end
           end
 
@@ -164,43 +142,10 @@ module Gitlab
             IGNORED_ERRORS_REGEXP.match?(line)
           end
 
-          def execute_and_track_errors(cmd, decompress_rd)
-            errors = []
-
-            Open3.popen3(ENV, *cmd) do |stdin, stdout, stderr, thread|
-              stdin.binmode
-
-              out_reader = Thread.new do
-                data = stdout.read
-                $stdout.write(data) # rubocop:disable Rails/Output
-              end
-
-              err_reader = Thread.new do
-                until (raw_line = stderr.gets).nil?
-                  warn(raw_line)
-                  errors << raw_line unless ignore_error?(raw_line)
-                end
-              end
-
-              begin
-                IO.copy_stream(decompress_rd, stdin)
-              rescue Errno::EPIPE
-              end
-
-              stdin.close
-              [thread, out_reader, err_reader].each(&:join)
-              [thread.value, errors]
-            end
-          end
-
           private
 
-          def decompression_cmd
-            Utils::Compression.decompression_command.cmd_args.flatten.first
-          end
-
-          def report_success(success)
-            Gitlab::Backup::Cli::Output.print_tag(success ? :success : :failure)
+          def report_finish_status(status)
+            Gitlab::Backup::Cli::Output.print_tag(status ? :success : :failure)
           end
 
           def drop_tables(database_name)
@@ -225,8 +170,8 @@ module Gitlab
             result
           end
 
-          def pg_restore_cmd(database)
-            ['psql', database]
+          def pg_restore_cmd(database, pg_env)
+            Shell::Command.new('psql', database, env: pg_env)
           end
 
           def each_database(destination_dir, &block)

gems/gitlab-backup-cli/spec/gitlab/backup/cli/shell/command_spec.rb

@@ -17,6 +17,33 @@ RSpec.describe Gitlab::Backup::Cli::Shell::Command do
     end
   end
 
+  describe '#cmd_args' do
+    let(:cmd_args) { %w[ls -l] }
+
+    it 'returns a list of command args' do
+      cmd = command.new(*cmd_args)
+
+      expect(cmd.cmd_args).to eq(cmd_args)
+    end
+
+    context 'when with_env is true' do
+      it 'returns the same list of command args when no env is provided' do
+        cmd = command.new(*cmd_args)
+
+        expect(cmd.cmd_args(with_env: true)).to eq(cmd_args)
+      end
+
+      it 'returns a list of command args with the env hash as its first element' do
+        cmd = command.new(*cmd_args, env: envdata)
+
+        result = cmd.cmd_args(with_env: true)
+
+        expect(result.first).to eq(envdata)
+        expect(result[1..]).to eq(cmd_args)
+      end
+    end
+  end
+
   describe '#capture' do
     it 'returns stdout from executed command' do
       expected_output = 'my custom content'
@@ -48,5 +75,80 @@ RSpec.describe Gitlab::Backup::Cli::Shell::Command do
 
       expect(result.duration).to be > 0.1
     end
+
+    it 'sets the provided env variables as part of process execution' do
+      result = command.new("echo \"variable value ${CUSTOM}\"", env: envdata).capture
+
+      expect(result.stdout.chomp).to eq('variable value data')
+    end
+  end
+
+  describe '#run_single_pipeline!' do
+    it 'runs without any exceptions' do
+      expect { command.new('true').run_single_pipeline! }.not_to raise_exception
+    end
+
+    it 'sets env variables from provided commands as part of pipeline execution' do
+      echo_command = command.new("echo \"variable value ${CUSTOM}\"", env: envdata)
+      read_io, write_io = IO.pipe
+
+      echo_command.run_single_pipeline!(output: write_io)
+      write_io.close
+      output = read_io.read.chomp
+      read_io.close
+
+      expect(output).to eq('variable value data')
+    end
+
+    it 'accepts stdin and stdout redirection' do
+      echo_command = command.new(%(ruby -e "print 'stdin is : ' + STDIN.readline"))
+      input_r, input_w = IO.pipe
+      input_w.sync = true
+      input_w.print 'my custom content'
+      input_w.close
+
+      output_r, output_w = IO.pipe
+
+      result = echo_command.run_single_pipeline!(input: input_r, output: output_w)
+
+      input_r.close
+      output_w.close
+      output = output_r.read
+      output_r.close
+
+      expect(result.status).to be_success
+      expect(output).to match(/stdin is : my custom content/)
+    end
+
+    it 'returns a Command::SinglePipelineResult' do
+      result = command.new('true').run_single_pipeline!
+
+      expect(result).to be_a(Gitlab::Backup::Cli::Shell::Command::SinglePipelineResult)
+    end
+
+    context 'with Pipeline::Status' do
+      it 'includes stderr from the executed pipeline' do
+        expected_output = 'my custom error content'
+        err_command = command.new("echo #{expected_output} > /dev/stderr")
+
+        result = err_command.run_single_pipeline!
+
+        expect(result.stderr.chomp).to eq(expected_output)
+      end
+
+      it 'executed pipelines returns a Process::Status in the status field' do
+        result = command.new('true').run_single_pipeline!
+
+        expect(result.status).to be_a(Process::Status)
+        expect(result.status).to respond_to(:exited?, :termsig, :stopsig, :exitstatus, :success?, :pid)
+      end
+
+      it 'includes a list of Process::Status that handles exit signals' do
+        result = command.new('false').run_single_pipeline!
+
+        expect(result.status).to satisfy { |status| !status.success? }
+        expect(result.status).to satisfy { |status| status.exitstatus == 1 }
+      end
+    end
   end
 end

gems/gitlab-backup-cli/spec/gitlab/backup/cli/shell/pipeline_spec.rb

@@ -6,6 +6,9 @@ RSpec.describe Gitlab::Backup::Cli::Shell::Pipeline do
   let(:sort_command) { command.new('sort') }
   let(:true_command) { command.new('true') }
   let(:false_command) { command.new('false') }
+  let(:envdata) do
+    { 'CUSTOM' => 'data' }
+  end
 
   subject(:pipeline) { described_class }
 
@@ -28,6 +31,18 @@ RSpec.describe Gitlab::Backup::Cli::Shell::Pipeline do
       expect(result).to be_a(Gitlab::Backup::Cli::Shell::Pipeline::Result)
     end
 
+    it 'sets env variables from provided commands as part of pipeline execution' do
+      echo_command = command.new("echo \"variable value ${CUSTOM}\"", env: envdata)
+      read_io, write_io = IO.pipe
+
+      pipeline.new(true_command, echo_command).run!(output: write_io)
+      write_io.close
+      output = read_io.read.chomp
+      read_io.close
+
+      expect(output).to eq('variable value data')
+    end
+
     context 'with Pipeline::Status' do
       it 'includes stderr from the executed pipeline' do
         expected_output = 'my custom error content'