From fe53f9c3582f3a13e4e38d3804aaf53c3829b3b9 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 22 Sep 2020 00:09:59 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- .rubocop_todo.yml | 3 - .../list/components/packages_sort.vue | 2 +- .../javascripts/packages/list/constants.js | 28 +- app/assets/javascripts/packages/list/utils.js | 5 +- .../projects/feature_flags/_errors.html.haml | 4 - ...-packages-in-the-package-registry-list.yml | 5 + changelogs/unreleased/rails-save-bang-35.yml | 5 + doc/development/prometheus.md | 10 + doc/development/reference_processing.md | 16 ++ doc/development/telemetry/snowplow.md | 5 + .../testing_guide/best_practices.md | 9 +- lib/api/project_import.rb | 2 - lib/backup/repositories.rb | 234 ++++++++++++++++ lib/backup/repository.rb | 265 ------------------ lib/tasks/gitlab/backup.rake | 4 +- locale/gitlab.pot | 3 + ...epository_spec.rb => repositories_spec.rb} | 96 +++---- .../features/packages_shared_examples.rb | 4 +- spec/tasks/gitlab/backup_rake_spec.rb | 4 +- 19 files changed, 333 insertions(+), 371 deletions(-) delete mode 100644 app/views/projects/feature_flags/_errors.html.haml create mode 100644 changelogs/unreleased/225175-change-the-default-order-for-packages-in-the-package-registry-list.yml create mode 100644 changelogs/unreleased/rails-save-bang-35.yml create mode 100644 lib/backup/repositories.rb delete mode 100644 lib/backup/repository.rb rename spec/lib/backup/{repository_spec.rb => repositories_spec.rb} (72%) diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index 1b16f357692..ff4911075d8 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -703,9 +703,6 @@ Rails/SaveBang: - 'ee/spec/models/burndown_spec.rb' - 'ee/spec/models/ci/pipeline_spec.rb' - 'ee/spec/models/ci/subscriptions/project_spec.rb' - - 'ee/spec/models/concerns/approver_migrate_hook_spec.rb' - - 'ee/spec/models/concerns/deprecated_approvals_before_merge_spec.rb' - - 'ee/spec/models/concerns/elastic/note_spec.rb' - 'ee/spec/models/ee/appearance_spec.rb' - 'ee/spec/models/ee/ci/job_artifact_spec.rb' - 'ee/spec/models/ee/protected_branch_spec.rb' diff --git a/app/assets/javascripts/packages/list/components/packages_sort.vue b/app/assets/javascripts/packages/list/components/packages_sort.vue index fa8f4f39d54..47e51bbdca5 100644 --- a/app/assets/javascripts/packages/list/components/packages_sort.vue +++ b/app/assets/javascripts/packages/list/components/packages_sort.vue @@ -51,7 +51,7 @@ export default { {{ item.label }} diff --git a/app/assets/javascripts/packages/list/constants.js b/app/assets/javascripts/packages/list/constants.js index 0ff8c86362d..366b674ae72 100644 --- a/app/assets/javascripts/packages/list/constants.js +++ b/app/assets/javascripts/packages/list/constants.js @@ -15,7 +15,7 @@ export const GROUP_PAGE_TYPE = 'groups'; export const LIST_KEY_NAME = 'name'; export const LIST_KEY_PROJECT = 'project_path'; export const LIST_KEY_VERSION = 'version'; -export const LIST_KEY_PACKAGE_TYPE = 'package_type'; +export const LIST_KEY_PACKAGE_TYPE = 'type'; export const LIST_KEY_CREATED_AT = 'created_at'; export const LIST_KEY_ACTIONS = 'actions'; @@ -23,47 +23,35 @@ export const LIST_LABEL_NAME = __('Name'); export const LIST_LABEL_PROJECT = __('Project'); export const LIST_LABEL_VERSION = __('Version'); export const LIST_LABEL_PACKAGE_TYPE = __('Type'); -export const LIST_LABEL_CREATED_AT = __('Created'); +export const LIST_LABEL_CREATED_AT = __('Published'); export const LIST_LABEL_ACTIONS = ''; -export const LIST_ORDER_BY_PACKAGE_TYPE = 'type'; - export const ASCENDING_ODER = 'asc'; export const DESCENDING_ORDER = 'desc'; // The following is not translated because it is used to build a JavaScript exception error message export const MISSING_DELETE_PATH_ERROR = 'Missing delete_api_path link'; -export const TABLE_HEADER_FIELDS = [ +export const SORT_FIELDS = [ { - key: LIST_KEY_NAME, - label: LIST_LABEL_NAME, orderBy: LIST_KEY_NAME, - class: ['text-left'], + label: LIST_LABEL_NAME, }, { - key: LIST_KEY_PROJECT, - label: LIST_LABEL_PROJECT, orderBy: LIST_KEY_PROJECT, - class: ['text-left'], + label: LIST_LABEL_PROJECT, }, { - key: LIST_KEY_VERSION, - label: LIST_LABEL_VERSION, orderBy: LIST_KEY_VERSION, - class: ['text-center'], + label: LIST_LABEL_VERSION, }, { - key: LIST_KEY_PACKAGE_TYPE, + orderBy: LIST_KEY_PACKAGE_TYPE, label: LIST_LABEL_PACKAGE_TYPE, - orderBy: LIST_ORDER_BY_PACKAGE_TYPE, - class: ['text-center'], }, { - key: LIST_KEY_CREATED_AT, - label: LIST_LABEL_CREATED_AT, orderBy: LIST_KEY_CREATED_AT, - class: ['text-center'], + label: LIST_LABEL_CREATED_AT, }, ]; diff --git a/app/assets/javascripts/packages/list/utils.js b/app/assets/javascripts/packages/list/utils.js index 98d78db8706..6a300d7bfe6 100644 --- a/app/assets/javascripts/packages/list/utils.js +++ b/app/assets/javascripts/packages/list/utils.js @@ -1,7 +1,6 @@ -import { LIST_KEY_PROJECT, TABLE_HEADER_FIELDS } from './constants'; +import { LIST_KEY_PROJECT, SORT_FIELDS } from './constants'; -export default isGroupPage => - TABLE_HEADER_FIELDS.filter(f => f.key !== LIST_KEY_PROJECT || isGroupPage); +export default isGroupPage => SORT_FIELDS.filter(f => f.key !== LIST_KEY_PROJECT || isGroupPage); /** * A small util function that works out if the delete action has deleted the diff --git a/app/views/projects/feature_flags/_errors.html.haml b/app/views/projects/feature_flags/_errors.html.haml deleted file mode 100644 index a32245640be..00000000000 --- a/app/views/projects/feature_flags/_errors.html.haml +++ /dev/null @@ -1,4 +0,0 @@ -#error_explanation - .alert.alert-danger - - @feature_flag.errors.full_messages.each do |message| - %p= message diff --git a/changelogs/unreleased/225175-change-the-default-order-for-packages-in-the-package-registry-list.yml b/changelogs/unreleased/225175-change-the-default-order-for-packages-in-the-package-registry-list.yml new file mode 100644 index 00000000000..63c800ee990 --- /dev/null +++ b/changelogs/unreleased/225175-change-the-default-order-for-packages-in-the-package-registry-list.yml @@ -0,0 +1,5 @@ +--- +title: Rename Created to Published in package sort dropdown +merge_request: 42677 +author: +type: changed diff --git a/changelogs/unreleased/rails-save-bang-35.yml b/changelogs/unreleased/rails-save-bang-35.yml new file mode 100644 index 00000000000..580b0cb1bf8 --- /dev/null +++ b/changelogs/unreleased/rails-save-bang-35.yml @@ -0,0 +1,5 @@ +--- +title: Fixes Rails/SaveBang cop for spec files in ee/spec/models/concerns/* +merge_request: 42839 +author: Rajendra Kadam +type: other diff --git a/doc/development/prometheus.md b/doc/development/prometheus.md index 902d4e6a1d0..b8cf5b997ba 100644 --- a/doc/development/prometheus.md +++ b/doc/development/prometheus.md @@ -47,3 +47,13 @@ using a Prometheus managed application in Kubernetes: ``` 1. Open `localhost:9090` in your browser to display the Prometheus user interface. + +## Script access to Prometheus + +You can script the access to Prometheus, extracting the name of the pod automatically like this: + +```shell +POD_INFORMATION=$(kubectl get pods -n gitlab-managed-apps | grep 'prometheus-prometheus-server') +POD_NAME=$(echo $POD_INFORMATION | awk '{print $1;}') +kubectl port-forward $POD_NAME 9090:9090 -n gitlab-managed-apps +``` diff --git a/doc/development/reference_processing.md b/doc/development/reference_processing.md index 527fb94f228..cf587043cae 100644 --- a/doc/development/reference_processing.md +++ b/doc/development/reference_processing.md @@ -77,6 +77,22 @@ a minimum implementation of `AbstractReferenceFilter` should define: and an identifier, find the object. For example, this in a reference filter for merge requests, this might be `project.merge_requests.where(iid: iid)`. +### Add a new reference prefix and filter + +For reference filters for new objects, use a prefix format following the pattern +`^#`, because: + +1. Varied single-character prefixes are hard for users to track. Especially for + lower-use object types, this can diminish value for the feature. +1. Suitable single-character prefixes are limited. +1. Following a consistent pattern allows users to infer the existence of new features. + +To add a reference prefix for a new object `apple`,which has both a name and ID, +format the reference as: + +- `^apple#123` for identification by ID. +- `^apple#"Granny Smith"` for identification by name. + ### Performance #### Find object optimization diff --git a/doc/development/telemetry/snowplow.md b/doc/development/telemetry/snowplow.md index f427d7d1488..d6e3f0de2c8 100644 --- a/doc/development/telemetry/snowplow.md +++ b/doc/development/telemetry/snowplow.md @@ -297,6 +297,11 @@ class Projects::CreateService < BaseService end ``` +### Unit testing + +Use the `expect_snowplow_event` helper when testing backend Snowplow events. See [testing best practices]( +https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#test-snowplow-events) for details. + ### Performance We use the [AsyncEmitter](https://github.com/snowplow/snowplow/wiki/Ruby-Tracker#52-the-asyncemitter-class) when tracking events, which allows for instrumentation calls to be run in a background thread. This is still an active area of development. diff --git a/doc/development/testing_guide/best_practices.md b/doc/development/testing_guide/best_practices.md index 6ef9be381b4..7bfbadae949 100644 --- a/doc/development/testing_guide/best_practices.md +++ b/doc/development/testing_guide/best_practices.md @@ -497,10 +497,9 @@ let_it_be(:project, refind: true) { create(:project) } ### Time-sensitive tests -[Timecop](https://github.com/travisjeffery/timecop) is available in our -Ruby-based tests for verifying things that are time-sensitive. Any test that -exercises or verifies something time-sensitive should make use of Timecop to -prevent transient test failures. +[`ActiveSupport::Testing::TimeHelpers`](https://api.rubyonrails.org/v6.0.3.1/classes/ActiveSupport/Testing/TimeHelpers.html) +can be used to verify things that are time-sensitive. Any test that exercises or verifies something time-sensitive +should make use of these helpers to prevent transient test failures. Example: @@ -508,7 +507,7 @@ Example: it 'is overdue' do issue = build(:issue, due_date: Date.tomorrow) - Timecop.freeze(3.days.from_now) do + travel_to(3.days.from_now) do expect(issue).to be_overdue end end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 9f43c3c7993..6ac7f02f305 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -4,8 +4,6 @@ module API class ProjectImport < Grape::API::Instance include PaginationParams - MAXIMUM_FILE_SIZE = 50.megabytes - helpers Helpers::ProjectsHelpers helpers Helpers::FileUploadHelpers helpers Helpers::RateLimiter diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb new file mode 100644 index 00000000000..0c5963e55ab --- /dev/null +++ b/lib/backup/repositories.rb @@ -0,0 +1,234 @@ +# frozen_string_literal: true + +require 'yaml' + +module Backup + class Repositories + attr_reader :progress + + def initialize(progress) + @progress = progress + end + + def dump(max_concurrency:, max_storage_concurrency:) + prepare + + if max_concurrency <= 1 && max_storage_concurrency <= 1 + return dump_consecutive + end + + if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? + raise Error, 'repositories.storages in gitlab.yml is misconfigured' + end + + semaphore = Concurrent::Semaphore.new(max_concurrency) + errors = Queue.new + + threads = Gitlab.config.repositories.storages.keys.map do |storage| + Thread.new do + Rails.application.executor.wrap do + dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) + rescue => e + errors << e + end + end + end + + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + threads.each(&:join) + end + + raise errors.pop unless errors.empty? + end + + def restore + Project.find_each(batch_size: 1000) do |project| + restore_repository(project, Gitlab::GlRepository::PROJECT) + restore_repository(project, Gitlab::GlRepository::WIKI) + end + + restore_object_pools + end + + private + + def restore_repository(container, type) + BackupRestore.new( + progress, + type.repository_for(container), + backup_repos_path + ).restore(always_create: type.project?) + end + + def backup_repos_path + File.join(Gitlab.config.backup.path, 'repositories') + end + + def prepare + FileUtils.rm_rf(backup_repos_path) + FileUtils.mkdir_p(Gitlab.config.backup.path) + FileUtils.mkdir(backup_repos_path, mode: 0700) + end + + def dump_consecutive + Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project| + dump_project(project) + end + end + + def dump_storage(storage, semaphore, max_storage_concurrency:) + errors = Queue.new + queue = InterlockSizedQueue.new(1) + + threads = Array.new(max_storage_concurrency) do + Thread.new do + Rails.application.executor.wrap do + while project = queue.pop + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + semaphore.acquire + end + + begin + dump_project(project) + rescue => e + errors << e + break + ensure + semaphore.release + end + end + end + end + end + + Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project| + break unless errors.empty? + + queue.push(project) + end + + raise errors.pop unless errors.empty? + ensure + queue.close + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + threads.each(&:join) + end + end + + def dump_project(project) + backup_repository(project, Gitlab::GlRepository::PROJECT) + backup_repository(project, Gitlab::GlRepository::WIKI) + end + + def backup_repository(container, type) + BackupRestore.new( + progress, + type.repository_for(container), + backup_repos_path + ).backup + end + + def restore_object_pools + PoolRepository.includes(:source_project).find_each do |pool| + progress.puts " - Object pool #{pool.disk_path}..." + + pool.source_project ||= pool.member_projects.first.root_of_fork_network + pool.state = 'none' + pool.save + + pool.schedule + end + end + + class BackupRestore + attr_accessor :progress, :repository, :backup_repos_path + + def initialize(progress, repository, backup_repos_path) + @progress = progress + @repository = repository + @backup_repos_path = backup_repos_path + end + + def backup + progress.puts " * #{display_repo_path} ... " + + if repository.empty? + progress.puts " * #{display_repo_path} ... " + "[SKIPPED]".color(:cyan) + return + end + + FileUtils.mkdir_p(repository_backup_path) + + repository.bundle_to_disk(path_to_bundle) + repository.gitaly_repository_client.backup_custom_hooks(custom_hooks_tar) + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue => e + progress.puts "[Failed] backing up #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + def restore(always_create: false) + progress.puts " * #{display_repo_path} ... " + + repository.remove rescue nil + + if File.exist?(path_to_bundle) + repository.create_from_bundle(path_to_bundle) + restore_custom_hooks + elsif always_create + repository.create_repository + end + + progress.puts " * #{display_repo_path} ... " + "[DONE]".color(:green) + + rescue => e + progress.puts "[Failed] restoring #{display_repo_path}".color(:red) + progress.puts "Error #{e}".color(:red) + end + + private + + def display_repo_path + "#{repository.full_path} (#{repository.disk_path})" + end + + def repository_backup_path + @repository_backup_path ||= File.join(backup_repos_path, repository.disk_path) + end + + def path_to_bundle + @path_to_bundle ||= File.join(backup_repos_path, repository.disk_path + '.bundle') + end + + def restore_custom_hooks + return unless File.exist?(custom_hooks_tar) + + repository.gitaly_repository_client.restore_custom_hooks(custom_hooks_tar) + end + + def custom_hooks_tar + File.join(repository_backup_path, "custom_hooks.tar") + end + end + + class InterlockSizedQueue < SizedQueue + extend ::Gitlab::Utils::Override + + override :pop + def pop(*) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + super + end + end + + override :push + def push(*) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + super + end + end + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb deleted file mode 100644 index eb0b230904e..00000000000 --- a/lib/backup/repository.rb +++ /dev/null @@ -1,265 +0,0 @@ -# frozen_string_literal: true - -require 'yaml' - -module Backup - class Repository - attr_reader :progress - - def initialize(progress) - @progress = progress - end - - def dump(max_concurrency:, max_storage_concurrency:) - prepare - - if max_concurrency <= 1 && max_storage_concurrency <= 1 - return dump_consecutive - end - - if Project.excluding_repository_storage(Gitlab.config.repositories.storages.keys).exists? - raise Error, 'repositories.storages in gitlab.yml is misconfigured' - end - - semaphore = Concurrent::Semaphore.new(max_concurrency) - errors = Queue.new - - threads = Gitlab.config.repositories.storages.keys.map do |storage| - Thread.new do - Rails.application.executor.wrap do - dump_storage(storage, semaphore, max_storage_concurrency: max_storage_concurrency) - rescue => e - errors << e - end - end - end - - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - - raise errors.pop unless errors.empty? - end - - def backup_project(project) - path_to_project_bundle = path_to_bundle(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .create_bundle(path_to_project_bundle) - - backup_custom_hooks(project) - rescue => e - progress_warn(project, e, 'Failed to backup repo') - end - - def backup_custom_hooks(project) - FileUtils.mkdir_p(project_backup_path(project)) - - custom_hooks_path = custom_hooks_tar(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .backup_custom_hooks(custom_hooks_path) - end - - def restore_custom_hooks(project) - return unless Dir.exist?(project_backup_path(project)) - return if Dir.glob("#{project_backup_path(project)}/custom_hooks*").none? - - custom_hooks_path = custom_hooks_tar(project) - Gitlab::GitalyClient::RepositoryService.new(project.repository) - .restore_custom_hooks(custom_hooks_path) - end - - def restore - Project.find_each(batch_size: 1000) do |project| - progress.print " * #{project.full_path} ... " - - restore_repo_success = - begin - try_restore_repository(project) - rescue => err - progress.puts "Error: #{err}".color(:red) - false - end - - if restore_repo_success - progress.puts "[DONE]".color(:green) - else - progress.puts "[Failed] restoring #{project.full_path} repository".color(:red) - end - - wiki = ProjectWiki.new(project) - wiki.repository.remove rescue nil - path_to_wiki_bundle = path_to_bundle(wiki) - - if File.exist?(path_to_wiki_bundle) - progress.print " * #{wiki.full_path} ... " - begin - wiki.repository.create_from_bundle(path_to_wiki_bundle) - restore_custom_hooks(wiki) - - progress.puts "[DONE]".color(:green) - rescue => e - progress.puts "[Failed] restoring #{wiki.full_path} wiki".color(:red) - progress.puts "Error #{e}".color(:red) - end - end - end - - restore_object_pools - end - - protected - - def try_restore_repository(project) - path_to_project_bundle = path_to_bundle(project) - project.repository.remove rescue nil - - if File.exist?(path_to_project_bundle) - project.repository.create_from_bundle(path_to_project_bundle) - restore_custom_hooks(project) - else - project.repository.create_repository - end - - true - end - - def path_to_bundle(project) - File.join(backup_repos_path, project.disk_path + '.bundle') - end - - def project_backup_path(project) - File.join(backup_repos_path, project.disk_path) - end - - def custom_hooks_tar(project) - File.join(project_backup_path(project), "custom_hooks.tar") - end - - def backup_repos_path - File.join(Gitlab.config.backup.path, 'repositories') - end - - def prepare - FileUtils.rm_rf(backup_repos_path) - FileUtils.mkdir_p(Gitlab.config.backup.path) - FileUtils.mkdir(backup_repos_path, mode: 0700) - end - - private - - def dump_consecutive - Project.includes(:route, :group, namespace: :owner).find_each(batch_size: 1000) do |project| - dump_project(project) - end - end - - def dump_storage(storage, semaphore, max_storage_concurrency:) - errors = Queue.new - queue = InterlockSizedQueue.new(1) - - threads = Array.new(max_storage_concurrency) do - Thread.new do - Rails.application.executor.wrap do - while project = queue.pop - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - semaphore.acquire - end - - begin - dump_project(project) - rescue => e - errors << e - break - ensure - semaphore.release - end - end - end - end - end - - Project.for_repository_storage(storage).includes(:route, :group, namespace: :owner).find_each(batch_size: 100) do |project| - break unless errors.empty? - - queue.push(project) - end - - raise errors.pop unless errors.empty? - ensure - queue.close - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - threads.each(&:join) - end - end - - def dump_project(project) - progress.puts " * #{display_repo_path(project)} ... " - - if project.hashed_storage?(:repository) - FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) - else - FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace - end - - if !empty_repo?(project) - backup_project(project) - progress.puts " * #{display_repo_path(project)} ... " + "[DONE]".color(:green) - else - progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED]".color(:cyan) - end - - wiki = ProjectWiki.new(project) - - if !empty_repo?(wiki) - backup_project(wiki) - progress.puts " * #{display_repo_path(project)} ... " + "[DONE] Wiki".color(:green) - else - progress.puts " * #{display_repo_path(project)} ... " + "[SKIPPED] Wiki".color(:cyan) - end - end - - def progress_warn(project, cmd, output) - progress.puts "[WARNING] Executing #{cmd}".color(:orange) - progress.puts "Ignoring error on #{display_repo_path(project)} - #{output}".color(:orange) - end - - def empty_repo?(project_or_wiki) - project_or_wiki.repository.expire_emptiness_caches - project_or_wiki.repository.empty? - end - - def display_repo_path(project) - project.hashed_storage?(:repository) ? "#{project.full_path} (#{project.disk_path})" : project.full_path - end - - def restore_object_pools - PoolRepository.includes(:source_project).find_each do |pool| - progress.puts " - Object pool #{pool.disk_path}..." - - pool.source_project ||= pool.member_projects.first.root_of_fork_network - pool.state = 'none' - pool.save - - pool.schedule - end - end - - class InterlockSizedQueue < SizedQueue - extend ::Gitlab::Utils::Override - - override :pop - def pop(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - - override :push - def push(*) - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - super - end - end - end - end -end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 2a3713ed85c..de2dfca8c1b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -107,7 +107,7 @@ namespace :gitlab do puts "GITLAB_BACKUP_MAX_CONCURRENCY and GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY must have a value of at least 1".color(:red) exit 1 else - Backup::Repository.new(progress).dump( + Backup::Repositories.new(progress).dump( max_concurrency: max_concurrency, max_storage_concurrency: max_storage_concurrency ) @@ -117,7 +117,7 @@ namespace :gitlab do task restore: :gitlab_environment do puts_time "Restoring repositories ...".color(:blue) - Backup::Repository.new(progress).restore + Backup::Repositories.new(progress).restore puts_time "done".color(:green) end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 8445bbed2d6..1f0ae857a25 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -20682,6 +20682,9 @@ msgstr "" msgid "Publish to status page" msgstr "" +msgid "Published" +msgstr "" + msgid "Published on status page" msgstr "" diff --git a/spec/lib/backup/repository_spec.rb b/spec/lib/backup/repositories_spec.rb similarity index 72% rename from spec/lib/backup/repository_spec.rb rename to spec/lib/backup/repositories_spec.rb index 718f38f9452..63f948e8dd4 100644 --- a/spec/lib/backup/repository_spec.rb +++ b/spec/lib/backup/repositories_spec.rb @@ -2,9 +2,7 @@ require 'spec_helper' -RSpec.describe Backup::Repository do - let_it_be(:project) { create(:project, :wiki_repo) } - +RSpec.describe Backup::Repositories do let(:progress) { StringIO.new } subject { described_class.new(progress) } @@ -12,7 +10,6 @@ RSpec.describe Backup::Repository do before do allow(progress).to receive(:puts) allow(progress).to receive(:print) - allow(FileUtils).to receive(:mv).and_return(true) allow_next_instance_of(described_class) do |instance| allow(instance).to receive(:progress).and_return(progress) @@ -24,11 +21,19 @@ RSpec.describe Backup::Repository do allow(Gitlab.config.repositories.storages).to receive(:keys).and_return(storage_keys) end - let_it_be(:projects) { create_list(:project, 5, :wiki_repo) + [project] } + let_it_be(:projects) { create_list(:project, 5, :repository, :wiki_repo) } let(:storage_keys) { %w[default test_second_storage] } context 'no concurrency' do + it 'creates repository bundle' do + subject.dump(max_concurrency: 1, max_storage_concurrency: 1) + + projects.each do |project| + expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle')) + end + end + it 'creates the expected number of threads' do expect(Thread).not_to receive(:new) @@ -58,12 +63,22 @@ RSpec.describe Backup::Repository do subject.dump(max_concurrency: 1, max_storage_concurrency: 1) end.count - create_list(:project, 2, :wiki_repo) + create_list(:project, 2, :repository, :wiki_repo) expect do subject.dump(max_concurrency: 1, max_storage_concurrency: 1) end.not_to exceed_query_limit(control_count) end + + context 'legacy storage' do + let_it_be(:project) { create(:project, :repository, :legacy_storage, :wiki_repo) } + + it 'creates repository bundle' do + subject.dump(max_concurrency: 1, max_storage_concurrency: 1) + + expect(File).to exist(File.join(Gitlab.config.backup.path, 'repositories', project.disk_path + '.bundle')) + end + end end [4, 10].each do |max_storage_concurrency| @@ -120,7 +135,7 @@ RSpec.describe Backup::Repository do subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) end.count - create_list(:project, 2, :wiki_repo) + create_list(:project, 2, :repository, :wiki_repo) expect do subject.dump(max_concurrency: 1, max_storage_concurrency: max_storage_concurrency) @@ -131,45 +146,32 @@ RSpec.describe Backup::Repository do end describe '#restore' do - let(:timestamp) { Time.utc(2017, 3, 22) } - let(:temp_dirs) do - Gitlab.config.repositories.storages.map do |name, storage| - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(storage.legacy_disk_path, '..', 'repositories.old.' + timestamp.to_i.to_s) - end - end - end - - around do |example| - Timecop.freeze(timestamp) { example.run } - end - - after do - temp_dirs.each { |path| FileUtils.rm_rf(path) } - end + let_it_be(:project) { create(:project, :wiki_repo) } describe 'command failure' do before do - # Allow us to set expectations on the project directly expect(Project).to receive(:find_each).and_yield(project) - expect(project.repository).to receive(:create_repository) { raise 'Fail in tests' } + + allow_next_instance_of(Repository) do |repository| + allow(repository).to receive(:create_repository) { raise 'Fail in tests' } + end end context 'hashed storage' do it 'shows the appropriate error' do subject.restore - expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository") + expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})") end end context 'legacy storage' do - let!(:project) { create(:project, :legacy_storage) } + let_it_be(:project) { create(:project, :legacy_storage) } it 'shows the appropriate error' do subject.restore - expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} repository") + expect(progress).to have_received(:puts).with("[Failed] restoring #{project.full_path} (#{project.disk_path})") end end end @@ -188,45 +190,15 @@ RSpec.describe Backup::Repository do end it 'cleans existing repositories' do - wiki_repository_spy = spy(:wiki) + expect(Repository).to receive(:new).twice.and_wrap_original do |method, *original_args| + repository = method.call(*original_args) - allow_next_instance_of(ProjectWiki) do |project_wiki| - allow(project_wiki).to receive(:repository).and_return(wiki_repository_spy) - end + expect(repository).to receive(:remove) - expect_next_instance_of(Repository) do |repo| - expect(repo).to receive(:remove) + repository end subject.restore - - expect(wiki_repository_spy).to have_received(:remove) - end - end - - describe '#empty_repo?' do - context 'for a wiki' do - let(:wiki) { create(:project_wiki) } - - it 'invalidates the emptiness cache' do - expect(wiki.repository).to receive(:expire_emptiness_caches).once - - subject.send(:empty_repo?, wiki) - end - - context 'wiki repo has content' do - let!(:wiki_page) { create(:wiki_page, wiki: wiki) } - - it 'returns true, regardless of bad cache value' do - expect(subject.send(:empty_repo?, wiki)).to be(false) - end - end - - context 'wiki repo does not have content' do - it 'returns true, regardless of bad cache value' do - expect(subject.send(:empty_repo?, wiki)).to be_truthy - end - end end end end diff --git a/spec/support/shared_examples/features/packages_shared_examples.rb b/spec/support/shared_examples/features/packages_shared_examples.rb index f201421e827..4d2e13aa5bc 100644 --- a/spec/support/shared_examples/features/packages_shared_examples.rb +++ b/spec/support/shared_examples/features/packages_shared_examples.rb @@ -84,11 +84,11 @@ RSpec.shared_examples 'shared package sorting' do let(:packages) { [package_two, package_one] } end - it_behaves_like 'correctly sorted packages list', 'Created' do + it_behaves_like 'correctly sorted packages list', 'Published' do let(:packages) { [package_two, package_one] } end - it_behaves_like 'correctly sorted packages list', 'Created', ascending: true do + it_behaves_like 'correctly sorted packages list', 'Published', ascending: true do let(:packages) { [package_one, package_two] } end end diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index a2cc2b12e5e..5d9694430e3 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -370,7 +370,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do end it 'has defaults' do - expect_next_instance_of(::Backup::Repository) do |instance| + expect_next_instance_of(::Backup::Repositories) do |instance| expect(instance).to receive(:dump) .with(max_concurrency: 1, max_storage_concurrency: 1) .and_call_original @@ -383,7 +383,7 @@ RSpec.describe 'gitlab:app namespace rake task', :delete do stub_env('GITLAB_BACKUP_MAX_CONCURRENCY', 5) stub_env('GITLAB_BACKUP_MAX_STORAGE_CONCURRENCY', 2) - expect_next_instance_of(::Backup::Repository) do |instance| + expect_next_instance_of(::Backup::Repositories) do |instance| expect(instance).to receive(:dump) .with(max_concurrency: 5, max_storage_concurrency: 2) .and_call_original