gitlab-ce/spec/lib/gitlab/database/partitioning_spec.rb

# frozen_string_literal: true

require 'spec_helper'

RSpec.describe Gitlab::Database::Partitioning, feature_category: :database do
  include Database::PartitioningHelpers
  include Database::TableSchemaHelpers

  let(:main_connection) { ApplicationRecord.connection }

  before do
    stub_feature_flags(disallow_database_ddl_feature_flags: false)
  end

  around do |example|
    previously_registered_models = described_class.registered_models.dup
    described_class.instance_variable_set(:@registered_models, Set.new)

    previously_registered_tables = described_class.registered_tables.dup
    described_class.instance_variable_set(:@registered_tables, Set.new)

    example.run

    described_class.instance_variable_set(:@registered_models, previously_registered_models)
    described_class.instance_variable_set(:@registered_tables, previously_registered_tables)
  end

  describe '.register_models' do
    context 'ensure that the registered models have partitioning strategy' do
      it 'fails when partitioning_strategy is not specified for the model' do
        model = Class.new(ApplicationRecord)
        expect { described_class.register_models([model]) }.to raise_error(/should have partitioning strategy defined/)
      end
    end
  end

  describe '.sync_partitions_ignore_db_error' do
    it 'calls sync_partitions' do
      expect(described_class).to receive(:sync_partitions).with(analyze: false)

      described_class.sync_partitions_ignore_db_error
    end

    [ActiveRecord::ActiveRecordError, PG::Error].each do |error|
      context "when #{error} is raised" do
        before do
          expect(described_class).to receive(:sync_partitions)
            .and_raise(error)
        end

        it 'ignores it' do
          described_class.sync_partitions_ignore_db_error
        end
      end
    end

    context 'when DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP is set' do
      before do
        stub_env('DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP', '1')
      end

      it 'does not call sync_partitions' do
        expect(described_class).not_to receive(:sync_partitions)

        described_class.sync_partitions_ignore_db_error
      end
    end
  end

  describe '.sync_partitions' do
    let(:ci_connection) { Ci::ApplicationRecord.connection }
    let(:table_names) { %w[_test_partitioning_test1 _test_partitioning_test2] }
    let(:models) do
      [
        Class.new(ApplicationRecord) do
          include PartitionedTable

          self.table_name = :_test_partitioning_test1
          partitioned_by :created_at, strategy: :monthly
        end,
        Class.new(Gitlab::Database::Partitioning::TableWithoutModel).tap do |klass|
          klass.table_name = :_test_partitioning_test2
          klass.partitioned_by(:created_at, strategy: :monthly)
          klass.limit_connection_names = %i[main]
        end
      ]
    end

    before do
      table_names.each do |table_name|
        execute_on_each_database(<<~SQL)
          CREATE TABLE #{table_name} (
            id serial not null,
            created_at timestamptz not null,
            PRIMARY KEY (id, created_at))
          PARTITION BY RANGE (created_at);
        SQL
      end
    end

    it 'manages partitions for each given model' do
      expect { described_class.sync_partitions(models) }
        .to change { find_partitions(table_names.first).size }.from(0)
        .and change { find_partitions(table_names.last).size }.from(0)
    end

    context 'for analyze' do
      let(:analyze_regex) { /ANALYZE / }
      let(:analyze) { true }

      shared_examples_for 'not running analyze' do
        specify do
          control = ActiveRecord::QueryRecorder.new { described_class.sync_partitions(analyze: analyze) }
          expect(control.occurrences).not_to include(analyze_regex)
        end
      end

      context 'when analyze_interval is not set' do
        it_behaves_like 'not running analyze'

        context 'when analyze is set to false' do
          it_behaves_like 'not running analyze'
        end
      end

      context 'when analyze_interval is set' do
        let(:models) do
          [
            Class.new(ApplicationRecord) do
              include PartitionedTable

              self.table_name = :_test_partitioning_test1
              partitioned_by :created_at, strategy: :monthly, analyze_interval: 1.week
            end,
            Class.new(Gitlab::Database::Partitioning::TableWithoutModel).tap do |klass|
              klass.table_name = :_test_partitioning_test2
              klass.partitioned_by(:created_at, strategy: :monthly, analyze_interval: 1.week)
              klass.limit_connection_names = %i[main]
            end
          ]
        end

        it 'runs analyze' do
          control = ActiveRecord::QueryRecorder.new { described_class.sync_partitions(models, analyze: analyze) }
          expect(control.occurrences).to include(analyze_regex)
        end

        context 'analyze is false' do
          let(:analyze) { false }

          it_behaves_like 'not running analyze'
        end
      end
    end

    context 'with multiple databases' do
      it 'creates partitions in each database' do
        skip_if_shared_database(:ci)

        expect { described_class.sync_partitions(models) }
          .to change { find_partitions(table_names.first, conn: main_connection).size }.from(0)
          .and change { find_partitions(table_names.last, conn: main_connection).size }.from(0)
          .and change { find_partitions(table_names.first, conn: ci_connection).size }.from(0)
          .and change { find_partitions(table_names.last, conn: ci_connection).size }.from(0)
      end

      it 'does not create partitions in each database if restricted' do
        skip_if_shared_database(:ci)

        expect { described_class.sync_partitions(models, owner_db_only: true) }
          .to change { find_partitions(table_names.first, conn: main_connection).size }.from(0)
          .and change { find_partitions(table_names.last, conn: main_connection).size }.from(0)
          .and change { find_partitions(table_names.first, conn: ci_connection).size }.by_at_most(0)
          .and change { find_partitions(table_names.last, conn: ci_connection).size }.by_at_most(0)
      end
    end

    context 'without ci database' do
      it 'only creates partitions for main database' do
        skip_if_database_exists(:ci)

        allow(Gitlab::Database::Partitioning::PartitionManager).to receive(:new).and_call_original

        # Also, in the case where `ci` database is shared with `main` database,
        # check that we do not run PartitionManager again for ci connection as
        # that is redundant.
        expect(Gitlab::Database::Partitioning::PartitionManager).not_to receive(:new)
          .with(anything, connection: ci_connection).and_call_original

        expect { described_class.sync_partitions(models) }
          .to change { find_partitions(table_names.first, conn: main_connection).size }.from(0)
          .and change { find_partitions(table_names.last, conn: main_connection).size }.from(0)
      end
    end

    context 'when no partitioned models are given' do
      it 'manages partitions for each registered model' do
        described_class.register_models([models.first])
        described_class.register_tables(
          [
            {
              table_name: table_names.last,
              partitioned_column: :created_at,
              strategy: :monthly
            }
          ])

        expect { described_class.sync_partitions }
          .to change { find_partitions(table_names.first).size }.from(0)
          .and change { find_partitions(table_names.last).size }.from(0)
      end
    end

    context 'when only a specific database is requested' do
      let(:ci_model) do
        Class.new(Ci::ApplicationRecord) do
          include PartitionedTable

          self.table_name = :_test_partitioning_test3
          partitioned_by :created_at, strategy: :monthly
        end
      end

      before do
        skip_if_shared_database(:ci)

        (table_names + [:_test_partitioning_test3]).each do |table_name|
          execute_on_each_database("DROP TABLE IF EXISTS #{table_name}")

          execute_on_each_database(<<~SQL)
            CREATE TABLE #{table_name} (
              id serial not null,
              created_at timestamptz not null,
              PRIMARY KEY (id, created_at))
            PARTITION BY RANGE (created_at);
          SQL
        end
      end

      after do
        (table_names + [:_test_partitioning_test3]).each do |table_name|
          ci_connection.execute("DROP TABLE IF EXISTS #{table_name}")
        end
      end

      it 'manages partitions for models for the given database', :aggregate_failures do
        expect { described_class.sync_partitions([models.first, ci_model], only_on: 'ci') }
          .to change { find_partitions(ci_model.table_name, conn: ci_connection).size }.from(0)

        expect(find_partitions(models.first.table_name, conn: main_connection).size).to eq(0)
        expect(find_partitions(models.first.table_name, conn: ci_connection).size).to eq(0)
        expect(find_partitions(ci_model.table_name, conn: main_connection).size).to eq(0)
      end
    end

    context 'when partition_manager_sync_partitions feature flag is disabled' do
      before do
        described_class.register_models(models)
        stub_feature_flags(partition_manager_sync_partitions: false)
      end

      it 'skips sync_partitions' do
        expect(described_class::PartitionManager).not_to receive(:new)
        expect(described_class).to receive(:sync_partitions)
          .and_call_original

        described_class.sync_partitions(models)
      end
    end

    context 'when disallow_database_ddl_feature_flags feature flag is enabled' do
      before do
        described_class.register_models(models)
        stub_feature_flags(disallow_database_ddl_feature_flags: true)
      end

      it 'skips sync_partitions' do
        expect(described_class::PartitionManager).not_to receive(:new)
        expect(described_class).to receive(:sync_partitions).and_call_original

        described_class.sync_partitions(models)
      end
    end
  end

  describe '.report_metrics' do
    let(:model1) { double('model') }
    let(:model2) { double('model') }

    let(:partition_monitoring_class) { described_class::PartitionMonitoring }

    context 'when no partitioned models are given' do
      it 'reports metrics for each registered model' do
        expect_next_instance_of(partition_monitoring_class) do |partition_monitor|
          expect(partition_monitor).to receive(:report_metrics_for_model).with(model1)
          expect(partition_monitor).to receive(:report_metrics_for_model).with(model2)
        end

        expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
          .with(described_class.__send__(:registered_models))
          .and_yield(model1)
          .and_yield(model2)

        described_class.report_metrics
      end
    end

    context 'when partitioned models are given' do
      it 'reports metrics for each given model' do
        expect_next_instance_of(partition_monitoring_class) do |partition_monitor|
          expect(partition_monitor).to receive(:report_metrics_for_model).with(model1)
          expect(partition_monitor).to receive(:report_metrics_for_model).with(model2)
        end

        expect(Gitlab::Database::EachDatabase).to receive(:each_model_connection)
          .with([model1, model2])
          .and_yield(model1)
          .and_yield(model2)

        described_class.report_metrics([model1, model2])
      end
    end
  end

  describe '.drop_detached_partitions' do
    let(:table_names) { %w[_test_detached_test_partition1 _test_detached_test_partition2] }

    before do
      table_names.each do |table_name|
        connection.create_table("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}")

        Postgresql::DetachedPartition.create!(table_name: table_name, drop_after: 1.year.ago)
      end
    end

    it 'drops detached partitions for each database' do
      expect(Gitlab::Database::EachDatabase).to receive(:each_connection).and_yield

      expect { described_class.drop_detached_partitions }
        .to change { Postgresql::DetachedPartition.count }.from(2).to(0)
        .and change { table_exists?(table_names.first) }.from(true).to(false)
        .and change { table_exists?(table_names.last) }.from(true).to(false)
    end

    context 'when the feature flag is disabled' do
      before do
        stub_feature_flags(partition_manager_sync_partitions: false)
      end

      it 'does not call the DetachedPartitionDropper' do
        expect(Gitlab::Database::Partitioning::DetachedPartitionDropper).not_to receive(:new)

        described_class.drop_detached_partitions
      end
    end

    context 'when the feature disallow DDL feature flags is enabled' do
      before do
        stub_feature_flags(disallow_database_ddl_feature_flags: true)
      end

      it 'does not call the DetachedPartitionDropper' do
        expect(Gitlab::Database::Partitioning::DetachedPartitionDropper).not_to receive(:new)

        described_class.drop_detached_partitions
      end
    end

    def table_exists?(table_name)
      table_oid(table_name).present?
    end
  end
end