gitlabhq/lib/gitlab/sidekiq_middleware.rb

# frozen_string_literal: true

module Gitlab
  # The SidekiqMiddleware class is responsible for configuring the
  # middleware stacks used in the client and server middlewares
  module SidekiqMiddleware
    class Client
      def self.middlewares
        [
          # ConcurrencyLimit::Resume needs to be first and before Labkit and ConcurrencyLimit::Client
          ::Gitlab::SidekiqMiddleware::ConcurrencyLimit::Resume,
          ::Gitlab::SidekiqMiddleware::WorkerContext::Client, # needs to be before the Labkit middleware
          ::Labkit::Middleware::Sidekiq::Client,
          # Sidekiq Client Middleware should be placed before DuplicateJobs::Client middleware,
          # so we can store WAL location before we deduplicate the job.
          ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware,
          ::Gitlab::SidekiqMiddleware::PauseControl::Client,
          ::Gitlab::SidekiqMiddleware::ConcurrencyLimit::Client,
          # NOTE: Everything from DuplicateJobs::Client to DuplicateJobs::Server must yield
          # no returning or job interception as it will leave the duplicate job redis key
          # dangling and errorneously deduplicating future jobs until key expires.
          ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client,
          ::Gitlab::SidekiqStatus::ClientMiddleware,
          ::Gitlab::SidekiqMiddleware::AdminMode::Client,
          # Size limiter should be placed at the bottom, but before the metrics middleware
          # NOTE: this raises error after the duplicate jobs middleware but is acceptable
          # since jobs that are too large should not be enqueued.
          ::Gitlab::SidekiqMiddleware::SizeLimiter::Client,
          ::Gitlab::SidekiqMiddleware::ClientMetrics,
          ::Gitlab::SidekiqMiddleware::Identity::Passthrough
        ]
      end

      # The result of this method should be passed to
      # Sidekiq's `config.client_middleware` method
      # eg: `config.client_middleware(&Gitlab::SidekiqMiddleware::Client.configurator)`
      def self.configurator
        ->(chain) do
          middlewares.each { |middleware| chain.add(middleware) }
        end
      end
    end

    class Server
      def self.middlewares(metrics: true, arguments_logger: true, skip_jobs: true)
        [
          # Size limiter should be placed at the top
          ::Gitlab::SidekiqMiddleware::SizeLimiter::Server,
          ::Gitlab::SidekiqMiddleware::ShardAwarenessValidator,
          ::Gitlab::SidekiqMiddleware::Monitor,

          # Labkit wraps the job in the `Labkit::Context` resurrected from
          # the job-hash. We need properties from the context for
          # recording metrics, so this needs to be before
          # `::Gitlab::SidekiqMiddleware::ServerMetrics` (if we're using
          # that).
          ::Labkit::Middleware::Sidekiq::Server,
          ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware,

          ::Gitlab::QueryLimiting.enabled_for_env? ? ::Gitlab::QueryLimiting::SidekiqMiddleware : nil,

          metrics ? ::Gitlab::SidekiqMiddleware::ServerMetrics : nil,

          arguments_logger ? ::Gitlab::SidekiqMiddleware::ArgumentsLogger : nil,
          ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata,
          ::Gitlab::SidekiqMiddleware::BatchLoader,
          ::Gitlab::SidekiqMiddleware::InstrumentationLogger,
          ::Gitlab::SidekiqMiddleware::SetIpAddress,
          ::Gitlab::SidekiqMiddleware::AdminMode::Server,
          ::Gitlab::SidekiqMiddleware::QueryAnalyzer,
          ::Gitlab::SidekiqVersioning::Middleware,
          ::Gitlab::SidekiqStatus::ServerMiddleware,
          ::Gitlab::SidekiqMiddleware::WorkerContext::Server,
          ::ClickHouse::MigrationSupport::SidekiqMiddleware,
          # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware,
          # so we can compare the latest WAL location against replica
          # NOTE: Everything from DuplicateJobs::Client to DuplicateJobs::Server must yield
          # no returning or job interception as it will leave the duplicate job redis key
          # dangling and errorneously deduplicating future jobs until key expires.
          # Any middlewares after DuplicateJobs::Server can return/intercept jobs.
          ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server,
          ::Gitlab::SidekiqMiddleware::PauseControl::Server,
          ::Gitlab::SidekiqMiddleware::Throttling::Server,
          ::Gitlab::SidekiqMiddleware::ConcurrencyLimit::Server,
          skip_jobs ? ::Gitlab::SidekiqMiddleware::SkipJobs : nil,
          ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware,
          ::Gitlab::SidekiqMiddleware::ResourceUsageLimit::Server,
          ::Gitlab::SidekiqMiddleware::Identity::Restore
        ].compact
      end

      # The result of this method should be passed to
      # Sidekiq's `config.server_middleware` method
      # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware::Server.configurator)`
      def self.configurator(metrics: true, arguments_logger: true, skip_jobs: true)
        ->(chain) do
          middlewares(metrics: metrics, arguments_logger: arguments_logger, skip_jobs: skip_jobs).each do |middleware|
            chain.add(middleware)
          end
          ::Gitlab::SidekiqMiddleware::ServerMetrics.initialize_process_metrics if metrics
        end
      end
    end
  end
end

Gitlab::SidekiqMiddleware::Client.prepend_mod
Gitlab::SidekiqMiddleware::Server.prepend_mod