# frozen_string_literal: true require 'fileutils' require 'cgi' require_relative '../config/environment' require_relative 'helpers/groups' require_relative 'helpers/milestones' require_relative 'helpers/git_diff_parser' require_relative 'helpers/ai_editor' require_relative 'prompts/remove_feature_flags' module Keeps # This is an implementation of a ::Gitlab::Housekeeper::Keep. This keep will locate any feature flag definition file # that were added at least ` milestones` ago and remove the definition file. # # You can run it individually with: # # ``` # bundle exec gitlab-housekeeper -d \ # -k Keeps::DeleteOldFeatureFlags # ``` class DeleteOldFeatureFlags < ::Gitlab::Housekeeper::Keep CUTOFF_MILESTONE_FOR_DISABLED_FLAG = 12 CUTOFF_MILESTONE_FOR_ENABLED_FLAG = 4 Error = Class.new(StandardError) GREP_IGNORE = [ 'locale/', 'db/structure.sql' ].freeze ROLLOUT_ISSUE_URL_REGEX = %r{\Ahttps://gitlab\.com/(?.*)/-/issues/(?\d+)\z} API_ISSUE_URL = "https://gitlab.com/api/v4/projects/%s/issues/%s" FEATURE_FLAG_LOG_ISSUES_URL = "https://gitlab.com/gitlab-com/gl-infra/feature-flag-log/-/issues/?search=%s&sort=created_date&state=all&label_name%%5B%%5D=host%%3A%%3Agitlab.com" MISSING_URL_PLACEHOLDER = '(missing URL)' def each_change each_feature_flag do |feature_flag| change = prepare_change(feature_flag) yield(change) if change end end private def can_remove_ff?(feature_flag, identifiers, latest_feature_flag_status) if feature_flag.milestone.nil? @logger.puts "#{feature_flag.name} has no milestone set!" return false end if latest_feature_flag_status.nil? @logger.puts "#{feature_flag.name} cannot be removed as we cannot get the status from the rollout issue." return false end if latest_feature_flag_status == :conditional @logger.puts "#{feature_flag.name} cannot be removed as it is partially rolled out." return false end cutoff = if latest_feature_flag_status == :disabled CUTOFF_MILESTONE_FOR_DISABLED_FLAG else CUTOFF_MILESTONE_FOR_ENABLED_FLAG end unless milestones_helper.before_cuttoff?(milestone: feature_flag.milestone, milestones_ago: cutoff) @logger.puts "#{feature_flag.name} cannot be removed as it is after the cutoff." return false end unless matches_filter_identifiers?(identifiers) @logger.puts "#{feature_flag.name} cannot be removed as it is not matching passed filter." return false end true end # rubocop:disable Gitlab/DocumentationLinks/HardcodedUrl -- Not running inside rails application def build_description(feature_flag, latest_feature_flag_status) <<~MARKDOWN This feature flag was introduced in #{feature_flag.milestone}, which is more than #{latest_feature_flag_status == :enabled ? CUTOFF_MILESTONE_FOR_ENABLED_FLAG : CUTOFF_MILESTONE_FOR_DISABLED_FLAG} milestones ago. As part of our process we want to ensure [feature flags don't stay too long in the codebase](https://docs.gitlab.com/ee/development/feature_flags/#types-of-feature-flags). Rollout issue: #{feature_flag_rollout_issue_url(feature_flag)}
Remaining mentions of the feature flag (click to expand) ``` #{feature_flag_grep(feature_flag.name)} ```
**Currently the feature flag is `#{latest_feature_flag_status}` on production** It is possible that this MR will still need some changes to remove references to the feature flag in the code. At the moment the `gitlab-housekeeper` is not always capable of removing all references so you must check the diff and pipeline failures to confirm if there are any issues. It is the responsibility of ~"#{feature_flag.group}" to push those changes to this branch. If they are already removing this feature flag in another merge request then they can just close this merge request. ## TODO for the reviewers before merging this MR - [ ] See the status of the rollout by checking #{feature_flag_rollout_issue_url(feature_flag)}, #{format(FEATURE_FLAG_LOG_ISSUES_URL, feature_flag_name: feature_flag.name)} - [ ] Verify the feature flag status via chatops by running `/chatops run feature get #{feature_flag.name}`. - [ ] [Search for references to `#{feature_flag.name.split('_').map(&:capitalize).join}` in frontend part of code](https://gitlab.com/search?project_id=278964&scope=blobs&search=#{feature_flag.name.split('_').map(&:capitalize).join}®ex=false) - [ ] [Search for references to `#{feature_flag.name}` in code](https://gitlab.com/search?project_id=278964&scope=blobs&search=#{feature_flag.name}®ex=false) - [ ] Check if we need to remove any Gem or other related code by looking at the changes in #{feature_flag.introduced_by_url} MARKDOWN end # rubocop:enable Gitlab/DocumentationLinks/HardcodedUrl def prepare_change(feature_flag) identifiers = [self.class.name.demodulize, feature_flag.name] latest_feature_flag_status = get_latest_feature_flag_status(feature_flag) return unless can_remove_ff?(feature_flag, identifiers, latest_feature_flag_status) change = ::Gitlab::Housekeeper::Change.new change.changelog_type = 'removed' change.title = "Delete the `#{feature_flag.name}` feature flag" change.identifiers = identifiers FileUtils.rm(feature_flag.path) change.changed_files = [feature_flag.path] applied = apply_patch_or_ask_ai(feature_flag, change) unless applied @logger.puts "#{feature_flag.name} aborting because change not applied." change.abort! return change end change.description = build_description(feature_flag, latest_feature_flag_status) change.labels = [ 'automation:feature-flag-removal', 'maintenance::removal', 'feature flag', feature_flag.group ] change.reviewers = assignees(feature_flag.rollout_issue_url) if change.reviewers.empty? group_data = groups_helper.group_for_group_label(feature_flag.group) change.reviewers = groups_helper.pick_reviewer(group_data, change.identifiers) if group_data end change end def feature_flag_grep(feature_flag_name) Gitlab::Housekeeper::Shell.execute( 'git', 'grep', '--heading', '--line-number', '--break', feature_flag_name, '--', *(GREP_IGNORE.map { |path| ":^#{path}" }) ) rescue ::Gitlab::Housekeeper::Shell::Error # git grep returns error status if nothing is found end def ai_patch(feature_flag, change) files_mentioning_feature_flag(feature_flag.name).each do |file| flag_enabled = get_latest_feature_flag_status(feature_flag) == :enabled user_message = remove_feature_flag_prompts.fetch(feature_flag, file, flag_enabled) return false unless user_message applied = ai_helper.ask_for_and_apply_patch(user_message, file) return false unless applied unless ::Gitlab::Housekeeper::Shell.rubocop_autocorrect(file) @logger.puts "#{feature_flag.name} aborting because autocorrect failed for file #{file}" change.changed_files << file # Adding file so debugging becomes easier. change.abort! return change end change.changed_files << file end end def apply_patch_or_ask_ai(feature_flag, change) return apply_patch(feature_flag, change) if patch_exists?(feature_flag) ai_patch(feature_flag, change) end def apply_patch(feature_flag, change) change.changed_files << patch_path(feature_flag) change.changed_files += extract_changed_files_from_patch(feature_flag) begin Gitlab::Housekeeper::Shell.execute('git', 'apply', patch_path(feature_flag)) rescue ::Gitlab::Housekeeper::Shell::Error return false end FileUtils.rm(patch_path(feature_flag)) true end def patch_exists?(feature_flag) File.file?(patch_path(feature_flag)) end def patch_path(feature_flag) feature_flag.path.sub(/.yml$/, '.patch') end def extract_changed_files_from_patch(feature_flag) git_diff_parser_helper.all_changed_files(File.read(patch_path(feature_flag))) end def feature_flag_rollout_issue_url(feature_flag) feature_flag.rollout_issue_url || MISSING_URL_PLACEHOLDER end def assignees(rollout_issue_url) rollout_issue = get_rollout_issue(rollout_issue_url) return unless rollout_issue && rollout_issue[:assignees] rollout_issue[:assignees].pluck(:username) # rubocop:disable CodeReuse/ActiveRecord -- We need to collect the usernames from array of hashes. end def get_rollout_issue(rollout_issue_url) matches = ROLLOUT_ISSUE_URL_REGEX.match(rollout_issue_url) return unless matches # rubocop:disable Gitlab/HttpV2 -- Not running inside rails application response = Gitlab::HTTP_V2.try_get( format(API_ISSUE_URL, project_path: CGI.escape(matches[:project_path]), issue_iid: matches[:issue_iid]) ) # rubocop:enable Gitlab/HttpV2 unless (200..299).cover?(response.code) raise Error, "Failed with response code: #{response.code} and body:\n#{response.body}" end Gitlab::Json.parse(response.body, symbolize_names: true) end def get_latest_feature_flag_status(feature_flag) return :enabled if feature_flag.default_enabled rollout_issue_url = feature_flag_rollout_issue_url(feature_flag) if rollout_issue_url == MISSING_URL_PLACEHOLDER @logger.puts "Can't fetch ff status for #{feature_flag.name} due to absence of rollout issue." return end rollout_issue = get_rollout_issue(rollout_issue_url) return unless rollout_issue state_label = rollout_issue[:labels].find { |label| label.start_with?('feature flag state::') } if state_label.nil? @logger.puts( "Can't fetch ff status for #{feature_flag.name} due to absence of feature flag state label on rollout issue." ) return end case state_label when 'feature flag state::enabled' :enabled when 'feature flag state::disabled' :disabled when 'feature flag state::rolling out' :conditional end end def each_feature_flag all_feature_flag_files.map do |f| feature_definition = Feature::Definition.new(f, YAML.safe_load_file(f, permitted_classes: [Symbol], symbolize_names: true)) next unless global_search_flag?(feature_definition) yield(feature_definition) end end def global_search_flag?(feature_flag) feature_flag.group == 'group::global search' end def all_feature_flag_files Dir.glob("{,ee/}config/feature_flags/{development,gitlab_com_derisk,ops,beta}/*.yml") end def remove_feature_flag_prompts @remove_feature_flag_prompts ||= ::Keeps::Prompts::RemoveFeatureFlags.new(@logger) end def groups_helper @groups_helper ||= ::Keeps::Helpers::Groups.new end def milestones_helper @milestones_helper ||= ::Keeps::Helpers::Milestones.new end def git_diff_parser_helper @git_diff_parser_helper ||= ::Keeps::Helpers::GitDiffParser.new end def ai_helper ::Keeps::Helpers::AiEditor.new end def files_mentioning_feature_flag(feature_flag_name) files = Gitlab::Housekeeper::Shell.execute( 'git', 'grep', '--name-only', '-i', "feature.*#{feature_flag_name}", '--', *(GREP_IGNORE.map { |path| ":^#{path}" }) ) return [] unless files files.split("\n") end end end