From 91763dcb693cd1eb13b88c96fd344b0743e7415e Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Mon, 21 Jul 2025 00:11:49 +0000 Subject: [PATCH] Add latest changes from gitlab-org/gitlab@master --- ...ci_gitlab_hosted_runner_monthly_usages.yml | 3 +-- db/docs/cluster_agent_migrations.yml | 2 +- db/docs/cluster_groups.yml | 2 +- db/docs/cluster_projects.yml | 2 +- db/docs/deployment_clusters.yml | 2 +- db/docs/excluded_merge_requests.yml | 3 +-- db/docs/group_deploy_keys.yml | 2 +- db/docs/group_deploy_keys_groups.yml | 2 +- db/docs/scim_oauth_access_tokens.yml | 4 ++-- db/docs/uploads_9ba88c4165.yml | 4 ++-- .../backup_restore/migrate_to_new_server.md | 3 ++- doc/api/model_registry.md | 6 +++-- .../database/no_cross_db_foreign_keys_spec.rb | 1 + ...new_tables_with_gitlab_main_schema_spec.rb | 3 ++- spec/lib/gitlab/database/sharding_key_spec.rb | 22 ++++++++++++++----- 15 files changed, 38 insertions(+), 23 deletions(-) diff --git a/db/docs/ci_gitlab_hosted_runner_monthly_usages.yml b/db/docs/ci_gitlab_hosted_runner_monthly_usages.yml index 5ce6f6137ec..8e336a3cd82 100644 --- a/db/docs/ci_gitlab_hosted_runner_monthly_usages.yml +++ b/db/docs/ci_gitlab_hosted_runner_monthly_usages.yml @@ -9,5 +9,4 @@ description: Per month CI usage data at the runner and project level meant to st introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/171336 milestone: '17.9' gitlab_schema: gitlab_ci -exempt_from_sharding: true -notes: Exempted from sharding because Dedicated only. +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/553104 diff --git a/db/docs/cluster_agent_migrations.yml b/db/docs/cluster_agent_migrations.yml index 652acd22a91..4649c06ca0c 100644 --- a/db/docs/cluster_agent_migrations.yml +++ b/db/docs/cluster_agent_migrations.yml @@ -7,7 +7,7 @@ feature_categories: description: Tracks progress when migrating to the GitLab agent from a certificate based cluster introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179324 milestone: '17.9' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main exempt_from_sharding: true notes: Exempted because parent table is scheduled for removal table_size: small diff --git a/db/docs/cluster_groups.yml b/db/docs/cluster_groups.yml index d55c5f15109..d1e1f02891a 100644 --- a/db/docs/cluster_groups.yml +++ b/db/docs/cluster_groups.yml @@ -7,7 +7,7 @@ feature_categories: description: "(Deprecated) Join table between 'clusters' and 'namespaces'" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/0e15eec86d83cbdfefe17966bf5c02e4d419a34d milestone: '11.5' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main exempt_from_sharding: true notes: Exempted because Table is scheduled for removal table_size: small diff --git a/db/docs/cluster_projects.yml b/db/docs/cluster_projects.yml index 7fbf65a2cd7..08cb749b4f1 100644 --- a/db/docs/cluster_projects.yml +++ b/db/docs/cluster_projects.yml @@ -7,7 +7,7 @@ feature_categories: description: "(Deprecated) Join table between 'clusters' and 'projects'" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/d0cff7f5855f91b5479f9fdaa39d8d95ec691a9e milestone: '10.2' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main exempt_from_sharding: true notes: Exempted because Table is scheduled for removal table_size: small diff --git a/db/docs/deployment_clusters.yml b/db/docs/deployment_clusters.yml index 44fd4c83ead..1c0df0056d7 100644 --- a/db/docs/deployment_clusters.yml +++ b/db/docs/deployment_clusters.yml @@ -7,7 +7,7 @@ feature_categories: description: "(Deprecated) Join table between `deployments` and `clusters`" introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24235 milestone: '12.8' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main exempt_from_sharding: true notes: Exempted because Table is scheduled for removal table_size: small diff --git a/db/docs/excluded_merge_requests.yml b/db/docs/excluded_merge_requests.yml index 5655b4ecb78..0c801725af9 100644 --- a/db/docs/excluded_merge_requests.yml +++ b/db/docs/excluded_merge_requests.yml @@ -5,6 +5,5 @@ feature_categories: description: Store MR ids for exclusion during migrations introduced_by_url: milestone: '17.10' -gitlab_schema: gitlab_main_cell -exempt_from_sharding: true +gitlab_schema: gitlab_main_cell_local table_size: small diff --git a/db/docs/group_deploy_keys.yml b/db/docs/group_deploy_keys.yml index 5c1f1047a69..2032c9ea3ae 100644 --- a/db/docs/group_deploy_keys.yml +++ b/db/docs/group_deploy_keys.yml @@ -7,7 +7,7 @@ feature_categories: description: https://docs.gitlab.com/ee/user/project/deploy_keys/ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/30886 milestone: '13.1' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main exempt_from_sharding: true notes: Exempted because Table is empty and will be removed - https://gitlab.com/gitlab-org/gitlab/-/issues/408933 table_size: small diff --git a/db/docs/group_deploy_keys_groups.yml b/db/docs/group_deploy_keys_groups.yml index 61619d3c53c..ec847d30a3f 100644 --- a/db/docs/group_deploy_keys_groups.yml +++ b/db/docs/group_deploy_keys_groups.yml @@ -7,7 +7,7 @@ feature_categories: description: https://docs.gitlab.com/ee/user/project/deploy_keys/ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/32901 milestone: '13.2' -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main exempt_from_sharding: true notes: Exempted because Table is empty and will be removed - https://gitlab.com/gitlab-org/gitlab/-/issues/408933 table_size: small diff --git a/db/docs/scim_oauth_access_tokens.yml b/db/docs/scim_oauth_access_tokens.yml index 847b89c594b..d204bf75744 100644 --- a/db/docs/scim_oauth_access_tokens.yml +++ b/db/docs/scim_oauth_access_tokens.yml @@ -8,6 +8,6 @@ description: Bearer token used for authenticating SCIM requests. This is table for Instance SCIM introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/e9b2253fe3538234d1c4d173c4549a955233d836 milestone: '11.10' -gitlab_schema: gitlab_main_cell -exempt_from_sharding: true +gitlab_schema: gitlab_main +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/553107 table_size: small diff --git a/db/docs/uploads_9ba88c4165.yml b/db/docs/uploads_9ba88c4165.yml index b84476360cf..115883dbe40 100644 --- a/db/docs/uploads_9ba88c4165.yml +++ b/db/docs/uploads_9ba88c4165.yml @@ -14,6 +14,6 @@ description: For tracking blob metadata. Single table inheritance is used to rel this table to many other tables. Partitioned by model_type. introduced_by_url: milestone: '17.11' -table_size: small gitlab_schema: gitlab_main_cell -exempt_from_sharding: true +sharding_key_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/398199 +table_size: small diff --git a/doc/administration/backup_restore/migrate_to_new_server.md b/doc/administration/backup_restore/migrate_to_new_server.md index 2de714b7bb4..e29de7f09e4 100644 --- a/doc/administration/backup_restore/migrate_to_new_server.md +++ b/doc/administration/backup_restore/migrate_to_new_server.md @@ -116,10 +116,11 @@ To prepare the new server: sudo gitlab-backup create ``` -1. Disable the following GitLab services and prevent unintentional restarts by adding the following to the bottom of `/etc/gitlab/gitlab.rb`: +1. After the backup is complete, disable the following GitLab services and prevent unintentional restarts by adding the following to the bottom of `/etc/gitlab/gitlab.rb`: ```ruby alertmanager['enable'] = false + gitaly['enable'] = false gitlab_exporter['enable'] = false gitlab_pages['enable'] = false gitlab_workhorse['enable'] = false diff --git a/doc/api/model_registry.md b/doc/api/model_registry.md index 7ca11edb836..a76e39a7af0 100644 --- a/doc/api/model_registry.md +++ b/doc/api/model_registry.md @@ -37,7 +37,8 @@ Parameters: | `filename` | string | yes | Filename | ```shell -curl --header "Authorization: Bearer " "https://gitlab.example.com/api/v4/projects/:id/packages/ml_models/:model_version_id/files/(*path/):filename +curl --header "Authorization: Bearer " \ + --url "https://gitlab.example.com/api/v4/projects/:id/packages/ml_models/:model_version_id/files/(*path/):filename" ``` The response contains the file contents. @@ -45,5 +46,6 @@ The response contains the file contents. For example, the following command returns the file `foo.txt` for the model version with an ID of `2` and project with an ID of `1`. ```shell -curl --header "Authorization: Bearer " "https://gitlab.example.com/api/v4/projects/1/packages/ml_models/2/files/foo.txt +curl --header "Authorization: Bearer " \ + --url "https://gitlab.example.com/api/v4/projects/1/packages/ml_models/2/files/foo.txt" ``` diff --git a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb index 7ac2e141f28..edaa5ab527c 100644 --- a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb @@ -9,6 +9,7 @@ RSpec.describe 'cross-database foreign keys', feature_category: :database do # should be added as a comment along with the name of the column. let!(:allowed_cross_database_foreign_keys) do keys = [ + 'excluded_merge_requests.merge_request_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/517248 'zoekt_indices.zoekt_enabled_namespace_id', 'zoekt_repositories.project_id', 'zoekt_replicas.zoekt_enabled_namespace_id', diff --git a/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb b/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb index e4db702919e..793d6544c5e 100644 --- a/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb +++ b/spec/lib/gitlab/database/no_new_tables_with_gitlab_main_schema_spec.rb @@ -30,7 +30,8 @@ RSpec.describe 'new tables with gitlab_main schema', feature_category: :cell do 'instance_integrations', # gitlab_main_clusterwide now deprecated 'ldap_admin_role_links', # gitlab_main_clusterwide now deprecated 'user_audit_events', # gitlab_main_clusterwide now deprecated - 'user_permission_export_upload_uploads' # gitlab_main_clusterwide now deprecated + 'user_permission_export_upload_uploads', # gitlab_main_clusterwide now deprecated + 'cluster_agent_migrations' # awaiting table removal ] end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index 38aacffa5e5..2bbef7c6ab6 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -9,7 +9,9 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :organizatio # the table name to remove this once a decision has been made. let(:allowed_to_be_missing_sharding_key) do [ - 'web_hook_logs_daily' # temporary copy of web_hook_logs + 'web_hook_logs_daily', # temporary copy of web_hook_logs + 'ci_gitlab_hosted_runner_monthly_usages', # Dedicated only table, to be sharded + 'uploads_9ba88c4165' # https://gitlab.com/gitlab-org/gitlab/-/issues/398199 ] end @@ -381,6 +383,18 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :organizatio end end + it 'does not allow tables in sharded schemas to be permanently exempted', :aggregate_failures do + sharded_schemas = Gitlab::Database + .all_gitlab_schemas + .select { |s| Gitlab::Database::GitlabSchema.require_sharding_key?(s) } + + tables_exempted_from_sharding.each do |entry| + expect(entry.gitlab_schema).not_to be_in(sharded_schemas), + "#{entry.table_name} is in a schema (#{entry.gitlab_schema}) " \ + "that requires sharding so is not allowed to be exempted from sharding" + end + end + it 'does not allow tables with FK references to be permanently exempted', :aggregate_failures do tables_exempted_from_sharding_table_names = tables_exempted_from_sharding.map(&:table_name) @@ -460,7 +474,6 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :organizatio def tables_missing_sharding_key(starting_from_milestone:) ::Gitlab::Database::Dictionary.entries.filter_map do |entry| entry.table_name if entry.sharding_key.blank? && - !entry.exempt_from_sharding? && entry.milestone_greater_than_or_equal_to?(starting_from_milestone) && ::Gitlab::Database::GitlabSchema.require_sharding_key?(entry.gitlab_schema) end @@ -469,10 +482,9 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :organizatio def tables_missing_sharding_key_or_sharding_in_progress ::Gitlab::Database::Dictionary.entries.filter_map do |entry| entry.table_name if entry.sharding_key.blank? && - !entry.exempt_from_sharding? && - ::Gitlab::Database::GitlabSchema.require_sharding_key?(entry.gitlab_schema) && entry.sharding_key_issue_url.blank? && - entry.desired_sharding_key.blank? + entry.desired_sharding_key.blank? && + ::Gitlab::Database::GitlabSchema.require_sharding_key?(entry.gitlab_schema) end end