diff --git a/db/docs/batched_background_migrations/backfill_bulk_import_trackers_namespace_id.yml b/db/docs/batched_background_migrations/backfill_bulk_import_trackers_namespace_id.yml new file mode 100644 index 00000000000..3f9b8625381 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_bulk_import_trackers_namespace_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillBulkImportTrackersNamespaceId +description: Backfills sharding key `bulk_import_trackers.namespace_id` from `bulk_import_entities`. +feature_category: importers +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/180437 +milestone: '17.9' +queued_migration_version: 20250205195337 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/backfill_bulk_import_trackers_organization_id.yml b/db/docs/batched_background_migrations/backfill_bulk_import_trackers_organization_id.yml new file mode 100644 index 00000000000..47da388cce6 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_bulk_import_trackers_organization_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillBulkImportTrackersOrganizationId +description: Backfills sharding key `bulk_import_trackers.organization_id` from `bulk_import_entities`. +feature_category: importers +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/180437 +milestone: '17.9' +queued_migration_version: 20250205195342 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/batched_background_migrations/backfill_bulk_import_trackers_project_id.yml b/db/docs/batched_background_migrations/backfill_bulk_import_trackers_project_id.yml new file mode 100644 index 00000000000..5818bb46d28 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_bulk_import_trackers_project_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillBulkImportTrackersProjectId +description: Backfills sharding key `bulk_import_trackers.project_id` from `bulk_import_entities`. +feature_category: importers +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/180437 +milestone: '17.9' +queued_migration_version: 20250205195332 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/bulk_import_trackers.yml b/db/docs/bulk_import_trackers.yml index abc33025383..8a3c762f64c 100644 --- a/db/docs/bulk_import_trackers.yml +++ b/db/docs/bulk_import_trackers.yml @@ -35,3 +35,7 @@ desired_sharding_key: sharding_key: organization_id belongs_to: entity table_size: small +desired_sharding_key_migration_job_name: +- BackfillBulkImportTrackersProjectId +- BackfillBulkImportTrackersNamespaceId +- BackfillBulkImportTrackersOrganizationId diff --git a/db/migrate/20250205195328_add_project_id_to_bulk_import_trackers.rb b/db/migrate/20250205195328_add_project_id_to_bulk_import_trackers.rb new file mode 100644 index 00000000000..c207e7fe04a --- /dev/null +++ b/db/migrate/20250205195328_add_project_id_to_bulk_import_trackers.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProjectIdToBulkImportTrackers < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def change + add_column :bulk_import_trackers, :project_id, :bigint + end +end diff --git a/db/migrate/20250205195333_add_namespace_id_to_bulk_import_trackers.rb b/db/migrate/20250205195333_add_namespace_id_to_bulk_import_trackers.rb new file mode 100644 index 00000000000..cfbbb047d69 --- /dev/null +++ b/db/migrate/20250205195333_add_namespace_id_to_bulk_import_trackers.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToBulkImportTrackers < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def change + add_column :bulk_import_trackers, :namespace_id, :bigint + end +end diff --git a/db/migrate/20250205195338_add_organization_id_to_bulk_import_trackers.rb b/db/migrate/20250205195338_add_organization_id_to_bulk_import_trackers.rb new file mode 100644 index 00000000000..690ae008f6e --- /dev/null +++ b/db/migrate/20250205195338_add_organization_id_to_bulk_import_trackers.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddOrganizationIdToBulkImportTrackers < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def change + add_column :bulk_import_trackers, :organization_id, :bigint + end +end diff --git a/db/post_migrate/20250205195329_index_bulk_import_trackers_on_project_id.rb b/db/post_migrate/20250205195329_index_bulk_import_trackers_on_project_id.rb new file mode 100644 index 00000000000..8b987ff2cff --- /dev/null +++ b/db/post_migrate/20250205195329_index_bulk_import_trackers_on_project_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexBulkImportTrackersOnProjectId < Gitlab::Database::Migration[2.2] + milestone '17.9' + disable_ddl_transaction! + + INDEX_NAME = 'index_bulk_import_trackers_on_project_id' + + def up + add_concurrent_index :bulk_import_trackers, :project_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :bulk_import_trackers, INDEX_NAME + end +end diff --git a/db/post_migrate/20250205195330_add_bulk_import_trackers_project_id_fk.rb b/db/post_migrate/20250205195330_add_bulk_import_trackers_project_id_fk.rb new file mode 100644 index 00000000000..c45741ffc76 --- /dev/null +++ b/db/post_migrate/20250205195330_add_bulk_import_trackers_project_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddBulkImportTrackersProjectIdFk < Gitlab::Database::Migration[2.2] + milestone '17.9' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_trackers, :projects, column: :project_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_trackers, column: :project_id + end + end +end diff --git a/db/post_migrate/20250205195331_add_bulk_import_trackers_project_id_trigger.rb b/db/post_migrate/20250205195331_add_bulk_import_trackers_project_id_trigger.rb new file mode 100644 index 00000000000..553764cc7c9 --- /dev/null +++ b/db/post_migrate/20250205195331_add_bulk_import_trackers_project_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddBulkImportTrackersProjectIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def up + install_sharding_key_assignment_trigger( + table: :bulk_import_trackers, + sharding_key: :project_id, + parent_table: :bulk_import_entities, + parent_sharding_key: :project_id, + foreign_key: :bulk_import_entity_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :bulk_import_trackers, + sharding_key: :project_id, + parent_table: :bulk_import_entities, + parent_sharding_key: :project_id, + foreign_key: :bulk_import_entity_id + ) + end +end diff --git a/db/post_migrate/20250205195332_queue_backfill_bulk_import_trackers_project_id.rb b/db/post_migrate/20250205195332_queue_backfill_bulk_import_trackers_project_id.rb new file mode 100644 index 00000000000..cb5ca0101fc --- /dev/null +++ b/db/post_migrate/20250205195332_queue_backfill_bulk_import_trackers_project_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillBulkImportTrackersProjectId < Gitlab::Database::Migration[2.2] + milestone '17.9' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillBulkImportTrackersProjectId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :bulk_import_trackers, + :id, + :project_id, + :bulk_import_entities, + :project_id, + :bulk_import_entity_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :bulk_import_trackers, + :id, + [ + :project_id, + :bulk_import_entities, + :project_id, + :bulk_import_entity_id + ] + ) + end +end diff --git a/db/post_migrate/20250205195334_index_bulk_import_trackers_on_namespace_id.rb b/db/post_migrate/20250205195334_index_bulk_import_trackers_on_namespace_id.rb new file mode 100644 index 00000000000..ef14632e191 --- /dev/null +++ b/db/post_migrate/20250205195334_index_bulk_import_trackers_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexBulkImportTrackersOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.9' + disable_ddl_transaction! + + INDEX_NAME = 'index_bulk_import_trackers_on_namespace_id' + + def up + add_concurrent_index :bulk_import_trackers, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :bulk_import_trackers, INDEX_NAME + end +end diff --git a/db/post_migrate/20250205195335_add_bulk_import_trackers_namespace_id_fk.rb b/db/post_migrate/20250205195335_add_bulk_import_trackers_namespace_id_fk.rb new file mode 100644 index 00000000000..9cdae150c51 --- /dev/null +++ b/db/post_migrate/20250205195335_add_bulk_import_trackers_namespace_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddBulkImportTrackersNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.9' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_trackers, :namespaces, column: :namespace_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_trackers, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20250205195336_add_bulk_import_trackers_namespace_id_trigger.rb b/db/post_migrate/20250205195336_add_bulk_import_trackers_namespace_id_trigger.rb new file mode 100644 index 00000000000..3121a8ed978 --- /dev/null +++ b/db/post_migrate/20250205195336_add_bulk_import_trackers_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddBulkImportTrackersNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def up + install_sharding_key_assignment_trigger( + table: :bulk_import_trackers, + sharding_key: :namespace_id, + parent_table: :bulk_import_entities, + parent_sharding_key: :namespace_id, + foreign_key: :bulk_import_entity_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :bulk_import_trackers, + sharding_key: :namespace_id, + parent_table: :bulk_import_entities, + parent_sharding_key: :namespace_id, + foreign_key: :bulk_import_entity_id + ) + end +end diff --git a/db/post_migrate/20250205195337_queue_backfill_bulk_import_trackers_namespace_id.rb b/db/post_migrate/20250205195337_queue_backfill_bulk_import_trackers_namespace_id.rb new file mode 100644 index 00000000000..f8c53aa5346 --- /dev/null +++ b/db/post_migrate/20250205195337_queue_backfill_bulk_import_trackers_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillBulkImportTrackersNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.9' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillBulkImportTrackersNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :bulk_import_trackers, + :id, + :namespace_id, + :bulk_import_entities, + :namespace_id, + :bulk_import_entity_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :bulk_import_trackers, + :id, + [ + :namespace_id, + :bulk_import_entities, + :namespace_id, + :bulk_import_entity_id + ] + ) + end +end diff --git a/db/post_migrate/20250205195339_index_bulk_import_trackers_on_organization_id.rb b/db/post_migrate/20250205195339_index_bulk_import_trackers_on_organization_id.rb new file mode 100644 index 00000000000..653644f73cc --- /dev/null +++ b/db/post_migrate/20250205195339_index_bulk_import_trackers_on_organization_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexBulkImportTrackersOnOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.9' + disable_ddl_transaction! + + INDEX_NAME = 'index_bulk_import_trackers_on_organization_id' + + def up + add_concurrent_index :bulk_import_trackers, :organization_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :bulk_import_trackers, INDEX_NAME + end +end diff --git a/db/post_migrate/20250205195340_add_bulk_import_trackers_organization_id_fk.rb b/db/post_migrate/20250205195340_add_bulk_import_trackers_organization_id_fk.rb new file mode 100644 index 00000000000..8d941dad749 --- /dev/null +++ b/db/post_migrate/20250205195340_add_bulk_import_trackers_organization_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddBulkImportTrackersOrganizationIdFk < Gitlab::Database::Migration[2.2] + milestone '17.9' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :bulk_import_trackers, :organizations, column: :organization_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :bulk_import_trackers, column: :organization_id + end + end +end diff --git a/db/post_migrate/20250205195341_add_bulk_import_trackers_organization_id_trigger.rb b/db/post_migrate/20250205195341_add_bulk_import_trackers_organization_id_trigger.rb new file mode 100644 index 00000000000..efcda5638a8 --- /dev/null +++ b/db/post_migrate/20250205195341_add_bulk_import_trackers_organization_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddBulkImportTrackersOrganizationIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def up + install_sharding_key_assignment_trigger( + table: :bulk_import_trackers, + sharding_key: :organization_id, + parent_table: :bulk_import_entities, + parent_sharding_key: :organization_id, + foreign_key: :bulk_import_entity_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :bulk_import_trackers, + sharding_key: :organization_id, + parent_table: :bulk_import_entities, + parent_sharding_key: :organization_id, + foreign_key: :bulk_import_entity_id + ) + end +end diff --git a/db/post_migrate/20250205195342_queue_backfill_bulk_import_trackers_organization_id.rb b/db/post_migrate/20250205195342_queue_backfill_bulk_import_trackers_organization_id.rb new file mode 100644 index 00000000000..1bd2bb05ae0 --- /dev/null +++ b/db/post_migrate/20250205195342_queue_backfill_bulk_import_trackers_organization_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillBulkImportTrackersOrganizationId < Gitlab::Database::Migration[2.2] + milestone '17.9' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillBulkImportTrackersOrganizationId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :bulk_import_trackers, + :id, + :organization_id, + :bulk_import_entities, + :organization_id, + :bulk_import_entity_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :bulk_import_trackers, + :id, + [ + :organization_id, + :bulk_import_entities, + :organization_id, + :bulk_import_entity_id + ] + ) + end +end diff --git a/db/schema_migrations/20250205195328 b/db/schema_migrations/20250205195328 new file mode 100644 index 00000000000..9de7379ea92 --- /dev/null +++ b/db/schema_migrations/20250205195328 @@ -0,0 +1 @@ +b8a30c0db8f83cb888097c01152b6a3541defa420a9f8078dc09ada0a9695eb9 \ No newline at end of file diff --git a/db/schema_migrations/20250205195329 b/db/schema_migrations/20250205195329 new file mode 100644 index 00000000000..7c6ae2a18ee --- /dev/null +++ b/db/schema_migrations/20250205195329 @@ -0,0 +1 @@ +925ba50be28ac7b22f83aa43c73b4679954eb468ed4f4bfbcd09abbf3353424e \ No newline at end of file diff --git a/db/schema_migrations/20250205195330 b/db/schema_migrations/20250205195330 new file mode 100644 index 00000000000..06f878f6a37 --- /dev/null +++ b/db/schema_migrations/20250205195330 @@ -0,0 +1 @@ +943813224e6b779fff90283e8eec99de0ab88f28df420652dcbb713fcf004d5f \ No newline at end of file diff --git a/db/schema_migrations/20250205195331 b/db/schema_migrations/20250205195331 new file mode 100644 index 00000000000..cd569d66fe0 --- /dev/null +++ b/db/schema_migrations/20250205195331 @@ -0,0 +1 @@ +49171333e5cc7286a28f8200e48c167557f4df02cba6afd6d7b4ad249d938325 \ No newline at end of file diff --git a/db/schema_migrations/20250205195332 b/db/schema_migrations/20250205195332 new file mode 100644 index 00000000000..cffbae83ac9 --- /dev/null +++ b/db/schema_migrations/20250205195332 @@ -0,0 +1 @@ +27548f6fff9e5970419f798d0ff91382271132407de7f01dff4954f0c11fe736 \ No newline at end of file diff --git a/db/schema_migrations/20250205195333 b/db/schema_migrations/20250205195333 new file mode 100644 index 00000000000..3f6ac3b25ec --- /dev/null +++ b/db/schema_migrations/20250205195333 @@ -0,0 +1 @@ +7cdb2c54a1a416648c3cb146a4b15e45dcd15d66d794c8d2e233c29059aee9c4 \ No newline at end of file diff --git a/db/schema_migrations/20250205195334 b/db/schema_migrations/20250205195334 new file mode 100644 index 00000000000..6982b9ec14f --- /dev/null +++ b/db/schema_migrations/20250205195334 @@ -0,0 +1 @@ +1a21a6da1c8121e0dbc80d786c5af9ee211ced3e6f9ccba3795367c2b7aed113 \ No newline at end of file diff --git a/db/schema_migrations/20250205195335 b/db/schema_migrations/20250205195335 new file mode 100644 index 00000000000..a0a479d94c2 --- /dev/null +++ b/db/schema_migrations/20250205195335 @@ -0,0 +1 @@ +c79465ade4a13805156011a5c1dc3e67d1a88e2c8d5e181e5020c900d3bb3ba1 \ No newline at end of file diff --git a/db/schema_migrations/20250205195336 b/db/schema_migrations/20250205195336 new file mode 100644 index 00000000000..6e5939423f8 --- /dev/null +++ b/db/schema_migrations/20250205195336 @@ -0,0 +1 @@ +bb49cae69540f0232c68de6481bfecb572334cf41d209297c30ae3e1f3803869 \ No newline at end of file diff --git a/db/schema_migrations/20250205195337 b/db/schema_migrations/20250205195337 new file mode 100644 index 00000000000..ce989ecde92 --- /dev/null +++ b/db/schema_migrations/20250205195337 @@ -0,0 +1 @@ +945832c7c245d5b6dc3f4baee904d7af5cfdf8cfc8bfec5e5357b95c6a0a6150 \ No newline at end of file diff --git a/db/schema_migrations/20250205195338 b/db/schema_migrations/20250205195338 new file mode 100644 index 00000000000..7bfb6da4540 --- /dev/null +++ b/db/schema_migrations/20250205195338 @@ -0,0 +1 @@ +24123383db8e997a430de887a38eef718618764b266ab6332362d9c76b02f411 \ No newline at end of file diff --git a/db/schema_migrations/20250205195339 b/db/schema_migrations/20250205195339 new file mode 100644 index 00000000000..5b3f52c00c8 --- /dev/null +++ b/db/schema_migrations/20250205195339 @@ -0,0 +1 @@ +1469e3cd4588aa0d723311fd4ab5247bec43a827f78e204f6f73e6df0bafc68d \ No newline at end of file diff --git a/db/schema_migrations/20250205195340 b/db/schema_migrations/20250205195340 new file mode 100644 index 00000000000..e15b9f89717 --- /dev/null +++ b/db/schema_migrations/20250205195340 @@ -0,0 +1 @@ +3a3232971da7bfde107ea0bd4bbea53420e3183657956615523bdaa68e07f7ca \ No newline at end of file diff --git a/db/schema_migrations/20250205195341 b/db/schema_migrations/20250205195341 new file mode 100644 index 00000000000..d3e6939aec9 --- /dev/null +++ b/db/schema_migrations/20250205195341 @@ -0,0 +1 @@ +155fc5abf343dc9270d2896eb7bbc50f86ee0be63d64009cfbcc1c6471a177a2 \ No newline at end of file diff --git a/db/schema_migrations/20250205195342 b/db/schema_migrations/20250205195342 new file mode 100644 index 00000000000..fb33053adb2 --- /dev/null +++ b/db/schema_migrations/20250205195342 @@ -0,0 +1 @@ +d1ecb0740c5d2d21e0e78e9652b7a70cb003f311066fc4bf8b5298a123fa4097 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 30f3ba52967..85456a46da9 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2363,6 +2363,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_765cae42cd77() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."organization_id" IS NULL THEN + SELECT "organization_id" + INTO NEW."organization_id" + FROM "bulk_import_entities" + WHERE "bulk_import_entities"."id" = NEW."bulk_import_entity_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_77d9fbad5b12() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -2472,6 +2488,22 @@ BEGIN END; $$; +CREATE FUNCTION trigger_7f84f9c7b945() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."project_id" IS NULL THEN + SELECT "project_id" + INTO NEW."project_id" + FROM "bulk_import_entities" + WHERE "bulk_import_entities"."id" = NEW."bulk_import_entity_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_81b4c93e7133() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -3496,6 +3528,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_eeb25d23ab2d() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "bulk_import_entities" + WHERE "bulk_import_entities"."id" = NEW."bulk_import_entity_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_efb9d354f05a() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -9758,6 +9806,9 @@ CREATE TABLE bulk_import_trackers ( source_objects_count bigint DEFAULT 0 NOT NULL, fetched_objects_count bigint DEFAULT 0 NOT NULL, imported_objects_count bigint DEFAULT 0 NOT NULL, + project_id bigint, + namespace_id bigint, + organization_id bigint, CONSTRAINT check_2d45cae629 CHECK ((char_length(relation) <= 255)), CONSTRAINT check_40aeaa600b CHECK ((char_length(next_page) <= 255)), CONSTRAINT check_603f91cb06 CHECK ((char_length(jid) <= 255)), @@ -31566,6 +31617,12 @@ CREATE INDEX index_bulk_import_failures_on_bulk_import_entity_id ON bulk_import_ CREATE INDEX index_bulk_import_failures_on_correlation_id_value ON bulk_import_failures USING btree (correlation_id_value); +CREATE INDEX index_bulk_import_trackers_on_namespace_id ON bulk_import_trackers USING btree (namespace_id); + +CREATE INDEX index_bulk_import_trackers_on_organization_id ON bulk_import_trackers USING btree (organization_id); + +CREATE INDEX index_bulk_import_trackers_on_project_id ON bulk_import_trackers USING btree (project_id); + CREATE INDEX index_bulk_imports_on_updated_at_and_id_for_stale_status ON bulk_imports USING btree (updated_at, id) WHERE (status = ANY (ARRAY[0, 1])); CREATE INDEX index_bulk_imports_on_user_id ON bulk_imports USING btree (user_id); @@ -38086,6 +38143,8 @@ CREATE TRIGGER trigger_70d3f0bba1de BEFORE INSERT OR UPDATE ON compliance_framew CREATE TRIGGER trigger_740afa9807b8 BEFORE INSERT OR UPDATE ON subscription_user_add_on_assignments FOR EACH ROW EXECUTE FUNCTION trigger_740afa9807b8(); +CREATE TRIGGER trigger_765cae42cd77 BEFORE INSERT OR UPDATE ON bulk_import_trackers FOR EACH ROW EXECUTE FUNCTION trigger_765cae42cd77(); + CREATE TRIGGER trigger_77d9fbad5b12 BEFORE INSERT OR UPDATE ON packages_debian_project_distribution_keys FOR EACH ROW EXECUTE FUNCTION trigger_77d9fbad5b12(); CREATE TRIGGER trigger_78c85ddc4031 BEFORE INSERT OR UPDATE ON issue_emails FOR EACH ROW EXECUTE FUNCTION trigger_78c85ddc4031(); @@ -38102,6 +38161,8 @@ CREATE TRIGGER trigger_7e2eed79e46e BEFORE INSERT OR UPDATE ON abuse_reports FOR CREATE TRIGGER trigger_7f41427eda69 BEFORE UPDATE OF pre_receive_secret_detection_enabled ON application_settings FOR EACH ROW EXECUTE FUNCTION function_for_trigger_7f41427eda69(); +CREATE TRIGGER trigger_7f84f9c7b945 BEFORE INSERT OR UPDATE ON bulk_import_trackers FOR EACH ROW EXECUTE FUNCTION trigger_7f84f9c7b945(); + CREATE TRIGGER trigger_7fbecfcdf89a BEFORE UPDATE OF secret_push_protection_enabled ON project_security_settings FOR EACH ROW EXECUTE FUNCTION function_for_trigger_7fbecfcdf89a(); CREATE TRIGGER trigger_81b4c93e7133 BEFORE INSERT OR UPDATE ON pages_deployment_states FOR EACH ROW EXECUTE FUNCTION trigger_81b4c93e7133(); @@ -38240,6 +38301,8 @@ CREATE TRIGGER trigger_ec1934755627 BEFORE INSERT OR UPDATE ON alert_management_ CREATE TRIGGER trigger_ed554313ca66 BEFORE INSERT OR UPDATE ON protected_branch_unprotect_access_levels FOR EACH ROW EXECUTE FUNCTION trigger_ed554313ca66(); +CREATE TRIGGER trigger_eeb25d23ab2d BEFORE INSERT OR UPDATE ON bulk_import_trackers FOR EACH ROW EXECUTE FUNCTION trigger_eeb25d23ab2d(); + CREATE TRIGGER trigger_efb9d354f05a BEFORE INSERT OR UPDATE ON incident_management_issuable_escalation_statuses FOR EACH ROW EXECUTE FUNCTION trigger_efb9d354f05a(); CREATE TRIGGER trigger_eff80ead42ac BEFORE INSERT OR UPDATE ON ci_unit_test_failures FOR EACH ROW EXECUTE FUNCTION trigger_eff80ead42ac(); @@ -38471,6 +38534,9 @@ ALTER TABLE ONLY member_approvals ALTER TABLE ONLY cluster_agent_migrations ADD CONSTRAINT fk_13af035625 FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE; +ALTER TABLE ONLY bulk_import_trackers + ADD CONSTRAINT fk_13c8d30bfb FOREIGN KEY (organization_id) REFERENCES organizations(id) ON DELETE CASCADE; + ALTER TABLE ONLY project_control_compliance_statuses ADD CONSTRAINT fk_13fb61bcfa FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; @@ -38660,6 +38726,9 @@ ALTER TABLE ONLY agent_group_authorizations ALTER TABLE ONLY deployment_approvals ADD CONSTRAINT fk_2d060dfc73 FOREIGN KEY (deployment_id) REFERENCES deployments(id) ON DELETE CASCADE; +ALTER TABLE ONLY bulk_import_trackers + ADD CONSTRAINT fk_2d0e051bc3 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY audit_events_instance_external_audit_event_destinations ADD CONSTRAINT fk_2d3ebd0fbc FOREIGN KEY (stream_destination_id) REFERENCES audit_events_instance_external_streaming_destinations(id) ON DELETE SET NULL; @@ -39455,6 +39524,9 @@ ALTER TABLE ONLY packages_conan_recipe_revisions ALTER TABLE ONLY epics ADD CONSTRAINT fk_9d480c64b2 FOREIGN KEY (start_date_sourcing_epic_id) REFERENCES epics(id) ON DELETE SET NULL; +ALTER TABLE ONLY bulk_import_trackers + ADD CONSTRAINT fk_9dc0581779 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY user_group_callouts ADD CONSTRAINT fk_9dc8b9d4b2 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; diff --git a/lib/gitlab/background_migration/backfill_bulk_import_trackers_namespace_id.rb b/lib/gitlab/background_migration/backfill_bulk_import_trackers_namespace_id.rb new file mode 100644 index 00000000000..ee833f39988 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_bulk_import_trackers_namespace_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillBulkImportTrackersNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_bulk_import_trackers_namespace_id + feature_category :importers + end + end +end diff --git a/lib/gitlab/background_migration/backfill_bulk_import_trackers_organization_id.rb b/lib/gitlab/background_migration/backfill_bulk_import_trackers_organization_id.rb new file mode 100644 index 00000000000..93468fdd99f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_bulk_import_trackers_organization_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillBulkImportTrackersOrganizationId < BackfillDesiredShardingKeyJob + operation_name :backfill_bulk_import_trackers_organization_id + feature_category :importers + end + end +end diff --git a/lib/gitlab/background_migration/backfill_bulk_import_trackers_project_id.rb b/lib/gitlab/background_migration/backfill_bulk_import_trackers_project_id.rb new file mode 100644 index 00000000000..cfa05fef5a2 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_bulk_import_trackers_project_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillBulkImportTrackersProjectId < BackfillDesiredShardingKeyJob + operation_name :backfill_bulk_import_trackers_project_id + feature_category :importers + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_namespace_id_spec.rb new file mode 100644 index 00000000000..98a3c6142a3 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillBulkImportTrackersNamespaceId, + feature_category: :importers, + schema: 20250205195333 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :bulk_import_trackers } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :bulk_import_entities } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :bulk_import_entity_id } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_organization_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_organization_id_spec.rb new file mode 100644 index 00000000000..ad376abff82 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_organization_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillBulkImportTrackersOrganizationId, + feature_category: :importers, + schema: 20250205195338 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :bulk_import_trackers } + let(:backfill_column) { :organization_id } + let(:backfill_via_table) { :bulk_import_entities } + let(:backfill_via_column) { :organization_id } + let(:backfill_via_foreign_key) { :bulk_import_entity_id } + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_project_id_spec.rb new file mode 100644 index 00000000000..111ffa4d9a1 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_bulk_import_trackers_project_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillBulkImportTrackersProjectId, + feature_category: :importers, + schema: 20250205195328 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :bulk_import_trackers } + let(:backfill_column) { :project_id } + let(:backfill_via_table) { :bulk_import_entities } + let(:backfill_via_column) { :project_id } + let(:backfill_via_foreign_key) { :bulk_import_entity_id } + end +end diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index 9485cc143ed..82ac41ba774 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -201,7 +201,8 @@ RSpec.describe 'new tables missing sharding_key', feature_category: :cell do "oauth_access_grants" => "https://gitlab.com/gitlab-org/gitlab/-/issues/496717", "oauth_openid_requests" => "https://gitlab.com/gitlab-org/gitlab/-/issues/496717", "oauth_device_grants" => "https://gitlab.com/gitlab-org/gitlab/-/issues/496717", - "uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199" + "uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", + "bulk_import_trackers" => "https://gitlab.com/gitlab-org/gitlab/-/issues/517823" } has_lfk = ->(lfks) { lfks.any? { |k| k.options[:column] == 'organization_id' && k.to_table == 'organizations' } } diff --git a/spec/migrations/20250205195332_queue_backfill_bulk_import_trackers_project_id_spec.rb b/spec/migrations/20250205195332_queue_backfill_bulk_import_trackers_project_id_spec.rb new file mode 100644 index 00000000000..1c1bf4c6a83 --- /dev/null +++ b/spec/migrations/20250205195332_queue_backfill_bulk_import_trackers_project_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillBulkImportTrackersProjectId, feature_category: :importers do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :bulk_import_trackers, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :project_id, + :bulk_import_entities, + :project_id, + :bulk_import_entity_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20250205195337_queue_backfill_bulk_import_trackers_namespace_id_spec.rb b/spec/migrations/20250205195337_queue_backfill_bulk_import_trackers_namespace_id_spec.rb new file mode 100644 index 00000000000..c1157199b87 --- /dev/null +++ b/spec/migrations/20250205195337_queue_backfill_bulk_import_trackers_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillBulkImportTrackersNamespaceId, feature_category: :importers do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :bulk_import_trackers, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :bulk_import_entities, + :namespace_id, + :bulk_import_entity_id + ] + ) + } + end + end +end diff --git a/spec/migrations/20250205195342_queue_backfill_bulk_import_trackers_organization_id_spec.rb b/spec/migrations/20250205195342_queue_backfill_bulk_import_trackers_organization_id_spec.rb new file mode 100644 index 00000000000..e2903366ad8 --- /dev/null +++ b/spec/migrations/20250205195342_queue_backfill_bulk_import_trackers_organization_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillBulkImportTrackersOrganizationId, feature_category: :importers do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :bulk_import_trackers, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :organization_id, + :bulk_import_entities, + :organization_id, + :bulk_import_entity_id + ] + ) + } + end + end +end