Test both PostgreSQL and MySQL for the win.

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: "dev.gitlab.org:5005/gitlab/gitlab-build-images:ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1"
+image: "registry.gitlab.com/godfat/gitlab-build-images:ruby-2.3.3-git-2.7-phantomjs-2.1-node-7.1-postgresql-9.6"
 
 cache:
   key: "ruby-233"
@@ -9,19 +9,14 @@ variables:
   MYSQL_ALLOW_EMPTY_PASSWORD: "1"
   RAILS_ENV: "test"
   SIMPLECOV: "true"
-  SETUP_DB: "true"
-  USE_BUNDLE_INSTALL: "true"
   GIT_DEPTH: "20"
   PHANTOMJS_VERSION: "2.1.1"
   GET_SOURCES_ATTEMPTS: "3"
 
 before_script:
-  - source ./scripts/prepare_build.sh
-  - cp config/gitlab.yml.example config/gitlab.yml
   - bundle --version
-  - '[ "$USE_BUNDLE_INSTALL" != "true" ] || retry bundle install --without postgres production --jobs $(nproc) --clean $FLAGS'
+  - . scripts/utils.sh
-  - retry gem install knapsack
+  - ./scripts/prepare_build.sh
-  - '[ "$SETUP_DB" != "true" ] || bundle exec rake db:drop db:create db:schema:load db:migrate add_limits_mysql'
 
 stages:
 - prepare
@@ -48,19 +43,27 @@ stages:
     paths:
     - knapsack/
 
-.use-db: &use-db
+.use-pg: &use-pg
+  services:
+    - postgres:latest
+    - redis:alpine
+  variables:
+    GITLAB_DATABASE: "postgresql"
+
+.use-mysql: &use-mysql
   services:
     - mysql:latest
     - redis:alpine
+  variables:
+    GITLAB_DATABASE: "mysql"
 
 .rspec-knapsack: &rspec-knapsack
   stage: test
   <<: *dedicated-runner
-  <<: *use-db
   script:
     - JOB_NAME=( $CI_JOB_NAME )
-    - export CI_NODE_INDEX=${JOB_NAME[1]}
+    - export CI_NODE_INDEX=${JOB_NAME[-2]}
-    - export CI_NODE_TOTAL=${JOB_NAME[2]}
+    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
     - export KNAPSACK_REPORT_PATH=knapsack/rspec_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
     - cp knapsack/rspec_report.json ${KNAPSACK_REPORT_PATH}
@@ -73,14 +76,21 @@ stages:
     - knapsack/
     - tmp/capybara/
 
+.rspec-knapsack-pg: &rspec-knapsack-pg
+  <<: *rspec-knapsack
+  <<: *use-pg
+
+.rspec-knapsack-mysql: &rspec-knapsack-mysql
+  <<: *rspec-knapsack
+  <<: *use-mysql
+
 .spinach-knapsack: &spinach-knapsack
   stage: test
   <<: *dedicated-runner
-  <<: *use-db
   script:
     - JOB_NAME=( $CI_JOB_NAME )
-    - export CI_NODE_INDEX=${JOB_NAME[1]}
+    - export CI_NODE_INDEX=${JOB_NAME[-2]}
-    - export CI_NODE_TOTAL=${JOB_NAME[2]}
+    - export CI_NODE_TOTAL=${JOB_NAME[-1]}
     - export KNAPSACK_REPORT_PATH=knapsack/spinach_node_${CI_NODE_INDEX}_${CI_NODE_TOTAL}_report.json
     - export KNAPSACK_GENERATE_REPORT=true
     - cp knapsack/spinach_report.json ${KNAPSACK_REPORT_PATH}
@@ -93,6 +103,14 @@ stages:
     - knapsack/
     - tmp/capybara/
 
+.spinach-knapsack-pg: &spinach-knapsack-pg
+  <<: *spinach-knapsack
+  <<: *use-pg
+
+.spinach-knapsack-mysql: &spinach-knapsack-mysql
+  <<: *spinach-knapsack
+  <<: *use-mysql
+
 # Prepare and merge knapsack tests
 
 knapsack:
@@ -105,7 +123,7 @@ knapsack:
     - '[[ -f knapsack/spinach_report.json ]] || echo "{}" > knapsack/spinach_report.json'
 
 setup-test-env:
-  <<: *use-db
+  <<: *use-pg
   <<: *dedicated-runner
   stage: prepare
   script:
@@ -136,37 +154,69 @@ update-knapsack:
     - master@gitlab/gitlabhq
     - master@gitlab/gitlab-ee
 
-rspec 0 20: *rspec-knapsack
+rspec pg 0 20: *rspec-knapsack-pg
-rspec 1 20: *rspec-knapsack
+rspec pg 1 20: *rspec-knapsack-pg
-rspec 2 20: *rspec-knapsack
+rspec pg 2 20: *rspec-knapsack-pg
-rspec 3 20: *rspec-knapsack
+rspec pg 3 20: *rspec-knapsack-pg
-rspec 4 20: *rspec-knapsack
+rspec pg 4 20: *rspec-knapsack-pg
-rspec 5 20: *rspec-knapsack
+rspec pg 5 20: *rspec-knapsack-pg
-rspec 6 20: *rspec-knapsack
+rspec pg 6 20: *rspec-knapsack-pg
-rspec 7 20: *rspec-knapsack
+rspec pg 7 20: *rspec-knapsack-pg
-rspec 8 20: *rspec-knapsack
+rspec pg 8 20: *rspec-knapsack-pg
-rspec 9 20: *rspec-knapsack
+rspec pg 9 20: *rspec-knapsack-pg
-rspec 10 20: *rspec-knapsack
+rspec pg 10 20: *rspec-knapsack-pg
-rspec 11 20: *rspec-knapsack
+rspec pg 11 20: *rspec-knapsack-pg
-rspec 12 20: *rspec-knapsack
+rspec pg 12 20: *rspec-knapsack-pg
-rspec 13 20: *rspec-knapsack
+rspec pg 13 20: *rspec-knapsack-pg
-rspec 14 20: *rspec-knapsack
+rspec pg 14 20: *rspec-knapsack-pg
-rspec 15 20: *rspec-knapsack
+rspec pg 15 20: *rspec-knapsack-pg
-rspec 16 20: *rspec-knapsack
+rspec pg 16 20: *rspec-knapsack-pg
-rspec 17 20: *rspec-knapsack
+rspec pg 17 20: *rspec-knapsack-pg
-rspec 18 20: *rspec-knapsack
+rspec pg 18 20: *rspec-knapsack-pg
-rspec 19 20: *rspec-knapsack
+rspec pg 19 20: *rspec-knapsack-pg
 
-spinach 0 10: *spinach-knapsack
+rspec mysql 0 20: *rspec-knapsack-mysql
-spinach 1 10: *spinach-knapsack
+rspec mysql 1 20: *rspec-knapsack-mysql
-spinach 2 10: *spinach-knapsack
+rspec mysql 2 20: *rspec-knapsack-mysql
-spinach 3 10: *spinach-knapsack
+rspec mysql 3 20: *rspec-knapsack-mysql
-spinach 4 10: *spinach-knapsack
+rspec mysql 4 20: *rspec-knapsack-mysql
-spinach 5 10: *spinach-knapsack
+rspec mysql 5 20: *rspec-knapsack-mysql
-spinach 6 10: *spinach-knapsack
+rspec mysql 6 20: *rspec-knapsack-mysql
-spinach 7 10: *spinach-knapsack
+rspec mysql 7 20: *rspec-knapsack-mysql
-spinach 8 10: *spinach-knapsack
+rspec mysql 8 20: *rspec-knapsack-mysql
-spinach 9 10: *spinach-knapsack
+rspec mysql 9 20: *rspec-knapsack-mysql
+rspec mysql 10 20: *rspec-knapsack-mysql
+rspec mysql 11 20: *rspec-knapsack-mysql
+rspec mysql 12 20: *rspec-knapsack-mysql
+rspec mysql 13 20: *rspec-knapsack-mysql
+rspec mysql 14 20: *rspec-knapsack-mysql
+rspec mysql 15 20: *rspec-knapsack-mysql
+rspec mysql 16 20: *rspec-knapsack-mysql
+rspec mysql 17 20: *rspec-knapsack-mysql
+rspec mysql 18 20: *rspec-knapsack-mysql
+rspec mysql 19 20: *rspec-knapsack-mysql
+
+spinach pg 0 10: *spinach-knapsack-pg
+spinach pg 1 10: *spinach-knapsack-pg
+spinach pg 2 10: *spinach-knapsack-pg
+spinach pg 3 10: *spinach-knapsack-pg
+spinach pg 4 10: *spinach-knapsack-pg
+spinach pg 5 10: *spinach-knapsack-pg
+spinach pg 6 10: *spinach-knapsack-pg
+spinach pg 7 10: *spinach-knapsack-pg
+spinach pg 8 10: *spinach-knapsack-pg
+spinach pg 9 10: *spinach-knapsack-pg
+
+spinach mysql 0 10: *spinach-knapsack-mysql
+spinach mysql 1 10: *spinach-knapsack-mysql
+spinach mysql 2 10: *spinach-knapsack-mysql
+spinach mysql 3 10: *spinach-knapsack-mysql
+spinach mysql 4 10: *spinach-knapsack-mysql
+spinach mysql 5 10: *spinach-knapsack-mysql
+spinach mysql 6 10: *spinach-knapsack-mysql
+spinach mysql 7 10: *spinach-knapsack-mysql
+spinach mysql 8 10: *spinach-knapsack-mysql
+spinach mysql 9 10: *spinach-knapsack-mysql
 
 # Other generic tests
 .ruby-static-analysis: &ruby-static-analysis
@@ -217,29 +267,43 @@ rake ee_compat_check:
     paths:
       - ee_compat_check/patches/*.patch
 
-rake db:migrate:reset:
+.db-migrate-reset: &db-migrate-reset
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   script:
     - bundle exec rake db:migrate:reset
 
-rake db:rollback:
+pg rake db:migrate:reset:
+  <<: *db-migrate-reset
+  <<: *use-pg
+
+mysql rake db:migrate:reset:
+  <<: *db-migrate-reset
+  <<: *use-mysql
+
+.db-rollback: &db-rollback
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
   script:
     - bundle exec rake db:rollback STEP=120
     - bundle exec rake db:migrate
 
-rake db:seed_fu:
+pg rake db:rollback:
-  stage: test
+  <<: *db-rollback
-  <<: *use-db
+  <<: *use-pg
-  <<: *dedicated-runner
+
-  variables:
+mysql rake db:rollback:
+  <<: *db-rollback
+  <<: *use-mysql
+
+.db-seed_fu-variables: &db-seed_fu-variables
   SIZE: "1"
   SETUP_DB: "false"
   RAILS_ENV: "development"
+
+.db-seed_fu: &db-seed_fu
+  stage: test
+  <<: *dedicated-runner
   script:
     - git clone https://gitlab.com/gitlab-org/gitlab-test.git
        /home/git/repositories/gitlab-org/gitlab-test.git
@@ -250,6 +314,20 @@ rake db:seed_fu:
     paths:
       - log/development.log
 
+pg rake db:seed_fu:
+  <<: *db-seed_fu
+  <<: *use-pg
+  variables:
+    <<: *db-seed_fu-variables
+    GITLAB_DATABASE: "postgresql"
+
+mysql rake db:seed_fu:
+  <<: *db-seed_fu
+  <<: *use-mysql
+  variables:
+    <<: *db-seed_fu-variables
+    GITLAB_DATABASE: "mysql"
+
 rake gitlab:assets:compile:
   stage: test
   <<: *dedicated-runner
@@ -275,7 +353,7 @@ rake karma:
       - vendor/ruby
       - node_modules
   stage: test
-  <<: *use-db
+  <<: *use-pg
   <<: *dedicated-runner
   variables:
     BABEL_ENV: "coverage"
@@ -314,12 +392,9 @@ bundler:audit:
   script:
     - "bundle exec bundle-audit check --update"
 
-migration paths:
+.migration-paths: &migration-paths
   stage: test
-  <<: *use-db
   <<: *dedicated-runner
-  variables:
-    SETUP_DB: "false"
   only:
     - master@gitlab-org/gitlab-ce
     - master@gitlab-org/gitlab-ee
@@ -330,13 +405,27 @@ migration paths:
     - git checkout -f FETCH_HEAD
     - cp config/resque.yml.example config/resque.yml
     - sed -i 's/localhost/redis/g' config/resque.yml
-    - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3
+    - bundle install --without production --jobs $(nproc) $FLAGS --retry=3
     - bundle exec rake db:drop db:create db:schema:load db:seed_fu
     - git checkout $CI_COMMIT_SHA
-    - bundle install --without postgres production --jobs $(nproc) $FLAGS --retry=3
+    - bundle install --without production --jobs $(nproc) $FLAGS --retry=3
     - source scripts/prepare_build.sh
     - bundle exec rake db:migrate
 
+pg migration paths:
+  <<: *migration-paths
+  <<: *use-pg
+  variables:
+    SETUP_DB: "false"
+    GITLAB_DATABASE: "postgresql"
+
+mysql migration paths:
+  <<: *migration-paths
+  <<: *use-mysql
+  variables:
+    SETUP_DB: "false"
+    GITLAB_DATABASE: "mysql"
+
 coverage:
   stage: post-test
   services: []
@@ -388,7 +477,7 @@ trigger_docs:
   before_script:
     - apk update && apk add curl
   variables:
-    GIT_STRATEGY: none
+    GIT_STRATEGY: "none"
   cache: {}
   artifacts: {}
   script:

config/database.yml.postgresql

@@ -21,6 +21,7 @@ development:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 #
 # Staging specific
@@ -32,6 +33,7 @@ staging:
   pool: 5
   username: postgres
   password:
+  # host: localhost
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
@@ -43,3 +45,4 @@ test: &test
   pool: 5
   username: postgres
   password:
+  # host: localhost

scripts/prepare_build.sh

@@ -1,25 +1,21 @@
 #!/bin/sh
 
-retry() {
+. scripts/utils.sh
-    if eval "$@"; then
-        return 0
-    fi
 
-    for i in 2 1; do
+export SETUP_DB=${SETUP_DB:-true}
-        sleep 3s
+export GITLAB_DATABASE=${GITLAB_DATABASE:-postgresql}
-        echo "Retrying $i..."
+export USE_BUNDLE_INSTALL=${USE_BUNDLE_INSTALL:-true}
-        if eval "$@"; then
-            return 0
-        fi
-    done
-    return 1
-}
 
 if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then
-    cp config/database.yml.mysql config/database.yml
+    cp config/database.yml.$GITLAB_DATABASE config/database.yml
+
+    if [ "$GITLAB_DATABASE" = 'postgresql' ]; then
+        sed -i 's/# host:.*/host: postgres/g' config/database.yml
+    else # assume it's mysql
         sed -i 's/username:.*/username: root/g' config/database.yml
         sed -i 's/password:.*/password:/g' config/database.yml
         sed -i 's/# socket:.*/host: mysql/g' config/database.yml
+    fi
 
     cp config/resque.yml.example config/resque.yml
     sed -i 's/localhost/redis/g' config/resque.yml
@@ -28,8 +24,24 @@ if [ -f /.dockerenv ] || [ -f ./dockerinit ]; then
 else
     rnd=$(awk 'BEGIN { srand() ; printf("%d\n",rand()*5) }')
     export PATH="$HOME/bin:/usr/local/bin:/usr/bin:/bin"
-    cp config/database.yml.mysql config/database.yml
+    cp config/database.yml.$GITLAB_DATABASE config/database.yml
     sed "s/username\:.*$/username\: runner/" -i config/database.yml
     sed "s/password\:.*$/password\: 'password'/" -i config/database.yml
     sed "s/gitlabhq_test/gitlabhq_test_$rnd/" -i config/database.yml
 fi
+
+cp config/gitlab.yml.example config/gitlab.yml
+
+if [ "$USE_BUNDLE_INSTALL" != "false" ]; then
+    retry bundle install --without production --jobs $(nproc) --clean $FLAGS
+fi
+
+retry gem install knapsack
+
+if [ "$SETUP_DB" != "false" ]; then
+    bundle exec rake db:drop db:create db:schema:load db:migrate
+
+    if [ "$GITLAB_DATABASE" = "mysql" ]; then
+        bundle exec rake add_limits_mysql
+    fi
+fi

scripts/utils.sh

@@ -0,0 +1,14 @@
+retry() {
+    if eval "$@"; then
+        return 0
+    fi
+
+    for i in 2 1; do
+        sleep 3s
+        echo "Retrying $i..."
+        if eval "$@"; then
+            return 0
+        fi
+    done
+    return 1
+}