2025-07-20 16:56:36 +00:00 · 2024-12-03 17:17:17 -08:00
parent 9f5adf0ce4
commit d1ba623677
4 changed files with 144 additions and 398 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -40,10 +40,10 @@ default:
 # submodules (a commit in this repo does not affect their builds anyway) and
 # many components that are otherwise slow to build.
 variables:
-  CMAKE_FLAGS: "-DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_TOKUDB=NO -DWITH_WSREP=OFF"
+  CMAKE_FLAGS: "-DCPACK_RPM_DEBUGINFO_PACKAGE=OFF -DWITH_SSL=system -DPLUGIN_COLUMNSTORE=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_S3=NO -DPLUGIN_MROONGA=NO -DPLUGIN_CONNECT=NO -DPLUGIN_TOKUDB=NO -DWITH_WSREP=OFF"
  # Major version dictates which branches share the same ccache. E.g. 10.6-abc
  # and 10.6-xyz will have the same cache.
-  MARIADB_MAJOR_VERSION: "11.1"
+  MARIADB_MAJOR_VERSION: "11.8"
  # NOTE! Currently ccache is only used on the Centos 9 build. As each job has
  # sufficiently different environments they are unable to benefit from each
  # other's ccaches. As each build generates about 1 GB of ccache, having
@ -59,9 +59,6 @@ variables:
  #
  # Most steps don't need the source code, only artifacts
  GIT_STRATEGY: none
-  # Hack to satisfy directory name length requirement by CPackRPM in CMake 3.x
-  # https://cmake.org/cmake/help/v3.7/module/CPackRPM.html#variable:CPACK_RPM_BUILD_SOURCE_DIRS_PREFIX
-  GIT_CLONE_PATH: $CI_BUILDS_DIR/CPACK_BUILD_SOURCE_DIRS_LONG_NAME_REQUIREMENT

 # Define once, use many times
 .rpm_listfiles: &rpm_listfiles
@ -183,7 +180,7 @@ fedora-sanitizer:
    GIT_SUBMODULE_STRATEGY: normal
  script:
    - yum install -y yum-utils rpm-build openssl-devel clang
-    - yum install -y libasan libtsan libubsan
+    - yum install -y libasan libtsan
    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
    - yum-builddep -y mariadb-server
    - mkdir builddir; cd builddir
@ -207,7 +204,9 @@ fedora-sanitizer:
      - builddir/_CPack_Packages/Linux/RPM/SPECS/
  parallel:
    matrix:
-      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES, -DWITH_UBSAN=YES]
+      - SANITIZER: [-DWITH_ASAN=YES, -DWITH_TSAN=YES]
+      # Removed in 2025 due to failing build: -DWITH_UBSAN=YES
+      # Removed in 2023 commit f4ce1e4: -DWITH_MSAN=YES

 centos9:
  stage: build
@ -249,19 +248,20 @@ centos9:
    paths:
      - .ccache

-centos7:
+amazonlinux2:
  stage: build
-  image: centos:7
+  image: amazonlinux:2
  variables:
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: normal
  script:
-    # This repository does not have any .spec files, so install dependencies based on Fedora spec file
+    # This repository does not have any .spec files, so install dependencies
+    # based on Amazon Linux spec file with a few extra ones, as Amazon Linux 2
+    # is quite old and these are added in newer MariaDB releases.
+    - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel pcre2-devel cmake3
    - yum-builddep -y mariadb-server
-    # ..with a few extra ones, as CentOS 7 is very old and these are added in newer MariaDB releases
-    - yum install -y yum-utils rpm-build gcc gcc-c++ bison libxml2-devel libevent-devel openssl-devel pcre2-devel
    - mkdir builddir; cd builddir
-    - cmake -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
+    - cmake3 -DRPM=$CI_JOB_NAME $CMAKE_FLAGS .. 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    - make package -j 2 2>&1 | tee -a ../build-$CI_JOB_NAME-$CI_COMMIT_REF_SLUG.log
    # @TODO: Don't use -j without the limit of 2 on Gitlab.com as builds just
    # get stuck when running multi-proc and out of memory, see https://jira.mariadb.org/browse/MDEV-25968
@ -285,17 +285,20 @@ centos7:
    cd /usr/share/mariadb-test
    # Skip failing tests
    echo "
-      main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
      main.flush_logs_not_windows : query 'flush logs' succeeded - should have failed with error ER_CANT_CREATE_FILE (1004)
-      main.mysql_upgrade_noengine : upgrade output order does not match the expected
      main.func_math              : MDEV-20966 - Wrong error code
+      main.mysql_upgrade_noengine : upgrade output order does not match the expected
+      main.mysqldump : Field separator argument is not what is expected; check the manual when executing 'SELECT INTO OUTFILE'
+      main.rpl_mysql_upgrade_slave_repo_check : skip for now as regression has gone unnoticed for a long time
+      main.rpl_mysqldump_slave : skip for now as regression has gone unnoticed for a long time
+      main.ssl_timeout : skip for now as regression has gone unnoticed for a long time
+      main.upgrade_MDEV-23102-1 : skip for now as regression has gone unnoticed for a long time
+      main.upgrade_MDEV-23102-2 : skip for now as regression has gone unnoticed for a long time
    " > skiplist
    $SIMULATE_TIME ./mtr --suite=main --force --parallel=auto --xml-report=$CI_PROJECT_DIR/junit.xml --skip-test-list=skiplist $MTR_FLAGS

 mysql-test-run:
  stage: test
-  dependencies:
-    - fedora
  needs:
    - fedora
  script:
@ -310,16 +313,14 @@ mysql-test-run:
 # sanitizer errors separate from functional test failures. Currently, there is no way to run the same
 # job for different dependencies.
 #
-# Additionally, for each sanitizer MTR job, we enable --force-restart so that 
-# sanitizer errors can be traced to individual tests. The difference in test 
+# Additionally, for each sanitizer MTR job, we enable --force-restart so that
+# sanitizer errors can be traced to individual tests. The difference in test
 # suite runtime as a result of this flag is negligable (~30s for the entire test suite).
 # (see https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_MYSQL_TEST_RUN_PL.html)
 mysql-test-run-asan:
  stage: test
  variables:
    MTR_FLAGS: "--force-restart"
-  dependencies:
-    - "fedora-sanitizer: [-DWITH_ASAN=YES]"
  needs:
    - "fedora-sanitizer: [-DWITH_ASAN=YES]"
  script:
@ -335,8 +336,6 @@ mysql-test-run-tsan:
  stage: test
  variables:
    MTR_FLAGS: "--force-restart"
-  dependencies:
-    - "fedora-sanitizer: [-DWITH_TSAN=YES]"
  needs:
    - "fedora-sanitizer: [-DWITH_TSAN=YES]"
  script:
@ -348,49 +347,8 @@ mysql-test-run-tsan:
      junit:
        - junit.xml

-mysql-test-run-ubsan:
-  stage: test
-  variables:
-    MTR_FLAGS: "--force-restart"
-  dependencies:
-    - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
-  needs:
-    - "fedora-sanitizer: [-DWITH_UBSAN=YES]"
-  script:
-    - *mysql-test-run-def
-  allow_failure: true
-  artifacts:
-    when: always  # Also show results when tests fail
-    reports:
-      junit:
-        - junit.xml
-
-mysql-test-run-future:
-  stage: test
-  variables:
-    MTR_FLAGS: "--max-test-fail=0"
-  dependencies:
-    - fedora
-  needs:
-    - fedora
-  script:
-    - yum install -y faketime
-    - *mysql-test-run-def
-  artifacts:
-    when: always  # Also show results when tests fail
-    reports:
-      junit:
-        - junit.xml
-  parallel:
-    matrix:
-    # The reason we test in two different future times is for the ability to separate between 
-    # failures pertaining to general time expiry issues and failures specific to the 32-bit time/Y2038 problem
-      - SIMULATE_TIME: ["faketime 2038-01-20",  "faketime 2038-01-18"]
-
 rpmlint:
  stage: test
-  dependencies:
-    - fedora
  needs:
    - fedora
  script:
@ -408,8 +366,6 @@ rpmlint:

 fedora install:
  stage: test
-  dependencies:
-    - fedora
  needs:
    - fedora
  script:
@ -453,15 +409,18 @@ cppcheck:
    # --template: output format
    # --force: check large directories without warning
    # -i<directory>: ignore this directory when scanning
-    # -I<directory>: include path, reduces false positives 
+    # -I<directory>: include path, reduces false positives
    #                related to inability to resolve symbols
    # -j: run multiple cppcheck threads
+    #
+    # Note: Scope reduced heavily with -i to keep cppcheck run under 3h
+    #
    # Use newline to escape colon in yaml
    - >
      cppcheck --template="{file}:{line}\n{code}\n{severity}: {message}" --force --check-level=exhaustive
      client dbug extra include libmariadb libmysqld libservices mysql-test mysys mysys_ssl pcre plugin
      strings tests unittest vio wsrep-lib sql sql-common storage
-      -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore
+      -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore -istorage/connect -istorage/myisam -istorage/perfschema
      -Iinclude -Istorage/innobase/include
      --output-file=initial-cppcheck_output.txt -j $(nproc)
    # when including {code} in the cppcheck template, some more pre-processing needs to be done
@ -483,9 +442,9 @@ cppcheck:
      cat preprocessed-cppcheck_sorted.txt | grep -v "There is an unknown macro here somewhere" > results-cppcheck_all_findings.txt
      sed 's/:[0-9]\+//' results-cppcheck_all_findings.txt > preprocessed_final-cppcheck_no_line_nums.txt
    # Only print new issues not found in ignore list
-    - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed)."
+    - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed):"
    - diff --changed-group-format='%>' --unchanged-group-format='' preprocessed_final-cppcheck_no_line_nums.txt tests/code_quality/cppcheck_ignorelist.txt || true
-    - echo "Problems found by cppcheck that were not in ignore list."
+    - echo "Problems found by cppcheck that were not in ignore list:"
    - diff --changed-group-format='%<' --unchanged-group-format='' preprocessed_final-cppcheck_no_line_nums.txt tests/code_quality/cppcheck_ignorelist.txt > results-cppcheck_new_findings.txt || true
    - cat results-cppcheck_new_findings.txt && test ! -s results-cppcheck_new_findings.txt
  artifacts:
@ -499,7 +458,7 @@ cppcheck:
      - results-cppcheck_all_findings.txt
      - results-cppcheck_new_findings.txt

-flawfinder: 
+flawfinder:
  allow_failure: true
  stage: sast
  needs: []
@ -530,11 +489,9 @@ flawfinder:
      - flawfinder_new_findings.txt
      - flawfinder-all-vulnerabilities.html
      - flawfinder-min-level5.json
-      
+
 mini-benchmark:
  stage: test
-  dependencies:
-    - fedora
  needs:
    - fedora
  script:
@ -562,91 +519,25 @@ mini-benchmark:
      metrics:
        - metrics.txt

-cppcheck: 
-  stage: sast
-  needs: []
-  variables:
-    GIT_STRATEGY: fetch
-    GIT_SUBMODULE_STRATEGY: normal
-  script:
-    - yum install -y cppcheck diffutils
-    # --template: use a single-line template
-    # --force: check large directories without warning
-    # -i<directory>: ignore this directory when scanning
-    # -j: run multiple cppcheck threads
-    # Use newline to escape colon in yaml
-    - >
-      cppcheck --template="{file}:{line}: {severity}: {message}" --force
-      client dbug extra include libmariadb libmysqld libservices mysql-test mysys mysys_ssl pcre plugin
-      strings tests unittest vio wsrep-lib sql sql-common storage
-      -istorage/mroonga -istorage/tokudb -istorage/spider -istorage/rocksdb -iextra/ -ilibmariadb/ -istorage/columnstore
-      --output-file=cppcheck.txt -j $(nproc)
-    # Parallel jobs may output findings in an nondeterministic order. Sort to match ignorelist.
-    - cat cppcheck.txt | sort > cppcheck_sorted.txt
-    # Remove line numbers for diff
-    - sed 's/:[^:]*:/:/' cppcheck_sorted.txt > cppcheck_sorted_no_line_numbers.txt
-    # Only print new issues not found in ignore list
-    - echo "Problems found in ignore list that were not discovered by cppcheck (may have been fixed)."
-    - diff --changed-group-format='%>' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt || true
-    - echo "Problems found by cppcheck that were not in ignore list."
-    - diff --changed-group-format='%<' --unchanged-group-format='' cppcheck_sorted_no_line_numbers.txt tests/code_quality/cppcheck_ignorelist.txt > lines_not_ignored.txt || true
-    - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
-  artifacts:
-    when: always
-    paths:
-      - cppcheck_sorted.txt
-
-flawfinder:
-  stage: sast
-  needs: []
-  variables:
-    GIT_STRATEGY: fetch
-    GIT_SUBMODULE_STRATEGY: normal
-  script:
-    - yum install -y python3 python3-pip jq diffutils git
-    - pip install flawfinder
-    - flawfinder --falsepositive --quiet --html . > flawfinder-all-vulnerabilities.html
-    - cat flawfinder-all-vulnerabilities.html | grep "Hits ="
-    - flawfinder --falsepositive --quiet --minlevel=5 --sarif . > flawfinder-output.json
-    # FlawFinder's --sarif output will display all vulnerabilities despite having --minlevel=5 specified.
-    # Therefore, we postprocess the results with jq and filter out findings where the vulnerability level is less than 5.
-    # Also in the SARIF output format, the vulnerabilities are ranked as 0.2/0.4/0.6/0.8/1.0 which correspond to the --minlevel=1/2/3/4/5 of FlawFinder.
-    # Additionally, we sort the results because individual findings are consistent across different runs, but their ordering may not be.
-    # Vulnerabilities can also be ignored in-line (/* Flawfinder: ignore */), but this option was chosen as to not clutter the codebase.
-    - jq 'del(.runs[] | .tool | .driver | .rules) | del(.runs[] | .results[] | select(.rank < 1)) | del(.runs[] | .results[] | .locations[] | .physicalLocation | .region | .startLine) | .runs[0].results|=sort_by(.fingerprints)' flawfinder-output.json > flawfinder-min-level5.json
-    # Diff against known vulnerabilities, but ignore the line number.
-    - echo "Problems found in ignore list that were not discovered by flawfinder (may have been fixed)."
-    - diff --changed-group-format='%>' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json || true
-    - echo "Problems found by flawfinder that were not in ignore list."
-    - diff --changed-group-format='%<' --unchanged-group-format='' flawfinder-min-level5.json tests/code_quality/flawfinder_ignorelist.json > lines_not_ignored.txt || true
-    - cat lines_not_ignored.txt && test ! -s lines_not_ignored.txt
-  artifacts:
-    when: always
-    paths:
-      - flawfinder-all-vulnerabilities.html
-      - flawfinder-min-level5.json
-      
 fedora upgrade-from:
  stage: test
  image: fedora:latest
  variables:
    GIT_STRATEGY: pull
-  dependencies:
-    - fedora
  needs:
    - fedora
  parallel:
    matrix:
      # Get latest versions of all major versions
-      - UPGRADE_PATH: 
-        - "10.6>11.0"
+      - UPGRADE_PATH:
+        - "10.6>11.4"
        - "10.3"
        - "10.4"
        - "10.4.8" # Test unexpected table rebuilds as in https://jira.mariadb.org/browse/MDEV-28727
        - "10.5"
        - "10.6"
        - "10.11"
-        - "11.0"
+        - "11.4"
  script:
    - |
      if [[ $UPGRADE_PATH == *">"* ]]; then