wget2/.gitlab-ci.yml

# we utilize the images generated by the build-images project, to
# speed up CI runs. We also use ccache and store config.cache
# to speed up compilation. We include a version number in cache
# name to allow expiration of old caches.

stages:
  - tarball
  - stage1-testing
  - stage2-tarball

cache:
  key: "$CI_JOB_NAME-wget2"
  paths:
    - cache/

before_script:
  # CCache Config
  - mkdir -p cache
  - export CCACHE_BASEDIR=${PWD}
  - export CCACHE_DIR=${PWD}/cache
  - echo $CCACHE_DIR
  - export CC="ccache gcc"

after_script:
  # somehow after_script looses environment
  - export CCACHE_BASEDIR=${PWD}
  - export CCACHE_DIR=${PWD}/cache
  - if which ccache >/dev/null; then ccache -s; fi

variables:
  BUILD_IMAGES_PROJECT: gnuwget/build-images
  DEBIAN_BUILD: buildenv-debian-stable
  DEBIAN_TESTING_BUILD: buildenv-debian-testing
  DEBIAN_STABLEi386_BUILD: buildenv-debian-stable-i386
  DEBIAN_UNSTABLE_BUILD: buildenv-debian-unstable
  DEBIAN_HTTP2_BUILD: buildenv-mhd-http2
  FEDORA_BUILD: buildenv-fedora
  CENTOS7_BUILD: buildenv-centos7
  MINGW_BUILD: buildenv-mingw
  MINGW_BUILD_STATIC: buildenv-mingw-static
  ALPINE_BUILD: buildenv-alpine
  ARCH_BUILD: buildenv-arch
  BASIC_BUILD: buildenv-basic
  TARBALL_BUILD: buildenv-tarball
  GET_SOURCES_ATTEMPTS: "3"
  GIT_DEPTH: "5"
  CONFIGURE_BASE_FLAGS: --enable-assert --cache-file cache/config.cache
  CFLAGS_DEFAULT: -O1 -g -ggdb3

sast:
  stage: tarball
  # SAST customization: https://docs.gitlab.com/ee/user/application_security/sast/#customizing-the-sast-settings
  variables:
    SAST_EXCLUDED_PATHS: examples, fuzz, tests, unit-tests
include:
  - template: Security/SAST.gitlab-ci.yml


# We organize the CI runners as thus:
#   1. Debian Stretch Build:
#       * ASan and UBSan builds
#       * make syntax-check
#       * make distcheck
#
#   2. Fedora Build
#       * LLVM/Clang Build
#       * Valgrind Tests
#
#   3. CentOS7 Build
#       * Build with musl / newlib
#       * Minimal Build (All features disabled)
#
#   This way we split most of the different build options across different
#   platforms as well. Other builds we would like to have:
#
#   1. Windows
#   2. OSX
#   3. Documentation Generation and Gitlab pages
#   4. Code coverage stats

Full+VPATH/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  script:
    - contrib/spell-checker
    - export CFLAGS="$CFLAGS_DEFAULT"
    - sed -i 's/dist-lzip//g' configure.ac
    - ./bootstrap --skip-po && touch .manywarnings
    - mkdir vpath && cd vpath
    - ../configure $CONFIGURE_BASE_FLAGS --enable-manylibs --cache-file ../cache/config.cache
    - make syntax-check
    - make -j$(nproc)
    - make -j$(nproc) distcheck
    - mv wget2-*.gz ..
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log
  artifacts:
    expire_in: 2 weeks
    when: on_success
    paths:
      - wget2-*.gz


# Debian Testing
# - Testing dash to detects bashims in configure.ac
Debian/Testing:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - CONFIG_SHELL=/bin/dash dash ./configure $CONFIGURE_BASE_FLAGS -q 2>&1 >/dev/null |
        grep -v "libhsts was not found" | grep . && exit 1
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

# Debian Unstable
Debian/Unstable:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_UNSTABLE_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

Debian/WolfSSL:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_UNSTABLE_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS --with-ssl=wolfssl
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

Debian/OpenSSL:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_UNSTABLE_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS --with-ssl=openssl
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

# In this build we combine
#  * gcc
#  * check, syntax-check
Minimal/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - sed -i 's/dist-lzip//g' configure.ac
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS --disable-doc
        --disable-xattr --with-ssl=none --without-libpsl --without-libnghttp2 --without-bzip2
        --without-gpgme --without-zlib --without-lzma --without-brotlidec --without-libidn2
        --without-libidn --without-libpcre2 --without-libpcre --without-plugin-support
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"
  tags:
    - shared
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

# In this build we combine
#  * clang
#  * ASan, UBSan
#  * check, syntax-check
Sanitizers/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_UNSTABLE_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - export CC="ccache clang"
    - export UBSAN_OPTIONS=print_stacktrace=1:report_error_type=1
    - export LSAN_OPTIONS=suppressions="$PWD/tests/clang-asan-suppressions"
    - export ASAN_SYMBOLIZER_PATH=/usr/bin/llvm-symbolizer
    - ./configure $CONFIGURE_BASE_FLAGS --enable-fsanitize-asan --enable-fsanitize-ubsan
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

Scan-Build/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_UNSTABLE_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - CC=clang ./configure $CONFIGURE_BASE_FLAGS
    - make -C lib -j$(nproc)
    - scan-build -v --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability --status-bugs -o scan-build make -j$(nproc)
    - scan-build -v --use-cc=clang --force-analyze-debug-code --show-description -analyzer-config stable-report-filename=true -enable-checker valist,nullability --status-bugs -o scan-build make -j$(nproc) check
  tags:
    - shared
    - linux
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - scan-build/*

Valgrind/Debian:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po
    - ./configure $CONFIGURE_BASE_FLAGS --enable-valgrind-tests
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

Valgrind/i386:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_STABLEi386_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po
    - ./configure $CONFIGURE_BASE_FLAGS --enable-valgrind-tests --without-gpgme
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

Simple/Fedora:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) -C lib
    - make -j$(nproc) check CFLAGS="$CFLAGS_DEFAULT -Werror"
  tags:
    - shared
    - linux
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

Alpine:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$ALPINE_BUILD
  variables:
    GIT_STRATEGY: none
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - tar xf wget2-*.gz
    - cd wget2-*
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) check
  tags:
    - shared
    - linux
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  dependencies:
    - Build-Tarball
  needs: ["Build-Tarball"]
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

#Centos7 build:
#  stage: stage1-testing
#  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$CENTOS7_BUILD
#  script:
#  - sed -i 's/AM_GNU_GETTEXT_VERSION(0.19.3)/AM_GNU_GETTEXT_VERSION(0.18.2)/g' configure.ac
#  - ./bootstrap && ./configure --enable-gcc-warnings --disable-doc && make -j$(nproc) check
#  tags:
#  - shared
#  artifacts:
#    expire_in: 2 weeks
#    when: on_failure
#    paths:
#      - tests/*.log
#      - compat_reports/

pages:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_TESTING_BUILD
  script:
    - mkdir -p public
    - apt-get -y install libcapture-tiny-perl libdatetime-perl
    - ./bootstrap --skip-po
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc) check-coverage

# Tarball
    - make dist
    - mv wget2-*.gz public/wget2-latest.tar.gz

# Test suite coverage report
    - coverage=$(grep -m 1 'headerCovTableEntry[a-zA-Z].*[0-9]*' lcov/index.html|sed 's/^.*>\([0-9]\+\.[0-9]\+\).*/\1/')
    - coverage=$(printf %.2f $coverage)
    - inum=$(echo $coverage|cut -d'.' -f1)
    - if [ -z "$inum" ]; then inum="0"; fi
    - contrib/make-coverage-badge "coverage" "$coverage%"
    - mv badge.svg public/coverage.svg
    - rm -rf public/coverage
    - mv lcov public/coverage

# Online Documentation
    - rm -rf public/reference
    - mv docs/html public/reference

# Coverage report for all our fuzz corpora
    - make -j$(nproc) fuzz-coverage
    - coverage=$(grep -m 1 'headerCovTableEntry[a-zA-Z].*[0-9]*' lcov/index.html|sed 's/^.*>\([0-9]\+\.[0-9]\+\).*/\1/')
    - coverage=$(printf %.2f $coverage)
    - inum=$(echo $coverage|cut -d'.' -f1)
    - if [ -z "$inum" ]; then inum="0"; fi
    - contrib/make-coverage-badge "fuzz-coverage" "$coverage%"
    - mv badge.svg public/fuzz-coverage.svg
    - rm -rf public/fuzz-coverage
    - mv lcov public/fuzz-coverage
  tags:
    - shared
    - linux
  artifacts:
    when: on_success
    paths:
      - public
  only:
    - master

MinGW64:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$MINGW_BUILD
  script:
    - ./bootstrap --skip-po
    - - export CFLAGS="$CFLAGS_DEFAULT"
    - export CC="ccache $PREFIX-gcc"
    - export GCCLIB=$(dirname $(find /usr/lib/gcc/$PREFIX -name libgcc_s_seh-1.dll|grep posix))
    - export WINEPATH="$WINEPATH;/usr/$PREFIX/bin;/usr/$PREFIX/lib;$PWD/libwget/.libs;$GCCLIB"
    - echo "WINEPATH=$WINEPATH"
    - ./configure $CONFIGURE_BASE_FLAGS --build=x86_64-pc-linux-gnu --host=$PREFIX --enable-shared
    - make -j$(nproc)
    - make check -j$(nproc) LOG_COMPILER=wine
  tags:
    - shared
    - linux
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  allow_failure: true
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./config.h
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log

MinGW64-static:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$MINGW_BUILD_STATIC
  script:
    - unset CC
    - ./bootstrap --skip-po
    - LDFLAGS="-Wl,-Bstatic,--whole-archive -lwinpthread -Wl,--no-whole-archive"
      CFLAGS="$CFLAGS_DEFAULT -DNGHTTP2_STATICLIB"
      ./configure $CONFIGURE_BASE_FLAGS --build=x86_64-pc-linux-gnu --host=$PREFIX --disable-shared --enable-static --without-gpgme --enable-threads=windows
    - make -j$(nproc)
    - mv src/wget2.exe .
  tags:
    - shared
    - linux
  allow_failure: true
  artifacts:
    when: on_success
    paths:
      - wget2.exe
  only:
    - master

Arch:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$ARCH_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - sed -i 's/dist-lzip//g' configure.ac
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS
    - make -j$(nproc)
    - make check -j$(nproc)
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log


# Test building from tarball in a non-dev environment
Tarball:
  stage: stage2-tarball
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$BASIC_BUILD
  variables:
    GIT_STRATEGY: none
  script:
    # remove all files from git repo
    - mv wget2-*.gz /tmp
    - rm -rf * .git* .travis*
    - mv /tmp/wget2-*.gz .
    # unpack and build from tarball
    - tar xvf wget2-*.gz
    - cd wget2-*
    - unset CC
    - ./configure $CONFIGURE_BASE_FLAGS --with-ssl=none --disable-manywarnings
    - make -j$(nproc)
    - make -j$(nproc) check
    # check if essential files are accidentally cleaned
    - make clean
    - make -j$(nproc)
  tags:
    - shared
    - linux
  dependencies:
    - Full+VPATH/Debian
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - wget2-*/*.log
      - wget2-*/tests/*.log
      - wget2-*/unit-tests/*.log
      - wget2-*/fuzz/*.log


CommitCheck:
  stage: tarball
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$BASIC_BUILD
  before_script:
    - /bin/true
  script:
    - ./contrib/commit-check
  after_script:
    - /bin/true
  allow_failure: true
  cache:
      paths:
      policy: push

# Create the tarball for further testing.
# Runs syntax-check, so no need to do this later on.
Build-Tarball:
  stage: tarball
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$TARBALL_BUILD
  cache:
    key: "$CI_JOB_NAME"
    paths:
      - cache/
  script:
    - rmdir gnulib && mv /gnulib . && git submodule update gnulib
    - ./bootstrap
    - mkdir vpath && cd vpath
    - export CC=gcc
    - export CFLAGS="-O0"
    - ../configure --cache-file ../cache/config.cache
      --with-ssl=none
    - make -j$(nproc) syntax-check
    - make -j$(nproc) -C docs
    - make -j$(nproc) dist-gzip
    - mv wget2-*.gz ..
  tags:
    - shared
    - linux
  except:
    - coverity-scan@gnuwget/wget
  dependencies:
    - CommitCheck
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - vpath/fuzz/*.log
      - vpath/tests/*.log
      - vpath/unit-tests/*.log
  artifacts:
    expire_in: 2 weeks
    when: on_success
    paths:
      - wget2-*.gz


Debian/HTTP2:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$DEBIAN_HTTP2_BUILD
  script:
    - export CFLAGS="$CFLAGS_DEFAULT"
    - ./bootstrap --skip-po && touch .manywarnings
    - ./configure $CONFIGURE_BASE_FLAGS --enable-code-coverage
    - make -j$(nproc)
    - make check -j$(nproc)
    - make code-coverage-capture
  tags:
    - shared
    - linux
  allow_failure: true
  except:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 2 weeks
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - unit-tests/*.log
      - tests/*.log
  artifacts:
    expire_in: 2 weeks
    when: on_success
    paths:
      - wget2-*-coverage

# That is a specific runner that we cannot enable universally.
# We restrict it to builds under the $BUILD_IMAGES_PROJECT project.
# The OpenSSL version on this machine is too old to be supported.
FreeBSD.x86_64:
  stage: stage1-testing
  image:
  script:
    - export CC="ccache clang"
    - sed -i.bak 's/dist-lzip//g' configure.ac
    - sed -i.bak '/^python/d' bootstrap.conf
    - sed -i.bak '/^lzip/d' bootstrap.conf
    - sed -i.bak '/^makeinfo/d' bootstrap.conf
    - git config --global http.sslVerify false
    - ./bootstrap --skip-po
    - LIBS="-L/usr/local/lib" ./configure --cache-file cache/config.cache --disable-doc --with-ssl=none
    - gmake -j$(sysctl hw.ncpu | awk '{print $2}')
    - gmake check -j$(sysctl hw.ncpu | awk '{print $2}')
  tags:
    - freebsd
  only:
    - branches@gnuwget/wget2
  allow_failure: true
  except:
    - tags
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 1 week
    when: on_failure
    paths:
      - ./*.log
      - fuzz/*.log
      - tests/*.log
      - unit-tests/*.log

coverity:
  stage: stage1-testing
  image: $CI_REGISTRY/$BUILD_IMAGES_PROJECT:$FEDORA_BUILD
  script:
    - wget https://scan.coverity.com/download/linux64 --post-data "token=$COVERITY_SCAN_TOKEN&project=$COVERITY_SCAN_PROJECT_NAME" -O /tmp/coverity_tool.tgz
    - tar xfz /tmp/coverity_tool.tgz
    - ./bootstrap --skip-po
    - CFLAGS="-g -Og" ./configure --cache-file cache/config.cache --disable-doc
    - cov-analysis-linux64-*/bin/cov-build --dir cov-int make -j$(nproc)
    - tar cfz cov-int.tar.gz cov-int
    - curl https://scan.coverity.com/builds?project=$COVERITY_SCAN_PROJECT_NAME
      --form token=$COVERITY_SCAN_TOKEN --form email=tim.ruehsen@gmx.de
      --form file=@cov-int.tar.gz --form version="`git describe --tags`"
      --form description="CI build"
  tags:
    - shared
    - linux
  only:
    - coverity-scan@gnuwget/wget2
  artifacts:
    expire_in: 1 week
    when: on_failure
    paths:
      - cov-int/*.txt