gitlab-ce/scripts/lint-doc.sh

#!/usr/bin/env bash
set -o pipefail

COLOR_RED="\e[31m"
COLOR_GREEN="\e[32m"
COLOR_RESET="\e[39m"

cd "$(dirname "$0")/.." || exit 1
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Linting documents at path $(pwd) as $(whoami)...${COLOR_RESET}\n"
ERRORCODE=0

# Use long options (e.g. --header instead of -H) for curl examples in documentation.
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for cURL short options...${COLOR_RESET}\n"
if grep --extended-regexp --recursive --color=auto 'curl (.+ )?-[^- ].*' doc/ >/dev/null 2>&1;
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: Short options for curl should not be used in documentation!${COLOR_RESET}"
  printf " Use long options (for example, --header instead of -H):\n" >&2
  grep --extended-regexp --recursive --color=auto 'curl (.+ )?-[^- ].*' doc
  ((ERRORCODE++))
fi

# Documentation pages need front matter for tracking purposes.
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking documentation for front matter...${COLOR_RESET}\n"
if ! scripts/lint-docs-metadata.sh
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: These documentation pages need front matter!${COLOR_RESET}"
  printf " See https://docs.gitlab.com/ee/development/documentation/index.html#stage-and-group-metadata for how to add it.\n" >&2
  ((ERRORCODE++))
fi

# Test for non-standard spaces (NBSP, NNBSP, ZWSP) in documentation.
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for non-standard spaces...${COLOR_RESET}\n"
if grep --extended-regexp --binary-file=without-match --recursive '[  ​]' doc/ >/dev/null 2>&1;
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: Non-standard spaces (NBSP, NNBSP, ZWSP) should not be used in documentation!${COLOR_RESET}"
  printf " https://docs.gitlab.com/ee/development/documentation/styleguide/index.html#spaces-between-words\n"
  printf "Replace with standard spaces:\n" >&2
  # Find the spaces, then add color codes with sed to highlight each NBSP or NNBSP in the output.
  # shellcheck disable=SC1018
  grep --extended-regexp --binary-file=without-match --recursive --color=auto '[  ]' doc \
       | sed -e ''/ /s//"$(printf "\033[0;101m \033[0m")"/'' -e ''/ /s//"$(printf "\033[0;101m \033[0m")"/''
  ((ERRORCODE++))
fi

# Ensure that the CHANGELOG.md does not contain duplicate versions
DUPLICATE_CHANGELOG_VERSIONS=$(grep --extended-regexp '^## .+' CHANGELOG.md | sed -E 's| \(.+\)||' | sort -r | uniq -d)
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for CHANGELOG.md duplicate entries...${COLOR_RESET}\n"
if [ "${DUPLICATE_CHANGELOG_VERSIONS}" != "" ]
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: Duplicate versions in CHANGELOG.md:${COLOR_RESET}\n" >&2
  echo "${DUPLICATE_CHANGELOG_VERSIONS}" >&2
  ((ERRORCODE++))
fi

# Make sure no files in doc/ are executable
EXEC_PERM_COUNT=$(find doc/ -type f -perm 755 | wc -l)
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking $(pwd)/doc for executable permissions...${COLOR_RESET}\n"
if [ "${EXEC_PERM_COUNT}" -ne 0 ]
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: Executable permissions should not be used in documentation!${COLOR_RESET} Use 'chmod 644' on these files:\n" >&2
  find doc -type f -perm 755
  ((ERRORCODE++))
fi

# Do not use 'README.md', instead use 'index.md'
# Number of 'README.md's as of 2021-08-17
NUMBER_READMES=0
FIND_READMES=$(find doc/ -name "README.md" | wc -l)
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for new README.md files...${COLOR_RESET}\n"
if [ "${FIND_READMES}" -ne $NUMBER_READMES ]
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: The number of README.md files has changed!${COLOR_RESET} Use index.md instead of README.md.\n" >&2
  printf "If removing a README.md file, update NUMBER_READMES in lint-doc.sh.\n" >&2
  printf "https://docs.gitlab.com/ee/development/documentation/site_architecture/folder_structure.html#work-with-directories-and-files\n"
  ((ERRORCODE++))
fi

# Do not use dashes (-) in directory names, use underscores (_) instead.
# Number of directories with dashes as of 2021-09-17
NUMBER_DASHES=2
FIND_DASHES=$(find doc -type d -name "*-*" | wc -l)
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for directory names containing dashes...${COLOR_RESET}\n"
if [ "${FIND_DASHES}" -ne $NUMBER_DASHES ]
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: The number of directory names containing dashes has changed!${COLOR_RESET} Use underscores instead of dashes for the directory names.\n" >&2
  printf "If removing a directory containing dashes, update NUMBER_DASHES in lint-doc.sh.\n" >&2
  printf "https://docs.gitlab.com/ee/development/documentation/site_architecture/folder_structure.html#work-with-directories-and-files\n"
   ((ERRORCODE++))
fi

# Do not use uppercase letters in directory and file names, use all lowercase instead.
# (find always returns 0, so we use the grep hack https://serverfault.com/a/225827)
FIND_UPPERCASE_DIRS=$(find doc -type d -name "*[[:upper:]]*")
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for directory names containing an uppercase letter...${COLOR_RESET}\n"
if echo "${FIND_UPPERCASE_DIRS}" | grep . &>/dev/null
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: Found one or more directories with an uppercase letter in their name!${COLOR_RESET} Use lowercase instead of uppercase for the directory names.\n" >&2
  printf "https://docs.gitlab.com/ee/development/documentation/site_architecture/folder_structure.html#work-with-directories-and-files\n" >&2
  echo "${FIND_UPPERCASE_DIRS}"
  ((ERRORCODE++))
fi

FIND_UPPERCASE_FILES=$(find doc -type f -name "*[[:upper:]]*.md")
# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Checking for file names containing an uppercase letter...${COLOR_RESET}\n"
if echo "${FIND_UPPERCASE_FILES}" | grep . &>/dev/null
then
  # shellcheck disable=2059
  printf "${COLOR_RED}ERROR: Found one or more file names with an uppercase letter in their name!${COLOR_RESET} Use lowercase instead of uppercase for the file names.\n" >&2
  printf "https://docs.gitlab.com/ee/development/documentation/site_architecture/folder_structure.html#work-with-directories-and-files\n" >&2
  echo "${FIND_UPPERCASE_FILES}"
  ((ERRORCODE++))
fi

# Run Vale and Markdownlint only on changed files. Only works on merged results
# pipelines, so first checks if a merged results CI variable is present. If not present,
# runs test on all files.
if [ -z "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" ]
then
  MD_DOC_PATH=${MD_DOC_PATH:-doc}
  # shellcheck disable=2059
  printf "${COLOR_GREEN}INFO: Merge request pipeline (detached) detected. Running Markdownlint and Vale on all files...${COLOR_RESET}\n"
else
  MERGE_BASE=$(git merge-base "${CI_MERGE_REQUEST_TARGET_BRANCH_SHA}" "${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}")
  if git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" | grep -E "\.vale|\.markdownlint|lint-doc\.sh|docs\.gitlab-ci\.yml"
  then
    MD_DOC_PATH=${MD_DOC_PATH:-doc}
    # shellcheck disable=2059
    printf "${COLOR_GREEN}INFO: Vale, Markdownlint, lint-doc.sh, or pipeline configuration changed. Testing all files.${COLOR_RESET}\n"
  else
    MD_DOC_PATH=$(git diff --diff-filter=d --name-only "${MERGE_BASE}..${CI_MERGE_REQUEST_SOURCE_BRANCH_SHA}" -- 'doc/*.md')
    if [ -n "${MD_DOC_PATH}" ]
    then
      # shellcheck disable=2059
      printf "${COLOR_GREEN}INFO: Merged results pipeline detected. Testing only the following files:${COLOR_RESET}\n${MD_DOC_PATH}"
    fi
  fi
fi

function run_locally_or_in_container() {
  local cmd=$1
  local args=$2
  local registry_url="registry.gitlab.com/gitlab-org/gitlab-docs/lint-markdown:alpine-3.16-vale-2.22.0-markdownlint-0.32.2-markdownlint2-0.6.0"

  if hash "${cmd}" 2>/dev/null
  then
    # shellcheck disable=2059
    printf "${COLOR_GREEN}INFO: Found locally-installed ${cmd}! Running...${COLOR_RESET}\n"
    $cmd $args
  # When using software like Rancher Desktop, both nerdctl and docker binaries are available
  # but only one is configured. To check which one to use, we need to probe each runtime
  elif (hash nerdctl 2>/dev/null) && (nerdctl info > /dev/null 2>&1)
  then
    # shellcheck disable=2059
    printf "${COLOR_GREEN}INFO: Found nerdctl! Using linting image to run ${cmd}...${COLOR_RESET}\n"
    nerdctl run -t -v "${PWD}:/gitlab" -w /gitlab --rm ${registry_url} ${cmd} ${args}
  elif (hash docker 2>/dev/null) && (docker info > /dev/null 2>&1)
  then
    # shellcheck disable=2059
    printf "${COLOR_GREEN}INFO: Found docker! Using linting image to run ${cmd}...${COLOR_RESET}\n"
    docker run -t -v "${PWD}:/gitlab" -w /gitlab --rm ${registry_url} ${cmd} ${args}
  else
    # shellcheck disable=2059
    printf "${COLOR_RED}ERROR: '${cmd}' not found!${COLOR_RESET} Install '${cmd}' locally, or install a container runtime (docker or nerdctl) and try again.\n" >&2
    ((ERRORCODE++))
  fi

  if [ $? -ne 0 ]
  then
    # shellcheck disable=2059
    printf "${COLOR_RED}ERROR: '${cmd}' failed with errors!${COLOR_RESET}\n" >&2
    ((ERRORCODE++))
  fi
}

# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Linting markdown style...${COLOR_RESET}\n"
if [ -z "${MD_DOC_PATH}" ]
then
  # shellcheck disable=2059
  printf "${COLOR_GREEN}INFO: Merged results pipeline detected, but no markdown files found. Skipping.${COLOR_RESET}\n"
else
  if ! yarn markdownlint --rules doc/.markdownlint/rules ${MD_DOC_PATH};
  then
    # shellcheck disable=2059
    printf "${COLOR_RED}ERROR: Markdownlint failed with errors!${COLOR_RESET}\n" >&2
    ((ERRORCODE++))
  fi
fi

# shellcheck disable=2059
printf "${COLOR_GREEN}INFO: Looking for Vale to lint prose, either installed locally or available in documentation linting image...${COLOR_RESET}\n"
run_locally_or_in_container 'vale' "--minAlertLevel error --output=doc/.vale/vale.tmpl ${MD_DOC_PATH}"

if [ "$ERRORCODE" -ne 0 ]
then
  # shellcheck disable=2059
  printf "\n${COLOR_RED}ERROR: lint test(s) failed! Review the log carefully to see full listing.${COLOR_RESET}\n"
  exit 1
else
  # shellcheck disable=2059
  printf "\n${COLOR_GREEN}INFO: Linting passed.${COLOR_RESET}\n"
  exit 0
fi