check-style-clang-format.py

Go to the documentation of this file.

#!/usr/bin/env python3
 
# Copyright (c) 2022 Eduardo Nuno Almeida.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation;
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#
# Author: Eduardo Nuno Almeida <enmsa@outlook.pt> [INESC TEC and FEUP, Portugal]
 
"""
Check and apply the ns-3 coding style to all files in the PATH argument.
 
The coding style is defined with the clang-format tool, whose definitions are in
the ".clang-format" file. This script performs the following checks / fixes:
- Check / fix local #include headers with "ns3/" prefix. Respects clang-format guards.
- Check / apply clang-format. Respects clang-format guards.
- Check / trim trailing whitespace. Always checked.
- Check / replace tabs with spaces. Respects clang-format guards.
 
This script can be applied to all text files in a given path or to individual files.
 
NOTE: The formatting check requires clang-format (version >= 14) to be found on the path.
Trimming of trailing whitespace and conversion of tabs to spaces (via the "--no-formatting"
option) do not depend on clang-format.
"""
 
import argparse
import concurrent.futures
import itertools
import os
import re
import shutil
import subprocess
import sys
 
from typing import Callable, Dict, List, Tuple
 
 
CLANG_FORMAT_VERSIONS = [
    16,
    15,
    14,
]
 
CLANG_FORMAT_GUARD_ON = '// clang-format on'
CLANG_FORMAT_GUARD_OFF = '// clang-format off'
 
DIRECTORIES_TO_SKIP = [
    '__pycache__',
    '.vscode',
    'bindings',
    'build',
    'cmake-cache',
    'testpy-output',
]
 
# List of files entirely copied from elsewhere that should not be checked,
# in order to optimize the performance of this script
FILES_TO_SKIP = [
    'valgrind.h',
]
 
FILE_EXTENSIONS_TO_CHECK_FORMATTING = [
    '.c',
    '.cc',
    '.h',
]
 
FILE_EXTENSIONS_TO_CHECK_INCLUDE_PREFIXES = FILE_EXTENSIONS_TO_CHECK_FORMATTING
 
FILE_EXTENSIONS_TO_CHECK_WHITESPACE = [
    '.c',
    '.cc',
    '.click',
    '.cmake',
    '.conf',
    '.css',
    '.dot',
    '.gnuplot',
    '.gp',
    '.h',
    '.html',
    '.js',
    '.json',
    '.m',
    '.md',
    '.mob',
    '.ns_params',
    '.ns_movements',
    '.params',
    '.pl',
    '.plt',
    '.py',
    '.rst',
    '.seqdiag',
    '.sh',
    '.txt',
    '.yml',
]
 
FILES_TO_CHECK_WHITESPACE = [
    'Makefile',
    'ns3',
]
 
FILE_EXTENSIONS_TO_CHECK_TABS = [
    '.c',
    '.cc',
    '.h',
    '.md',
    '.py',
    '.rst',
    '.sh',
    '.yml',
]
TAB_SIZE = 4
 
 
 
def should_analyze_directory(dirpath: str) -> bool:
    """
    Check whether a directory should be analyzed.
 
    @param dirpath Directory path.
    @return Whether the directory should be analyzed.
    """
 
    _, directory = os.path.split(dirpath)
 
    return not (directory in DIRECTORIES_TO_SKIP or
                (directory.startswith('.') and directory != '.'))
 
 
def should_analyze_file(path: str,
                        files_to_check: List[str],
                        file_extensions_to_check: List[str],
                        ) -> bool:
    """
    Check whether a file should be analyzed.
 
    @param path Path to the file.
    @param files_to_check List of files that shall be checked.
    @param file_extensions_to_check List of file extensions that shall be checked.
    @return Whether the file should be analyzed.
    """
 
    filename = os.path.split(path)[1]
 
    if filename in FILES_TO_SKIP:
        return False
 
    basename, extension = os.path.splitext(filename)
 
    return (basename in files_to_check or
            extension in file_extensions_to_check)
 
 
def find_files_to_check_style(path: str) -> Tuple[List[str], List[str], List[str], List[str]]:
    """
    Find all files to be checked in a given path.
 
    @param path Path to check.
    @return Tuple [List of files to check include prefixes,
                   List of files to check formatting,
                   List of files to check trailing whitespace,
                   List of files to check tabs].
    """
 
    files_to_check: List[str] = []
    abs_path = os.path.abspath(os.path.expanduser(path))
 
    if os.path.isfile(abs_path):
        files_to_check = [path]
 
    elif os.path.isdir(abs_path):
        for dirpath, dirnames, filenames in os.walk(path, topdown=True):
            if not should_analyze_directory(dirpath):
                # Remove directory and its subdirectories
                dirnames[:] = []
                continue
 
            files_to_check.extend([os.path.join(dirpath, f) for f in filenames])
 
    else:
        raise ValueError(f'Error: {path} is not a file nor a directory')
 
    files_to_check.sort()
 
    files_to_check_include_prefixes: List[str] = []
    files_to_check_formatting: List[str] = []
    files_to_check_whitespace: List[str] = []
    files_to_check_tabs: List[str] = []
 
    for f in files_to_check:
        if should_analyze_file(f, [], FILE_EXTENSIONS_TO_CHECK_INCLUDE_PREFIXES):
            files_to_check_include_prefixes.append(f)
 
        if should_analyze_file(f, [], FILE_EXTENSIONS_TO_CHECK_FORMATTING):
            files_to_check_formatting.append(f)
 
        if should_analyze_file(f, FILES_TO_CHECK_WHITESPACE, FILE_EXTENSIONS_TO_CHECK_WHITESPACE):
            files_to_check_whitespace.append(f)
 
        if should_analyze_file(f, [], FILE_EXTENSIONS_TO_CHECK_TABS):
            files_to_check_tabs.append(f)
 
    return (
        files_to_check_include_prefixes,
        files_to_check_formatting,
        files_to_check_whitespace,
        files_to_check_tabs,
    )
 
 
def find_clang_format_path() -> str:
    """
    Find the path to one of the supported versions of clang-format.
    If no supported version of clang-format is found, raise an exception.
 
    @return Path to clang-format.
    """
 
    # Find exact version
    for version in CLANG_FORMAT_VERSIONS:
        clang_format_path = shutil.which(f'clang-format-{version}')
 
        if clang_format_path:
            return clang_format_path
 
    # Find default version and check if it is supported
    clang_format_path = shutil.which('clang-format')
 
    if clang_format_path:
        process = subprocess.run(
            [clang_format_path, '--version'],
            capture_output=True,
            text=True,
            check=True,
        )
 
        version = process.stdout.strip().split(' ')[-1]
        major_version = int(version.split('.')[0])
 
        if major_version in CLANG_FORMAT_VERSIONS:
            return clang_format_path
 
    # No supported version of clang-format found
    raise RuntimeError(
        f'Could not find any supported version of clang-format installed on this system. '
        f'List of supported versions: {CLANG_FORMAT_VERSIONS}.'
    )
 
 
 
def check_style_clang_format(path: str,
                             enable_check_include_prefixes: bool,
                             enable_check_formatting: bool,
                             enable_check_whitespace: bool,
                             enable_check_tabs: bool,
                             fix: bool,
                             verbose: bool,
                             n_jobs: int = 1,
                             ) -> bool:
    """
    Check / fix the coding style of a list of files.
 
    @param path Path to the files.
    @param enable_check_include_prefixes Whether to enable checking #include headers from the same module with the "ns3/" prefix.
    @param enable_check_formatting Whether to enable checking code formatting.
    @param enable_check_whitespace Whether to enable checking trailing whitespace.
    @param enable_check_tabs Whether to enable checking tabs.
    @param fix Whether to fix (True) or just check (False) the file.
    @param verbose Show the lines that are not compliant with the style.
    @param n_jobs Number of parallel jobs.
    @return Whether all files are compliant with all enabled style checks.
    """
 
    (files_to_check_include_prefixes,
     files_to_check_formatting,
     files_to_check_whitespace,
     files_to_check_tabs) = find_files_to_check_style(path)
 
    check_include_prefixes_successful = True
    check_formatting_successful = True
    check_whitespace_successful = True
    check_tabs_successful = True
 
    if enable_check_include_prefixes:
        check_include_prefixes_successful = check_style_file(
            files_to_check_include_prefixes,
            check_include_prefixes_file,
            '#include headers from the same module with the "ns3/" prefix',
            fix,
            verbose,
            n_jobs,
        )
 
        print('')
 
    if enable_check_formatting:
        check_formatting_successful = check_style_file(
            files_to_check_formatting,
            check_formatting_file,
            'bad code formatting',
            fix,
            verbose,
            n_jobs,
            clang_format_path=find_clang_format_path(),
        )
 
        print('')
 
    if enable_check_whitespace:
        check_whitespace_successful = check_style_file(
            files_to_check_whitespace,
            check_trailing_whitespace_file,
            'trailing whitespace',
            fix,
            verbose,
            n_jobs,
        )
 
        print('')
 
    if enable_check_tabs:
        check_tabs_successful = check_style_file(
            files_to_check_tabs,
            check_tabs_file,
            'tabs',
            fix,
            verbose,
            n_jobs,
        )
 
    return all([
        check_include_prefixes_successful,
        check_formatting_successful,
        check_whitespace_successful,
        check_tabs_successful,
    ])
 
 
def check_style_file(filenames: List[str],
                     check_style_file_function: Callable,
                     style_check_str: str,
                     fix: bool,
                     verbose: bool,
                     n_jobs: int,
                     **kwargs,
                     ) -> bool:
    """
    Check / fix style of a list of files.
 
    @param filename Name of the file to be checked.
    @param check_style_file_function Function used to check the file.
    @param style_check_str Description of the check to be performed.
    @param fix Whether to fix (True) or just check (False) the file (True).
    @param verbose Show the lines that are not compliant with the style.
    @param n_jobs Number of parallel jobs.
    @param kwargs Additional keyword arguments to the check_style_file_function.
    @return Whether all files are compliant with the style.
    """
 
    # Check files
    non_compliant_files: List[str] = []
    files_verbose_infos: Dict[str, List[str]] = {}
 
    with concurrent.futures.ProcessPoolExecutor(n_jobs) as executor:
        non_compliant_files_results = executor.map(
            check_style_file_function,
            filenames,
            itertools.repeat(fix),
            itertools.repeat(verbose),
            *[arg if isinstance(arg, list) else itertools.repeat(arg) for arg in kwargs.values()],
        )
 
        for (filename, is_file_compliant, verbose_infos) in non_compliant_files_results:
            if not is_file_compliant:
                non_compliant_files.append(filename)
 
            if verbose:
                files_verbose_infos[filename] = verbose_infos
 
    # Output results
    if not non_compliant_files:
        print(f'- No files detected with {style_check_str}')
        return True
 
    else:
        n_non_compliant_files = len(non_compliant_files)
 
        if fix:
            print(f'- Fixed {style_check_str} in the files ({n_non_compliant_files}):')
        else:
            print(f'- Detected {style_check_str} in the files ({n_non_compliant_files}):')
 
        for f in non_compliant_files:
            if verbose:
                print(*[f'    {l}' for l in files_verbose_infos[f]], sep='\n')
            else:
                print(f'    - {f}')
 
        # If all files were fixed, there are no more non-compliant files
        return fix
 
 
 
def check_include_prefixes_file(filename: str,
                                fix: bool,
                                verbose: bool,
                                ) -> Tuple[str, bool, List[str]]:
    """
    Check / fix #include headers from the same module with the "ns3/" prefix in a file.
 
    @param filename Name of the file to be checked.
    @param fix Whether to fix (True) or just check (False) the style of the file (True).
    @param verbose Show the lines that are not compliant with the style.
    @return Tuple [Filename,
                   Whether the file is compliant with the style (before the check),
                   Verbose information].
    """
 
    is_file_compliant = True
    clang_format_enabled = True
 
    verbose_infos: List[str] = []
 
    with open(filename, 'r', encoding='utf-8') as f:
        file_lines = f.readlines()
 
    for (i, line) in enumerate(file_lines):
 
        # Check clang-format guards
        line_stripped = line.strip()
 
        if line_stripped == CLANG_FORMAT_GUARD_ON:
            clang_format_enabled = True
        elif line_stripped == CLANG_FORMAT_GUARD_OFF:
            clang_format_enabled = False
 
        if (not clang_format_enabled and
                line_stripped not in (CLANG_FORMAT_GUARD_ON, CLANG_FORMAT_GUARD_OFF)):
            continue
 
        # Check if the line is an #include and extract its header file
        header_file = re.findall(r'^#include ["<]ns3/(.*\.h)[">]', line_stripped)
 
        if not header_file:
            continue
 
        # Check if the header file belongs to the same module and remove the "ns3/" prefix
        header_file = header_file[0]
        parent_path = os.path.split(filename)[0]
 
        if not os.path.exists(os.path.join(parent_path, header_file)):
            continue
 
        is_file_compliant = False
        file_lines[i] = line_stripped.replace(
            f'ns3/{header_file}', header_file).replace('<', '"').replace('>', '"') + '\n'
 
        if verbose:
            header_index = len('#include "')
 
            verbose_infos.extend([
                f'{filename}:{i + 1}:{header_index + 1}: error: #include headers from the same module with the "ns3/" prefix detected',
                f'    {line_stripped}',
                f'    {"":{header_index}}^',
            ])
 
        # Optimization: If running in non-verbose check mode, only one error is needed to check that the file is not compliant
        if not fix and not verbose:
            break
 
    # Update file with the fixed lines
    if fix and not is_file_compliant:
        with open(filename, 'w', encoding='utf-8') as f:
            f.writelines(file_lines)
 
    return (filename, is_file_compliant, verbose_infos)
 
 
def check_formatting_file(filename: str,
                          fix: bool,
                          verbose: bool,
                          clang_format_path: str,
                          ) -> Tuple[str, bool, List[str]]:
    """
    Check / fix the coding style of a file with clang-format.
 
    @param filename Name of the file to be checked.
    @param fix Whether to fix (True) or just check (False) the style of the file (True).
    @param verbose Show the lines that are not compliant with the style.
    @param clang_format_path Path to clang-format.
    @return Tuple [Filename,
                   Whether the file is compliant with the style (before the check),
                   Verbose information].
    """
 
    verbose_infos: List[str] = []
 
    # Check if the file is well formatted
    process = subprocess.run(
        [
            clang_format_path,
            filename,
            '-style=file',
            '--dry-run',
            '--Werror',
            # Optimization: In non-verbose mode, only one error is needed to check that the file is not compliant
            f'--ferror-limit={0 if verbose else 1}',
        ],
        check=False,
        capture_output=True,
        text=True,
    )
 
    is_file_compliant = (process.returncode == 0)
 
    if verbose:
        verbose_infos = process.stderr.splitlines()
 
    # Fix file
    if fix and not is_file_compliant:
        process = subprocess.run(
            [
                clang_format_path,
                filename,
                '-style=file',
                '-i',
            ],
            check=False,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
        )
 
    return (filename, is_file_compliant, verbose_infos)
 
 
def check_trailing_whitespace_file(filename: str,
                                   fix: bool,
                                   verbose: bool,
                                   ) -> Tuple[str, bool, List[str]]:
    """
    Check / fix trailing whitespace in a file.
 
    @param filename Name of the file to be checked.
    @param fix Whether to fix (True) or just check (False) the style of the file (True).
    @param verbose Show the lines that are not compliant with the style.
    @return Tuple [Filename,
                   Whether the file is compliant with the style (before the check),
                   Verbose information].
    """
 
    is_file_compliant = True
    verbose_infos: List[str] = []
 
    with open(filename, 'r', encoding='utf-8') as f:
        file_lines = f.readlines()
 
    # Check if there are trailing whitespace and fix them
    for (i, line) in enumerate(file_lines):
        line_fixed = line.rstrip() + '\n'
 
        if line_fixed == line:
            continue
 
        is_file_compliant = False
        file_lines[i] = line_fixed
 
        if verbose:
            line_fixed_stripped_expanded = line_fixed.rstrip().expandtabs(TAB_SIZE)
 
            verbose_infos.extend([
                f'{filename}:{i + 1}:{len(line_fixed_stripped_expanded) + 1}: error: Trailing whitespace detected',
                f'    {line_fixed_stripped_expanded}',
                f'    {"":{len(line_fixed_stripped_expanded)}}^',
            ])
 
        # Optimization: If running in non-verbose check mode, only one error is needed to check that the file is not compliant
        if not fix and not verbose:
            break
 
    # Update file with the fixed lines
    if fix and not is_file_compliant:
        with open(filename, 'w', encoding='utf-8') as f:
            f.writelines(file_lines)
 
    return (filename, is_file_compliant, verbose_infos)
 
 
def check_tabs_file(filename: str,
                    fix: bool,
                    verbose: bool,
                    ) -> Tuple[str, bool, List[str]]:
    """
    Check / fix tabs in a file.
 
    @param filename Name of the file to be checked.
    @param fix Whether to fix (True) or just check (False) the style of the file (True).
    @param verbose Show the lines that are not compliant with the style.
    @return Tuple [Filename,
                   Whether the file is compliant with the style (before the check),
                   Verbose information].
    """
 
    is_file_compliant = True
    clang_format_enabled = True
 
    verbose_infos: List[str] = []
 
    with open(filename, 'r', encoding='utf-8') as f:
        file_lines = f.readlines()
 
    for (i, line) in enumerate(file_lines):
 
        # Check clang-format guards
        line_stripped = line.strip()
 
        if line_stripped == CLANG_FORMAT_GUARD_ON:
            clang_format_enabled = True
        elif line_stripped == CLANG_FORMAT_GUARD_OFF:
            clang_format_enabled = False
 
        if (not clang_format_enabled and
                line_stripped not in (CLANG_FORMAT_GUARD_ON, CLANG_FORMAT_GUARD_OFF)):
            continue
 
        # Check if there are tabs and fix them
        tab_index = line.find('\t')
 
        if tab_index == -1:
            continue
 
        is_file_compliant = False
        file_lines[i] = line.expandtabs(TAB_SIZE)
 
        if verbose:
            verbose_infos.extend([
                f'{filename}:{i + 1}:{tab_index + 1}: error: Tab detected',
                f'    {line.rstrip()}',
                f'    {"":{tab_index}}^',
            ])
 
        # Optimization: If running in non-verbose check mode, only one error is needed to check that the file is not compliant
        if not fix and not verbose:
            break
 
    # Update file with the fixed lines
    if fix and not is_file_compliant:
        with open(filename, 'w', encoding='utf-8') as f:
            f.writelines(file_lines)
 
    return (filename, is_file_compliant, verbose_infos)
 
 
 
if __name__ == '__main__':
 
    parser = argparse.ArgumentParser(
        description='Check and apply the ns-3 coding style to all files in a given PATH. '
        'The script checks the formatting of the file with clang-format. '
        'Additionally, it checks #include headers from the same module with the "ns3/" prefix, '
        'the presence of trailing whitespace and tabs. '
        'Formatting, local #include "ns3/" prefixes and tabs checks respect clang-format guards. '
        'When used in "check mode" (default), the script checks if all files are well '
        'formatted and do not have trailing whitespace nor tabs. '
        'If it detects non-formatted files, they will be printed and this process exits with a '
        'non-zero code. When used in "fix mode", this script automatically fixes the files.')
 
    parser.add_argument('path', action='store', type=str,
                        help='Path to the files to check')
 
    parser.add_argument('--no-include-prefixes', action='store_true',
                        help='Do not check / fix #include headers from the same module with the "ns3/" prefix')
 
    parser.add_argument('--no-formatting', action='store_true',
                        help='Do not check / fix code formatting')
 
    parser.add_argument('--no-whitespace', action='store_true',
                        help='Do not check / fix trailing whitespace')
 
    parser.add_argument('--no-tabs', action='store_true',
                        help='Do not check / fix tabs')
 
    parser.add_argument('--fix', action='store_true',
                        help='Fix coding style issues detected in the files')
 
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Show the lines that are not well-formatted')
 
    parser.add_argument('-j', '--jobs', type=int, default=max(1, os.cpu_count() - 1),
                        help='Number of parallel jobs')
 
    args = parser.parse_args()
 
    try:
        all_checks_successful = check_style_clang_format(
            path=args.path,
            enable_check_include_prefixes=(not args.no_include_prefixes),
            enable_check_formatting=(not args.no_formatting),
            enable_check_whitespace=(not args.no_whitespace),
            enable_check_tabs=(not args.no_tabs),
            fix=args.fix,
            verbose=args.verbose,
            n_jobs=args.jobs,
        )
 
    except Exception as e:
        print(e)
        sys.exit(1)
 
    if all_checks_successful:
        sys.exit(0)
    else:
        sys.exit(1)