CI: Refactor generic whitespace checks linting setup.

.github/workflows/sub_lint.yml

@@ -276,103 +276,32 @@ jobs:
         if: inputs.checkLineendings && always()
         continue-on-error: ${{ inputs.lineendingsFailSilent }}
         run: |
-          lineendings=0
+          python3 tools/lint/generic_checks.py \
-          for file in ${{ inputs.changedFiles }}
+            --lineendings-check \
-          do
+            --files "${{ inputs.changedFiles }}" \
-            # Check for DOS or MAC line endings
+            --report-file "${{ env.reportdir }}${{ env.reportfilename }}" \
-            if [[ $(file -b $file) =~ "with CR" ]]
+            --log-dir "${{ env.logdir }}"
-            then
+
-              echo "::warning file=$file,title=$file::File has non Unix line endings"
-              echo "$file has non Unix line endings" >> ${{ env.logdir }}lineendings.log
-              ((lineendings=lineendings+1))
-            fi
-          done
-          echo "Found $lineendings line ending errors"
-          # Write the report
-          if [ $lineendings -gt 0 ]
-          then
-            echo "<details><summary>:information_source: Found $lineendings problems with line endings</summary>" >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
-            cat ${{ env.logdir }}lineendings.log >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo "</details>" >> ${{env.reportdir}}${{ env.reportfilename }}
-          else
-            echo ":heavy_check_mark: No line ending problem found  " >> ${{env.reportdir}}${{ env.reportfilename }}
-          fi
-          echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
-          # Exit the step with appropriate code
-          [ $lineendings -eq 0 ]
       - name: Check for trailing whitespaces
         if: inputs.checkWhitespace && always()
         continue-on-error: ${{ inputs.whitespaceFailSilent }}
         run: |
-          whitespaceErrors=0
+          python3 tools/lint/generic_checks.py \
-          exclude="*[.md]"
+            --whitespace-check \
-          for file in ${{ inputs.changedFiles }}
+            --files "${{ inputs.changedFiles }}" \
-          do
+            --report-file "${{ env.reportdir }}${{ env.reportfilename }}" \
-            # Check for trailing whitespaces
+            --log-dir "${{ env.logdir }}"
-            grep -nIHE --exclude="$exclude" " $" $file | sed -e "s/$/<-- trailing whitespace/" >> ${{ env.logdir }}whitespace.log || true
+
-          done
-          # Write the Log to the console with the Problem Matchers
-          if [ -f ${{ env.logdir }}whitespace.log ]
-          then
-            echo "::add-matcher::${{ runner.workspace }}/FreeCAD/.github/problemMatcher/grepMatcherWarning.json"
-            cat ${{ env.logdir }}whitespace.log
-            echo "::remove-matcher owner=grepMatcher-warning::"
-            whitespaceErrors=$(wc -l < ${{ env.logdir }}whitespace.log)
-          fi
-          echo "Found $whitespaceErrors whitespace errors"
-          # Write the report
-          if [ $whitespaceErrors -gt 0 ]
-          then
-            echo "<details><summary>:information_source: Found $whitespaceErrors trailing whitespace</summary>" >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
-            cat ${{ env.logdir }}whitespace.log >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo "</details>" >> ${{env.reportdir}}${{ env.reportfilename }}
-          else
-            echo ":heavy_check_mark: No trailing whitespace found  " >> ${{env.reportdir}}${{ env.reportfilename }}
-          fi
-          echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
-          # Exit the step with appropriate code
-          [ $whitespaceErrors -eq 0 ]
       - name: Check for Tab usage
         if: inputs.checkTabs && always()
         continue-on-error: ${{ inputs.tabsFailSilent }}
         run: |
-          tabErrors=0
+          python3 tools/lint/generic_checks.py \
-          exclude="*[.md]"
+            --tabs-check \
-          # Check for Tab usage
+            --files "${{ inputs.changedFiles }}" \
-          for file in ${{ steps.changed-files.outputs.all_changed_files }}
+            --report-file "${{ env.reportdir }}${{ env.reportfilename }}" \
-          do
+            --log-dir "${{ env.logdir }}"
-            grep -nIHE --exclude="$exclude" $'\t' $file | sed -e "s/$/ <-- contains tab/" >> ${{ env.logdir }}tab.log || true
+
-          done
-          # Write the Log to the console with the Problem Matchers
-          if [ -f ${{ env.logdir }}tab.log ]
-          then
-            echo "::add-matcher::${{ runner.workspace }}/FreeCAD/.github/problemMatcher/grepMatcherWarning.json"
-            cat ${{ env.logdir }}tab.log
-            echo "::remove-matcher owner=grepMatcher-warning::"
-            tabErrors=$(wc -l < ${{ env.logdir }}tab.log)
-          fi
-          echo "Found $tabErrors tab errors"
-          # Write the report
-          if [ $tabErrors -gt 0 ]; then
-            echo "<details><summary>:information_source: Found $tabErrors tabs, better to use spaces</summary>" >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
-            cat ${{ env.logdir }}tab.log >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
-            echo "</details>" >> ${{env.reportdir}}${{ env.reportfilename }}
-          else
-            echo ":heavy_check_mark: No tabs found" >> ${{env.reportdir}}${{ env.reportfilename }}
-          fi
-          echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
-          # Exit the step with appropriate code
-          [ $tabErrors -eq 0 ]
-    # Run Python lints
       - name: Pylint
         if: inputs.checkPylint && inputs.changedPythonFiles != '' && always()
         continue-on-error: ${{ inputs.pylintFailSilent }}

tools/lint/generic_checks.py

@@ -0,0 +1,165 @@
+import argparse
+import glob
+import os
+from utils import (
+    add_common_arguments,
+    init_environment,
+    write_file,
+    emit_problem_matchers,
+)
+
+
+def check_line_endings(file_paths):
+    """Check if files have non-Unix (CRLF or CR) line endings.
+
+    Returns a dict mapping file path to an issue description.
+    """
+    issues = {}
+    for path in file_paths:
+        try:
+            with open(path, "rb") as f:
+                content = f.read()
+                if b"\r\n" in content or (b"\r" in content and b"\n" not in content):
+                    issues[path] = "File has non-Unix line endings."
+        except Exception as e:
+            issues[path] = f"Error reading file: {e}"
+    return issues
+
+
+def check_trailing_whitespace(file_paths):
+    """Check for trailing whitespace in files.
+
+    Returns a dict mapping file paths to a list of line numbers with trailing whitespace.
+    """
+    issues = {}
+    for path in file_paths:
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                lines = f.readlines()
+                error_lines = []
+                for idx, line in enumerate(lines, start=1):
+                    if line.rstrip("\n") != line.rstrip():
+                        error_lines.append(idx)
+                if error_lines:
+                    issues[path] = error_lines
+        except Exception as e:
+            issues[path] = f"Error reading file: {e}"
+    return issues
+
+
+def check_tabs(file_paths):
+    """Check for usage of tab characters in files.
+
+    Returns a dict mapping file paths to a list of line numbers containing tabs.
+    """
+    issues = {}
+    for path in file_paths:
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                lines = f.readlines()
+                tab_lines = []
+                for idx, line in enumerate(lines, start=1):
+                    if "\t" in line:
+                        tab_lines.append(idx)
+                if tab_lines:
+                    issues[path] = tab_lines
+        except Exception as e:
+            issues[path] = f"Error reading file: {e}"
+    return issues
+
+
+def format_report(section_title, issues):
+    """Format a report section with a Markdown details block."""
+    report = []
+    if issues:
+        count = len(issues)
+        report.append(
+            f"<details><summary>:information_source: Found {count} issue(s) in {section_title}</summary>\n"
+        )
+        report.append("```")
+        for file, details in issues.items():
+            report.append(f"{file}: {details}")
+        report.append("```\n</details>\n")
+    else:
+        report.append(f":heavy_check_mark: No issues found in {section_title}\n")
+    return "\n".join(report)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Run formatting lint checks.")
+    add_common_arguments(parser)
+    parser.add_argument(
+        "--lineendings-check",
+        action="store_true",
+        help="Enable check for non-Unix line endings.",
+    )
+    parser.add_argument(
+        "--whitespace-check",
+        action="store_true",
+        help="Enable check for trailing whitespace.",
+    )
+    parser.add_argument(
+        "--tabs-check", action="store_true", help="Enable check for tab characters."
+    )
+    args = parser.parse_args()
+    init_environment(args)
+
+    file_list = glob.glob(args.files, recursive=True)
+    file_list = [f for f in file_list if os.path.isfile(f)]
+
+    report_sections = []
+
+    # Check non-Unix line endings.
+    if args.lineendings_check:
+        le_issues = check_line_endings(file_list)
+        for file, detail in le_issues.items():
+            print(f"::warning file={file},title={file}::{detail}")
+        report_sections.append(format_report("Non-Unix Line Endings", le_issues))
+
+    # Check trailing whitespace.
+    if args.whitespace_check:
+        ws_issues = check_trailing_whitespace(file_list)
+        if ws_issues:
+            ws_output_lines = []
+            for file, details in ws_issues.items():
+                if isinstance(details, list):
+                    for line in details:
+                        ws_output_lines.append(f"{file}:{line}: trailing whitespace")
+                else:
+                    ws_output_lines.append(f"{file}: {details}")
+            ws_output = "\n".join(ws_output_lines)
+
+            ws_log_file = os.path.join(args.log_dir, "whitespace.log")
+            write_file(ws_log_file, ws_output)
+            emit_problem_matchers(
+                ws_log_file, "grepMatcherWarning.json", "grepMatcher-warning"
+            )
+        report_sections.append(format_report("Trailing Whitespace", ws_issues))
+
+    # Check tab usage.
+    if args.tabs_check:
+        tab_issues = check_tabs(file_list)
+        if tab_issues:
+            tab_output_lines = []
+            for file, details in tab_issues.items():
+                if isinstance(details, list):
+                    for line in details:
+                        tab_output_lines.append(f"{file}:{line}: contains tab")
+                else:
+                    tab_output_lines.append(f"{file}: {details}")
+            tab_output = "\n".join(tab_output_lines)
+
+            tab_log_file = os.path.join(args.log_dir, "tabs.log")
+            write_file(tab_log_file, tab_output)
+            emit_problem_matchers(
+                tab_log_file, "grepMatcherWarning.json", "grepMatcher-warning"
+            )
+        report_sections.append(format_report("Tab Usage", tab_issues))
+
+    report_content = "\n".join(report_sections)
+    write_file(args.report_file, report_content)
+    print("Lint report generated at:", args.report_file)
+
+
+if __name__ == "__main__":
+    main()

tools/lint/utils.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+import os
+import sys
+import logging
+import subprocess
+import argparse
+from typing import Tuple
+
+
+def run_command(cmd, check=False) -> Tuple[str, str, int]:
+    """
+    Run a command using subprocess.run and return stdout, stderr, and exit code.
+    """
+    try:
+        result = subprocess.run(
+            cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=check, text=True
+        )
+        return result.stdout, result.stderr, result.returncode
+    except subprocess.CalledProcessError as e:
+        return e.stdout, e.stderr, e.returncode
+
+
+def setup_logger(verbose: bool):
+    """
+    Setup the logging level based on the verbose flag.
+    """
+    level = logging.DEBUG if verbose else logging.INFO
+    logging.basicConfig(level=level, format="%(levelname)s: %(message)s")
+
+
+def write_file(file_path: str, content: str):
+    """Write content to the specified file."""
+    with open(file_path, "w", encoding="utf-8") as f:
+        f.write(content)
+    logging.info("Wrote file: %s", file_path)
+
+
+def append_file(file_path: str, content: str):
+    """Append content to the specified file."""
+    with open(file_path, "a", encoding="utf-8") as f:
+        f.write(content + "\n")
+    logging.info("Appended content to file: %s", file_path)
+
+
+def emit_problem_matchers(log_path: str, matcher_filename: str, remove_owner: str):
+    """
+    Emit GitHub Actions problem matcher commands using the given matcher file.
+
+    This function will:
+      1. Check if the log file exists.
+      2. Use the RUNNER_WORKSPACE environment variable to construct the matcher path.
+      3. Print the add-matcher command, then the log content, then the remove-matcher command.
+    """
+    if os.path.isfile(log_path):
+        runner_workspace = os.getenv("RUNNER_WORKSPACE")
+        matcher_path = os.path.join(
+            runner_workspace, "FreeCAD", ".github", "problemMatcher", matcher_filename
+        )
+        print(f"::add-matcher::{matcher_path}")
+        with open(log_path, "r", encoding="utf-8") as f:
+            sys.stdout.write(f.read())
+        print(f"::remove-matcher owner={remove_owner}::")
+
+
+def add_common_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
+    """
+    Add common command-line arguments shared between tools.
+    """
+    parser.add_argument(
+        "--files",
+        required=True,
+        help="A space-separated list or glob pattern of files to check.",
+    )
+    parser.add_argument(
+        "--log-dir", required=True, help="Directory where log files will be written."
+    )
+    parser.add_argument(
+        "--report-file",
+        required=True,
+        help="Path to the Markdown report file to append results.",
+    )
+    parser.add_argument("--verbose", action="store_true", help="Enable verbose output.")
+    return parser
+
+
+def init_environment(args: argparse.Namespace):
+    """
+    Perform common initialization tasks:
+      - Set up logging.
+      - Create the log directory.
+      - Create the directory for the report file.
+    """
+    setup_logger(args.verbose)
+    os.makedirs(args.log_dir, exist_ok=True)
+    os.makedirs(os.path.dirname(args.report_file), exist_ok=True)