kindred/create
1
1
Fork 0
Code Issues 53 Pull Requests Actions Packages Projects 9 Releases 2 Wiki Activity

Merge pull request #19933 from tritao/ci-initial-lint-refactor

Browse Source 
CI: Refactor generic whitespace checks linting setup.
This commit is contained in:
Chris Hennes
2025-03-04 15:26:10 -06:00
committed by GitHub
parent f956c7cd4a d93c3233fe
commit da12de5d5e
3 changed files with 272 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

107
.github/workflows/sub_lint.yml vendored
View File

@@ -276,103 +276,32 @@ jobs:
if: inputs.checkLineendings && always()
continue-on-error: ${{ inputs.lineendingsFailSilent }}
run: |
lineendings=0
for file in ${{ inputs.changedFiles }}
do
# Check for DOS or MAC line endings
if [[ $(file -b $file) =~ "with CR" ]]
then
echo "::warning file=$file,title=$file::File has non Unix line endings"
echo "$file has non Unix line endings" >> ${{ env.logdir }}lineendings.log
((lineendings=lineendings+1))
fi
done
echo "Found $lineendings line ending errors"
# Write the report
if [ $lineendings -gt 0 ]
then
echo "<details><summary>:information_source: Found $lineendings problems with line endings</summary>" >> ${{env.reportdir}}${{ env.reportfilename }}
echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
cat ${{ env.logdir }}lineendings.log >> ${{env.reportdir}}${{ env.reportfilename }}
echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
echo "</details>" >> ${{env.reportdir}}${{ env.reportfilename }}
else
echo ":heavy_check_mark: No line ending problem found " >> ${{env.reportdir}}${{ env.reportfilename }}
fi
echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
# Exit the step with appropriate code
[ $lineendings -eq 0 ]
python3 tools/lint/generic_checks.py \
--lineendings-check \
--files "${{ inputs.changedFiles }}" \
--report-file "${{ env.reportdir }}${{ env.reportfilename }}" \
--log-dir "${{ env.logdir }}"
- name: Check for trailing whitespaces
if: inputs.checkWhitespace && always()
continue-on-error: ${{ inputs.whitespaceFailSilent }}
run: |
whitespaceErrors=0
exclude="*[.md]"
for file in ${{ inputs.changedFiles }}
do
# Check for trailing whitespaces
grep -nIHE --exclude="$exclude" " $" $file | sed -e "s/$/<-- trailing whitespace/" >> ${{ env.logdir }}whitespace.log || true
done
# Write the Log to the console with the Problem Matchers
if [ -f ${{ env.logdir }}whitespace.log ]
then
echo "::add-matcher::${{ runner.workspace }}/FreeCAD/.github/problemMatcher/grepMatcherWarning.json"
cat ${{ env.logdir }}whitespace.log
echo "::remove-matcher owner=grepMatcher-warning::"
whitespaceErrors=$(wc -l < ${{ env.logdir }}whitespace.log)
fi
echo "Found $whitespaceErrors whitespace errors"
# Write the report
if [ $whitespaceErrors -gt 0 ]
then
echo "<details><summary>:information_source: Found $whitespaceErrors trailing whitespace</summary>" >> ${{env.reportdir}}${{ env.reportfilename }}
echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
cat ${{ env.logdir }}whitespace.log >> ${{env.reportdir}}${{ env.reportfilename }}
echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
echo "</details>" >> ${{env.reportdir}}${{ env.reportfilename }}
else
echo ":heavy_check_mark: No trailing whitespace found " >> ${{env.reportdir}}${{ env.reportfilename }}
fi
echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
# Exit the step with appropriate code
[ $whitespaceErrors -eq 0 ]
python3 tools/lint/generic_checks.py \
--whitespace-check \
--files "${{ inputs.changedFiles }}" \
--report-file "${{ env.reportdir }}${{ env.reportfilename }}" \
--log-dir "${{ env.logdir }}"
- name: Check for Tab usage
if: inputs.checkTabs && always()
continue-on-error: ${{ inputs.tabsFailSilent }}
run: |
tabErrors=0
exclude="*[.md]"
# Check for Tab usage
for file in ${{ steps.changed-files.outputs.all_changed_files }}
do
grep -nIHE --exclude="$exclude" $'\t' $file | sed -e "s/$/ <-- contains tab/" >> ${{ env.logdir }}tab.log || true
done
# Write the Log to the console with the Problem Matchers
if [ -f ${{ env.logdir }}tab.log ]
then
echo "::add-matcher::${{ runner.workspace }}/FreeCAD/.github/problemMatcher/grepMatcherWarning.json"
cat ${{ env.logdir }}tab.log
echo "::remove-matcher owner=grepMatcher-warning::"
tabErrors=$(wc -l < ${{ env.logdir }}tab.log)
fi
echo "Found $tabErrors tab errors"
# Write the report
if [ $tabErrors -gt 0 ]; then
echo "<details><summary>:information_source: Found $tabErrors tabs, better to use spaces</summary>" >> ${{env.reportdir}}${{ env.reportfilename }}
echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
cat ${{ env.logdir }}tab.log >> ${{env.reportdir}}${{ env.reportfilename }}
echo '```' >> ${{env.reportdir}}${{ env.reportfilename }}
echo "</details>" >> ${{env.reportdir}}${{ env.reportfilename }}
else
echo ":heavy_check_mark: No tabs found" >> ${{env.reportdir}}${{ env.reportfilename }}
fi
echo "" >> ${{env.reportdir}}${{ env.reportfilename }}
# Exit the step with appropriate code
[ $tabErrors -eq 0 ]
# Run Python lints
python3 tools/lint/generic_checks.py \
--tabs-check \
--files "${{ inputs.changedFiles }}" \
--report-file "${{ env.reportdir }}${{ env.reportfilename }}" \
--log-dir "${{ env.logdir }}"
- name: Pylint
if: inputs.checkPylint && inputs.changedPythonFiles != '' && always()
continue-on-error: ${{ inputs.pylintFailSilent }}

159
tools/lint/generic_checks.py Executable file
View File

@@ -0,0 +1,159 @@
import argparse
import glob
import os
from utils import (
add_common_arguments,
init_environment,
write_file,
emit_problem_matchers,
)
def check_line_endings(file_paths):
"""Check if files have non-Unix (CRLF or CR) line endings.
Returns a dict mapping file path to an issue description.
"""
issues = {}
for path in file_paths:
try:
with open(path, "rb") as f:
content = f.read()
if b"\r\n" in content or (b"\r" in content and b"\n" not in content):
issues[path] = "File has non-Unix line endings."
except Exception as e:
issues[path] = f"Error reading file: {e}"
return issues
def check_trailing_whitespace(file_paths):
"""Check for trailing whitespace in files.
Returns a dict mapping file paths to a list of line numbers with trailing whitespace.
"""
issues = {}
for path in file_paths:
try:
with open(path, "r", encoding="utf-8") as f:
lines = files.read().splitlines()
error_lines = [idx if line.endswith(" ") for idx, line in enumerate(lines, start=1)]
if error_lines:
issues[path] = error_lines
except Exception as e:
issues[path] = f"Error reading file: {e}"
return issues
def check_tabs(file_paths):
"""Check for usage of tab characters in files.
Returns a dict mapping file paths to a list of line numbers containing tabs.
"""
issues = {}
for path in file_paths:
try:
with open(path, "r", encoding="utf-8") as f:
lines = f.readlines()
tab_lines = [idx if "\t" in line for idx, line in enumerate(lines, start=1)]
if tab_lines:
issues[path] = tab_lines
except Exception as e:
issues[path] = f"Error reading file: {e}"
return issues
def format_report(section_title, issues):
"""Format a report section with a Markdown details block."""
report = []
if issues:
count = len(issues)
report.append(
f"<details><summary>:information_source: Found {count} issue(s) in {section_title}</summary>\n"
)
report.append("```")
for file, details in issues.items():
report.append(f"{file}: {details}")
report.append("```\n</details>\n")
else:
report.append(f":heavy_check_mark: No issues found in {section_title}\n")
return "\n".join(report)
def main():
parser = argparse.ArgumentParser(description="Run formatting lint checks.")
add_common_arguments(parser)
parser.add_argument(
"--lineendings-check",
action="store_true",
help="Enable check for non-Unix line endings.",
)
parser.add_argument(
"--whitespace-check",
action="store_true",
help="Enable check for trailing whitespace.",
)
parser.add_argument(
"--tabs-check", action="store_true", help="Enable check for tab characters."
)
args = parser.parse_args()
init_environment(args)
file_list = glob.glob(args.files, recursive=True)
file_list = [f for f in file_list if os.path.isfile(f)]
report_sections = []
# Check non-Unix line endings.
if args.lineendings_check:
le_issues = check_line_endings(file_list)
for file, detail in le_issues.items():
print(f"::warning file={file},title={file}::{detail}")
report_sections.append(format_report("Non-Unix Line Endings", le_issues))
# Check trailing whitespace.
if args.whitespace_check:
ws_issues = check_trailing_whitespace(file_list)
if ws_issues:
ws_output_lines = []
for file, details in ws_issues.items():
if isinstance(details, list):
for line in details:
ws_output_lines.append(f"{file}:{line}: trailing whitespace")
else:
ws_output_lines.append(f"{file}: {details}")
ws_output = "\n".join(ws_output_lines)
ws_log_file = os.path.join(args.log_dir, "whitespace.log")
write_file(ws_log_file, ws_output)
emit_problem_matchers(
ws_log_file, "grepMatcherWarning.json", "grepMatcher-warning"
)
report_sections.append(format_report("Trailing Whitespace", ws_issues))
# Check tab usage.
if args.tabs_check:
tab_issues = check_tabs(file_list)
if tab_issues:
tab_output_lines = []
for file, details in tab_issues.items():
if isinstance(details, list):
for line in details:
tab_output_lines.append(f"{file}:{line}: contains tab")
else:
tab_output_lines.append(f"{file}: {details}")
tab_output = "\n".join(tab_output_lines)
tab_log_file = os.path.join(args.log_dir, "tabs.log")
write_file(tab_log_file, tab_output)
emit_problem_matchers(
tab_log_file, "grepMatcherWarning.json", "grepMatcher-warning"
)
report_sections.append(format_report("Tab Usage", tab_issues))
report_content = "\n".join(report_sections)
write_file(args.report_file, report_content)
print("Lint report generated at:", args.report_file)
if __name__ == "__main__":
main()

95
tools/lint/utils.py Normal file
View File

@@ -0,0 +1,95 @@
#!/usr/bin/env python3
import os
import sys
import logging
import subprocess
import argparse
from typing import Tuple
def run_command(cmd, check=False) -> Tuple[str, str, int]:
"""
Run a command using subprocess.run and return stdout, stderr, and exit code.
"""
try:
result = subprocess.run(
cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=check, text=True
)
return result.stdout, result.stderr, result.returncode
except subprocess.CalledProcessError as e:
return e.stdout, e.stderr, e.returncode
def setup_logger(verbose: bool):
"""
Setup the logging level based on the verbose flag.
"""
level = logging.DEBUG if verbose else logging.INFO
logging.basicConfig(level=level, format="%(levelname)s: %(message)s")
def write_file(file_path: str, content: str):
"""Write content to the specified file."""
with open(file_path, "w", encoding="utf-8") as f:
f.write(content)
logging.info("Wrote file: %s", file_path)
def append_file(file_path: str, content: str):
"""Append content to the specified file."""
with open(file_path, "a", encoding="utf-8") as f:
f.write(content + "\n")
logging.info("Appended content to file: %s", file_path)
def emit_problem_matchers(log_path: str, matcher_filename: str, remove_owner: str):
"""
Emit GitHub Actions problem matcher commands using the given matcher file.
This function will:
1. Check if the log file exists.
2. Use the RUNNER_WORKSPACE environment variable to construct the matcher path.
3. Print the add-matcher command, then the log content, then the remove-matcher command.
"""
if os.path.isfile(log_path):
runner_workspace = os.getenv("RUNNER_WORKSPACE")
matcher_path = os.path.join(
runner_workspace, "FreeCAD", ".github", "problemMatcher", matcher_filename
)
print(f"::add-matcher::{matcher_path}")
with open(log_path, "r", encoding="utf-8") as f:
sys.stdout.write(f.read())
print(f"::remove-matcher owner={remove_owner}::")
def add_common_arguments(parser: argparse.ArgumentParser) -> argparse.ArgumentParser:
"""
Add common command-line arguments shared between tools.
"""
parser.add_argument(
"--files",
required=True,
help="A space-separated list or glob pattern of files to check.",
)
parser.add_argument(
"--log-dir", required=True, help="Directory where log files will be written."
)
parser.add_argument(
"--report-file",
required=True,
help="Path to the Markdown report file to append results.",
)
parser.add_argument("--verbose", action="store_true", help="Enable verbose output.")
return parser
def init_environment(args: argparse.Namespace):
"""
Perform common initialization tasks:
- Set up logging.
- Create the log directory.
- Create the directory for the report file.
"""
setup_logger(args.verbose)
os.makedirs(args.log_dir, exist_ok=True)
os.makedirs(os.path.dirname(args.report_file), exist_ok=True)