#!/usr/bin/env python """ You should use scripts/generate_docs.sh and scripts/verify_docs.sh instead of using this. If the PWNDBG_GEN_DOC_JUST_VERIFY environment variable is set, then : Exit with non-zero exit status if the docs/commands/ files aren't up to date with the sources. Don't modify anything. If it isn't, this fixes up the docs/commands/ files to be up to date with the (argparse) information from the sources. """ from __future__ import annotations import os # We need to patch shutil.get_terminal_size() because otherwise argparse will output # .format_usage() based on terminal width which may be different for different users. # I tried every other solution, it doesn't work :). import shutil shutil.get_terminal_size = lambda fallback=(80, 24): os.terminal_size((80, 24)) import re import sys from typing import Dict from typing import Tuple from mdutils.mdutils import MdUtils import pwndbg.commands from pwndbg.commands import CommandObj from scripts._gen_docs_generic import verify_existence autogen_end_marker1 = "\n" autogen_end_marker2 = "\n" def category_to_folder_name(category) -> str: folder = category.lower() folder = re.sub(r"[ /]", "_", folder) # replace all spaces and / with _ # Don't allow wacky characters for folder names. If you hit this assert, feel free # to update the regex above to sanitize the category name. assert all(c.isalnum() or c == "_" for c in folder) return folder def extract_sources() -> Tuple[Dict[str, CommandObj], Dict[str, list[str]]]: """ Extract the sources. Returns: (A dictionary that maps the filenames of .md files to the corresponding CommandObj objects, A dictionary that maps a category name to a list of filenames for commands that belong to the category). """ filename_to_source: Dict[str, CommandObj] = {} category_to_filename: Dict[str, list[str]] = {} # FIXME: If an lldb-only command is added this (or pwndbg.commands) will need to be # revamped. # This depends on pwndbg.commands.load_commands() importing every command :) # `obj` iterates over all modules in pwndbg.commands (among other stuff). for obj_name in dir(pwndbg.commands): # Get the (potential) module by name. mod = getattr(pwndbg.commands, obj_name) # Iterate over everything in the module, which includes the command functions. for fn_name in dir(mod): fn = getattr(mod, fn_name) if not isinstance(fn, pwndbg.commands.CommandObj): continue # This object is a command! category = fn.category name = fn.command_name if category is None: # Should never be reached since CommandObj.__init__() will throw the error first. print( f"ERROR: Command function {fn_name} in {obj_name} does not have an assigned category." ) sys.exit(4) cat_folder = category_to_folder_name(category.value) filename = base_path + f"{cat_folder}/{name}.md" # Should be using join but whatever. filename_to_source[filename] = fn if category.value not in category_to_filename: category_to_filename[category.value] = [] category_to_filename[category.value].append(filename) assert filename_to_source assert category_to_filename return filename_to_source, category_to_filename def convert_to_markdown(filename: str, command: CommandObj) -> str: parser = command.parser assert parser name = command.command_name description = command.description assert description formatter = parser._get_formatter() if not description: print(f"ERROR: Command {name} ({filename}) does not have a description.") sys.exit(5) mdFile = MdUtils(filename) # title mdFile.new_header(level=1, title=name) # usage mdFile.insert_code(parser.format_usage(), language="text") # description mdFile.new_paragraph(description + "\n") # aliases if command.aliases: alias_txt = "Alias" + ("es" if len(command.aliases) > 1 else "") + ":" mdFile.write(f"\n**{alias_txt}** " + ", ".join(command.aliases) + "\n") used_actions = {} positionals = ["Positional Argument", "Help"] optionals = ["Short", "Long", "Help"] # positional arguments if parser._positionals._group_actions: for action in parser._positionals._group_actions: # The formatter decides if the default should be shown. param_help = formatter._expand_help(action) list_of_str = [action.dest, param_help] this_id = id(action) if this_id in used_actions: continue used_actions[this_id] = True positionals.extend(list_of_str) mdFile.write("### Positional arguments\n") positionals = [di if di is None else di.replace("\n", " ") for di in positionals] mdFile.new_table( columns=2, rows=len(positionals) // 2, text=positionals, text_align="left", ) mdFile.write("\n") # optional arguments if parser._option_string_actions: for k in parser._option_string_actions: action = parser._option_string_actions[k] # The formatter decides if the default should be shown. param_help = formatter._expand_help(action) list_of_str = ["", "", param_help] this_id = id(action) if this_id in used_actions: continue used_actions[this_id] = True for opt in action.option_strings: # --, long option if len(opt) > 1 and opt[1] in parser.prefix_chars: list_of_str[1] = opt # short opt elif len(opt) > 0 and opt[0] in parser.prefix_chars: list_of_str[0] = opt optionals.extend(list_of_str) mdFile.write("### Optional arguments\n") optionals = [di if di is None else di.replace("\n", " ") for di in optionals] mdFile.new_table( columns=3, rows=len(optionals) // 3, text=optionals, text_align="left", ) mdFile.write("\n") if command.examples: # Put the examples into a code block so they are formatted sensically. mdFile.write("### Examples\n```text\n" + command.examples + "\n```\n") if command.notes: # The author of the note should make sure it make sense in markdown. mdFile.write("### Notes\n" + command.notes + "\n") if command.pure_epilog: mdFile.write("### Extra\n" + command.pure_epilog + "\n") autogen_warning = "" return autogen_warning + "\n" + mdFile.get_md_text() def convert_all_to_markdown( filename_to_command: Dict[str, CommandObj], ) -> Dict[str, str]: result = {} for file in filename_to_command: result[file] = convert_to_markdown(file, filename_to_command[file]) return result def generate_index( filename_to_command: Dict[str, CommandObj], category_to_filename: Dict[str, list[str]], ) -> str: mdFile = MdUtils("docs/commands/index.md") mdFile.new_header(level=1, title="Commands") for cat in sorted(category_to_filename): mdFile.new_header(level=2, title=f"{cat}") items = [] for filename in sorted(category_to_filename[cat]): cmd = filename_to_command[filename] name = cmd.command_name short_desc = cmd.description.splitlines()[0] folder = category_to_folder_name(cat) items.append(f" [{name}]({folder}/{name}.md) - {short_desc}") mdFile.new_list(items=items) index_autogen_warning = ( "\n" ) return index_autogen_warning + mdFile.get_md_text() def verify_files(filename_to_markdown: Dict[str, str]) -> str | None: """ Verify all the markdown files are up to date with the sources. Returns: None if everything is up-to-date. A string containing the error message if something is not. """ for filename, markdown in filename_to_markdown.items(): print(f"Checking {filename} ..") if not os.path.exists(filename): return f"File {filename} does not exist." file_data = "" with open(filename, "r") as file: file_data = file.readlines() markdown = [x + "\n" for x in markdown.splitlines()] mkdlen = len(markdown) if len(file_data) < (mkdlen + 3): return ( f"File {filename} is too short. Expected {mkdlen + 3} lines, got {len(file_data)}." ) if not ( file_data[mkdlen + 1] == autogen_end_marker1 and file_data[mkdlen + 2] == autogen_end_marker2 ): return f'Expected autogenerated end markers in {filename} @ lines {mkdlen} and {mkdlen+1}. Instead found "{file_data[mkdlen]}" and "{file_data[mkdlen+1]}".' for i in range(mkdlen): if file_data[i] != markdown[i]: return f'File {filename} differs from autogenerated on line {i}.\nFile: "{file_data[i]}".\nAutogenerated: "{markdown[i]}".' return None def update_files(filename_to_markdown: Dict[str, str]): """ Fix files so they are up to date with the sources. This also creates new files/directories if needed. """ for filename, markdown in filename_to_markdown.items(): print(f"Updating {filename} ..") if not os.path.exists(filename): # Simple case, just create the file and write it. os.makedirs(os.path.dirname(filename), exist_ok=True) with open(filename, "w") as file: file.write(markdown + "\n" + autogen_end_marker1 + autogen_end_marker2) continue # Need to find the marker in the file, and edit only above that part. with open(filename, "r+") as file: file_data = file.readlines() marker_idx = -1 for i in reversed(range(len(file_data))): if file_data[i] == autogen_end_marker2: if i == 0 or file_data[i - 1] != autogen_end_marker1: print( f"ERROR: In file {filename} found the second autogen marker, but couldn't find the first ({autogen_end_marker1})." ) sys.exit(6) marker_idx = i - 1 break if marker_idx == -1: print( f"ERROR: In file {filename} couldn't find autogen marker ({autogen_end_marker2})." ) sys.exit(7) handwritten_doc = "".join(file_data[marker_idx:]) # Includes the autogen markers final = markdown + "\n" + handwritten_doc file.seek(0) file.write(final) file.truncate() def file_has_handwritten(filename: str) -> bool: """ Returns if a file has a hand-written part. Also returns true if the autogen markers are malformed or don't exist. """ with open(filename, "r+") as file: file_data = file.readlines() marker_idx = -1 for i in reversed(range(len(file_data))): if file_data[i] == autogen_end_marker2: if i == 0 or file_data[i - 1] != autogen_end_marker1: return True marker_idx = i - 1 break if marker_idx == -1: return True if len(file_data) == marker_idx + 2: # there is nothing after the markers return False handwritten_doc = "".join(file_data[marker_idx + 2 :]) if handwritten_doc.strip(): # There is some non-whitespace after the markers return True # There is only whitespace after the markers, we won't # complain about this. return False base_path = "docs/commands/" # Must have trailing slash. # ==== Start ==== if len(sys.argv) > 1: print("This script doesn't accept any arguments.") print("See top of the file for usage.") sys.exit(1) just_verify = False if os.getenv("PWNDBG_GEN_DOC_JUST_VERIFY"): just_verify = True print("\n==== Command Documentation ====") extracted, cat_to_names = extract_sources() markdowned = convert_all_to_markdown(extracted) markdowned[base_path + "index.md"] = generate_index(extracted, cat_to_names) if just_verify: print("Checking if all files are in place..") missing, extra = verify_existence(list(markdowned.keys()), base_path) if missing or extra: print("To fix this please run ./scripts/generate_docs.sh.") sys.exit(2) print("Every file is where it should be!") print("Verifying contents...") err = verify_files(markdowned) if err: print("VERIFICATION FAILED. The files differ from what would be auto-generated.") print("Error:", err) print("Please run ./scripts/generate_docs.sh from project root and commit the changes.") sys.exit(3) print("Verification successful!") else: print("Updating files...") update_files(markdowned) print("Update successful.") missing, extra = verify_existence(list(markdowned.keys()), base_path) assert not missing and "Some files are missing, which should be impossible." if extra: print("Take care! Deleting these extra files:") not_deleted = [] for e in extra: if file_has_handwritten(e): not_deleted.append(e) else: print(e) os.remove(e) if not_deleted: print("\nSome files were not auto-deleted as they contain a hand-written part") print("(or the markers for the hand-written part are malformed). Please delete") print("them manually, probably after transferring the hand-written part to a") print("new file.") print(f"Files ({len(not_deleted)}):") print("\n".join(not_deleted)) exit(18) else: print("Deleted successfully.")