Skip to content

Commit

Permalink
docs: Cleanup docs json for better parsing
Browse files Browse the repository at this point in the history 
Change-Id: I9484541223d5e1e81a5ac240b26636b71991f9d6
  • Loading branch information
@aMayzner
aMayzner committed Oct 28, 2024
1 parent fbccf99 commit a9e55d1
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 68 deletions.
42 changes: 23 additions & 19 deletions infra/perfetto.dev/src/gen_stdlib_docs_md.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,8 +98,8 @@ def __init__(self, package_name: str, module_dict: Dict):
self.objs, self.funs, self.view_funs, self.macros = [], [], [], []

# Views/tables
for data in module_dict['tables/views']:
if not data['cols'].items():
for data in module_dict['data_objects']:
if not data['cols']:
continue

obj_summary = (
Expand All @@ -110,7 +110,8 @@ def __init__(self, package_name: str, module_dict: Dict):
content.append(data['desc'])

table = [_md_table_header(['Column', 'Type', 'Description'])]
for name, info in data['cols'].items():
for info in data['cols']:
name = info["name"]
table.append(
f'{name} | {info["type"]} | {_escape(info["desc"])}')
content.append('\n\n')
Expand All @@ -121,22 +122,22 @@ def __init__(self, package_name: str, module_dict: Dict):
self.objs.append('\n\n')

# Functions
for data in module_dict['functions']:
obj_summary = f'''{_bold(data['name'])} -> {data['return_type']}. {data['summary_desc']}\n\n'''
for d in module_dict['functions']:
summary = f'''{_bold(d['name'])} -> {d['return_type']}. {d['summary_desc']}\n\n'''
content = []
if (data['summary_desc'] != data['desc']):
content.append(data['desc'])
if (d['summary_desc'] != d['desc']):
content.append(d['desc'])

content.append(
f"Returns {data['return_type']}: {data['return_desc']}\n\n")
if data['args']:
f"Returns {d['return_type']}: {d['return_desc']}\n\n")
if d['args']:
content.append(_md_table_header(['Argument', 'Type', 'Description']))
for name, arg_dict in data['args'].items():
for arg_dict in d['args']:
content.append(
f'''{name} | {arg_dict['type']} | {_escape(arg_dict['desc'])}'''
f'''{arg_dict['name']} | {arg_dict['type']} | {_escape(arg_dict['desc'])}'''
)

self.funs.append(_md_rolldown(obj_summary, '\n'.join(content)))
self.funs.append(_md_rolldown(summary, '\n'.join(content)))
self.funs.append('\n\n')

# Table functions
Expand All @@ -148,16 +149,17 @@ def __init__(self, package_name: str, module_dict: Dict):

if data['args']:
args_table = [_md_table_header(['Argument', 'Type', 'Description'])]
for name, arg_dict in data['args'].items():
for arg_dict in data['args']:
args_table.append(
f'''{name} | {arg_dict['type']} | {_escape(arg_dict['desc'])}'''
f'''{arg_dict['name']} | {arg_dict['type']} | {_escape(arg_dict['desc'])}'''
)
content.append('\n'.join(args_table))
content.append('\n\n')

content.append(_md_table_header(['Column', 'Type', 'Description']))
for name, column in data['cols'].items():
content.append(f'{name} | {column["type"]} | {column["desc"]}')
for column in data['cols']:
content.append(
f'{column["name"]} | {column["type"]} | {column["desc"]}')

self.view_funs.append(_md_rolldown(obj_summary, '\n'.join(content)))
self.view_funs.append('\n\n')
Expand All @@ -173,9 +175,9 @@ def __init__(self, package_name: str, module_dict: Dict):
f'''Returns: {data['return_type']}, {data['return_desc']}\n\n''')
if data['args']:
table = [_md_table_header(['Argument', 'Type', 'Description'])]
for name, arg_dict in data['args'].items():
for arg_dict in data['args']:
table.append(
f'''{name} | {arg_dict['type']} | {_escape(arg_dict['desc'])}'''
f'''{arg_dict['name']} | {arg_dict['type']} | {_escape(arg_dict['desc'])}'''
)
content.append('\n'.join(table))

Expand Down Expand Up @@ -274,7 +276,9 @@ def main():

# Fetch the modules from json documentation.
packages: Dict[str, PackageMd] = {}
for package_name, modules in stdlib_json.items():
for package in stdlib_json:
package_name = package["name"]
modules = package["modules"]
# Remove 'common' when it has been removed from the code.
if package_name not in ['deprecated', 'common']:
package = PackageMd(package_name, modules)
Expand Down
108 changes: 59 additions & 49 deletions tools/gen_stdlib_docs_json.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ def main():

sql_outputs[relpath] = f.read()

modules = defaultdict(list)
packages = defaultdict(list)
# Add documentation from each file
for path, sql in sql_outputs.items():
package = path.split("/")[0]
package_name = path.split("/")[0]
module_name = path.split(".sql")[0].replace("/", ".")

docs = parse_file(path, sql)
Expand All @@ -86,71 +86,81 @@ def main():
print(e)
return 1

file_dict = {
module_dict = {
'module_name':
module_name,
'tables/views': [{
'name': table.name,
'desc': table.desc,
'summary_desc': _summary_desc(table.desc),
'type': table.type,
'cols': {
col_name: {
'type': col.type,
'desc': col.description,
} for (col_name, col) in table.cols.items()
},
'data_objects': [{
'name':
table.name,
'desc':
table.desc,
'summary_desc':
_summary_desc(table.desc),
'type':
table.type,
'cols': [{
'name': col_name,
'type': col.type,
'desc': col.description
} for (col_name, col) in table.cols.items()]
} for table in docs.table_views],
'functions': [{
'name': function.name,
'desc': function.desc,
'summary_desc': _summary_desc(function.desc),
'args': {
arg_name: {
'type': arg.type,
'desc': arg.description,
} for (arg_name, arg) in function.args.items()
},
'args': [{
'name': arg_name,
'type': arg.type,
'desc': arg.description,
} for (arg_name, arg) in function.args.items()],
'return_type': function.return_type,
'return_desc': function.return_desc,
} for function in docs.functions],
'table_functions': [{
'name': function.name,
'desc': function.desc,
'summary_desc': _summary_desc(function.desc),
'args': {
arg_name: {
'type': arg.type,
'desc': arg.description,
} for (arg_name, arg) in function.args.items()
},
'cols': {
col_name: {
'type': col.type,
'desc': col.description,
} for (col_name, col) in function.cols.items()
},
'name':
function.name,
'desc':
function.desc,
'summary_desc':
_summary_desc(function.desc),
'args': [{
'name': arg_name,
'type': arg.type,
'desc': arg.description,
} for (arg_name, arg) in function.args.items()],
'cols': [{
'name': col_name,
'type': col.type,
'desc': col.description
} for (col_name, col) in function.cols.items()]
} for function in docs.table_functions],
'macros': [{
'name': macro.name,
'desc': macro.desc,
'summary_desc': _summary_desc(macro.desc),
'return_desc': macro.return_desc,
'return_type': macro.return_type,
'args': {
arg_name: {
'type': arg.type,
'desc': arg.description,
} for (arg_name, arg) in macro.args.items()
},
'name':
macro.name,
'desc':
macro.desc,
'summary_desc':
_summary_desc(macro.desc),
'return_desc':
macro.return_desc,
'return_type':
macro.return_type,
'args': [{
'name': arg_name,
'type': arg.type,
'desc': arg.description,
} for (arg_name, arg) in macro.args.items()],
} for macro in docs.macros],
}
modules[package].append(file_dict)
packages[package_name].append(module_dict)

indent = None if args.minify else 4
packages_list = [{
"name": name,
"modules": modules
} for name, modules in packages.items()]

with open(args.json_out, 'w+') as f:
json.dump(modules, f, indent=indent)
json.dump(packages_list, f, indent=None if args.minify else 4)

return 0

Expand Down

0 comments on commit a9e55d1

Please sign in to comment.