Cherry pick PR #2081: Make devsite generation scripts public (#2230)

Refer to the original PR: #2081

b/236404667

Co-authored-by: Oscar Vestlie <[email protected]>

cobalt/site/README.md

@@ -0,0 +1,22 @@
+# Regenerating developer site contents
+
+1.  Update documents and images. Note that files in `cobalt/site/gen` and
+    `cobalt/site/reference` are autogenerated and changes in these directories
+    will be overwritten by the steps below.
+
+2.  From the root of your repository run the following to build the docsite
+    Docker image.
+
+    ```shell
+    $ docker build -t docsite --build-arg UID=$(id -u) --build-arg GID=$(id -g) cobalt/site/docker
+    ```
+
+3.  Run it to regenerate the site.
+
+    ```shell
+    $ docker run -it --mount type=bind,source=$(pwd),target=/code docsite
+    ```
+
+4.  (Optional) To preview the changes you must commit the generated files and
+    create a Pull Request. That pull request can be previewed in staging by
+    following the direction for deploying the developer site.

cobalt/site/docker/Dockerfile

@@ -0,0 +1,65 @@
+# Copyright 2021 The Cobalt Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG FROM_IMAGE
+FROM ${FROM_IMAGE:-gcr.io/cloud-marketplace-containers/google/debian10}
+
+RUN apt update -qqy \
+    && apt install -qqy --no-install-recommends \
+        curl \
+        doxygen \
+        git \
+        python3 \
+        unzip \
+    && apt-get clean autoclean \
+    && apt-get autoremove -y --purge \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && rm -rf /var/lib/{apt,dpkg,cache,log}
+
+# Tell GN that we are building with Docker.
+ENV IS_DOCKER=1
+
+# Set python path for gn.
+ENV PYTHONPATH=/code
+
+# Mark the source directory safe.
+RUN git config --global --add safe.directory /code
+
+# === Get GN via CIPD
+ARG GN_SHA256SUM="af7b2dcb3905bca56655e12131b365f1cba8e159db80d2022330c4f522fab2ef  /tmp/gn.zip"
+ARG GN_HASH=r3styzkFvKVmVeEhMbNl8cuo4VnbgNICIzDE9SL6su8C
+RUN curl --location --silent --output /tmp/gn.zip \
+    "https://chrome-infra-packages.appspot.com/dl/gn/gn/linux-amd64/+/${GN_HASH}" \
+    && echo ${GN_SHA256SUM} | sha256sum --check \
+    && unzip /tmp/gn.zip -d /usr/local/bin \
+    && rm /tmp/gn.zip
+RUN chmod a+x /usr/local/bin/gn
+
+# We create and use a non-root user explicitly so that the generated and
+# modified files maintain the same permissions as the user that launched the
+# Docker container.
+ARG USER
+ARG UID
+ARG GID
+RUN addgroup --group --gid "${GID}" defaultgroup \
+    && adduser --disabled-password --gecos '' --uid "${UID}" --gid "${GID}" defaultuser
+
+# Create an out directory for gn. Its name is hardcoded in the docsite script.
+RUN mkdir /project_out_dir \
+    && chown ${USER:-defaultuser}:defaultgroup /project_out_dir
+
+# Once the directory has been created we can switch to the new user.
+USER ${USER:-defaultuser}
+
+CMD /code/cobalt/site/scripts/generate_site.py

cobalt/site/scripts/cobalt_configuration_public.py

@@ -0,0 +1,162 @@
+# Copyright 2023 The Cobalt Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to generate Starboard Configuration Reference."""
+import environment
+import os
+import re
+import sys
+
+
+def print_doc_header(doc, title):
+  doc.write('Project: /youtube/cobalt/_project.yaml\n')
+  doc.write('Book: /youtube/cobalt/_book.yaml\n\n')
+  doc.write(f'# {title}\n\n')
+
+
+def print_doc_section_header(doc, section, section_headers):
+  doc.write('## ' + section + '\n\n')
+  if section in section_headers:
+    doc.write(section_headers[section] + '\n\n')
+
+
+def print_section_property_table_header(doc):
+  doc.write('| Properties |\n')
+  doc.write('| :--- |\n')
+
+
+def print_property(doc, prop):
+  comment = prop['comment']
+  comment = comment.replace('__', '__')
+  if prop['value']:
+    line_break = ''
+    value = prop['value'].replace('|', r'\|')
+    if len(value) > 30:
+      line_break = '<br>'
+    if comment[-8:] != '<br><br>':
+      comment += '<br><br>'
+    comment += ('The default value in the Stub implementation is ' +
+                line_break + '`' + prop['value'] + '`')
+  elif prop['undefined']:
+    if comment[-8:] != '<br><br>':
+      comment += '<br><br>'
+    comment += 'By default, this property is undefined.'
+  if comment[0:8] != '<br><br>':
+    doc.write('| **`' + prop['name'] + '`**<br><br>' + comment + ' |\n')
+  else:
+    doc.write('| **`' + prop['name'] + '`**' + comment + ' |\n')
+
+
+def create_reference_doc(site_path, properties, section_headers):
+  reference_doc_path = os.path.join(site_path, 'docs', 'reference', 'starboard',
+                                    'configuration-public.md')
+  environment.make_dirs(os.path.dirname(reference_doc_path))
+  with open(reference_doc_path, 'w', encoding='utf8') as doc:
+    print_doc_header(doc, 'Starboard Configuration Reference Guide')
+
+    for section in sorted(properties):
+      if len(properties[section]) > 0:
+        print_doc_section_header(doc, section, section_headers)
+
+        print_section_property_table_header(doc)
+
+        for p in range(0, len(properties[section])):
+          print_property(doc, properties[section][p])
+
+        doc.write('\n\n')
+
+
+def main(source_dir, output_dir=None):
+  config_dir = environment.get_stub_platform_dir(source_dir)
+  file_path = os.path.join(config_dir, 'configuration_public.h')
+  with open(file_path, 'r', encoding='utf8') as file_contents:
+    file_lines = file_contents.readlines()
+
+  # parse .proto files
+  comment = ''
+  in_line_item = ''
+  section = ''
+  properties = {}
+  section_headers = {}
+  last_thing_was_a_header = False
+  for original_line in file_lines:
+    line = original_line.strip()
+    if line[0:7] == '// --- ':
+      section = line[7:].split(' -')[0]
+      properties[section] = []
+      last_thing_was_a_header = True
+    elif section and (line[0:8] == '#define ' or line[0:7] == '#undef '):
+
+      if in_line_item:
+        if comment:
+          comment += '</li></' + in_line_item + '>'
+        in_line_item = ''
+      last_thing_was_a_header = False
+      prop_array = line.split(' ')
+      prop = {'comment': '', 'name': '', 'value': '', 'undefined': False}
+      if line[0:7] == '#undef ':
+        prop['undefined'] = True
+      if len(prop_array) > 1:
+        prop['name'] = prop_array[1]
+      if len(prop_array) > 2:
+        prop['value'] = ' '.join(prop_array[2:])
+      if comment:
+        prop['comment'] = comment.strip()
+      if '(' in prop['name'] and ')' not in prop['name']:
+        new_string = ' '.join(prop_array[1:])
+        new_prop_array = new_string.split(')')
+        prop['name'] = new_prop_array[0] + ')'
+        new_value = ')'.join(new_prop_array[1:])
+        prop['value'] = new_value.strip()
+      properties[section].append(prop)
+      comment = ''
+    elif section and line[0:2] == '//':
+      ol_item_regex = re.compile(r'^\d\. ')
+      comment_text = line[2:].strip()
+      is_ol_item = re.search(ol_item_regex, comment_text)
+      if (is_ol_item or comment_text.strip()[0:2] == '- ' or
+          comment_text.strip()[0:2] == '* '):
+        # Replace '* ' at beginning of comment with '<li>'
+        # Strip whitespace before '*' and after '*" up to start of text
+        if not in_line_item:
+          if is_ol_item:
+            comment_text = '<ol><li>' + comment_text.strip()[2:].strip()
+            in_line_item = 'ol'
+          else:
+            comment_text = '<ul><li>' + comment_text.strip()[1:].strip()
+            in_line_item = 'ul'
+        else:
+          if is_ol_item:
+            comment_text = '</li><li>' + comment_text.strip()[2:].strip()
+          else:
+            comment_text = '</li><li>' + comment_text.strip()[1:].strip()
+      comment += ' ' + comment_text
+    elif comment and line == '':
+      if last_thing_was_a_header:
+        section_headers[section] = comment
+      last_thing_was_a_header = False
+      comment = ''
+      if comment[-8:] != '<br><br>':
+        comment += '<br><br>'
+
+  if output_dir:
+    site_path = environment.get_site_dir(output_dir)
+  else:
+    site_path = environment.get_site_dir(source_dir)
+  create_reference_doc(site_path, properties, section_headers)
+  return 0
+
+
+if __name__ == '__main__':
+  options = environment.parse_arguments(__doc__, sys.argv[1:])
+  sys.exit(main(options.source, options.out))

cobalt/site/scripts/cobalt_documentation.py

@@ -0,0 +1,61 @@
+# Copyright 2023 The Cobalt Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Module to copy Cobalt doc folders to cobalt site directory."""
+import environment
+import os
+import shutil
+import sys
+
+_COBALT_DOC_LOCATIONS = [
+    'cobalt/doc/',
+    'starboard/doc/',
+    'starboard/build/doc/',
+    'starboard/tools/doc/',
+]
+
+
+def write_header(doc):
+  doc.write('Project: /youtube/cobalt/_project.yaml\n')
+  doc.write('Book: /youtube/cobalt/_book.yaml\n\n')
+
+
+def copy_doc_locations(source_dir, output_dir=None):
+  if output_dir:
+    site_path = environment.get_site_dir(output_dir)
+  else:
+    site_path = environment.get_site_dir(source_dir)
+
+  gen_dir = os.path.join(site_path, 'docs', 'gen')
+  if os.path.exists(gen_dir):
+    shutil.rmtree(gen_dir)
+
+  for location in _COBALT_DOC_LOCATIONS:
+    shutil.copytree(
+        os.path.join(source_dir, location), os.path.join(gen_dir, location))
+
+  for root, _, files in os.walk(gen_dir):
+    for filename in files:
+      if not filename.endswith('.md'):
+        continue
+      filename = os.path.join(root, filename)
+      with open(filename, encoding='utf8') as f:
+        lines = f.readlines()
+      with open(filename, 'w', encoding='utf8') as f:
+        write_header(f)
+        f.writelines(lines)
+
+
+if __name__ == '__main__':
+  out = sys.argv[2] if len(sys.argv) == 3 else None
+  copy_doc_locations(sys.argv[1], out)