add doxygen doc build configurations (#2191)

Co-authored-by: Nikolay Tyukaev <ntyukaev_lo@jenkins.inn.intel.com>
This commit is contained in:
Nikolay Tyukaev 2020-09-22 12:37:10 +03:00 committed by GitHub
parent 261bd3de6b
commit f828b16f40
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 3106 additions and 46 deletions

View File

@ -45,3 +45,6 @@ ie_dependent_option (ENABLE_AVX2 "Enable AVX2 optimizations" ON "X86_64 OR X86"
ie_dependent_option (ENABLE_AVX512F "Enable AVX512 optimizations" ON "X86_64 OR X86" OFF)
ie_dependent_option (ENABLE_PROFILING_ITT "ITT tracing of IE and plugins internals" ON "NOT CMAKE_CROSSCOMPILING" OFF)
# Documentation build
ie_option (ENABLE_DOCS "build docs using Doxygen" OFF)

View File

@ -2,59 +2,186 @@
# SPDX-License-Identifier: Apache-2.0
#
add_subdirectory(examples)
if(NOT ENABLE_DOCKER)
add_subdirectory(examples)
# Detect nGraph
find_package(ngraph QUIET)
if(NOT ngraph_FOUND)
set(ngraph_DIR ${CMAKE_BINARY_DIR}/ngraph)
endif()
# Detect InferenceEngine
find_package(InferenceEngine QUIET)
if(NOT InferenceEngine_FOUND)
set(InferenceEngine_DIR ${CMAKE_BINARY_DIR})
endif()
add_subdirectory(template_extension)
set(all_docs_targets
ie_docs_examples
template_extension
templatePlugin TemplateBehaviorTests TemplateFunctionalTests)
foreach(target_name IN LISTS all_docs_targets)
if (TARGET ${target_name})
set_target_properties(${target_name} PROPERTIES FOLDER docs)
# Detect nGraph
find_package(ngraph QUIET)
if(NOT ngraph_FOUND)
set(ngraph_DIR ${CMAKE_BINARY_DIR}/ngraph)
endif()
endforeach()
# OpenVINO docs
# Detect InferenceEngine
find_package(InferenceEngine QUIET)
if(NOT InferenceEngine_FOUND)
set(InferenceEngine_DIR ${CMAKE_BINARY_DIR})
endif()
set(OPENVINO_DOCS_PATH "" CACHE PATH "Path to openvino-documentation local repository")
set(args "")
add_subdirectory(template_extension)
if(OPENVINO_DOCS_PATH)
set(args "${args} ovinodoc_path:${OPENVINO_DOCS_PATH}")
set(all_docs_targets
ie_docs_examples
template_extension
templatePlugin TemplateBehaviorTests TemplateFunctionalTests)
foreach(target_name IN LISTS all_docs_targets)
if (TARGET ${target_name})
set_target_properties(${target_name} PROPERTIES FOLDER docs)
endif()
endforeach()
endif()
file(GLOB_RECURSE docs_files "${OpenVINO_MAIN_SOURCE_DIR}/docs")
file(GLOB_RECURSE include_files "${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/include")
file(GLOB_RECURSE ovino_files "${OPENVINO_DOCS_PATH}")
function(build_docs)
add_custom_target(ie_docs
COMMAND ./build_docs.sh ${args}
WORKING_DIRECTORY "${OpenVINO_MAIN_SOURCE_DIR}/docs/build_documentation"
COMMENT "Generating OpenVINO documentation"
SOURCES ${docs_files} ${include_files} ${ovino_files}
VERBATIM)
set_target_properties(ie_docs PROPERTIES FOLDER docs)
find_package(Doxygen REQUIRED dot)
find_package(Python3 COMPONENTS Interpreter)
find_package(LATEX)
find_program(browser NAMES xdg-open)
if(browser)
add_custom_target(ie_docs_open
COMMAND ${browser} "${OpenVINO_MAIN_SOURCE_DIR}/doc/html/index.html"
DEPENDS ie_docs
COMMENT "Open OpenVINO documentation"
VERBATIM)
set_target_properties(ie_docs_open PROPERTIES FOLDER docs)
if(NOT DOXYGEN_FOUND)
message(FATAL_ERROR "Doxygen is required to build the documentation")
endif()
if(NOT Python3_FOUND)
message(FATAL_ERROR "Python3 is required to build the documentation")
endif()
if(NOT LATEX_FOUND)
message(FATAL_ERROR "LATEX is required to build the documentation")
endif()
set(DOCS_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR})
set(DOXYGEN_DIR ${OpenVINO_MAIN_SOURCE_DIR}/docs/doxygen)
set(IE_SOURCE_DIR ${OpenVINO_MAIN_SOURCE_DIR}/inference-engine)
set(PYTHON_API_IN ${IE_SOURCE_DIR}/ie_bridges/python/src/openvino/inference_engine/ie_api.pyx)
set(PYTHON_API_OUT ${DOCS_BINARY_DIR}/python_api/ie_api.pyx)
set(C_API ${IE_SOURCE_DIR}/ie_bridges/c/include)
set(PLUGIN_API_DIR ${DOCS_BINARY_DIR}/IE_PLUGIN_DG)
# Preprocessing scripts
set(DOXY_MD_FILTER ${DOXYGEN_DIR}/doxy_md_filter.py)
set(PYX_FILTER ${DOXYGEN_DIR}/pyx_filter.py)
file(GLOB_RECURSE doc_source_files
LIST_DIRECTORIES true
RELATIVE ${OpenVINO_MAIN_SOURCE_DIR}
${OpenVINO_MAIN_SOURCE_DIR}/docs/*.md
${OpenVINO_MAIN_SOURCE_DIR}/docs/*.png
${OpenVINO_MAIN_SOURCE_DIR}/docs/*.gif
${OpenVINO_MAIN_SOURCE_DIR}/docs/*.jpg
${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.md
${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.png
${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.gif
${OpenVINO_MAIN_SOURCE_DIR}/inference-engine/*.jpg
)
configure_file(${PYTHON_API_IN} ${PYTHON_API_OUT} @ONLY)
set(IE_CONFIG_SOURCE ${DOXYGEN_DIR}/ie_docs.config)
set(C_CONFIG_SOURCE ${DOXYGEN_DIR}/ie_c_api.config)
set(PY_CONFIG_SOURCE ${DOXYGEN_DIR}/ie_py_api.config)
set(PLUGIN_CONFIG_SOURCE ${DOXYGEN_DIR}/ie_plugin_api.config)
set(IE_CONFIG_BINARY ${DOCS_BINARY_DIR}/ie_docs.config)
set(C_CONFIG_BINARY ${DOCS_BINARY_DIR}/ie_c_api.config)
set(PY_CONFIG_BINARY ${DOCS_BINARY_DIR}/ie_py_api.config)
set(PLUGIN_CONFIG_BINARY ${DOCS_BINARY_DIR}/ie_plugin_api.config)
set(IE_LAYOUT_SOURCE ${DOXYGEN_DIR}/ie_docs.xml)
set(C_LAYOUT_SOURCE ${DOXYGEN_DIR}/ie_c_api.xml)
set(PY_LAYOUT_SOURCE ${DOXYGEN_DIR}/ie_py_api.xml)
set(PLUGIN_LAYOUT_SOURCE ${DOXYGEN_DIR}/ie_plugin_api.xml)
set(IE_LAYOUT_BINARY ${DOCS_BINARY_DIR}/ie_docs.xml)
set(C_LAYOUT_BINARY ${DOCS_BINARY_DIR}/ie_c_api.xml)
set(PY_LAYOUT_BINARY ${DOCS_BINARY_DIR}/ie_py_api.xml)
set(PLUGIN_LAYOUT_BINARY ${DOCS_BINARY_DIR}/ie_plugin_api.xml)
# Tables of contents
configure_file(${IE_LAYOUT_SOURCE} ${IE_LAYOUT_BINARY} @ONLY)
configure_file(${C_LAYOUT_SOURCE} ${C_LAYOUT_BINARY} @ONLY)
configure_file(${PY_LAYOUT_SOURCE} ${PY_LAYOUT_BINARY} @ONLY)
configure_file(${PLUGIN_LAYOUT_SOURCE} ${PLUGIN_LAYOUT_BINARY} @ONLY)
# Doxygen config files
configure_file(${IE_CONFIG_SOURCE} ${IE_CONFIG_BINARY} @ONLY)
configure_file(${C_CONFIG_SOURCE} ${C_CONFIG_BINARY} @ONLY)
configure_file(${PY_CONFIG_SOURCE} ${PY_CONFIG_BINARY} @ONLY)
configure_file(${PLUGIN_CONFIG_SOURCE} ${PLUGIN_CONFIG_BINARY} @ONLY)
# Preprocessing scripts
set(DOXY_MD_FILTER ${DOXYGEN_DIR}/doxy_md_filter.py)
set(PYX_FILTER ${DOXYGEN_DIR}/pyx_filter.py)
add_custom_target(c_api
COMMAND ${DOXYGEN_EXECUTABLE} ${C_CONFIG_BINARY}
WORKING_DIRECTORY ${DOCS_BINARY_DIR}
COMMENT "Generating C API Reference"
VERBATIM
)
add_custom_target(py_api
COMMAND ${DOXYGEN_EXECUTABLE} ${PY_CONFIG_BINARY}
WORKING_DIRECTORY ${DOCS_BINARY_DIR}
COMMENT "Generating Python API Reference"
VERBATIM
)
add_custom_target(plugin_api
COMMAND ${DOXYGEN_EXECUTABLE} ${PLUGIN_CONFIG_BINARY}
WORKING_DIRECTORY ${DOCS_BINARY_DIR}
COMMENT "Generating Plugin API Reference"
VERBATIM
)
add_custom_target(preprocess_docs
COMMENT "Pre-process docs"
VERBATIM
)
add_custom_target(ie_docs
DEPENDS preprocess_docs
COMMAND ${DOXYGEN_EXECUTABLE} ${IE_CONFIG_BINARY}
WORKING_DIRECTORY ${DOCS_BINARY_DIR}
VERBATIM
)
add_custom_target(openvino_docs
DEPENDS c_api py_api ie_docs plugin_api
COMMENT "Generating OpenVINO documentation"
VERBATIM
)
add_custom_command(TARGET py_api
PRE_BUILD
COMMAND ${Python3_EXECUTABLE} ${PYX_FILTER} ${PYTHON_API_OUT}
COMMENT "Pre-process Python API."
)
foreach(source_file ${doc_source_files})
add_custom_command(TARGET preprocess_docs
PRE_BUILD
COMMAND ${CMAKE_COMMAND} -E copy ${OpenVINO_MAIN_SOURCE_DIR}/${source_file} ${DOCS_BINARY_DIR}/${source_file})
endforeach()
add_custom_command(TARGET preprocess_docs
PRE_BUILD
COMMAND ${Python3_EXECUTABLE} ${DOXY_MD_FILTER} ${DOCS_BINARY_DIR}
COMMENT "Pre-process markdown and image links."
)
set_target_properties(ie_docs PROPERTIES FOLDER docs)
find_program(browser NAMES xdg-open)
if(browser)
add_custom_target(ie_docs_open
COMMAND ${browser} "${OpenVINO_MAIN_SOURCE_DIR}/doc/html/index.html"
DEPENDS ie_docs
COMMENT "Open OpenVINO documentation"
VERBATIM)
set_target_properties(ie_docs_open PROPERTIES FOLDER docs)
endif()
endfunction()
if (ENABLE_DOCS)
build_docs()
endif()

View File

@ -0,0 +1,94 @@
import os
import re
import glob
import argparse
def get_label(file):
"""
Read lines of a file and try to find a doxygen label.
If the label is not found return None.
"""
for line in file:
label = re.search(r'\{\#(.+)\}', line)
if label:
return label.group(1)
return
def replace_links(content, items, folder, labels, docs_folder):
"""
Replace markdown links with doxygen labels.
"""
for item in items:
link = item[0]
ext = item[1]
link_path = os.path.abspath(os.path.join(folder, link))
if os.path.exists(link_path):
if ext == 'md':
if link_path in labels:
label = labels.get(link_path)
else:
with open(link_path, 'r', encoding='utf-8') as file:
lines = []
i = 0
while i < 5:
try:
lines.append(next(file))
except StopIteration:
break
i += 1
label = get_label(lines)
labels[link_path] = label
if label:
content = content.replace(link, '@ref ' + label)
else:
rel_path = os.path.relpath(link_path, docs_folder).replace('\\', '/')
content = content.replace(link, rel_path)
return content
def process_github_md_links(content, items):
"""
This is a workaround to support github markdown links in doxygen 1.8.12.
"""
for item in items:
orig = item[0]
link_name = item[1]
link_url = item[2]
html_link = '<a href="{}">{}</a>'.format(link_url, link_name)
content = content.replace(orig, html_link)
return content
def process(docs_folder):
"""
Recursively find markdown files in docs_folder and
replace links to markdown files with doxygen labels (ex. @ref label_name).
"""
labels = dict() # store labels in dictionary
md_files = glob.glob(os.path.join(docs_folder, '**/*.md'), recursive=True)
for md_file in md_files:
md_folder = os.path.dirname(md_file)
with open(md_file, 'r', encoding='utf-8') as f:
content = f.read()
inline_links = set(re.findall(r'!?\[.*?\]\(([\w\/\-\.]+\.(md|png|jpg|gif))\)', content, flags=re.IGNORECASE))
github_md_links = set(re.findall(r'(\[(.+?)\]\((https:[\w\.\/-]+?\.md)\))', content, flags=re.IGNORECASE))
reference_links = set(re.findall(r'\[.+\]\:\s*?([\w\/\-\.]+\.(md|png|jpg|gif))', content, flags=re.IGNORECASE))
content = replace_links(content, inline_links, md_folder, labels, docs_folder)
content = replace_links(content, reference_links, md_folder, labels, docs_folder)
content = process_github_md_links(content, github_md_links)
if inline_links or reference_links or github_md_links:
with open(md_file, 'w', encoding='utf-8') as f:
f.write(content)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('docs', type=str, help='Path to a folder containing .md files.')
args = parser.parse_args()
process(args.docs)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,25 @@
@INCLUDE = @IE_CONFIG_BINARY@
EXCLUDE_SYMBOLS = INFERENCE_ENGINE_C_API_EXTERN \
INFERENCE_ENGINE_C_API \
IE_NODISCARD
PREDEFINED = "__attribute__(x)=" \
"__VA_ARGS__=" \
"INFERENCE_ENGINE_C_API_EXTERN=" \
"INFERENCE_ENGINE_C_API=" \
"IE_NODISCARD=" \
"__cdecl=" \
"__declspec(x)=" \
"__GNUC__=" \
"_WIN32"
FILE_PATTERNS = *.h
LAYOUT_FILE = @C_LAYOUT_BINARY@
INPUT = @C_API@
HTML_OUTPUT = ie_c_api
GENERATE_TAGFILE = @DOCS_BINARY_DIR@/ie_c_api.tag

2596
docs/doxygen/ie_docs.config Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,51 @@
@INCLUDE = @IE_CONFIG_BINARY@
LAYOUT_FILE = @PLUGIN_LAYOUT_BINARY@
HTML_OUTPUT = ie_plugin_api
GENERATE_TAGFILE = @DOCS_BINARY_DIR@/ie_plugin_api.tag
EXTRACT_LOCAL_CLASSES = NO
INPUT = @DOCS_BINARY_DIR@/docs/IE_PLUGIN_DG \
@IE_SOURCE_DIR@/src/plugin_api
FILE_PATTERNS = *.c \
*.cpp \
*.c++ \
*.h \
*.hpp \
*.md
EXCLUDE_PATTERNS = cnn_network_ngraph_impl.hpp \
ie_imemory_state_internal.hpp \
ie_memory_state_internal.hpp \
ie_memory_state_base.hpp \
convert_function_to_cnn_network.hpp \
generic_ie.hpp
EXCLUDE_SYMBOLS =
EXAMPLE_PATH = @CMAKE_CURRENT_SOURCE_DIR@/template_plugin/src \
@CMAKE_CURRENT_SOURCE_DIR@/template_plugin/include \
@CMAKE_CURRENT_SOURCE_DIR@/template_plugin/src/CMakeLists.txt \
@CMAKE_CURRENT_SOURCE_DIR@/template_plugin/tests/functional/CMakeLists.txt \
@CMAKE_CURRENT_SOURCE_DIR@/examples
EXAMPLE_PATTERNS = *.cpp \
*.hpp
ENUM_VALUES_PER_LINE = 1
EXPAND_ONLY_PREDEF = YES
PREDEFINED = INFERENCE_ENGINE_API \
INFERENCE_ENGINE_API_CPP \
INFERENCE_ENGINE_API_CLASS \
INFERENCE_ENGINE_DEPRECATED \
IE_SUPPRESS_DEPRECATED_START \
IE_SUPPRESS_DEPRECATED_END \
IE_SUPPRESS_DEPRECATED_START_WIN \
IE_SUPPRESS_DEPRECATED_END_WIN \
IE_THREAD=IE_THREAD_TBB

View File

@ -0,0 +1,35 @@
@INCLUDE = @IE_CONFIG_BINARY@
EXCLUDE_SYMBOLS = ie_api::BlobBuffer \
*impl* \
*device_name* \
*num_requests* \
*exec_net* \
*c_config* \
*ie_core_impl* \
*plugin_impl* \
*extension_str* \
*buffer* \
*__cinit__*
PREDEFINED = "__attribute__(x)=" \
"__VA_ARGS__=" \
"INFERENCE_ENGINE_C_API_EXTERN=" \
"INFERENCE_ENGINE_C_API=" \
"IE_NODISCARD=" \
"__cdecl=" \
"__declspec(x)=" \
"__GNUC__=" \
"_WIN32"
EXTENSION_MAPPING = pyx=Python
FILE_PATTERNS = *.pyx
LAYOUT_FILE = @PY_LAYOUT_BINARY@
INPUT = @PYTHON_API_OUT@
HTML_OUTPUT = ie_python_api
GENERATE_TAGFILE = @DOCS_BINARY_DIR@/ie_python_api.tag

129
docs/doxygen/pyx_filter.py Normal file
View File

@ -0,0 +1,129 @@
import re
import argparse
def process_pyx(pyx_file):
"""
Convert .pyx file to a more readable format for doxygen.
"""
with open(pyx_file, 'r') as f:
source = f.readlines()
idx = 0
while idx < len(source):
line = source[idx]
striped_line = line.lstrip()
tabs = ' ' * (len(line) - len(striped_line)) # Keep indentation
striped_line = striped_line.rstrip()
if striped_line == '@property': # Python functions wrapped with @property decorator
new_getter = convert_getter(source, idx)
if new_getter:
indent = tabs + ' ' * 4
new_func, comments, shift = new_getter
func_name = re.search(r'def\s+?([A-Za-z0-9_]+)\s*?\(', new_func).group(1)
source[idx + 1] = tabs + new_func + '\n'
for i in range(shift):
source.pop(idx + 2)
# This is a workaround to help Doxygen understand "@property" functions as class properties.
for comm in comments:
source.insert(idx + 2, '{indent}{comment}\n'.format(indent=indent, comment=comm))
idx += 1
source.insert(idx + 2, '{indent}self.{func_name} = {func_name}\n'.format(
indent=indent,
func_name=func_name
))
idx += 1
if re.search(r'c?p?def.+\(', striped_line): # Convert cython functions to python format
new_sign = get_signature(source, idx)
if new_sign:
new_func, shift = new_sign
args = re.search(r'\((.+)\)', new_func)
if args:
new_func = new_func.replace(args.group(1), process_args(args.group(1))).replace('cpdef', 'def')
source[idx] = tabs + new_func + '\n'
for i in range(shift):
source.pop(idx + 1)
if '__cinit__' in striped_line: # Doxygen only interprets "__init__" constructors
source[idx] = source[idx].replace('__cinit__', '__init__')
idx += 1
with open(pyx_file, 'w') as f:
f.writelines(source)
def process_args(str_args):
"""
Convert function arguments to the doxygen readable format.
"""
args = re.sub(r'\[.*?\]', r'', str_args)
args = re.sub(r'\(.*?\)', r'', args)
args = args.split(',')
for idx, arg in enumerate(args):
arg = arg.replace('&', '').strip()
if arg.startswith('const'):
arg = arg.replace('const', '').strip()
if ':' in arg:
arg = arg.split(':')[0]
match = re.match(r'^[\w\.]+\s+(\w.+)', arg)
if match:
arg = match.group(1)
args[idx] = arg.strip()
return ', '.join(args)
def convert_getter(source, start):
"""
Process a function that is wrapped with @property decorator
"""
current = source[start + 1].strip()
if not current.startswith('def'): # Base Case
return
new_sign = get_signature(source, start + 1)
if new_sign:
new_func, shift = new_sign
new_func += ':'
# get comments
comments = []
if start > 1:
idx = start - 1
while source[idx].lstrip().startswith('#') and idx >= 0:
comments.append(source[idx].strip())
idx -= 1
comments.reverse()
return new_func, comments, shift
def get_signature(source, start):
"""
Get function signature and process it
"""
match = re.search(r'c?p?def.+\(', source[start].strip())
if not match:
return
start_j = match.span()[1]
open_brackets = 1
new_sign = match.group()
for i in range(start, len(source)):
line = source[i].strip()
for j in range(start_j, len(line)):
char = line[j]
if char == ')':
open_brackets -= 1
if char == '(':
open_brackets += 1
new_sign += char
if not open_brackets:
return new_sign + ':\n', i - start
start_j = 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument('pyx_file', type=str, nargs='+', help='Path to a .pyx file.')
args = parser.parse_args()
for pyx in args.pyx_file:
process_pyx(pyx)
if __name__ == '__main__':
main()