snippets_translate: Add a way of using snippets from Python

Split a helper off the snippet extraction function taking the comment pattern and let it return a dict by id to make it possible to replace snippets by id. Prototypically use it for the modelview tutorial. Fixes: PYSIDE-1984 Task-number: PYSIDE-1952 Pick-to: 6.3 6.2 Change-Id: I05dbc3e36825761fe2968d6507880cd6f588682d Reviewed-by: Christian Tismer <tismer@stackless.com>
author: Friedemann Kleint <Friedemann.Kleint@qt.io> 2022-08-08 15:07:12 +0200
committer: Friedemann Kleint <Friedemann.Kleint@qt.io> 2022-08-11 09:12:56 +0200
commit: c4a266e38fe5bdce707ad6b123fa88bb4f10dff3 (patch)
tree: a7cc96a47dec82f5b11093ad9837cc221b8c6c8e
parent: 89e5d35c0f002fcb1aac3b1205402ce306074255 (diff)
10 files changed, 162 insertions, 32 deletions
diff --git a/examples/widgets/tutorials/modelview/1_readonly.py b/examples/widgets/tutorials/modelview/1_readonly.py
index 3bd16e78f..9dc923260 100644
--- a/examples/widgets/tutorials/modelview/1_readonly.py
+++ b/examples/widgets/tutorials/modelview/1_readonly.py
@@ -9,6 +9,7 @@ from PySide6.QtWidgets import QApplication, QTableView
 """PySide6 port of the widgets/tutorials/modelview/1_readonly example from Qt v6.x"""
 
 
+#! [1]
 class MyModel(QAbstractTableModel):
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -25,6 +26,7 @@ class MyModel(QAbstractTableModel):
             column = index.column() + 1
             return f"Row{row}, Column{column}"
         return None
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/2_formatting.py b/examples/widgets/tutorials/modelview/2_formatting.py
index b029ef846..73c993e5f 100644
--- a/examples/widgets/tutorials/modelview/2_formatting.py
+++ b/examples/widgets/tutorials/modelview/2_formatting.py
@@ -20,6 +20,7 @@ class MyModel(QAbstractTableModel):
     def columnCount(self, parent=None):
         return 3
 
+#! [1]
     def data(self, index, role=Qt.DisplayRole):
         row = index.row()
         col = index.column()
@@ -52,6 +53,7 @@ class MyModel(QAbstractTableModel):
                 return Qt.Checked
 
         return None
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/3_changingmodel.py b/examples/widgets/tutorials/modelview/3_changingmodel.py
index 55c004bbc..e277dd1e8 100644
--- a/examples/widgets/tutorials/modelview/3_changingmodel.py
+++ b/examples/widgets/tutorials/modelview/3_changingmodel.py
@@ -10,12 +10,14 @@ from PySide6.QtWidgets import QApplication, QTableView
 
 
 class MyModel(QAbstractTableModel):
+#! [1]
     def __init__(self, parent=None):
         super().__init__(parent)
         self._timer = QTimer(self)
         self._timer.setInterval(1000)
         self._timer.timeout.connect(self.timer_hit)
         self._timer.start()
+#! [1]
 
     def rowCount(self, parent=None):
         return 2
@@ -23,20 +25,23 @@ class MyModel(QAbstractTableModel):
     def columnCount(self, parent=None):
         return 3
 
+#! [2]
     def data(self, index, role=Qt.DisplayRole):
         row = index.row()
         col = index.column()
         if role == Qt.DisplayRole and row == 0 and col == 0:
             return QTime.currentTime().toString()
         return None
+#! [2]
 
+#! [3]
     @Slot()
     def timer_hit(self):
         # we identify the top left cell
         top_left = self.createIndex(0, 0)
         # emit a signal to make the view reread identified data
         self.dataChanged.emit(top_left, top_left, [Qt.DisplayRole])
-
+#! [3]
 
 if __name__ == '__main__':
     app = QApplication(sys.argv)
diff --git a/examples/widgets/tutorials/modelview/4_headers.py b/examples/widgets/tutorials/modelview/4_headers.py
index 97ab164c0..3feef17bf 100644
--- a/examples/widgets/tutorials/modelview/4_headers.py
+++ b/examples/widgets/tutorials/modelview/4_headers.py
@@ -26,10 +26,12 @@ class MyModel(QAbstractTableModel):
             return f"Row{row}, Column{column}"
         return None
 
+#! [1]
     def headerData(self, section, orientation, role):
         if role == Qt.DisplayRole and orientation == Qt.Horizontal:
             return ["first", "second", "third"][section]
         return None
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/5_edit.py b/examples/widgets/tutorials/modelview/5_edit.py
index e1dfdecec..1a4481fc9 100644
--- a/examples/widgets/tutorials/modelview/5_edit.py
+++ b/examples/widgets/tutorials/modelview/5_edit.py
@@ -33,6 +33,7 @@ class MyModel(QAbstractTableModel):
             return self._grid_data[index.row()][index.column()]
         return None
 
+#! [1]
     def setData(self, index, value, role):
         if role != Qt.EditRole or not self.checkIndex(index):
             return False
@@ -42,9 +43,12 @@ class MyModel(QAbstractTableModel):
         result = " ".join(chain(*self._grid_data))
         self.editCompleted.emit(result)
         return True
+#! [1]
 
+#! [2]
     def flags(self, index):
         return Qt.ItemIsEditable | super().flags(index)
+#! [2]
 
 
 class MainWindow(QMainWindow):
diff --git a/examples/widgets/tutorials/modelview/6_treeview.py b/examples/widgets/tutorials/modelview/6_treeview.py
index 5371c303c..861f8d587 100644
--- a/examples/widgets/tutorials/modelview/6_treeview.py
+++ b/examples/widgets/tutorials/modelview/6_treeview.py
@@ -9,7 +9,7 @@ from PySide6.QtWidgets import QApplication, QMainWindow, QTreeView
 
 """PySide6 port of the widgets/tutorials/modelview/6_treeview example from Qt v6.x"""
 
-
+#! [1]
 class MainWindow(QMainWindow):
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -32,6 +32,7 @@ class MainWindow(QMainWindow):
     def prepare_row(self, first, second, third):
         return [QStandardItem(first), QStandardItem(second),
                 QStandardItem(third)]
+#! [1]
 
 
 if __name__ == '__main__':
diff --git a/examples/widgets/tutorials/modelview/7_selections.py b/examples/widgets/tutorials/modelview/7_selections.py
index 129480bb7..6c519c865 100644
--- a/examples/widgets/tutorials/modelview/7_selections.py
+++ b/examples/widgets/tutorials/modelview/7_selections.py
@@ -9,7 +9,7 @@ from PySide6.QtWidgets import QApplication, QMainWindow, QTreeView
 
 """PySide6 port of the widgets/tutorials/modelview/7_selections example from Qt v6.x"""
 
-
+#! [1]
 class MainWindow(QMainWindow):
     def __init__(self, parent=None):
         super().__init__(parent)
@@ -45,7 +45,9 @@ class MainWindow(QMainWindow):
         # selection changes shall trigger a slot
         selection_model = self._tree_view.selectionModel()
         selection_model.selectionChanged.connect(self.selection_changed_slot)
+#! [1]
 
+#! [2]
     @Slot(QItemSelection, QItemSelection)
     def selection_changed_slot(self, new_selection, old_selection):
         # get the text of the selected item
@@ -58,6 +60,7 @@ class MainWindow(QMainWindow):
             seek_root = seek_root.parent()
             hierarchy_level += 1
         self.setWindowTitle(f"{selected_text}, Level {hierarchy_level}")
+#! [2]
 
 
 if __name__ == '__main__':
diff --git a/tools/snippets_translate/main.py b/tools/snippets_translate/main.py
index a0743e013..3f8152a74 100644
--- a/tools/snippets_translate/main.py
+++ b/tools/snippets_translate/main.py
@@ -5,14 +5,28 @@ import logging
 import os
 import re
 import sys
-from argparse import ArgumentParser, Namespace
+from argparse import ArgumentParser, Namespace, RawDescriptionHelpFormatter
 from enum import Enum
 from pathlib import Path
 from textwrap import dedent
-from typing import List
+from typing import Dict, List
 
+from override import python_example_snippet_mapping
 from converter import snippet_translate
 
+HELP = """Converts Qt C++ code snippets to Python snippets.
+
+Ways to override Snippets:
+
+1) Complete snippets from local files:
+   To replace snippet "[1]" of "foo/bar.cpp", create a file
+   "sources/pyside6/doc/snippets/foo/bar_1.cpp.py" .
+2) Snippets extracted from Python examples:
+   To use snippets from Python examples, add markers ("#! [id]") to it
+   and an entry to _PYTHON_EXAMPLE_SNIPPET_MAPPING.
+"""
+
+
 # Logger configuration
 try:
     from rich.logging import RichHandler
@@ -37,11 +51,16 @@ log = logging.getLogger("snippets_translate")
 # Filter and paths configuration
 SKIP_END = (".pro", ".pri", ".cmake", ".qdoc", ".yaml", ".frag", ".qsb", ".vert", "CMakeLists.txt")
 SKIP_BEGIN = ("changes-", ".")
-SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]")
+CPP_SNIPPET_PATTERN = re.compile(r"//! ?\[([^]]+)\]")
+PYTHON_SNIPPET_PATTERN = re.compile(r"#! ?\[([^]]+)\]")
+
+ROOT_PATH = Path(__file__).parents[2]
+SOURCE_PATH = ROOT_PATH / "sources" / "pyside6" / "doc" / "snippets"
+
 
-SOURCE_PATH = Path(__file__).parents[2] / "sources" / "pyside6" / "doc" / "snippets"
 OVERRIDDEN_SNIPPET = "# OVERRIDDEN_SNIPPET"
 
+
 class FileStatus(Enum):
     Exists = 0
     New = 1
@@ -52,7 +71,9 @@ def get_parser() -> ArgumentParser:
     Returns a parser for the command line arguments of the script.
     See README.md for more information.
     """
-    parser = ArgumentParser(prog="snippets_translate")
+    parser = ArgumentParser(prog="snippets_translate",
+                            description=HELP,
+                            formatter_class=RawDescriptionHelpFormatter)
     parser.add_argument(
         "--qt",
         action="store",
@@ -163,32 +184,39 @@ def is_valid_file(x):
     return True
 
 
-def get_snippet_ids(line: str) -> List[str]:
+def get_snippet_ids(line: str, pattern: re.Pattern) -> List[str]:
     # Extract the snippet ids for a line '//! [1] //! [2]'
     result = []
-    for m in SNIPPET_PATTERN.finditer(line):
+    for m in pattern.finditer(line):
         result.append(m.group(1))
     return result
 
 
+def overriden_snippet_lines(lines: List[str], start_id: str) -> List[str]:
+    """Wrap an overridden snippet with marker and id lines."""
+    id_string = f"//! [{start_id}]"
+    result = [OVERRIDDEN_SNIPPET, id_string]
+    result.extend(lines)
+    result.append(id_string)
+    return result
+
+
 def get_snippet_override(start_id: str, rel_path: str) -> List[str]:
-    # Check if the snippet is overridden by a local file
+    """Check if the snippet is overridden by a local file under
+       sources/pyside6/doc/snippets."""
     file_start_id = start_id.replace(' ', '_')
     override_name = f"{rel_path.stem}_{file_start_id}{rel_path.suffix}.py"
     override_path = SOURCE_PATH / rel_path.parent / override_name
-    snippet = []
-    if override_path.is_file():
-        snippet.append(OVERRIDDEN_SNIPPET)
-        id_string = f"//! [{start_id}]"
-        snippet.append(id_string)
-        snippet.extend(override_path.read_text().splitlines())
-        snippet.append(id_string)
-    return snippet
+    if not override_path.is_file():
+        return []
+    lines = override_path.read_text().splitlines()
+    return overriden_snippet_lines(lines, start_id)
 
 
-def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
-    # Extract (potentially overlapping) snippets from a C++ file indicated by //! [1]
-    snippets: List[List[str]] = []
+def _get_snippets(lines: List[str], pattern: re.Pattern) -> Dict[str, List[str]]:
+    """Helper to extract (potentially overlapping) snippets from a C++ file
+       indicated by pattern ("//! [1]") and return them as a dict by <id>."""
+    snippets: Dict[str, List[str]] = {}
     snippet: List[str]
     done_snippets : List[str] = []
 
@@ -197,19 +225,14 @@ def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
         line = lines[i]
         i += 1
 
-        start_ids = get_snippet_ids(line)
+        start_ids = get_snippet_ids(line, pattern)
         while start_ids:
             # Start of a snippet
             start_id = start_ids.pop(0)
             if start_id in done_snippets:
                 continue
             done_snippets.append(start_id)
-            snippet = get_snippet_override(start_id, rel_path)
-            if snippet:
-                snippets.append(snippet)
-                continue
-
-            snippet.append(line)  # The snippet starts with this id
+            snippet = [line]  # The snippet starts with this id
 
             # Find the end of the snippet
             j = i
@@ -221,14 +244,46 @@ def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
                 snippet.append(l)
 
                 # Check if the snippet is complete
-                if start_id in get_snippet_ids(l):
+                if start_id in get_snippet_ids(l, pattern):
                     # End of snippet
-                    snippets.append(snippet)
+                    snippets[start_id] = snippet
                     break
 
     return snippets
 
 
+def get_python_example_snippet_override(start_id: str, rel_path: str) -> List[str]:
+    """Check if the snippet is overridden by a python example snippet."""
+    key = (os.fspath(rel_path), start_id)
+    value = python_example_snippet_mapping().get(key)
+    if not value:
+        return []
+    path, id = value
+    file_lines = path.read_text().splitlines()
+    snippet_dict = _get_snippets(file_lines, PYTHON_SNIPPET_PATTERN)
+    lines = snippet_dict.get(id)
+    if not lines:
+        raise RuntimeError(f'Snippet "{id}" not found in "{os.fspath(path)}"')
+    lines = lines[1:-1]  # Strip Python snippet markers
+    return overriden_snippet_lines(lines, start_id)
+
+
+def get_snippets(lines: List[str], rel_path: str) -> List[List[str]]:
+    """Extract (potentially overlapping) snippets from a C++ file indicated
+       by '//! [1]'."""
+    result = _get_snippets(lines, CPP_SNIPPET_PATTERN)
+    id_list = result.keys()
+    for snippet_id in id_list:
+        # Check file overrides and example overrides
+        snippet = get_snippet_override(snippet_id, rel_path)
+        if not snippet:
+            snippet = get_python_example_snippet_override(snippet_id, rel_path)
+        if snippet:
+            result[snippet_id] = snippet
+
+    return result.values()
+
+
 def get_license_from_file(filename):
     lines = []
     with open(filename, "r") as f:
diff --git a/tools/snippets_translate/override.py b/tools/snippets_translate/override.py
new file mode 100644
index 000000000..c9aa6d498
--- /dev/null
+++ b/tools/snippets_translate/override.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2022 The Qt Company Ltd.
+# SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+from pathlib import Path
+
+ROOT_PATH = Path(__file__).parents[2]
+EXAMPLES_PATH = ROOT_PATH / "examples"
+
+
+_PYTHON_EXAMPLE_SNIPPET_MAPPING = {
+    ("qtbase/examples/widgets/tutorials/modelview/1_readonly/mymodel.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "1_readonly.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/2_formatting/mymodel.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "2_formatting.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_QVariant"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "2"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_a"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/3_changingmodel/mymodel.cpp",
+     "quoting mymodel_b"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "3_changingmodel.py", "3"),
+    ("qtbase/examples/widgets/tutorials/modelview/4_headers/mymodel.cpp",
+     "quoting mymodel_c"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "4_headers.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp",
+     "quoting mymodel_e"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/5_edit/mymodel.cpp",
+     "quoting mymodel_f"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "5_edit.py", "2"),
+    ("qtbase/examples/widgets/tutorials/modelview/6_treeview/mainwindow.cpp",
+     "Quoting ModelView Tutorial"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "6_treeview.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp",
+     "quoting modelview_a"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "1"),
+    ("qtbase/examples/widgets/tutorials/modelview/7_selections/mainwindow.cpp",
+     "quoting modelview_b"):
+    (EXAMPLES_PATH / "widgets" / "tutorials" / "modelview" / "7_selections.py", "2")
+}
+
+
+_python_example_snippet_mapping = {}
+
+
+def python_example_snippet_mapping():
+    global _python_example_snippet_mapping
+    if not _python_example_snippet_mapping:
+        result = _PYTHON_EXAMPLE_SNIPPET_MAPPING
+        _python_example_snippet_mapping = result
+
+    return _python_example_snippet_mapping
diff --git a/tools/snippets_translate/snippets_translate.pyproject b/tools/snippets_translate/snippets_translate.pyproject
index 8016eb637..6073e9b89 100644
--- a/tools/snippets_translate/snippets_translate.pyproject
+++ b/tools/snippets_translate/snippets_translate.pyproject
@@ -1,3 +1,3 @@
 {
-    "files": ["main.py", "converter.py", "handlers.py", "tests/test_converter.py"]
+    "files": ["main.py", "converter.py", "handlers.py", "override.py", "tests/test_converter.py"]
 }
author	Friedemann Kleint <Friedemann.Kleint@qt.io>	2022-08-08 15:07:12 +0200
committer	Friedemann Kleint <Friedemann.Kleint@qt.io>	2022-08-11 09:12:56 +0200
commit	c4a266e38fe5bdce707ad6b123fa88bb4f10dff3 (patch)
tree	a7cc96a47dec82f5b11093ad9837cc221b8c6c8e
parent	89e5d35c0f002fcb1aac3b1205402ce306074255 (diff)