From 2d6dc8003929ddf961e45315678538e1e64c8eda Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Tue, 30 Mar 2021 11:22:56 +0300 Subject: [PATCH 01/10] Docs: Adds tag-based outline generator w/ glossary support --- Scripts/doc_outline_generator.py | 105 +++++++++++++++++++++++++++++++ Scripts/generate_doc_outline.sh | 8 +++ 2 files changed, 113 insertions(+) create mode 100755 Scripts/doc_outline_generator.py create mode 100755 Scripts/generate_doc_outline.sh diff --git a/Scripts/doc_outline_generator.py b/Scripts/doc_outline_generator.py new file mode 100755 index 000000000..33e2ba754 --- /dev/null +++ b/Scripts/doc_outline_generator.py @@ -0,0 +1,105 @@ +#!/usr/bin/python3 + +import re +from pathlib import Path +import sys + +if (len(sys.argv) != 4): + print("doc_outline_generator GIT_DIR SRC_DIR LINK_PREFIX") + sys.exit(-2) + +Base = Path(sys.argv[1]) +Root = Path(sys.argv[2]) +Prefix = sys.argv[3] + +Paths = [] + +for path in Root.rglob('*.c'): + Paths.append(path) + +for path in Root.rglob('*.cpp'): + Paths.append(path) + +for path in Root.rglob('*.cc'): + Paths.append(path) + +for path in Root.rglob('*.h'): + Paths.append(path) + +for path in Root.rglob('*.hpp'): + Paths.append(path) + +CategoryLabels = { } +MetaLabels = { } +GlossaryLabels = { } + +Meta = { } +Desc = { } + +for path in Paths: + with path.open() as file: + txt = file.read() + x = re.findall("\$info\$([^\$]*)\$end_info\$", txt) + if x: + for entry in x[0].strip().split("\n"): + name = entry.split(":")[0].strip(); + val = entry.split(":")[1].strip(); + if name == "category": + cat_name = val.split("~")[0].strip(); + cat_val = val.split("~")[1].strip(); + CategoryLabels[cat_name] = cat_val + elif name == "meta": + meta_name = val.split("~")[0].strip(); + meta_val = val.split("~")[1].strip(); + MetaLabels[meta_name] = meta_val + elif name == "glossary": + glossary_name = val.split("~")[0].strip(); + glossary_val = val.split("~")[1].strip(); + GlossaryLabels[glossary_name] = glossary_val + elif name == "tags": + for meta_name in val.split(","): + if meta_name.strip() not in Meta: + Meta[meta_name.strip()] = [] + Meta[meta_name.strip()].append(path) + elif name == "desc": + Desc[path] = val; + else: + print("Error") + sys.exit(-1) + +print("# " + Root.name) +print("") + +if (GlossaryLabels): + print ("## Glossary") + print("") + for item in GlossaryLabels.items(): + print("- " + item[0] + ": " + item[1]) + print("") + print("") + +Category = "" +for item in sorted(Meta.items()): + meta = item[0] + category = meta.split("|")[0] + topic = meta.split("|")[1] + if Category != category: + if Category != "": + print("") + print("") + Category = category + print("## " + Category) + if Category in CategoryLabels: + print(CategoryLabels[Category]) + print("") + + print("### " + topic) + if meta in MetaLabels: + print(MetaLabels[meta]) + + for path in sorted(item[1]): + if path in Desc: + print("- [" + path.name + "](" + Prefix + path.relative_to(Base).as_posix() + ")" + ": " + Desc[path]) + else: + print("- [" + path.name + "](" + Prefix + path.relative_to(Base).as_posix() + ")") + print("") \ No newline at end of file diff --git a/Scripts/generate_doc_outline.sh b/Scripts/generate_doc_outline.sh new file mode 100755 index 000000000..2b5941469 --- /dev/null +++ b/Scripts/generate_doc_outline.sh @@ -0,0 +1,8 @@ +#!/bin/env bash +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/External/FEXCore" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/ThunkLibs" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Scripts" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Common" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/CommonCore" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Tests" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/unittests" "../" From ceb7082e37ad58af12157f314c31dcae14fd468e Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Sun, 28 Mar 2021 13:49:19 +0300 Subject: [PATCH 02/10] Docs/Versioning: Add Changelog Generator --- Scripts/changelog_generator.py | 47 ++++++++++++++++++++++++++++++++++ Scripts/generate_changelog.sh | 7 +++++ 2 files changed, 54 insertions(+) create mode 100755 Scripts/changelog_generator.py create mode 100755 Scripts/generate_changelog.sh diff --git a/Scripts/changelog_generator.py b/Scripts/changelog_generator.py new file mode 100755 index 000000000..c5f8c1ffd --- /dev/null +++ b/Scripts/changelog_generator.py @@ -0,0 +1,47 @@ +#!/bin/env python3 +import sys + +import fileinput +import re + +Meta = { } +for line in sys.stdin.readlines(): + if detailed := re.findall("^([A-Za-z0-9]+)/([A-Za-z0-9]+):(.+)$", line): + detailed = detailed[0] + meta = detailed[0].strip() + "/" + detailed[1].strip() + if meta not in Meta: + Meta[meta] = [] + Meta[meta].append(detailed[2].strip()) + elif category := re.findall("(^[A-Za-z0-9]+):(.+)$", line): + category = category[0] + if category[0].strip() not in Meta: + Meta[category[0].strip()] = [] + Meta[category[0].strip()].append(category[1].strip()) + else: + if "_Misc" not in Meta: + Meta["_Misc"] = [] + Meta["_Misc"].append(line.strip()) + +print("# " + sys.argv[1]) + +Category = "" +Tag = "" +for item in sorted(Meta.items()): + if item[0] == "_Misc": + tag = "Misc" + else: + tag = item[0] + + category = tag.split("/")[0] + if category != Category: + Category = category + Tag = "" + print("") + print("## " + category) + if Tag != tag and tag != category: + Tag = tag + print("") + print("### " + tag.split("/")[1]) + + for change in item[1]: + print("- " + change) diff --git a/Scripts/generate_changelog.sh b/Scripts/generate_changelog.sh new file mode 100755 index 000000000..49cb400a4 --- /dev/null +++ b/Scripts/generate_changelog.sh @@ -0,0 +1,7 @@ +#!/bin/env bash +if [ "$#" -ne 2 ]; +then +echo "$0: " +exit -1 +fi +git log "$1..HEAD" --pretty="%b (%h)" --abbrev-commit --merges | Scripts/changelog_generator.py "$2" From bdd35e574302c3a41f735c5c5df92a0666a6667e Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Sun, 28 Mar 2021 17:17:11 +0300 Subject: [PATCH 03/10] docs: Update outline generator --- Scripts/doc_outline_generator.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Scripts/doc_outline_generator.py b/Scripts/doc_outline_generator.py index 33e2ba754..cd5831dfa 100755 --- a/Scripts/doc_outline_generator.py +++ b/Scripts/doc_outline_generator.py @@ -67,11 +67,11 @@ for path in Paths: print("Error") sys.exit(-1) -print("# " + Root.name) +print("## " + Root.name) print("") if (GlossaryLabels): - print ("## Glossary") + print ("### Glossary") print("") for item in GlossaryLabels.items(): print("- " + item[0] + ": " + item[1]) @@ -88,12 +88,12 @@ for item in sorted(Meta.items()): print("") print("") Category = category - print("## " + Category) + print("### " + Category) if Category in CategoryLabels: print(CategoryLabels[Category]) print("") - print("### " + topic) + print("#### " + topic) if meta in MetaLabels: print(MetaLabels[meta]) @@ -102,4 +102,4 @@ for item in sorted(Meta.items()): print("- [" + path.name + "](" + Prefix + path.relative_to(Base).as_posix() + ")" + ": " + Desc[path]) else: print("- [" + path.name + "](" + Prefix + path.relative_to(Base).as_posix() + ")") - print("") \ No newline at end of file + print("") From 350c33ada408f67ae9c971c37f91f63e692ba934 Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Sun, 28 Mar 2021 17:33:56 +0300 Subject: [PATCH 04/10] Docs: Update outline script --- Scripts/generate_doc_outline.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Scripts/generate_doc_outline.sh b/Scripts/generate_doc_outline.sh index 2b5941469..c5a4254be 100755 --- a/Scripts/generate_doc_outline.sh +++ b/Scripts/generate_doc_outline.sh @@ -1,4 +1,7 @@ #!/bin/env bash +echo \# `git describe --always` +echo + ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/External/FEXCore" "../" ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/ThunkLibs" "../" ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Scripts" "../" From 309139e2033e9f5deb05625fcabd599eb01a9e2b Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Sun, 28 Mar 2021 17:35:52 +0300 Subject: [PATCH 05/10] Scripts: Add generate_release script --- Scripts/generate_release.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 Scripts/generate_release.sh diff --git a/Scripts/generate_release.sh b/Scripts/generate_release.sh new file mode 100755 index 000000000..7ae7a07a2 --- /dev/null +++ b/Scripts/generate_release.sh @@ -0,0 +1,21 @@ +#!/bin/env bash + +if [ "$#" -ne 2 ]; +then +echo "$0: " +exit -1 +fi + + +echo "Tagging FEX-$2, previous release FEX-$1" +echo "Press cltr-c to cancel within 10 seconds" +sleep 10 + +git tag FEX-$2 -a -m "temporary" +Scripts/generate_doc_outline.sh > docs/SourceOutline.md +git commit docs/SourceOutline.md -m "Docs: Update for release FEX-$2" +git tad -d FEX-$2 +git tag FEX-2104 -a -m "$(Scripts/generate_changelog.sh FEX-$1 FEX-$2)" --edit + +echo "Inspect if everything went smooth via 'git log -6 FEX-$2' " +echo "if all is good, do 'git push FEX-$2:FEX-$2'" From 0dd02da57ccb7298ae0eee00f72b08f29243dcc0 Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Mon, 29 Mar 2021 13:16:55 +0300 Subject: [PATCH 06/10] Docs: Update outline scripts --- Scripts/doc_outline_generator.py | 30 +++++++++++++++++++++--------- Scripts/generate_doc_outline.sh | 10 ++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/Scripts/doc_outline_generator.py b/Scripts/doc_outline_generator.py index cd5831dfa..50358ae6a 100755 --- a/Scripts/doc_outline_generator.py +++ b/Scripts/doc_outline_generator.py @@ -42,19 +42,19 @@ for path in Paths: x = re.findall("\$info\$([^\$]*)\$end_info\$", txt) if x: for entry in x[0].strip().split("\n"): - name = entry.split(":")[0].strip(); - val = entry.split(":")[1].strip(); + name = entry.split(":", 1)[0].strip(); + val = entry.split(":", 1)[1].strip(); if name == "category": - cat_name = val.split("~")[0].strip(); - cat_val = val.split("~")[1].strip(); + cat_name = val.split("~", 1)[0].strip(); + cat_val = val.split("~", 1)[1].strip(); CategoryLabels[cat_name] = cat_val elif name == "meta": - meta_name = val.split("~")[0].strip(); - meta_val = val.split("~")[1].strip(); + meta_name = val.split("~", 1)[0].strip(); + meta_val = val.split("~", 1)[1].strip(); MetaLabels[meta_name] = meta_val elif name == "glossary": - glossary_name = val.split("~")[0].strip(); - glossary_val = val.split("~")[1].strip(); + glossary_name = val.split("~", 1)[0].strip(); + glossary_val = val.split("~", 1)[1].strip(); GlossaryLabels[glossary_name] = glossary_val elif name == "tags": for meta_name in val.split(","): @@ -67,7 +67,19 @@ for path in Paths: print("Error") sys.exit(-1) -print("## " + Root.name) + +Readme = None +if (Root / "README.md").is_file(): + Readme = Root / "README.md" + +if (Root / "Readme.md").is_file(): + Readme = Root / "Readme.md" + +print("## " + Root.relative_to(Base).as_posix()) + +if Readme: + print("See [" + Root.name + "/" + Readme.name + "](" + Prefix + Readme.relative_to(Base).as_posix() + ") for more details") + print("") if (GlossaryLabels): diff --git a/Scripts/generate_doc_outline.sh b/Scripts/generate_doc_outline.sh index c5a4254be..8b771e967 100755 --- a/Scripts/generate_doc_outline.sh +++ b/Scripts/generate_doc_outline.sh @@ -4,8 +4,10 @@ echo ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/External/FEXCore" "../" ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/ThunkLibs" "../" -./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Scripts" "../" -./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Common" "../" -./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/CommonCore" "../" ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Tests" "../" -./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/unittests" "../" + +# These don't have useful documentation at this point +#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Scripts" "../" +#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Common" "../" +#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/CommonCore" "../" +#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/unittests" "../" From 2b10b9792b64471dfb1bbef6b22375207d5da6ae Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Tue, 30 Mar 2021 11:19:01 +0300 Subject: [PATCH 07/10] Scripts: Fix generate_release, don't use markdown for changelogs --- Scripts/changelog_generator.py | 11 +++++++---- Scripts/generate_release.sh | 4 ++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Scripts/changelog_generator.py b/Scripts/changelog_generator.py index c5f8c1ffd..40d23659f 100755 --- a/Scripts/changelog_generator.py +++ b/Scripts/changelog_generator.py @@ -22,7 +22,7 @@ for line in sys.stdin.readlines(): Meta["_Misc"] = [] Meta["_Misc"].append(line.strip()) -print("# " + sys.argv[1]) +print(sys.argv[1]) Category = "" Tag = "" @@ -37,11 +37,14 @@ for item in sorted(Meta.items()): Category = category Tag = "" print("") - print("## " + category) + print("- " + category) if Tag != tag and tag != category: Tag = tag print("") - print("### " + tag.split("/")[1]) + print(" - " + tag.split("/")[1]) for change in item[1]: - print("- " + change) + if Tag == "": + print(" - " + change) + else: + print(" - " + change) diff --git a/Scripts/generate_release.sh b/Scripts/generate_release.sh index 7ae7a07a2..612c84e4e 100755 --- a/Scripts/generate_release.sh +++ b/Scripts/generate_release.sh @@ -15,7 +15,7 @@ git tag FEX-$2 -a -m "temporary" Scripts/generate_doc_outline.sh > docs/SourceOutline.md git commit docs/SourceOutline.md -m "Docs: Update for release FEX-$2" git tad -d FEX-$2 -git tag FEX-2104 -a -m "$(Scripts/generate_changelog.sh FEX-$1 FEX-$2)" --edit +git tag FEX-$2 -a -m "$(Scripts/generate_changelog.sh FEX-$1 FEX-$2)" --edit -echo "Inspect if everything went smooth via 'git log -6 FEX-$2' " +echo "Inspect if everything went smoothly via 'git log -6 FEX-$2' " echo "if all is good, do 'git push FEX-$2:FEX-$2'" From 458bebf598185174b07732ab3f69f848e96c73fe Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Tue, 30 Mar 2021 11:53:11 +0300 Subject: [PATCH 08/10] Scripts: Add a documentation comment in doc_outline_generator.py --- Scripts/doc_outline_generator.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/Scripts/doc_outline_generator.py b/Scripts/doc_outline_generator.py index 50358ae6a..501a5e2df 100755 --- a/Scripts/doc_outline_generator.py +++ b/Scripts/doc_outline_generator.py @@ -1,5 +1,22 @@ #!/usr/bin/python3 +# Tag Format +# $info$ +# glossary: ~ (Optional, registers or replaces a glossary entry) +# glossary: IR ~ Intermidiate Representation, a of storage for our high-level opcode representation +# glossary: SSA ~ Single Static Assignment, a form of representing IR in memory +# glossary: Basic Block ~ A block of instructions with no control flow, terminated by control flow +# glossary: Fragment ~ A Collection of basic blocks, possible an entire guest function or a fraction of it +# category: ~ (Optional, registers or replaces a category description) +# category: backend ~ Concerns itself with generating binary code from (optimized) IR +# meta: ~ (Optional, registers or replaces a meta, aka tag, description) +# meta: backend|arm64 ~ Arm64 Splatter backend +# tags: [, , ...] (Required if info tag exists) +# tags: backend|arm64 +# desc: (Optional) +# desc: Main glue logic of the arm64 splatter backend +# $end_info$ + import re from pathlib import Path import sys From 1fdd6fbb14804287d941ef25f93bfb56f5fd3784 Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Tue, 30 Mar 2021 11:55:39 +0300 Subject: [PATCH 09/10] Scripts: Add a documentation comment in changelog_generator.py --- Scripts/changelog_generator.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Scripts/changelog_generator.py b/Scripts/changelog_generator.py index 40d23659f..9bde2ffa6 100755 --- a/Scripts/changelog_generator.py +++ b/Scripts/changelog_generator.py @@ -4,6 +4,12 @@ import sys import fileinput import re +# Handles the following formats: +# +# -> goes in Misc category +# : -> Goes in +# /: -> Goes in / + Meta = { } for line in sys.stdin.readlines(): if detailed := re.findall("^([A-Za-z0-9]+)/([A-Za-z0-9]+):(.+)$", line): From 15537f8c8c5a993176e809b3a1939e3b8989ea3a Mon Sep 17 00:00:00 2001 From: Stefanos Kornilios Mitsis Poiitidis Date: Tue, 30 Mar 2021 11:58:18 +0300 Subject: [PATCH 10/10] Scripts: Also add unittests folder for doc outline --- Scripts/generate_doc_outline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Scripts/generate_doc_outline.sh b/Scripts/generate_doc_outline.sh index 8b771e967..4328ee6d1 100755 --- a/Scripts/generate_doc_outline.sh +++ b/Scripts/generate_doc_outline.sh @@ -5,9 +5,9 @@ echo ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/External/FEXCore" "../" ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/ThunkLibs" "../" ./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Tests" "../" +./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/unittests" "../" # These don't have useful documentation at this point #./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Scripts" "../" #./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/Common" "../" -#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/CommonCore" "../" -#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/unittests" "../" +#./Scripts/doc_outline_generator.py "`pwd`" "`pwd`/Source/CommonCore" "../" \ No newline at end of file