MaD: update generator scripts to support writing JSON data extensions

The `generate_mad.py` script accepts a flag that lets you choose between
YAML or JSON, but YAML is still the default. We can change the default
later, when CLI support for JSON is widely available.

`convert_extensions.py` also still hard-codes `.yml` as the output
extension/format.

Author: nickrolfe

misc/scripts/models-as-data/bulk_generate_mad.py

@@ -430,10 +430,11 @@ def download_and_decompress(analyzed_database: dict) -> str:
 def clean_up_mad_destination_for_project(config, name: str):
     target = pathlib.Path(config["destination"], name)
     if config.get("single-file", False):
-        target = target.with_suffix(".model.yml")
-        if target.exists():
-            print(f"Deleting existing MaD file at {target}")
-            target.unlink()
+        for extension in (".model.json", ".model.yml"):
+            model_target = target.with_suffix(extension)
+            if model_target.exists():
+                print(f"Deleting existing MaD file at {model_target}")
+                model_target.unlink()
     elif target.exists():
         print(f"Deleting existing MaD directory at {target}")
         shutil.rmtree(target, ignore_errors=True)

misc/scripts/models-as-data/data_extension_file.py

@@ -1,6 +1,7 @@
 import json
 
-# Simple class for representing a data extension file and writing it to disk in a pretty-printed YAML format.
+# Simple class for representing a data extension file and writing it to disk in a pretty-printed
+# YAML or JSON format.
 class DataExtensionFile:
     def __init__(self, pack):
          self.pack = pack
@@ -37,3 +38,31 @@ def write_yaml(self, f):
         f.write('extensions:\n')
         for predicate in sorted(self.predicates.keys()):
             f.write(self.yaml_for_predicate(predicate))
+
+    def json_for_row(self, row):
+        return "[" + ", ".join(json.dumps(value) for value in row) +"]"
+
+    def json_for_predicate(self, predicate):
+        return f"""    {{
+      "addsTo": {{
+        "pack": "{self.pack}",
+        "extensible": "{predicate}"
+      }},
+      "data": [
+        {',\n        '.join(self.json_for_row(row) for row in self.predicates[predicate])}
+      ]
+    }}"""
+
+    def write_json(self, f):
+        # It would be preferable to use a serializer like Python's json.dumps, but it can't
+        # pretty-print with one line per tuple.
+        f.write('// THIS FILE IS AN AUTO-GENERATED MODELS AS DATA FILE. DO NOT EDIT.\n')
+        f.write('{\n')
+        f.write('  "extensions": [\n')
+        for i, predicate in enumerate(self.predicates):
+            if i > 0:
+                f.write(',\n')
+            f.write(self.json_for_predicate(predicate))
+        f.write('\n')
+        f.write('  ]\n')
+        f.write('}')

misc/scripts/models-as-data/generate_mad.py

@@ -119,6 +119,12 @@ def make():
             "--single-file",
             help="Generate a single file with all models instead of separate files for each namespace, using provided argument as the base filename.",
         )
+        p.add_argument(
+            "--extension-format",
+            choices=["json", "yaml"],
+            default="yaml",
+            help="Format for the generated data extension files (default %(default)s)",
+        )
         generator = p.parse_args(namespace=Generator())
 
         if (
@@ -210,16 +216,18 @@ def run(self):
             print("Models as data extensions generated, but not written to file.")
             sys.exit(0)
 
+        file_extension = ".json" if self.extension_format == "json" else ".yml"
+
         if (
             self.with_sinks
             or self.with_sources
             or self.with_summaries
             or self.with_neutrals
         ):
-            merger.save(self.generated_frameworks,  ".model.yml")
+            merger.save(self.generated_frameworks,  f".model{file_extension}")
 
         if self.with_typebased_summaries:
-            type_based_extensions.save(self.generated_frameworks,  ".typebased.model.yml")
+            type_based_extensions.save(self.generated_frameworks,  f".typebased.model{file_extension}")
 
 
 if __name__ == "__main__":

misc/scripts/models-as-data/helpers.py

@@ -50,7 +50,10 @@ def save(self, dir, file_extension):
             )
             target = os.path.join(dir, f"{sanitized_namespace}{file_extension}")
             with open(target, "w") as f:
-                extension.write_yaml(f)
+                if file_extension.endswith(".json"):
+                    extension.write_json(f)
+                else:
+                    extension.write_yaml(f)
             print("Models as data extensions written to " + target)