Merge pull request #378 from numero-744/gen-features-using-rust-not-python

Use our beloved Rust instead of Python

embassy-stm32/gen_features.py

@@ -1,54 +0,0 @@
-import os
-import toml
-import yaml
-from glob import glob
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
-abspath = os.path.abspath(__file__)
-dname = os.path.dirname(abspath)
-os.chdir(dname)
-
-supported_families = [
-    "STM32F0",
-    'STM32F4',
-    'STM32G0',
-    'STM32L0',
-    'STM32L4',
-    'STM32H7',
-    'STM32WB55',
-    'STM32WL55',
-]
-
-# ======= load chip list
-features = {}
-for f in sorted(glob('../stm32-data/data/chips/*.yaml')):
-    # Use the filename to get the chip name. Ultra fast, we don't have to read YAML!
-    name = os.path.splitext(os.path.basename(f))[0]
-    if any((family in name for family in supported_families)):
-        name = name.lower()
-        # ======= load chip
-        with open(f, 'r') as f:
-            chip = yaml.load(f, Loader=SafeLoader)
-
-        if len(chip['cores']) > 1:
-            for core in chip['cores']:
-                features[name + "_" + core['name']] = ['stm32-metapac/' + name + '_' + core['name']]
-        else:
-            features[name] = ['stm32-metapac/' + name]
-
-# ========= Update Cargo features
-
-SEPARATOR_START = '# BEGIN GENERATED FEATURES\n'
-SEPARATOR_END = '# END GENERATED FEATURES\n'
-HELP = '# Generated by gen_features.py. DO NOT EDIT.\n'
-with open('Cargo.toml', 'r') as f:
-    cargo = f.read()
-before, cargo = cargo.split(SEPARATOR_START, maxsplit=1)
-_, after = cargo.split(SEPARATOR_END, maxsplit=1)
-cargo = before + SEPARATOR_START + HELP + toml.dumps(features) + SEPARATOR_END + after
-with open('Cargo.toml', 'w') as f:
-    f.write(cargo)

stm32-gen-features/.cargo/config.toml

@@ -0,0 +1,3 @@
+[profile.dev]
+opt-level = 3
+lto = false

stm32-gen-features/.gitignore

@@ -0,0 +1 @@
+/target

stm32-gen-features/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "gen_features"
+version = "0.1.0"
+authors = ["Côme ALLART <come.allart@netc.fr>"]
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+glob = "0.3.0"
+yaml-rust = "0.4.5"

stm32-gen-features/src/lib.rs

@@ -0,0 +1,213 @@
+//! FIXME discuss about which errors to print and when to panic
+
+use std::{iter::FilterMap, path::Path, slice::Iter};
+
+const SUPPORTED_FAMILIES: [&str; 8] = [
+    "stm32f0",
+    "stm32f4",
+    "stm32g0",
+    "stm32l0",
+    "stm32l4",
+    "stm32h7",
+    "stm32wb55",
+    "stm32wl55",
+];
+
+const SEPARATOR_START: &str = "# BEGIN GENERATED FEATURES\n";
+const SEPARATOR_END: &str = "# END GENERATED FEATURES\n";
+const HELP: &str = "# Generated by stm32-gen-features. DO NOT EDIT.\n";
+
+/// True if the chip named `name` is supported else false
+fn is_supported(name: &str) -> bool {
+    SUPPORTED_FAMILIES
+        .iter()
+        .any(|family| name.starts_with(family))
+}
+
+type SupportedIter<'a> = FilterMap<
+    Iter<'a, (String, Vec<String>)>,
+    fn(&(String, Vec<String>)) -> Option<(&String, &Vec<String>)>,
+>;
+trait FilterSupported {
+    fn supported(&self) -> SupportedIter;
+}
+impl FilterSupported for &[(String, Vec<String>)] {
+    /// Get a new Vec with only the supported chips
+    fn supported(&self) -> SupportedIter {
+        self.iter()
+            .filter_map(|(name, cores)| is_supported(name).then(|| (name, cores)))
+    }
+}
+
+/// Get the list of all the chips and their supported cores
+///
+/// Print errors to `stderr` when something is returned by the glob but is not in the returned
+/// [`Vec`]
+///
+/// This function is slow because all the yaml files are parsed.
+pub fn chip_names_and_cores() -> Vec<(String, Vec<String>)> {
+    glob::glob("../stm32-data/data/chips/*.yaml")
+        .unwrap()
+        .filter_map(|entry| entry.map_err(|e| eprintln!("{:?}", e)).ok())
+        .filter_map(|entry| {
+            if let Some(name) = entry.file_stem().and_then(|stem| stem.to_str()) {
+                Some((name.to_lowercase(), chip_cores(&entry)))
+            } else {
+                eprintln!("{:?} is not a regular file", entry);
+                None
+            }
+        })
+        .collect()
+}
+
+/// Get the list of the cores of a chip by its associated file
+///
+/// # Panic
+/// Panics if the file does not exist or if it contains yaml syntax errors.
+/// Panics if "cores" is not an array.
+fn chip_cores(path: &Path) -> Vec<String> {
+    let file_contents = std::fs::read_to_string(path).unwrap();
+    let doc = &yaml_rust::YamlLoader::load_from_str(&file_contents).unwrap()[0];
+    doc["cores"]
+        .as_vec()
+        .unwrap_or_else(|| panic!("{:?}:[cores] is not an array", path))
+        .iter()
+        .enumerate()
+        .map(|(i, core)| {
+            core["name"]
+                .as_str()
+                .unwrap_or_else(|| panic!("{:?}:[cores][{}][name] is not a string", path, i))
+                .to_owned()
+        })
+        .collect()
+}
+
+/// Generate data needed in `../embassy-stm32/Cargo.toml`
+///
+/// Print errors to `stderr` when something is returned by the glob but is not in the returned
+/// [`Vec`]
+///
+/// # Panic
+/// Panics if a file contains yaml syntax errors or if a value does not have a consistent type
+pub fn embassy_stm32_needed_data(names_and_cores: &[(String, Vec<String>)]) -> String {
+    let mut result = String::new();
+    for (chip_name, cores) in names_and_cores.supported() {
+        if cores.len() > 1 {
+            for core_name in cores.iter() {
+                result += &format!(
+                    "{chip}_{core} = [ \"stm32-metapac/{chip}_{core}\" ]\n",
+                    chip = chip_name,
+                    core = core_name
+                );
+            }
+        } else {
+            result += &format!("{chip} = [ \"stm32-metapac/{chip}\" ]\n", chip = chip_name);
+        }
+    }
+    result
+}
+
+/// Generate data needed in `../stm32-metapac/Cargo.toml`
+///
+/// Print errors to `stderr` when something is returned by the glob but is not in the returned
+/// [`Vec`]
+///
+/// # Panic
+/// Panics if a file contains yaml syntax errors or if a value does not have a consistent type
+pub fn stm32_metapac_needed_data(names_and_cores: &[(String, Vec<String>)]) -> String {
+    let mut result = String::new();
+    for (chip_name, cores) in names_and_cores {
+        if cores.len() > 1 {
+            for core_name in cores {
+                result += &format!("{}_{} = []\n", chip_name, core_name);
+            }
+        } else {
+            result += &format!("{} = []\n", chip_name);
+        }
+    }
+    result
+}
+
+/// Get contents before and after generated contents
+///
+/// # Panic
+/// Panics when a separator cound not be not found
+fn split_cargo_toml_contents(contents: &str) -> (&str, &str) {
+    let (before, remainder) = contents
+        .split_once(SEPARATOR_START)
+        .unwrap_or_else(|| panic!("missing \"{}\" tag", SEPARATOR_START));
+    let (_, after) = remainder
+        .split_once(SEPARATOR_END)
+        .unwrap_or_else(|| panic!("missing \"{}\" tag", SEPARATOR_END));
+
+    (before, after)
+}
+
+/// Generates new contents for Cargo.toml
+///
+/// # Panic
+/// Panics when a separator cound not be not found
+pub fn generate_cargo_toml_file(previous_text: &str, new_contents: &str) -> String {
+    let (before, after) = split_cargo_toml_contents(previous_text);
+    before.to_owned() + SEPARATOR_START + HELP + new_contents + SEPARATOR_END + after
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn stm32f407vg_is_supported() {
+        assert!(is_supported("stm32f407vg"))
+    }
+
+    #[test]
+    fn abcdef_is_not_supported() {
+        assert!(!is_supported("abcdef"))
+    }
+
+    #[test]
+    #[ignore]
+    fn stm32f407vg_yaml_file_exists_and_is_supported() {
+        assert!(chip_names_and_cores()
+            .as_slice()
+            .supported()
+            .into_iter()
+            .any(|(name, _)| { name == "stm32f407vg" }))
+    }
+
+    #[test]
+    fn keeps_text_around_separators() {
+        let initial = "\
+before
+# BEGIN GENERATED FEATURES
+# END GENERATED FEATURES
+after
+";
+
+        let expected = "\
+before
+# BEGIN GENERATED FEATURES
+# Generated by stm32-gen-features. DO NOT EDIT.
+a = [\"b\"]
+# END GENERATED FEATURES
+after
+";
+
+        let new_contents = String::from("a = [\"b\"]\n");
+        assert_eq!(generate_cargo_toml_file(initial, &new_contents), expected);
+    }
+
+    #[test]
+    #[should_panic]
+    fn does_not_generate_if_separators_are_missing() {
+        let initial = "\
+before
+# END GENERATED FEATURES
+after
+";
+
+        let new_contents = String::from("a = [\"b\"]\n");
+        generate_cargo_toml_file(initial, &new_contents);
+    }
+}

stm32-gen-features/src/main.rs

@@ -0,0 +1,26 @@
+use gen_features::{
+    chip_names_and_cores, embassy_stm32_needed_data, generate_cargo_toml_file,
+    stm32_metapac_needed_data,
+};
+
+fn main() {
+    let names_and_cores = chip_names_and_cores();
+    update_cargo_file(
+        "../embassy-stm32/Cargo.toml",
+        &embassy_stm32_needed_data(&names_and_cores),
+    );
+    update_cargo_file(
+        "../stm32-metapac/Cargo.toml",
+        &stm32_metapac_needed_data(&names_and_cores),
+    );
+}
+
+/// Update a Cargo.toml file
+///
+/// Update the content between "# BEGIN GENERATED FEATURES" and "# END GENERATED FEATURES"
+/// with the given content
+fn update_cargo_file(path: &str, new_contents: &str) {
+    let previous_text = std::fs::read_to_string(path).unwrap();
+    let new_text = generate_cargo_toml_file(&previous_text, new_contents);
+    std::fs::write(path, new_text).unwrap();
+}

stm32-metapac/Cargo.toml

@@ -20,7 +20,7 @@ rt = ["cortex-m-rt/device"]
 memory-x = []
 
 # BEGIN GENERATED FEATURES
-# Generated by gen_features.py. DO NOT EDIT.
+# Generated by stm32-gen-features. DO NOT EDIT.
 stm32f030c6 = []
 stm32f030c8 = []
 stm32f030cc = []

stm32-metapac/gen_features.py

@@ -1,47 +0,0 @@
-import xmltodict
-import yaml
-import re
-import json
-import os
-import re
-import toml
-from collections import OrderedDict
-from glob import glob
-
-try:
-    from yaml import CSafeLoader as SafeLoader
-except ImportError:
-    from yaml import SafeLoader
-
-abspath = os.path.abspath(__file__)
-dname = os.path.dirname(abspath)
-os.chdir(dname)
-
-# ======= load chip list
-
-features = {}
-
-for f in sorted(glob('../stm32-data/data/chips/*.yaml')):
-    # Use the filename to get the chip name. Ultra fast, we don't have to read YAML!
-    name = os.path.splitext(os.path.basename(f))[0].lower()
-    with open(f, 'r') as f:
-        chip = yaml.load(f, Loader=SafeLoader)
-    if len(chip['cores']) > 1:
-        for core in chip['cores']:
-            features[name + "_" + core['name']] = []
-    else:
-        features[name] = []
-
-# ========= Update Cargo features
-
-SEPARATOR_START = '# BEGIN GENERATED FEATURES\n'
-SEPARATOR_END = '# END GENERATED FEATURES\n'
-HELP = '# Generated by gen_features.py. DO NOT EDIT.\n'
-with open('Cargo.toml', 'r') as f:
-    cargo = f.read()
-before, cargo = cargo.split(SEPARATOR_START, maxsplit=1)
-_, after = cargo.split(SEPARATOR_END, maxsplit=1)
-cargo = before + SEPARATOR_START + HELP + \
-    toml.dumps(features) + SEPARATOR_END + after
-with open('Cargo.toml', 'w') as f:
-    f.write(cargo)