Update README, add d script

Makefile

@@ -1,39 +0,0 @@
-SHELL=/bin/bash
-
-clean:
-	rm -rf sources
-	rm -rf tmp
-
-metadata: sources/metadata/files.json sources/metadata/mcus.json
-
-sources/metadata/files.json:
-	mkdir -p sources/metadata
-	wget http://stmcufinder.com/API/getFiles.php -O sources/metadata/files.json
-
-sources/metadata/mcus.json:
-	mkdir -p sources/metadata
-	wget http://stmcufinder.com/API/getMCUsForMCUFinderPC.php -O sources/metadata/mcus.json	
-
-sources/files: metadata
-	mkdir -p sources/files
-	jq -r .Files[].URL < sources/metadata/files.json  | wget -P sources/files/ -N -i -
-
-svd: sources/.tmp/stm32-rs
-	mkdir -p sources/svd
-	ls -1 ./sources/.tmp/stm32-rs/svd/*.formatted | xargs basename | cut -f 1 -d . \
-	| awk '{print "sources/.tmp/stm32-rs/svd/" $$0 ".svd.formatted" " " "sources/svd/" $$0 ".svd"}' \
-	| xargs -n2 cp
-
-sources/.tmp/stm32-rs:
-	rm -rf ./sources/.tmp/stm32-rs
-	git clone https://github.com/stm32-rs/stm32-rs.git ./sources/.tmp/stm32-rs
-	cd ./sources/.tmp/stm32-rs && make svdformat
-
-mcu_dirs: svd metadata
-
-	ls -1 ./sources/.tmp/stm32-rs/svd/*.formatted | xargs basename | cut -f 1 -d . \
-	| awk '{print "./sources/.tmp/stm32-rs/svd/" $$0 ".svd.formatted" " sources/mcu/" $$0 "/" $$0 ".svd" }' \
-	| tr ' ' '\n' \
-	| xargs -n2 cp
-	
-	

README.md

@@ -1,4 +1,35 @@
-wget http://stmcufinder.com/API/getFiles.php -O sources/files.json
+# stm32-data
-wget http://stmcufinder.com/API/getMCUsForMCUFinderPC.php -O sources/mcus.json
 
-jq -r .Files[].URL < sources/files.json  | wget -P sources/ -N -i - 
+`stm32-data` is a project aiming to produce clean machine-readable data about the STM32 microcontroller
+families, including:
+
+- :heavy_check_mark: Base chip information
+    - RAM, flash
+    - Packages
+- :heavy_check_mark: Peripheral addresses and interrupts
+- :heavy_check_mark: Interrupts
+- :heavy_check_mark: GPIO AlternateFunction mappings (except F1)
+- :construction: Register blocks for all peripherals
+- :x: DMA stream mappings
+- :x: Per-package pinouts
+- :x: Links to applicable reference manuals, datasheets, appnotes PDFs.
+
+:heavy_check_mark: = done, :construction: = work in progress, :x: = to do
+
+## Data sources
+
+These are the data sources currently used.
+
+- STM32Cube database: describes all MCUs, with useful stuff like GPIO AF mappings, DMA stream mappings, pinouts...
+- stm32-rs SVDs: register blocks. YAMLs are extracted and manually cleaned up.
+
+## Generating the YAMLs
+
+- Run `./d download_all`
+- Run `python3 parse.py`
+
+This generates all the YAMLs in `data/` except those in `data/registers/`, which are manually extracted and cleaned up.
+
+## Extracting new register blocks
+
+TODO document

d

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -e
+cd $(dirname $0)
+
+case "$1" in
+    download_all)
+        ./d download_mcufinder
+        ./d download_svd
+        ./d download_headers
+        ./d download_cubedb
+    ;;
+    download_mcufinder)
+        mkdir -p sources/mcufinder
+        wget http://stmcufinder.com/API/getFiles.php -O sources/mcufinder/files.json
+        wget http://stmcufinder.com/API/getMCUsForMCUFinderPC.php -O sources/mcufinder/mcus.json	
+    ;;
+    download_pdf)
+	    jq -r .Files[].URL < sources/mcufinder/files.json  | wget -P sources/pdf/ -N -i -
+    ;;
+    download_svd)
+    	rm -rf ./sources/git/stm32-rs
+        git clone --depth 1 https://github.com/stm32-rs/stm32-rs.git ./sources/git/stm32-rs
+        (cd ./sources/git/stm32-rs; make svdformat)
+        mkdir -p sources/svd
+        for f in ./sources/git/stm32-rs/svd/*.formatted; do
+            base=$(basename $f | cut -f 1 -d .)
+            cp $f sources/svd/$base.svd
+        done
+    ;;
+    download_headers)
+        for f in F0 F1 F2 F3 F4 F7 H7 L0 L1 L4 L5 G0 G4 WB WL; do
+            rm -rf ./sources/git/STM32Cube$f
+            git clone --depth 1 https://github.com/STMicroelectronics/STM32Cube$f sources/git/STM32Cube$f
+        done
+        rm -rf sources/headers
+        mkdir -p sources/headers
+        cp sources/git/STM32Cube*/Drivers/CMSIS/Device/ST/STM32*/Include/*.h sources/headers
+        rm sources/headers/stm32??xx.h
+        rm sources/headers/system_*.h
+        rm sources/headers/partition_*.h
+    ;;
+    download_cubedb)
+        rm -rf sources/cubedb
+        git clone --depth 1 https://github.com/embassy-rs/stm32cube-database.git sources/cubedb
+    ;;
+    extract_all)
+        peri=$2
+        mkdir -p tmp/$peri
+
+        cargo build --release --manifest-path ../svd2rust/Cargo.toml
+
+        for f in `ls sources/svd`; do
+            f=${f#"stm32"}
+            f=${f%".svd"}
+            echo -n processing $f ...
+            RUST_LOG=info ../svd2rust/target/release/svd4rust extract-peripheral --svd sources/svd/stm32$f.svd --transform transform.yaml --peripheral $peri > tmp/$peri/$f.yaml 2> tmp/$peri/$f.yaml.out
+            if [ $? -ne 0 ]; then 
+                rm tmp/$peri/$f.yaml
+                echo FAIL
+            else
+                rm tmp/$peri/$f.yaml.out
+                echo OK
+            fi
+        done
+    ;;
+    *)
+        echo "unknown command"
+    ;;
+esac

extract-all.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-
-peri=$1
-mkdir -p tmp/$peri
-
-cargo build --release --manifest-path ../../svd2rust/Cargo.toml
-
-for f in `ls sources/svd`; do
-    f=${f#"stm32"}
-    f=${f%".svd"}
-    echo -n processing $f ...
-    RUST_LOG=info ../../svd2rust/target/release/svd4rust extract-peripheral --svd sources/svd/stm32$f.svd --transform transform.yaml --peripheral $peri > tmp/$peri/$f.yaml 2> tmp/$peri/$f.yaml.out
-    if [ $? -ne 0 ]; then 
-        rm tmp/$peri/$f.yaml
-        echo FAIL
-    else
-        rm tmp/$peri/$f.yaml.out
-        echo OK
-    fi
-done

parse.py

@@ -243,7 +243,7 @@ def parse_chips():
 
     chips = {}
 
-    for f in sorted(glob('sources/mcu/STM32*.xml')):
+    for f in sorted(glob('sources/cubedb/mcu/STM32*.xml')):
         if 'STM32MP' in f: continue
         print(f)
 
@@ -317,10 +317,10 @@ def parse_chips():
 
 def parse_gpio_af():
     os.makedirs('data/gpio_af', exist_ok=True)
-    for f in glob('sources/mcu/IP/GPIO-*_gpio_v1_0_Modes.xml'):
+    for f in glob('sources/cubedb/mcu/IP/GPIO-*_gpio_v1_0_Modes.xml'):
         if 'STM32F1' in f: continue
 
-        ff = f.removeprefix('sources/mcu/IP/GPIO-')
+        ff = f.removeprefix('sources/cubedb/mcu/IP/GPIO-')
         ff = ff.removesuffix('_gpio_v1_0_Modes.xml')
         print(ff)