diff --git a/rpde-filter-deletes.sh b/rpde-filter-deletes.sh new file mode 100755 index 0000000..c0b1856 --- /dev/null +++ b/rpde-filter-deletes.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh + +# required commands: jq + +jq 'group_by(.id) | map(select(.[-1].state == "updated")) | flatten(1)' rpde-walk.json > rpde-filter-deletes.json diff --git a/walk-rpde.sh b/rpde-walk.sh similarity index 73% rename from walk-rpde.sh rename to rpde-walk.sh index 6a19d9f..3e7c6f0 100755 --- a/walk-rpde.sh +++ b/rpde-walk.sh @@ -18,7 +18,7 @@ set -- $1 -1 while [ "$2" -ne "0" ] do page_padded=$(printf '%0*d\n' ${#max} $page) - set -- $(curl -L -sS "$1" | jq '.' | tee rpde-$page_padded.json | jq -r '.next, (.items | length)') + set -- $(curl -L -sS "$1" | jq '.' | tee rpde-walk-$page_padded.json | jq -r '.next, (.items | length)') printf 'got page with next url: %s, num items: %s\n' "$1" $2 case $1 in /*) set -- $base$1 $2 @@ -28,6 +28,6 @@ done if [ "$single_file" != "" ] then - jq -s '[.[].items[]]' rpde-*.json > rpde.json - rm -rf rpde-*.json + jq -s '[.[].items[]]' rpde-walk-*.json > rpde-walk.json + rm -rf rpde-walk-*.json fi