* release deployment comprehensive update (#1548)

tools/build-release.sh

@@ -5,15 +5,16 @@
 # $2 - JSON output file 
 function downloadAndMergePackageJSON()
 {
-	echo " --- Package JSON definition merge BEGIN ---"
+	echo
+	echo " ---Package JSON definition merge BEGIN--->"
 	
 	jsonLink=$1
 	jsonOut=$2
 	curlAuthToken=$3
 	outDirectory=$4
 	
-	echo "Remote package JSON: $jsonLink (source)"
+	echo " - remote package JSON: $jsonLink (source)"
-	echo "Current package JSON: $jsonOut (target)"
+	echo " - current package JSON: $jsonOut (target)"
 	
 	old_json=$outDirectory/oldJson.json
 	merged_json=$outDirectory/mergedJson.json
@@ -22,7 +23,7 @@ function downloadAndMergePackageJSON()
 	#echo "     Local tmp for remote JSON: $old_json"
 	#echo "     Merge output JSON: $merged_json"
 	
-	echo "Downloading JSON package definition: $jsonLink ..."
+	echo " - downloading JSON package definition: $jsonLink ..."
 
 	# Authentication through HTTP headers might fail on redirection due to bug in cURL (https://curl.haxx.se/docs/adv_2018-b3bf.html - headers are resent to the target location including the original authentication)
 	# Notes:
@@ -38,7 +39,7 @@ function downloadAndMergePackageJSON()
 	
 	#curl -L -o "$old_json" "$jsonLink"
 	
-	echo "Merging $old_json into $jsonOut ..."
+	echo " - merging $old_json into $jsonOut ..."
 	
 	echo
 	set +e
@@ -49,16 +50,17 @@ function downloadAndMergePackageJSON()
 	set -v
 	if [ ! -s $merged_json ]; then
 		rm -f "$merged_json"
-		echo "Nothing to merge ($merged_json empty), $jsonOut unchanged"
+		echo " Done: nothing to merge ($merged_json empty) => $jsonOut remains unchanged"
 	else
 		rm -f "$jsonOut"
 		mv "$merged_json" "$jsonOut"
-		echo "Data successfully merged to $jsonOut"
+		echo " Done: JSON data successfully merged to $jsonOut"
 	fi
 
 	rm -f "$old_json"
 	set +v
-	echo " --- Package JSON definition merge END ---"
+	echo " <---Package JSON definition merge END---"
+	echo
 }
 ############################################################
 
@@ -73,58 +75,65 @@ echo "==================================================================="
 echo "RELEASE PACKAGE PUBLISHING ARRANGEMENTS (GitHub/Arduino compliance)"
 echo "==================================================================="
 echo
+cmdLine=`basename $0 $@`
+echo "Cmdline: ${cmdLine}"
 
 # cURL authentication token
 while getopts ":a:,:d:" opt; do
   case $opt in
 	a)
 	  curlAuth=$OPTARG
-	  echo "ACCESS TOKEN: $curlAuth" >&2
+	  #echo " ACCESS TOKEN: $curlAuth" >&2
 	  ;;
 	d)
 	  releaseDir=$OPTARG
-	  echo "RELEASE OUTPUT DIRECTORY: $releaseDir" >&2
+	  #echo " RELEASE OUTPUT DIRECTORY: $releaseDir" >&2
 	  ;;
 	\?)
-	  echo "Invalid option: -$OPTARG" >&2
+	  echo "Error: invalid option -$OPTARG => aborting" >&2
 	  exit 1
 	  ;;
 	:)
-	  echo "Option -$OPTARG requires an argument." >&2
+	  echo "Error: option -$OPTARG requires an argument => aborting" >&2
 	  exit 1
 	  ;;
   esac
 done
 
 # where we at?
+echo
+echo "Prequisite check:"
 if [ -z "$TRAVIS_BUILD_DIR" ]; then
-	echo "Non-TravisCI environment"
+	echo " - non-TravisCI environment"
     cd "$( dirname ${BASH_SOURCE[0]} )"/..
 	bTravisRun=0
 else
-	echo "TravisCI run"
+	echo " - TravisCI run"
 	cd $TRAVIS_BUILD_DIR
 	bTravisRun=1
 fi
 
 # no tag, no love
 if [ -z "$TRAVIS_TAG" ] && [ $bTravisRun -eq 1 ]; then
-	echo "Non-tagged builds not supported in Travis CI environment, exiting"
+	echo "Warning: non-tagged builds not supported in Travis CI environment => exiting"
 	exit 0
 fi
 	
-currentDir=`pwd`
+echo
-echo "Current working directory: $currentDir"
+echo "Package build settings:"
+echo "======================="
 
-srcdir=$currentDir
+# source directory
+srcdir=`pwd`
+echo "Current working directory: ${srcdir}"
 
+# target directory for actual release fileset
 if [ -z "$releaseDir" ]; then
 	releaseDir=release
 fi
 echo "Release output directory: $releaseDir"
 
-
+# Git versions, branch names, tags
-# get current branch name and commit hash
 branch_name=""
 verx=""
 extent=""
@@ -132,15 +141,18 @@ extent=""
 if [ -z "$TRAVIS_TAG" ]; then	
 	branch_name=`git rev-parse --abbrev-ref HEAD 2>/dev/null`
 	ver=`sed -n -E 's/version=([0-9.]+)/\1/p' platform.txt`
-	verx=`git rev-parse --short=8 HEAD 2>/dev/null`
 else 
 	ver=$TRAVIS_TAG
 fi
+verx=`git rev-parse --short=8 HEAD 2>/dev/null`
+
+# Package name resolving (case-insensitive):
+# - unknown branch, master branch or branch in detached state (HEAD revision) use only the tag's name as version string (esp32-$TAG_NAME, eg 'esp32-1.0.0-RC1')
+# - all other branches use long-version string (esp32-$BRANCH_NAME-$GITREV_NUMBER_SHORT, eg 'esp32-idf_update-cde668da')
 
-# Package name (case-insensitive)
 shopt -s nocasematch
 
-if [ ! -z "$branch_name" ] && [ "$branch_name" != "master" ]; then
+if [ ! -z "$branch_name" ] && [ "$branch_name" != "master" ] && [ "$branch_name" != "head" ]; then
 	extent="-$branch_name-$verx"
 fi
 	
@@ -148,54 +160,57 @@ package_name=esp32-$ver$extent
 
 shopt -u nocasematch
 
-echo "Version: $ver"
+echo "Package version: $ver"
-echo "Branch name: $branch_name"
+echo "Git branch name: $branch_name"
-echo "Git revision (8B): $verx"
+echo "Git revision number (short): $verx"
-echo "Extension: $extent"	
+echo "Package name extension: $extent"	
 echo "Travis CI tag: $TRAVIS_TAG"
-echo "Package name: $package_name"
+echo "Release package name (unzipped): $package_name"
 
 # Set REMOTE_URL environment variable to the address where the package will be
 # available for download. This gets written into package json file.
 
 if [ -z "$REMOTE_URL" ]; then
     REMOTE_URL="http://localhost:8000"
-    echo "REMOTE_URL not defined, using default"
+	remoteEchoOut="${REMOTE_URL} (REMOTE_URL variable not defined, using default)"
+else 
+	remoteEchoOut="${REMOTE_URL}"
 fi
-
+echo "Target URL for download (JSON incl): ${remoteEchoOut}"
-echo "Remote: $REMOTE_URL"
 
 # Create directory for the package
 outdir=$releaseDir/$package_name
-echo "Temporary output directory: $outdir"
+echo "Local temp directory: $outdir"
 
 rm -rf $releaseDir
 mkdir -p $outdir
 
-# Copy package required stuff:
+# Copy files required for the package release:
+echo
+echo "Package build processing:"
+echo "========================="
+echo
+echo "Prepare files for the package main archive:"
+echo "-------------------------------------------"
+echo " - copying neccessary files from current Git repository..."
 
-# <package root>
+# <PACKAGE ROOT>
 cp -f $srcdir/boards.txt $outdir/
 cp -f $srcdir/platform.txt $outdir/
 cp -f $srcdir/programmers.txt $outdir/
 
-# <complete dirs> 
+# <COMPLETE DIRS> 
 #	cores/
 #	libraries/
 #	variants/
+#	tools/partitions/
 cp -Rf $srcdir/cores $outdir/
 cp -Rf $srcdir/libraries $outdir/
 cp -Rf $srcdir/variants $outdir/
+mkdir -p $outdir/tools
+cp -Rf $srcdir/tools/partitions $outdir/tools/
 
-# <dir & files>
+# <DIR & FILES>
-#	tools/partitions/
-mkdir -p $outdir/tools/partitions
-cp -f $srcdir/tools/partitions/boot_app0.bin $outdir/tools/partitions
-cp -f $srcdir/tools/partitions/default.csv $outdir/tools/partitions
-cp -f $srcdir/tools/partitions/minimal.csv $outdir/tools/partitions
-cp -f $srcdir/tools/partitions/min_spiffs.csv $outdir/tools/partitions
-cp -f $srcdir/tools/partitions/no_ota.csv $outdir/tools/partitions
-
 #	tools/sdk/
 cp -Rf $srcdir/tools/sdk $outdir/tools/
 
@@ -206,50 +221,53 @@ cp -f $srcdir/tools/esptool.py $outdir/tools/
 cp -f $srcdir/tools/gen_esp32part.py $outdir/tools/
 cp -f $srcdir/tools/gen_esp32part.exe $outdir/tools/
 
+echo " - cleaning *.DS_Store files..."
 find $outdir -name '*.DS_Store' -exec rm -f {} \;
 
 # Do some replacements in platform.txt file, which are required because IDE
 # handles tool paths differently when package is installed in hardware folder
+echo " - updating platform.txt..."
 cat $srcdir/platform.txt | \
 sed 's/runtime.tools.xtensa-esp32-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32-elf//g' | \
 sed 's/tools.esptool.path={runtime.platform.path}\/tools\/esptool/tools.esptool.path=\{runtime.tools.esptool.path\}/g' \
  > $outdir/platform.txt
 
 # Put core version and short hash of git version into core_version.h
-ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"`
+ver_define=`echo $ver | tr "[:lower:].\055" "[:upper:]_"`
-echo Ver define: $ver_define
+echo " - generating C/C++ header defines ($ver_define -> /cores/esp32/core_version.h)..."
-echo \#define ARDUINO_ESP32_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp32/core_version.h
+
+echo \#define ARDUINO_ESP32_GIT_VER 0x$verx >$outdir/cores/esp32/core_version.h
 echo \#define ARDUINO_ESP32_GIT_DESC `git describe --tags 2>/dev/null` >>$outdir/cores/esp32/core_version.h
 echo \#define ARDUINO_ESP32_RELEASE_$ver_define >>$outdir/cores/esp32/core_version.h
 echo \#define ARDUINO_ESP32_RELEASE \"$ver_define\" >>$outdir/cores/esp32/core_version.h
 
 # Store submodules' current versions
+echo " - getting submodule list (${releaseDir}/submodules.txt)..."
 git submodule status > $releaseDir/submodules.txt
 
 # remove all .git* files
+echo " - removing *.git files possibly fetched to package tempdir..."
 find $outdir -name '*.git*' -type f -delete
  
 # Zip the package
 package_name_zip=$package_name.zip
-
+echo " - creating package ZIP archive (${package_name_zip})..."
-echo "----------------------------------------------------------"
-echo "Making $package_name ZIP archive..."
-echo
 
 pushd $releaseDir >/dev/null
 
 zip -qr $package_name_zip $package_name
+if [ $? -ne 0 ]; then
+    echo "	!error: failed to create ${package_name_zip} (ZIP errno: $?) => aborting"
+	exit 1
+fi
 
-
+# Calculate SHA sum and size of ZIP archive
-echo "----------------------------------------------------------"
-echo "Making $package_name JSON definition file(s)..."
-echo
-
-# Calculate SHA sum and size
 sha=`shasum -a 256 $package_name_zip | cut -f 1 -d ' '`
 size=`/bin/ls -l $package_name_zip | awk '{print $5}'`	
-echo Size: $size
+echo "${package_name_zip} creation OK (size: $size B, sha2: $sha)"
-echo SHA-256: $sha
+echo
+
+echo "Making $package_name JSON definition file(s):"
 
 popd >/dev/null
 
@@ -261,20 +279,39 @@ shopt -s nocasematch
 if [[ $TRAVIS_TAG == *-RC* ]]; then
 	bIsPrerelease=1
 	package_name_json=$PACKAGE_JSON_DEV
-	echo "Package type: PRE-RELEASE, JSON def.file: $PACKAGE_JSON_DEV"
+	echo " - package type: PRE-RELEASE, JSON def.file: $PACKAGE_JSON_DEV"
 else
 	bIsPrerelease=0
 	package_name_json=$PACKAGE_JSON_REL
-	echo "Package type: RELEASE, JSON def.files: $PACKAGE_JSON_REL, $PACKAGE_JSON_DEV"
+	echo " - package type: RELEASE, JSON def.files: $PACKAGE_JSON_REL, $PACKAGE_JSON_DEV"
 fi
 shopt -u nocasematch
 
-# Generate JSON package definition
+# Cleanup temporary work dir
-echo
+rm -rf $outdir
-echo "----------------------------------------------------------"
-echo "Preparing current package definition ($package_name_json)..."
 
-# JSON contents
+# Get all previously released versions
+echo " - fetching previous (pre)release versions from GitHub..."
+
+releasesJson=$releaseDir/releases.json
+curl -sH "Authorization: token $curlAuth" https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases > $releasesJson
+
+set +e
+# Previous final release (prerelase == false)
+prev_release=$(jq -e -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson})
+# Previous release (possibly a pre-release)
+prev_any_release=$(jq -e -r '. | map(select(.draft == false)) | sort_by(.created_at | - fromdateiso8601)  | .[0].tag_name' ${releasesJson})
+# Previous pre-release
+prev_pre_release=$(jq -e -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601)  | .[0].tag_name' ${releasesJson})
+set -e
+
+rm -f "$releasesJson"
+
+echo " -- previous Release: $prev_release"
+echo " -- previous (?Pre-)release: $prev_any_release"
+echo " -- previous Pre-release: $prev_pre_release"
+
+# add generated items to JSON package-definition contents
 jq_arg=".packages[0].platforms[0].version = \"$ver\" | \
     .packages[0].platforms[0].url = \"$REMOTE_URL/$package_name_zip\" |\
     .packages[0].platforms[0].archiveFileName = \"$package_name_zip\""
@@ -283,42 +320,13 @@ jq_arg="$jq_arg |\
 	.packages[0].platforms[0].size = \"$size\" |\
 	.packages[0].platforms[0].checksum = \"SHA-256:$sha\""
 
-
-# Cleanup temporary work dir
-rm -rf $outdir
-
-
-# Get previous release name
-echo
-echo "----------------------------------------------------------"
-echo "Getting previous releases versions..."
-
-releasesJson=$releaseDir/releases.json
-
-curl -sH "Authorization: token $curlAuth" https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases > $releasesJson
-
-# Previous final release (prerelase == false)
-prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson})
-# Previous release (possibly a pre-release)
-prev_any_release=$(jq -r '. | map(select(.draft == false)) | sort_by(.created_at | - fromdateiso8601)  | .[0].tag_name' ${releasesJson})
-# Previous pre-release
-prev_pre_release=$(jq -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601)  | .[0].tag_name' ${releasesJson})
-
-rm -f "$releasesJson"
-
-echo "Previous release: $prev_release"
-echo "Previous (pre-?)release: $prev_any_release"
-echo "Previous pre-release: $prev_pre_release"
-
 # always get DEV version of JSON (included in both RC/REL)
-echo
+pkgJsonDev=$releaseDir/$PACKAGE_JSON_DEV
-echo "----------------------------------------------------------"
+echo " - generating/merging _DEV_ JSON file (${pkgJsonDev})..."
-echo "Generating $PACKAGE_JSON_DEV..."
-echo
 
-cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $releaseDir/$PACKAGE_JSON_DEV
+cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $pkgJsonDev
 if [ ! -z "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then
-	downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_any_release}/${PACKAGE_JSON_DEV}" "$releaseDir/${PACKAGE_JSON_DEV}" "${curlAuth}" "$releaseDir"
+	downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_any_release}/${PACKAGE_JSON_DEV}" "${pkgJsonDev}" "${curlAuth}" "$releaseDir"
 	
 	# Release notes: GIT log comments (prev_any_release, current_release>
 	git log --oneline $prev_any_release.. > $releaseDir/commits.txt
@@ -327,14 +335,12 @@ fi
 # for RELEASE run update REL JSON as well
 if [ $bIsPrerelease -eq 0 ]; then
 
-	echo
+	pkgJsonRel=$releaseDir/$PACKAGE_JSON_REL
-	echo "----------------------------------------------------------"
+	echo " - generating/merging _REL_ JSON file (${pkgJsonRel})..."
-	echo "Generating $PACKAGE_JSON_REL..."
-	echo
 
-	cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $releaseDir/$PACKAGE_JSON_REL
+	cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $pkgJsonRel
 	if [ ! -z "$prev_release" ] && [ "$prev_release" != "null" ]; then
-		downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_release}/${PACKAGE_JSON_REL}" "$releaseDir/${PACKAGE_JSON_REL}" "${curlAuth}" "$releaseDir"
+		downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_release}/${PACKAGE_JSON_REL}" "${pkgJsonRel}" "${curlAuth}" "$releaseDir"
 		
 		# Release notes: GIT log comments (prev_release, current_release>
 		git log --oneline $prev_release.. > $releaseDir/commits.txt
@@ -342,7 +348,17 @@ if [ $bIsPrerelease -eq 0 ]; then
 fi
 
 echo
-echo "=============================================================="
+echo "JSON definition file(s) creation OK"
-echo "Package '$package_name' ready for publishing, script finished."
+
-echo "=============================================================="
+echo
+echo "==================================================================="
+echo "Package preparation done ('$releaseDir' contents):"
+fileset=`ls -1 $releaseDir`
+echo -e $fileset
+
+echo
+echo "==================================================================="
+echo "==================================================================="
+echo "'$package_name' ready for publishing, processing completed."
+echo "==================================================================="
 echo

tools/deploy.sh

@@ -46,6 +46,16 @@ while getopts ":t:,:a:,:s:,:p:,:f:,:d:" opt; do
   esac
 done
 
+# use TravisCI env as default, if available
+if [ -z $varTagName ] && [ ! -z $TRAVIS_TAG ]; then
+	varTagName=$TRAVIS_TAG
+fi
+
+if [ -z $varTagName ]; then
+	echo "No tag name available => aborting"
+	exit 1
+fi
+
 #Check tag name for release/prerelease (prerelease tag contains '_RC' as for release-candidate. case-insensitive)
 shopt -s nocasematch
 if [ -z $varPrerelease ]; then
@@ -108,12 +118,23 @@ fi
 
 releaseNotes=$(perl -pe 's/\r?\n/\\n/' <<< ${releaseNotes})
 
-#JSON parameters to create a new release
+# Check possibly existing release for current tag 
-curlData="{\"tag_name\": \"$varTagName\",\"target_commitish\": \"master\",\"name\": \"v$varTagName\",\"body\": \"$releaseNotes\",\"draft\": false,\"prerelease\": $varPrerelease}"
+# (eg build invoked by Create New Release GHUI button -> GH default release pack created immediatelly including default assests)
+HTTP_RESPONSE=$(curl -L --silent --write-out "HTTPSTATUS:%{http_code}" https://api.github.com/repos/$varRepoSlug/releases/tags/$varTagName)
+HTTP_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g')
+HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
 
-#Create the release (initial source file assets created by GitHub)
+# if the release exists, append/update recent files to its assets vector
-releaseId=$(curl --data "$curlData" https://api.github.com/repos/$varRepoSlug/releases?access_token=$varAccessToken | jq -r '.id')
+if [ $HTTP_STATUS -eq 200 ]; then
-echo Release ID: $releaseId
+	releaseId=$(echo $HTTP_BODY | jq -r '.id')
+	echo "GH release exists for current $varTagName tag (id $releaseId)"
+#... or create a new release record
+else 
+	curlData="{\"tag_name\": \"$varTagName\",\"target_commitish\": \"master\",\"name\": \"v$varTagName\",\"body\": \"$releaseNotes\",\"draft\": false,\"prerelease\": $varPrerelease}"
+	#echo "DEBUG: curl --data \"${curlData}\" https://api.github.com/repos/${varRepoSlug}/releases?access_token=$varAccessToken | jq -r '.id'"
+	releaseId=$(curl --data "$curlData" https://api.github.com/repos/$varRepoSlug/releases?access_token=$varAccessToken | jq -r '.id')
+	echo "New GH release created for $varTagName tag (id $releaseId)"
+fi
 
 # Assets defined by dir contents
 if [ ! -z $varAssetsDir ]; then
@@ -133,7 +154,6 @@ if [ ! -z $varAssets ]; then
 	curlAuth="Authorization: token $varAccessToken"
 	for filename in $(echo $varAssets | tr ";" "\n")
 	do
-	  echo
 	  echo
 	  echo Uploading $filename...