diff --git a/tools/build-release.sh b/tools/build-release.sh index 76c57a73..0219bd89 100755 --- a/tools/build-release.sh +++ b/tools/build-release.sh @@ -5,15 +5,16 @@ # $2 - JSON output file function downloadAndMergePackageJSON() { - echo " --- Package JSON definition merge BEGIN ---" + echo + echo " ---Package JSON definition merge BEGIN--->" jsonLink=$1 jsonOut=$2 curlAuthToken=$3 outDirectory=$4 - echo "Remote package JSON: $jsonLink (source)" - echo "Current package JSON: $jsonOut (target)" + echo " - remote package JSON: $jsonLink (source)" + echo " - current package JSON: $jsonOut (target)" old_json=$outDirectory/oldJson.json merged_json=$outDirectory/mergedJson.json @@ -22,7 +23,7 @@ function downloadAndMergePackageJSON() #echo " Local tmp for remote JSON: $old_json" #echo " Merge output JSON: $merged_json" - echo "Downloading JSON package definition: $jsonLink ..." + echo " - downloading JSON package definition: $jsonLink ..." # Authentication through HTTP headers might fail on redirection due to bug in cURL (https://curl.haxx.se/docs/adv_2018-b3bf.html - headers are resent to the target location including the original authentication) # Notes: @@ -38,7 +39,7 @@ function downloadAndMergePackageJSON() #curl -L -o "$old_json" "$jsonLink" - echo "Merging $old_json into $jsonOut ..." + echo " - merging $old_json into $jsonOut ..." echo set +e @@ -49,16 +50,17 @@ function downloadAndMergePackageJSON() set -v if [ ! -s $merged_json ]; then rm -f "$merged_json" - echo "Nothing to merge ($merged_json empty), $jsonOut unchanged" + echo " Done: nothing to merge ($merged_json empty) => $jsonOut remains unchanged" else rm -f "$jsonOut" mv "$merged_json" "$jsonOut" - echo "Data successfully merged to $jsonOut" + echo " Done: JSON data successfully merged to $jsonOut" fi rm -f "$old_json" set +v - echo " --- Package JSON definition merge END ---" + echo " <---Package JSON definition merge END---" + echo } ############################################################ @@ -73,58 +75,65 @@ echo "===================================================================" echo "RELEASE PACKAGE PUBLISHING ARRANGEMENTS (GitHub/Arduino compliance)" echo "===================================================================" echo +cmdLine=`basename $0 $@` +echo "Cmdline: ${cmdLine}" # cURL authentication token while getopts ":a:,:d:" opt; do case $opt in a) curlAuth=$OPTARG - echo "ACCESS TOKEN: $curlAuth" >&2 + #echo " ACCESS TOKEN: $curlAuth" >&2 ;; d) releaseDir=$OPTARG - echo "RELEASE OUTPUT DIRECTORY: $releaseDir" >&2 + #echo " RELEASE OUTPUT DIRECTORY: $releaseDir" >&2 ;; \?) - echo "Invalid option: -$OPTARG" >&2 + echo "Error: invalid option -$OPTARG => aborting" >&2 exit 1 ;; :) - echo "Option -$OPTARG requires an argument." >&2 + echo "Error: option -$OPTARG requires an argument => aborting" >&2 exit 1 ;; esac done # where we at? +echo +echo "Prequisite check:" if [ -z "$TRAVIS_BUILD_DIR" ]; then - echo "Non-TravisCI environment" + echo " - non-TravisCI environment" cd "$( dirname ${BASH_SOURCE[0]} )"/.. bTravisRun=0 else - echo "TravisCI run" + echo " - TravisCI run" cd $TRAVIS_BUILD_DIR bTravisRun=1 fi # no tag, no love if [ -z "$TRAVIS_TAG" ] && [ $bTravisRun -eq 1 ]; then - echo "Non-tagged builds not supported in Travis CI environment, exiting" + echo "Warning: non-tagged builds not supported in Travis CI environment => exiting" exit 0 fi -currentDir=`pwd` -echo "Current working directory: $currentDir" +echo +echo "Package build settings:" +echo "=======================" -srcdir=$currentDir +# source directory +srcdir=`pwd` +echo "Current working directory: ${srcdir}" +# target directory for actual release fileset if [ -z "$releaseDir" ]; then releaseDir=release fi echo "Release output directory: $releaseDir" - -# get current branch name and commit hash +# Git versions, branch names, tags branch_name="" verx="" extent="" @@ -132,15 +141,18 @@ extent="" if [ -z "$TRAVIS_TAG" ]; then branch_name=`git rev-parse --abbrev-ref HEAD 2>/dev/null` ver=`sed -n -E 's/version=([0-9.]+)/\1/p' platform.txt` - verx=`git rev-parse --short=8 HEAD 2>/dev/null` else ver=$TRAVIS_TAG fi +verx=`git rev-parse --short=8 HEAD 2>/dev/null` + +# Package name resolving (case-insensitive): +# - unknown branch, master branch or branch in detached state (HEAD revision) use only the tag's name as version string (esp32-$TAG_NAME, eg 'esp32-1.0.0-RC1') +# - all other branches use long-version string (esp32-$BRANCH_NAME-$GITREV_NUMBER_SHORT, eg 'esp32-idf_update-cde668da') -# Package name (case-insensitive) shopt -s nocasematch -if [ ! -z "$branch_name" ] && [ "$branch_name" != "master" ]; then +if [ ! -z "$branch_name" ] && [ "$branch_name" != "master" ] && [ "$branch_name" != "head" ]; then extent="-$branch_name-$verx" fi @@ -148,54 +160,57 @@ package_name=esp32-$ver$extent shopt -u nocasematch -echo "Version: $ver" -echo "Branch name: $branch_name" -echo "Git revision (8B): $verx" -echo "Extension: $extent" +echo "Package version: $ver" +echo "Git branch name: $branch_name" +echo "Git revision number (short): $verx" +echo "Package name extension: $extent" echo "Travis CI tag: $TRAVIS_TAG" -echo "Package name: $package_name" +echo "Release package name (unzipped): $package_name" # Set REMOTE_URL environment variable to the address where the package will be # available for download. This gets written into package json file. if [ -z "$REMOTE_URL" ]; then REMOTE_URL="http://localhost:8000" - echo "REMOTE_URL not defined, using default" + remoteEchoOut="${REMOTE_URL} (REMOTE_URL variable not defined, using default)" +else + remoteEchoOut="${REMOTE_URL}" fi - -echo "Remote: $REMOTE_URL" +echo "Target URL for download (JSON incl): ${remoteEchoOut}" # Create directory for the package outdir=$releaseDir/$package_name -echo "Temporary output directory: $outdir" +echo "Local temp directory: $outdir" rm -rf $releaseDir mkdir -p $outdir -# Copy package required stuff: +# Copy files required for the package release: +echo +echo "Package build processing:" +echo "=========================" +echo +echo "Prepare files for the package main archive:" +echo "-------------------------------------------" +echo " - copying neccessary files from current Git repository..." -# +# cp -f $srcdir/boards.txt $outdir/ cp -f $srcdir/platform.txt $outdir/ cp -f $srcdir/programmers.txt $outdir/ -# +# # cores/ # libraries/ # variants/ +# tools/partitions/ cp -Rf $srcdir/cores $outdir/ cp -Rf $srcdir/libraries $outdir/ cp -Rf $srcdir/variants $outdir/ +mkdir -p $outdir/tools +cp -Rf $srcdir/tools/partitions $outdir/tools/ -# -# tools/partitions/ -mkdir -p $outdir/tools/partitions -cp -f $srcdir/tools/partitions/boot_app0.bin $outdir/tools/partitions -cp -f $srcdir/tools/partitions/default.csv $outdir/tools/partitions -cp -f $srcdir/tools/partitions/minimal.csv $outdir/tools/partitions -cp -f $srcdir/tools/partitions/min_spiffs.csv $outdir/tools/partitions -cp -f $srcdir/tools/partitions/no_ota.csv $outdir/tools/partitions - +# # tools/sdk/ cp -Rf $srcdir/tools/sdk $outdir/tools/ @@ -206,50 +221,53 @@ cp -f $srcdir/tools/esptool.py $outdir/tools/ cp -f $srcdir/tools/gen_esp32part.py $outdir/tools/ cp -f $srcdir/tools/gen_esp32part.exe $outdir/tools/ +echo " - cleaning *.DS_Store files..." find $outdir -name '*.DS_Store' -exec rm -f {} \; # Do some replacements in platform.txt file, which are required because IDE # handles tool paths differently when package is installed in hardware folder +echo " - updating platform.txt..." cat $srcdir/platform.txt | \ sed 's/runtime.tools.xtensa-esp32-elf-gcc.path={runtime.platform.path}\/tools\/xtensa-esp32-elf//g' | \ sed 's/tools.esptool.path={runtime.platform.path}\/tools\/esptool/tools.esptool.path=\{runtime.tools.esptool.path\}/g' \ > $outdir/platform.txt - + # Put core version and short hash of git version into core_version.h -ver_define=`echo $plain_ver | tr "[:lower:].\055" "[:upper:]_"` -echo Ver define: $ver_define -echo \#define ARDUINO_ESP32_GIT_VER 0x`git rev-parse --short=8 HEAD 2>/dev/null` >$outdir/cores/esp32/core_version.h +ver_define=`echo $ver | tr "[:lower:].\055" "[:upper:]_"` +echo " - generating C/C++ header defines ($ver_define -> /cores/esp32/core_version.h)..." + +echo \#define ARDUINO_ESP32_GIT_VER 0x$verx >$outdir/cores/esp32/core_version.h echo \#define ARDUINO_ESP32_GIT_DESC `git describe --tags 2>/dev/null` >>$outdir/cores/esp32/core_version.h echo \#define ARDUINO_ESP32_RELEASE_$ver_define >>$outdir/cores/esp32/core_version.h echo \#define ARDUINO_ESP32_RELEASE \"$ver_define\" >>$outdir/cores/esp32/core_version.h - + # Store submodules' current versions +echo " - getting submodule list (${releaseDir}/submodules.txt)..." git submodule status > $releaseDir/submodules.txt # remove all .git* files +echo " - removing *.git files possibly fetched to package tempdir..." find $outdir -name '*.git*' -type f -delete - + # Zip the package package_name_zip=$package_name.zip - -echo "----------------------------------------------------------" -echo "Making $package_name ZIP archive..." -echo +echo " - creating package ZIP archive (${package_name_zip})..." pushd $releaseDir >/dev/null zip -qr $package_name_zip $package_name +if [ $? -ne 0 ]; then + echo " !error: failed to create ${package_name_zip} (ZIP errno: $?) => aborting" + exit 1 +fi - -echo "----------------------------------------------------------" -echo "Making $package_name JSON definition file(s)..." -echo - -# Calculate SHA sum and size +# Calculate SHA sum and size of ZIP archive sha=`shasum -a 256 $package_name_zip | cut -f 1 -d ' '` size=`/bin/ls -l $package_name_zip | awk '{print $5}'` -echo Size: $size -echo SHA-256: $sha +echo "${package_name_zip} creation OK (size: $size B, sha2: $sha)" +echo + +echo "Making $package_name JSON definition file(s):" popd >/dev/null @@ -261,20 +279,39 @@ shopt -s nocasematch if [[ $TRAVIS_TAG == *-RC* ]]; then bIsPrerelease=1 package_name_json=$PACKAGE_JSON_DEV - echo "Package type: PRE-RELEASE, JSON def.file: $PACKAGE_JSON_DEV" + echo " - package type: PRE-RELEASE, JSON def.file: $PACKAGE_JSON_DEV" else bIsPrerelease=0 package_name_json=$PACKAGE_JSON_REL - echo "Package type: RELEASE, JSON def.files: $PACKAGE_JSON_REL, $PACKAGE_JSON_DEV" + echo " - package type: RELEASE, JSON def.files: $PACKAGE_JSON_REL, $PACKAGE_JSON_DEV" fi shopt -u nocasematch -# Generate JSON package definition -echo -echo "----------------------------------------------------------" -echo "Preparing current package definition ($package_name_json)..." +# Cleanup temporary work dir +rm -rf $outdir -# JSON contents +# Get all previously released versions +echo " - fetching previous (pre)release versions from GitHub..." + +releasesJson=$releaseDir/releases.json +curl -sH "Authorization: token $curlAuth" https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases > $releasesJson + +set +e +# Previous final release (prerelase == false) +prev_release=$(jq -e -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) +# Previous release (possibly a pre-release) +prev_any_release=$(jq -e -r '. | map(select(.draft == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) +# Previous pre-release +prev_pre_release=$(jq -e -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) +set -e + +rm -f "$releasesJson" + +echo " -- previous Release: $prev_release" +echo " -- previous (?Pre-)release: $prev_any_release" +echo " -- previous Pre-release: $prev_pre_release" + +# add generated items to JSON package-definition contents jq_arg=".packages[0].platforms[0].version = \"$ver\" | \ .packages[0].platforms[0].url = \"$REMOTE_URL/$package_name_zip\" |\ .packages[0].platforms[0].archiveFileName = \"$package_name_zip\"" @@ -283,42 +320,13 @@ jq_arg="$jq_arg |\ .packages[0].platforms[0].size = \"$size\" |\ .packages[0].platforms[0].checksum = \"SHA-256:$sha\"" - -# Cleanup temporary work dir -rm -rf $outdir - - -# Get previous release name -echo -echo "----------------------------------------------------------" -echo "Getting previous releases versions..." - -releasesJson=$releaseDir/releases.json - -curl -sH "Authorization: token $curlAuth" https://api.github.com/repos/$TRAVIS_REPO_SLUG/releases > $releasesJson - -# Previous final release (prerelase == false) -prev_release=$(jq -r '. | map(select(.draft == false and .prerelease == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) -# Previous release (possibly a pre-release) -prev_any_release=$(jq -r '. | map(select(.draft == false)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) -# Previous pre-release -prev_pre_release=$(jq -r '. | map(select(.draft == false and .prerelease == true)) | sort_by(.created_at | - fromdateiso8601) | .[0].tag_name' ${releasesJson}) - -rm -f "$releasesJson" - -echo "Previous release: $prev_release" -echo "Previous (pre-?)release: $prev_any_release" -echo "Previous pre-release: $prev_pre_release" - # always get DEV version of JSON (included in both RC/REL) -echo -echo "----------------------------------------------------------" -echo "Generating $PACKAGE_JSON_DEV..." -echo +pkgJsonDev=$releaseDir/$PACKAGE_JSON_DEV +echo " - generating/merging _DEV_ JSON file (${pkgJsonDev})..." -cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $releaseDir/$PACKAGE_JSON_DEV +cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $pkgJsonDev if [ ! -z "$prev_any_release" ] && [ "$prev_any_release" != "null" ]; then - downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_any_release}/${PACKAGE_JSON_DEV}" "$releaseDir/${PACKAGE_JSON_DEV}" "${curlAuth}" "$releaseDir" + downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_any_release}/${PACKAGE_JSON_DEV}" "${pkgJsonDev}" "${curlAuth}" "$releaseDir" # Release notes: GIT log comments (prev_any_release, current_release> git log --oneline $prev_any_release.. > $releaseDir/commits.txt @@ -327,14 +335,12 @@ fi # for RELEASE run update REL JSON as well if [ $bIsPrerelease -eq 0 ]; then - echo - echo "----------------------------------------------------------" - echo "Generating $PACKAGE_JSON_REL..." - echo + pkgJsonRel=$releaseDir/$PACKAGE_JSON_REL + echo " - generating/merging _REL_ JSON file (${pkgJsonRel})..." - cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $releaseDir/$PACKAGE_JSON_REL + cat $srcdir/package/package_esp32_index.template.json | jq "$jq_arg" > $pkgJsonRel if [ ! -z "$prev_release" ] && [ "$prev_release" != "null" ]; then - downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_release}/${PACKAGE_JSON_REL}" "$releaseDir/${PACKAGE_JSON_REL}" "${curlAuth}" "$releaseDir" + downloadAndMergePackageJSON "https://github.com/$TRAVIS_REPO_SLUG/releases/download/${prev_release}/${PACKAGE_JSON_REL}" "${pkgJsonRel}" "${curlAuth}" "$releaseDir" # Release notes: GIT log comments (prev_release, current_release> git log --oneline $prev_release.. > $releaseDir/commits.txt @@ -342,7 +348,17 @@ if [ $bIsPrerelease -eq 0 ]; then fi echo -echo "==============================================================" -echo "Package '$package_name' ready for publishing, script finished." -echo "==============================================================" +echo "JSON definition file(s) creation OK" + +echo +echo "===================================================================" +echo "Package preparation done ('$releaseDir' contents):" +fileset=`ls -1 $releaseDir` +echo -e $fileset + +echo +echo "===================================================================" +echo "===================================================================" +echo "'$package_name' ready for publishing, processing completed." +echo "===================================================================" echo diff --git a/tools/deploy.sh b/tools/deploy.sh index 5ce5a690..11f3f942 100644 --- a/tools/deploy.sh +++ b/tools/deploy.sh @@ -46,6 +46,16 @@ while getopts ":t:,:a:,:s:,:p:,:f:,:d:" opt; do esac done +# use TravisCI env as default, if available +if [ -z $varTagName ] && [ ! -z $TRAVIS_TAG ]; then + varTagName=$TRAVIS_TAG +fi + +if [ -z $varTagName ]; then + echo "No tag name available => aborting" + exit 1 +fi + #Check tag name for release/prerelease (prerelease tag contains '_RC' as for release-candidate. case-insensitive) shopt -s nocasematch if [ -z $varPrerelease ]; then @@ -108,12 +118,23 @@ fi releaseNotes=$(perl -pe 's/\r?\n/\\n/' <<< ${releaseNotes}) -#JSON parameters to create a new release -curlData="{\"tag_name\": \"$varTagName\",\"target_commitish\": \"master\",\"name\": \"v$varTagName\",\"body\": \"$releaseNotes\",\"draft\": false,\"prerelease\": $varPrerelease}" +# Check possibly existing release for current tag +# (eg build invoked by Create New Release GHUI button -> GH default release pack created immediatelly including default assests) +HTTP_RESPONSE=$(curl -L --silent --write-out "HTTPSTATUS:%{http_code}" https://api.github.com/repos/$varRepoSlug/releases/tags/$varTagName) +HTTP_BODY=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g') +HTTP_STATUS=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://') -#Create the release (initial source file assets created by GitHub) -releaseId=$(curl --data "$curlData" https://api.github.com/repos/$varRepoSlug/releases?access_token=$varAccessToken | jq -r '.id') -echo Release ID: $releaseId +# if the release exists, append/update recent files to its assets vector +if [ $HTTP_STATUS -eq 200 ]; then + releaseId=$(echo $HTTP_BODY | jq -r '.id') + echo "GH release exists for current $varTagName tag (id $releaseId)" +#... or create a new release record +else + curlData="{\"tag_name\": \"$varTagName\",\"target_commitish\": \"master\",\"name\": \"v$varTagName\",\"body\": \"$releaseNotes\",\"draft\": false,\"prerelease\": $varPrerelease}" + #echo "DEBUG: curl --data \"${curlData}\" https://api.github.com/repos/${varRepoSlug}/releases?access_token=$varAccessToken | jq -r '.id'" + releaseId=$(curl --data "$curlData" https://api.github.com/repos/$varRepoSlug/releases?access_token=$varAccessToken | jq -r '.id') + echo "New GH release created for $varTagName tag (id $releaseId)" +fi # Assets defined by dir contents if [ ! -z $varAssetsDir ]; then @@ -133,7 +154,6 @@ if [ ! -z $varAssets ]; then curlAuth="Authorization: token $varAccessToken" for filename in $(echo $varAssets | tr ";" "\n") do - echo echo echo Uploading $filename...