linker: allow multi-step link recipe (c.combine) (#2954)

* builder: RunRecipe: run either a set of recipes or a single recipe

Recipes must follow the 'prefix.NNN.suffix' pattern, where 'NNN' is a
number. When there is also a single 'prefix.suffix' property defined,
make sure to not include that in the list of recipes to run. However, if
no numbered recipes are found, use the single 'prefix.suffix' form if it
exists.

This allows to have both a single recipe and a set of numbered recipes
in the same build properties, for backwards compatibility.

Signed-off-by: Luca Burelli <l.burelli@arduino.cc>

* linker: switch 'recipe.c.combine.pattern' to a recipe

Enable the use of recipes for the `recipe.c.combine.pattern` command.
Allows for more flexibility in the build process.

Signed-off-by: Luca Burelli <l.burelli@arduino.cc>

* Removed useless cloning of buildProperties

* Added unit test for recipe finder

---------

Signed-off-by: Luca Burelli <l.burelli@arduino.cc>
Co-authored-by: Cristian Maglie <c.maglie@arduino.cc>

Author: pillo79

docs/platform-specification.md

@@ -302,6 +302,10 @@ compiler.libraries.ldflags=
 recipe.c.combine.pattern="{compiler.path}{compiler.c.elf.cmd}" {compiler.c.elf.flags} -mmcu={build.mcu} -o "{build.path}/{build.project_name}.elf" {object_files} {compiler.libraries.ldflags} "{archive_file_path}" "-L{build.path}" -lm
 ```
 
+If the linking process requires multiple steps, the recipe can be written using the **recipe.c.combine.NUMBER.pattern**
+syntax. In this case, each step will be executed in the order specified by the number. When multiple steps are defined,
+the **recipe.c.combine.pattern** property is ignored.
+
 #### Recipes for extraction of executable files and other binary data
 
 An arbitrary number of extra steps can be performed at the end of objects linking. These steps can be used to extract

internal/arduino/builder/builder.go

@@ -464,7 +464,7 @@ func (b *Builder) build() error {
 	}
 	b.Progress.CompleteStep()
 
-	if err := b.RunRecipe("recipe.objcopy.", ".pattern", true); err != nil {
+	if err := b.RunRecipe("recipe.objcopy", ".pattern", true); err != nil {
 		return err
 	}
 	b.Progress.CompleteStep()

internal/arduino/builder/linker.go

@@ -90,10 +90,5 @@ func (b *Builder) link() error {
 	properties.Set("archive_file_path", b.buildArtifacts.coreArchiveFilePath.String())
 	properties.Set("object_files", objectFileList)
 
-	command, err := b.prepareCommandForRecipe(properties, "recipe.c.combine.pattern", false)
-	if err != nil {
-		return err
-	}
-
-	return b.execCommand(command)
+	return b.RunRecipeWithProps("recipe.c.combine", ".pattern", properties, false)
 }

internal/arduino/builder/recipe.go

@@ -27,18 +27,18 @@ import (
 
 // RunRecipe fixdoc
 func (b *Builder) RunRecipe(prefix, suffix string, skipIfOnlyUpdatingCompilationDatabase bool) error {
+	return b.RunRecipeWithProps(prefix, suffix, b.buildProperties, skipIfOnlyUpdatingCompilationDatabase)
+}
+
+func (b *Builder) RunRecipeWithProps(prefix, suffix string, buildProperties *properties.Map, skipIfOnlyUpdatingCompilationDatabase bool) error {
 	logrus.Debugf("Looking for recipes like %s", prefix+"*"+suffix)
 
-	// TODO is it necessary to use Clone?
-	buildProperties := b.buildProperties.Clone()
 	recipes := findRecipes(buildProperties, prefix, suffix)
 
-	// TODO is it necessary to use Clone?
-	properties := buildProperties.Clone()
 	for _, recipe := range recipes {
 		logrus.Debugf("Running recipe: %s", recipe)
 
-		command, err := b.prepareCommandForRecipe(properties, recipe, false)
+		command, err := b.prepareCommandForRecipe(buildProperties, recipe, false)
 		if err != nil {
 			return err
 		}
@@ -60,12 +60,20 @@ func (b *Builder) RunRecipe(prefix, suffix string, skipIfOnlyUpdatingCompilation
 
 func findRecipes(buildProperties *properties.Map, patternPrefix string, patternSuffix string) []string {
 	var recipes []string
+
+	exactKey := patternPrefix + patternSuffix
+
 	for _, key := range buildProperties.Keys() {
-		if strings.HasPrefix(key, patternPrefix) && strings.HasSuffix(key, patternSuffix) && buildProperties.Get(key) != "" {
+		if key != exactKey && strings.HasPrefix(key, patternPrefix) && strings.HasSuffix(key, patternSuffix) && buildProperties.Get(key) != "" {
 			recipes = append(recipes, key)
 		}
 	}
 
+	// If no recipes were found, check if the exact key exists
+	if len(recipes) == 0 && buildProperties.Get(exactKey) != "" {
+		recipes = append(recipes, exactKey)
+	}
+
 	sort.Strings(recipes)
 
 	return recipes

internal/arduino/builder/recipe_test.go

@@ -0,0 +1,81 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package builder
+
+import (
+	"testing"
+
+	properties "github.com/arduino/go-properties-orderedmap"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRecipeFinder(t *testing.T) {
+	t.Run("NumberedRecipes", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.test", "test")
+		buildProperties.Set("recipe.1.test", "test2")
+		buildProperties.Set("recipe.2.test", "test3")
+		recipes := findRecipes(buildProperties, "recipe", ".test")
+		require.Equal(t, []string{"recipe.1.test", "recipe.2.test"}, recipes)
+	})
+	t.Run("NumberedRecipesWithGaps", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.test", "test")
+		buildProperties.Set("recipe.2.test", "test3")
+		buildProperties.Set("recipe.0.test", "test2")
+		recipes := findRecipes(buildProperties, "recipe", ".test")
+		require.Equal(t, []string{"recipe.0.test", "recipe.2.test"}, recipes)
+	})
+	t.Run("NumberedRecipesWithGapsAndDifferentLenghtNumbers", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.test", "test")
+		buildProperties.Set("recipe.12.test", "test3")
+		buildProperties.Set("recipe.2.test", "test2")
+		recipes := findRecipes(buildProperties, "recipe", ".test")
+		// The order is sorted alphabetically, not numerically
+		require.Equal(t, []string{"recipe.12.test", "recipe.2.test"}, recipes)
+	})
+	t.Run("NumberedRecipesWithGapsAndNumbers", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.test", "test")
+		buildProperties.Set("recipe.12.test", "test3")
+		buildProperties.Set("recipe.02.test", "test2")
+		buildProperties.Set("recipe.09.test", "test2")
+		recipes := findRecipes(buildProperties, "recipe", ".test")
+		require.Equal(t, []string{"recipe.02.test", "recipe.09.test", "recipe.12.test"}, recipes)
+	})
+	t.Run("UnnumberedRecipies", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.test", "test")
+		buildProperties.Set("recipe.a.test", "test3")
+		buildProperties.Set("recipe.b.test", "test2")
+		recipes := findRecipes(buildProperties, "recipe", ".test")
+		require.Equal(t, []string{"recipe.a.test", "recipe.b.test"}, recipes)
+	})
+	t.Run("ObjcopyRecipies/1", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.objcopy.eep.pattern", "test")
+		buildProperties.Set("recipe.objcopy.hex.pattern", "test")
+		recipes := findRecipes(buildProperties, "recipe.objcopy", ".pattern")
+		require.Equal(t, []string{"recipe.objcopy.eep.pattern", "recipe.objcopy.hex.pattern"}, recipes)
+	})
+	t.Run("ObjcopyRecipies/2", func(t *testing.T) {
+		buildProperties := properties.NewMap()
+		buildProperties.Set("recipe.objcopy.partitions.bin.pattern", "test")
+		recipes := findRecipes(buildProperties, "recipe.objcopy", ".pattern")
+		require.Equal(t, []string{"recipe.objcopy.partitions.bin.pattern"}, recipes)
+	})
+}