Auto-import libraries based on sketch profile. (#2951)

* Moved Result structure into his own package

This change is preparatory for next refactorings

* Moving sourceFile object in his own file

* Moved uniqueSourceFileQueue in his own file

* Moved includeCache in his own file and made it a field of detector

* Fixed comment

* Renamed variable (typo)

* Simplified handling of de-duplication of cache hit messages

This equivalent change moves the 'first' variable closer to his definition and use.

* Implemented a better include-cache

* Remove the old, no longer used, includeCache

* Refactored function

it will be useful in the next commits.

* Always add to the build the libraries requested in the active sketch profile.

This will improve sketch compile time.

* Fixed result conversion.

* Ensure stable ordering when libraries are auto-included

This avoid cache invalidation if recompiled.

Author: cmaglie

commands/instances.go

@@ -380,7 +380,7 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor
 				}
 				lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
 					Path:            libDir,
-					Location:        libraries.Unmanaged,
+					Location:        libraries.Profile,
 					IsSingleLibrary: true,
 				})
 				continue
@@ -428,7 +428,7 @@ func (s *arduinoCoreServerImpl) Init(req *rpc.InitRequest, stream rpc.ArduinoCor
 
 			lmb.AddLibrariesDir(librariesmanager.LibrariesDir{
 				Path:     libRoot,
-				Location: libraries.User,
+				Location: libraries.Profile,
 			})
 		}
 	}

commands/service_compile.go

@@ -105,10 +105,11 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 		return &cmderrors.CantOpenSketchError{Cause: err}
 	}
 
+	profile := pme.GetProfile()
 	fqbnIn := req.GetFqbn()
 	if fqbnIn == "" && sk != nil {
-		if pme.GetProfile() != nil {
-			fqbnIn = pme.GetProfile().FQBN
+		if profile != nil {
+			fqbnIn = profile.FQBN
 		} else {
 			fqbnIn = sk.GetDefaultFQBN()
 		}
@@ -224,7 +225,7 @@ func (s *arduinoCoreServerImpl) Compile(req *rpc.CompileRequest, stream rpc.Ardu
 	otherLibrariesDirs.Add(s.settings.LibrariesDir())
 
 	var libsManager *librariesmanager.LibrariesManager
-	if pme.GetProfile() != nil {
+	if profile != nil {
 		libsManager = lm
 	}
 

internal/arduino/builder/internal/detector/cache.go

@@ -0,0 +1,122 @@
+// This file is part of arduino-cli.
+//
+// Copyright 2024 ARDUINO SA (http://www.arduino.cc/)
+//
+// This software is released under the GNU General Public License version 3,
+// which covers the main part of arduino-cli.
+// The terms of this license can be found at:
+// https://www.gnu.org/licenses/gpl-3.0.en.html
+//
+// You can be released from the requirements of the above licenses by purchasing
+// a commercial license. Buying such a license is mandatory if you want to
+// modify or otherwise use the software for commercial activities involving the
+// Arduino software without disclosing the source code of your own applications.
+// To purchase a commercial license, send an email to license@arduino.cc.
+
+package detector
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/arduino/go-paths-helper"
+)
+
+type detectorCache struct {
+	curr    int
+	entries []*detectorCacheEntry
+}
+
+type detectorCacheEntry struct {
+	AddedIncludePath    *paths.Path `json:"added_include_path,omitempty"`
+	CompilingSourcePath *paths.Path `json:"compiling_source_path,omitempty"`
+	MissingIncludeH     *string     `json:"missing_include_h,omitempty"`
+}
+
+func (e *detectorCacheEntry) String() string {
+	if e.AddedIncludePath != nil {
+		return "Added include path: " + e.AddedIncludePath.String()
+	}
+	if e.CompilingSourcePath != nil {
+		return "Compiling source path: " + e.CompilingSourcePath.String()
+	}
+	if e.MissingIncludeH != nil {
+		if *e.MissingIncludeH == "" {
+			return "No missing include files detected"
+		}
+		return "Missing include file: " + *e.MissingIncludeH
+	}
+	return "No operation"
+}
+
+func (e *detectorCacheEntry) Equals(entry *detectorCacheEntry) bool {
+	return e.String() == entry.String()
+}
+
+func newDetectorCache() *detectorCache {
+	return &detectorCache{}
+}
+
+func (c *detectorCache) String() string {
+	res := ""
+	for _, entry := range c.entries {
+		res += fmt.Sprintln(entry)
+	}
+	return res
+}
+
+// Load reads a saved cache from the given file.
+// If the file do not exists, it does nothing.
+func (c *detectorCache) Load(cacheFile *paths.Path) error {
+	if exist, err := cacheFile.ExistCheck(); err != nil {
+		return err
+	} else if !exist {
+		return nil
+	}
+	data, err := cacheFile.ReadFile()
+	if err != nil {
+		return err
+	}
+	var entries []*detectorCacheEntry
+	if err := json.Unmarshal(data, &entries); err != nil {
+		return err
+	}
+	c.curr = 0
+	c.entries = entries
+	return nil
+}
+
+// Expect adds an entry to the cache and checks if it matches the next expected entry.
+func (c *detectorCache) Expect(entry *detectorCacheEntry) {
+	if c.curr < len(c.entries) {
+		if c.entries[c.curr].Equals(entry) {
+			// Cache hit, move to the next entry
+			c.curr++
+			return
+		}
+		// Cache mismatch, invalidate and cut the remainder of the cache
+		c.entries = c.entries[:c.curr]
+	}
+	c.curr++
+	c.entries = append(c.entries, entry)
+}
+
+// Peek returns the next cache entry to be expected or nil if the cache is fully consumed.
+func (c *detectorCache) Peek() *detectorCacheEntry {
+	if c.curr < len(c.entries) {
+		return c.entries[c.curr]
+	}
+	return nil
+}
+
+// Save writes the current cache to the given file.
+func (c *detectorCache) Save(cacheFile *paths.Path) error {
+	// Cut off the cache if it is not fully consumed
+	c.entries = c.entries[:c.curr]
+
+	data, err := json.MarshalIndent(c.entries, "", "  ")
+	if err != nil {
+		return err
+	}
+	return cacheFile.WriteFile(data)
+}