Merging changes from mover/Stage2 to mover/c2c-stage

.gitignore

@@ -23,6 +23,9 @@ vendor/
 ## Ignore the executables
 azcopy*
 
+## Do not ignore azcopy as a library code
+!azcopy/
+
 ## Dep tool
 vendor/
 
@@ -336,3 +339,6 @@ venv*
 # go files
 *.mod
 *.sum
+
+# ignore the latest_version.txt
+/latest_version.txt

ChangeLog.md

@@ -1,6 +1,29 @@
 
 # Change Log
 
+## Version 10.30.0
+### Breaking changes
+1. For transfers involving Azure Files (NFS or SMB), AzCopy will not auto create file shares.
+2. AzCopy binaries and latest version information will now be distributed from Github releases instead of the static website. ([#3014](https://github.com/Azure/azure-storage-azcopy/pull/3014))
+
+### New Features
+1. Azure Files NFS Support via REST. 
+   Support for transferring data between local Linux systems and Azure Files NFS using REST. To use this feature, please explicitly specify the `--from-to` flag. 
+   - Transfer from local Linux to Azure Files NFS. (`--from-to=LocalFileNFS`)
+   - Transfer from Azure Files NFS to local Linux. (`--from-to=FileNFSLocal`)
+   - Transfer between Azure Files NFS shares. (`--from-to=FileNFSFileNFS`)
+2. Added support to retry on copy source error code and status code for service to service copies. ([#3105](https://github.com/Azure/azure-storage-azcopy/pull/3105))
+3. Added support for service to service copies from Azure Files to Blob Storage using EntraID. ([#3053](https://github.com/Azure/azure-storage-azcopy/pull/3053))
+
+### Bug Fixes
+1. Fixed a bug where when copying a file that has already been deleted with `--trailing-dot=Disable` resulted in the wrong error instead of a 404. ([#3092](https://github.com/Azure/azure-storage-azcopy/pull/3092))
+
+### Supportability
+1. Removed the warning message when failing to create a container. This message can be misleading when there is insufficient permissions to create a container and the container already exists. ([#3045](https://github.com/Azure/azure-storage-azcopy/pull/3045))
+2. Improved the error message returned when block size is larger than bandwidth limit. ([#3051](https://github.com/Azure/azure-storage-azcopy/pull/3051))
+3. Warn user if transfer is going to exceed 10M objects. ([#3111](https://github.com/Azure/azure-storage-azcopy/pull/3111))
+4. Warn user if multiple AzCopy processes are running. ([#3128](https://github.com/Azure/azure-storage-azcopy/pull/3128))
+
 ## Version 10.30.0-preview.2
 
 ### Dependency updates

azcopy/client.go

@@ -0,0 +1,65 @@
+// Copyright © 2025 Microsoft <wastore@microsoft.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package azcopy
+
+import (
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
+	"github.com/Azure/azure-storage-azcopy/v10/ste"
+	"log"
+	"runtime"
+)
+
+type Client struct {
+	CurrentJobID common.JobID // TODO (gapra): In future this should only be set when there is a current job running. On complete, this should be cleared. It can also behave as something we can check to see if a current job is running
+}
+
+type ClientOptions struct {
+	CapMbps float64
+}
+
+func NewClient(opts ClientOptions) (Client, error) {
+	c := Client{}
+	common.InitializeFolders()
+	configureGoMaxProcs()
+	// Perform os specific initialization
+	azcopyMaxFileAndSocketHandles, err := processOSSpecificInitialization()
+	if err != nil {
+		log.Fatalf("initialization failed: %v", err)
+	}
+	// startup of the STE happens here, so that the startup can access the values of command line parameters that are defined for "root" command
+	concurrencySettings := ste.NewConcurrencySettings(azcopyMaxFileAndSocketHandles)
+	err = jobsAdmin.MainSTE(concurrencySettings, opts.CapMbps)
+	if err != nil {
+		return c, err
+	}
+	return c, nil
+}
+
+// Ensure we always have more than 1 OS thread running goroutines, since there are issues with having just 1.
+// (E.g. version check doesn't happen at login time, if have only one go proc. Not sure why that happens if have only one
+// proc. Is presumably due to the high CPU usage we see on login if only 1 CPU, even tho can't see any busy-wait in that code)
+func configureGoMaxProcs() {
+	isOnlyOne := runtime.GOMAXPROCS(0) == 1
+	if isOnlyOne {
+		runtime.GOMAXPROCS(2)
+	}
+}

cmd/root_unix.go → azcopy/client_unix.go

@@ -21,14 +21,11 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
-package cmd
+package azcopy
 
 import (
 	"math"
-	"path"
 	"syscall"
-
-	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
 
 // processOSSpecificInitialization changes the soft limit for file descriptor for process
@@ -67,11 +64,3 @@ func processOSSpecificInitialization() (int, error) {
 		return int(set.Cur), nil
 	}
 }
-
-// getAzCopyAppPath returns the path of Azcopy folder in local appdata.
-// Azcopy folder in local appdata contains all the files created by azcopy locally.
-func getAzCopyAppPath() string {
-	localAppData := common.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir())
-	azcopyAppDataFolder := path.Join(localAppData, ".azcopy")
-	return azcopyAppDataFolder
-}

cmd/root_windows.go → azcopy/client_windows.go

@@ -18,17 +18,13 @@
 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 // THE SOFTWARE.
 
-package cmd
+package azcopy
 
 import (
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/minio/minio-go/v7"
 	"math"
 	"net/http"
-	"path"
-	"strings"
-
-	"github.com/minio/minio-go/v7"
-
-	"github.com/Azure/azure-storage-azcopy/v10/common"
 )
 
 // processOSSpecificInitialization changes the soft limit for filedescriptor for process
@@ -44,14 +40,6 @@ func processOSSpecificInitialization() (int, error) {
 	return effectivelyUnlimited, nil
 }
 
-// getAzCopyAppPath returns the path of Azcopy in local appdata.
-func getAzCopyAppPath() string {
-	userProfile := common.GetEnvironmentVariable(common.EEnvironmentVariable.UserDir())
-	azcopyAppDataFolder := strings.ReplaceAll(path.Join(userProfile, ".azcopy"), "/", `\`)
-
-	return azcopyAppDataFolder
-}
-
 func init() {
 	//Catch everything that uses http.DefaultTransport with ieproxy.GetProxyFunc()
 	http.DefaultTransport.(*http.Transport).Proxy = common.GlobalProxyLookup

azcopy/jobsClean.go

@@ -0,0 +1,65 @@
+// Copyright © 2025 Microsoft <wastore@microsoft.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package azcopy
+
+import (
+	"fmt"
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
+)
+
+// TODO (gapra) : Consider adding an onDelete callback to CleanJobsOptions? That way we can print to console as we clean jobs.
+type CleanJobsOptions struct {
+	WithStatus *common.JobStatus // Default: All
+}
+
+type CleanJobsResult struct {
+	Count int            // Number of files cleaned
+	Jobs  []common.JobID // List of job IDs cleaned if WithStatus is not All, otherwise nil
+}
+
+// CleanJobs removes jobs with a specified status.
+// If WithStatus is All, it cleans all jobs and returns the count of jobs cleaned.
+// If WithStatus is not All, it cleans jobs with that status and returns the count of jobs cleaned and list of job IDs cleaned.
+func (c Client) CleanJobs(opts CleanJobsOptions) (result CleanJobsResult, err error) {
+	result = CleanJobsResult{}
+	status := common.IffNil(opts.WithStatus, common.EJobStatus.All())
+
+	if status == common.EJobStatus.All() {
+		result.Count, err = jobsAdmin.DeleteAllJobFilesExceptCurrent(c.CurrentJobID)
+	} else {
+		resp := jobsAdmin.ListJobs(status)
+		if resp.ErrorMessage != "" {
+			return result, fmt.Errorf("failed to list jobs due to error: %s", resp.ErrorMessage)
+		}
+		result.Jobs = []common.JobID{}
+		for _, job := range resp.JobIDDetails {
+			result.Jobs = append(result.Jobs, job.JobId)
+			count, err := jobsAdmin.RemoveSingleJobFiles(job.JobId)
+			if err != nil {
+				return result, fmt.Errorf("failed to remove job %s due to error: %w", job.JobId, err)
+			} else {
+				result.Count += count
+			}
+		}
+	}
+	return result, err
+}

azcopy/jobsList.go

@@ -0,0 +1,79 @@
+// Copyright © 2025 Microsoft <wastore@microsoft.com>
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package azcopy
+
+import (
+	"fmt"
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/Azure/azure-storage-azcopy/v10/jobsAdmin"
+	"sort"
+	"time"
+)
+
+type ListJobsOptions struct {
+	WithStatus *common.JobStatus // Default: All
+}
+
+type JobDetail struct {
+	JobID     common.JobID
+	StartTime time.Time
+	Status    common.JobStatus
+	Command   string
+}
+
+type ListJobsResponse struct {
+	Details []JobDetail
+}
+
+func (c Client) ListJobs(opts ListJobsOptions) (result ListJobsResponse, err error) {
+	status := common.IffNil(opts.WithStatus, common.EJobStatus.All())
+
+	resp := jobsAdmin.ListJobs(status)
+	if resp.ErrorMessage != "" {
+		return ListJobsResponse{}, fmt.Errorf("failed to list jobs due to error: %s", resp.ErrorMessage)
+	}
+
+	var details []JobDetail
+	for _, job := range resp.JobIDDetails {
+		details = append(details, JobDetail{
+			JobID:     job.JobId,
+			StartTime: time.Unix(0, job.StartTime),
+			Status:    job.JobStatus,
+			Command:   job.CommandString,
+		})
+	}
+
+	// before displaying the jobs, sort them accordingly so that they are displayed in a consistent way
+	sortJobs(details)
+
+	return ListJobsResponse{
+		Details: details,
+	}, nil
+}
+
+func sortJobs(jobsDetails []JobDetail) {
+	// sort the jobs so that the latest one is shown first
+	sort.Slice(jobsDetails, func(i, j int) bool {
+		// this function essentially asks whether i should be placed before j
+		// we say yes if the job i is more recent
+		return jobsDetails[i].StartTime.After(jobsDetails[j].StartTime)
+	})
+}

azcopy/jobsList_test.go

@@ -0,0 +1,43 @@
+package azcopy
+
+import (
+	"github.com/Azure/azure-storage-azcopy/v10/common"
+	"github.com/stretchr/testify/assert"
+	"testing"
+	"time"
+)
+
+func TestSortJobs(t *testing.T) {
+	a := assert.New(t)
+	// setup
+	job2 := JobDetail{
+		JobID:     common.NewJobID(),
+		StartTime: time.Now(),
+		Command:   "dummy2",
+	}
+
+	// sleep for a bit so that the time stamp is different
+	time.Sleep(time.Millisecond)
+	job1 := JobDetail{
+		JobID:     common.NewJobID(),
+		StartTime: time.Now(),
+		Command:   "dummy1",
+	}
+
+	// sleep for a bit so that the time stamp is different
+	time.Sleep(time.Millisecond)
+	job0 := JobDetail{
+		JobID:     common.NewJobID(),
+		StartTime: time.Now(),
+		Command:   "dummy0",
+	}
+	jobsList := []JobDetail{job2, job1, job0}
+
+	// act
+	sortJobs(jobsList)
+
+	// verify
+	a.Equal(job0, jobsList[0])
+	a.Equal(job1, jobsList[1])
+	a.Equal(job2, jobsList[2])
+}