internal/build: switch azure sdk from Azure/azure-storage-blob-go to...

internal/build: switch azure sdk from Azure/azure-storage-blob-go to Azure/azure-sdk-for-go/sdk/storage/azblob. (#24473)

* go.mod: update azure-storage-blob-go

update Azure/azure-storage-blob-go from v0.7.0 to v0.14.0.
relation #24396.

* internal/build: fix for breaking changes of azure-storage-blob-go

fix for breaking changes of update Azure/azure-storage-blob-go from v0.7.0 to v0.14.0.
relation #24396.

* internal/build: switch azure sdk from Azure/azure-storage-blob-go to Azure/azure-sdk-for-go/sdk/storage/azblob.

* internal/build refactor appending BlobItems

* internal/build: fix azure blobstore client to include container id
Co-authored-by: Péter Szilágyi <peterke@gmail.com>

build/ci.go

@@ -1238,21 +1238,21 @@ func doPurge(cmdline []string) {
 
 	// Iterate over the blobs, collect and sort all unstable builds
 	for i := 0; i < len(blobs); i++ {
-		if !strings.Contains(blobs[i].Name, "unstable") {
+		if !strings.Contains(*blobs[i].Name, "unstable") {
 			blobs = append(blobs[:i], blobs[i+1:]...)
 			i--
 		}
 	}
 	for i := 0; i < len(blobs); i++ {
 		for j := i + 1; j < len(blobs); j++ {
-			if blobs[i].Properties.LastModified.After(blobs[j].Properties.LastModified) {
+			if blobs[i].Properties.LastModified.After(*blobs[j].Properties.LastModified) {
 				blobs[i], blobs[j] = blobs[j], blobs[i]
 			}
 		}
 	}
 	// Filter out all archives more recent that the given threshold
 	for i, blob := range blobs {
-		if time.Since(blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
+		if time.Since(*blob.Properties.LastModified) < time.Duration(*limit)*24*time.Hour {
 			blobs = blobs[:i]
 			break
 		}

go.mod

@@ -3,9 +3,7 @@ module github.com/ethereum/go-ethereum
 go 1.15
 
 require (
-	github.com/Azure/azure-pipeline-go v0.2.2 // indirect
-	github.com/Azure/azure-storage-blob-go v0.7.0
-	github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0
 	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
 	github.com/VictoriaMetrics/fastcache v1.6.0
 	github.com/aws/aws-sdk-go-v2 v1.2.0
@@ -30,7 +28,7 @@ require (
 	github.com/golang/protobuf v1.4.3
 	github.com/golang/snappy v0.0.4
 	github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa
-	github.com/google/uuid v1.1.5
+	github.com/google/uuid v1.2.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/graph-gophers/graphql-go v1.3.0
 	github.com/hashicorp/go-bexpr v0.1.10
@@ -62,10 +60,9 @@ require (
 	github.com/tklauser/go-sysconf v0.3.5 // indirect
 	github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef
 	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2
-	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912
-	golang.org/x/text v0.3.6
+	golang.org/x/text v0.3.7
 	golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba
 	golang.org/x/tools v0.1.0
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce

internal/build/azure.go

@@ -19,10 +19,9 @@ package build
 import (
 	"context"
 	"fmt"
-	"net/url"
 	"os"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 )
 
 // AzureBlobstoreConfig is an authentication and configuration struct containing
@@ -49,15 +48,11 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig)
 	if err != nil {
 		return err
 	}
-
-	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
-
-	u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account))
-	service := azblob.NewServiceURL(*u, pipeline)
-
-	container := service.NewContainerURL(config.Container)
-	blockblob := container.NewBlockBlobURL(name)
-
+	u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container)
+	container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil)
+	if err != nil {
+		return err
+	}
 	// Stream the file to upload into the designated blobstore container
 	in, err := os.Open(path)
 	if err != nil {
@@ -65,49 +60,41 @@ func AzureBlobstoreUpload(path string, name string, config AzureBlobstoreConfig)
 	}
 	defer in.Close()
 
-	_, err = blockblob.Upload(context.Background(), in, azblob.BlobHTTPHeaders{}, azblob.Metadata{}, azblob.BlobAccessConditions{})
+	blockblob := container.NewBlockBlobClient(name)
+	_, err = blockblob.Upload(context.Background(), in, nil)
 	return err
 }
 
 // AzureBlobstoreList lists all the files contained within an azure blobstore.
-func AzureBlobstoreList(config AzureBlobstoreConfig) ([]azblob.BlobItem, error) {
-	credential := azblob.NewAnonymousCredential()
-	if len(config.Token) > 0 {
-		c, err := azblob.NewSharedKeyCredential(config.Account, config.Token)
-		if err != nil {
-			return nil, err
-		}
-		credential = c
+func AzureBlobstoreList(config AzureBlobstoreConfig) ([]*azblob.BlobItemInternal, error) {
+	// Create an authenticated client against the Azure cloud
+	credential, err := azblob.NewSharedKeyCredential(config.Account, config.Token)
+	if err != nil {
+		return nil, err
 	}
-	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
-
-	u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account))
-	service := azblob.NewServiceURL(*u, pipeline)
-
-	var allBlobs []azblob.BlobItem
-	// List all the blobs from the container and return them
-	container := service.NewContainerURL(config.Container)
-	nextMarker := azblob.Marker{}
-	for nextMarker.NotDone() {
-		res, err := container.ListBlobsFlatSegment(context.Background(), nextMarker, azblob.ListBlobsSegmentOptions{
-			MaxResults: 5000, // The server only gives max 5K items
-		})
-		if err != nil {
-			return nil, err
-		}
-		allBlobs = append(allBlobs, res.Segment.BlobItems...)
-		nextMarker = res.NextMarker
-
+	u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container)
+	container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil)
+	if err != nil {
+		return nil, err
+	}
+	var maxResults int32 = 5000
+	pager := container.ListBlobsFlat(&azblob.ContainerListBlobFlatSegmentOptions{
+		Maxresults: &maxResults,
+	})
+	var allBlobs []*azblob.BlobItemInternal
+	for pager.NextPage(context.Background()) {
+		res := pager.PageResponse()
+		allBlobs = append(allBlobs, res.ContainerListBlobFlatSegmentResult.Segment.BlobItems...)
 	}
-	return allBlobs, nil
+	return allBlobs, pager.Err()
 }
 
 // AzureBlobstoreDelete iterates over a list of files to delete and removes them
 // from the blobstore.
-func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []azblob.BlobItem) error {
+func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []*azblob.BlobItemInternal) error {
 	if *DryRunFlag {
 		for _, blob := range blobs {
-			fmt.Printf("would delete %s (%s) from %s/%s\n", blob.Name, blob.Properties.LastModified, config.Account, config.Container)
+			fmt.Printf("would delete %s (%s) from %s/%s\n", *blob.Name, blob.Properties.LastModified, config.Account, config.Container)
 		}
 		return nil
 	}
@@ -116,21 +103,18 @@ func AzureBlobstoreDelete(config AzureBlobstoreConfig, blobs []azblob.BlobItem)
 	if err != nil {
 		return err
 	}
-
-	pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{})
-
-	u, _ := url.Parse(fmt.Sprintf("https://%s.blob.core.windows.net", config.Account))
-	service := azblob.NewServiceURL(*u, pipeline)
-
-	container := service.NewContainerURL(config.Container)
-
+	u := fmt.Sprintf("https://%s.blob.core.windows.net/%s", config.Account, config.Container)
+	container, err := azblob.NewContainerClientWithSharedKey(u, credential, nil)
+	if err != nil {
+		return err
+	}
 	// Iterate over the blobs and delete them
 	for _, blob := range blobs {
-		blockblob := container.NewBlockBlobURL(blob.Name)
-		if _, err := blockblob.Delete(context.Background(), azblob.DeleteSnapshotsOptionInclude, azblob.BlobAccessConditions{}); err != nil {
+		blockblob := container.NewBlockBlobClient(*blob.Name)
+		if _, err := blockblob.Delete(context.Background(), &azblob.DeleteBlobOptions{}); err != nil {
 			return err
 		}
-		fmt.Printf("deleted  %s (%s)\n", blob.Name, blob.Properties.LastModified)
+		fmt.Printf("deleted  %s (%s)\n", *blob.Name, blob.Properties.LastModified)
 	}
 	return nil
 }