From 6cfbc809cf6d94283088127f05f9c98eeaa21098 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Thu, 21 Nov 2019 22:25:46 -0800 Subject: [PATCH] Add mirror command to compare metadata and copy (#2965) mirror now uses the newly introduced ListObjectsV2WithMetadata API in minio-go --- cmd/client-fs.go | 119 ++++++++++++++++++++----------------- cmd/client-fs_test.go | 8 +-- cmd/client-s3.go | 118 +++++++++++++++++++----------------- cmd/client-s3_test.go | 6 +- cmd/client-url.go | 3 +- cmd/client.go | 2 +- cmd/common-methods.go | 61 +++++++++++-------- cmd/complete.go | 2 +- cmd/cp-url.go | 2 +- cmd/difference.go | 22 ++++--- cmd/du-main.go | 2 +- cmd/find.go | 2 +- cmd/ls.go | 2 +- cmd/mirror-main.go | 79 +++++++++++++----------- cmd/policy-main.go | 2 +- cmd/rb-main.go | 4 +- cmd/rm-main.go | 2 +- cmd/share-download-main.go | 2 +- cmd/sql-main.go | 2 +- cmd/stat.go | 2 +- cmd/tree-main.go | 2 +- go.mod | 4 +- go.sum | 8 +-- 23 files changed, 251 insertions(+), 205 deletions(-) diff --git a/cmd/client-fs.go b/cmd/client-fs.go index c2587124fc..7572035adb 100644 --- a/cmd/client-fs.go +++ b/cmd/client-fs.go @@ -68,11 +68,11 @@ func fsNew(path string) (Client, *probe.Error) { }, nil } -func isNotSupported(err error) bool { - if err == nil { +func isNotSupported(e error) bool { + if e == nil { return false } - errno := err.(*xattr.Error) + errno := e.(*xattr.Error) if errno == nil { return false } @@ -83,13 +83,13 @@ func isNotSupported(err error) bool { // isIgnoredFile returns true if 'filename' is on the exclude list. func isIgnoredFile(filename string) bool { - matchFile := path.Base(filename) + matchFile := filepath.Base(filename) // OS specific ignore list. for _, ignoredFile := range ignoreFiles[runtime.GOOS] { - matched, err := filepath.Match(ignoredFile, matchFile) - if err != nil { - panic(err) + matched, e := filepath.Match(ignoredFile, matchFile) + if e != nil { + panic(e) } if matched { return true @@ -98,9 +98,9 @@ func isIgnoredFile(filename string) bool { // Default ignore list for all OSes. for _, ignoredFile := range ignoreFiles["default"] { - matched, err := filepath.Match(ignoredFile, matchFile) - if err != nil { - panic(err) + matched, e := filepath.Match(ignoredFile, matchFile) + if e != nil { + panic(e) } if matched { return true @@ -343,46 +343,55 @@ func (f *fsClient) put(reader io.Reader, size int64, metadata map[string][]strin } if len(metadata["mc-attrs"]) != 0 { - attr, err := parseAttribute(metadata["mc-attrs"][0]) - if err != nil { - return totalWritten, probe.NewError(err) + attr, e := parseAttribute(metadata["mc-attrs"][0]) + if e != nil { + return totalWritten, probe.NewError(e) } - mode, err := strconv.ParseUint(attr["mode"], 10, 32) - if err != nil { - return totalWritten, probe.NewError(err) + mode, e := strconv.ParseUint(attr["mode"], 10, 32) + if e != nil { + return totalWritten, probe.NewError(e) } + // Change the mode of file - if err := os.Chmod(objectPath, os.FileMode(mode)); err != nil { - return totalWritten, probe.NewError(err) + if e := os.Chmod(objectPath, os.FileMode(mode)); e != nil { + if !os.IsPermission(e) { + return totalWritten, probe.NewError(e) + } } - uid, err := strconv.Atoi(attr["uid"]) - if err != nil { - return totalWritten, probe.NewError(err) + uid, e := strconv.Atoi(attr["uid"]) + if e != nil { + return totalWritten, probe.NewError(e) } - gid, err := strconv.Atoi(attr["gid"]) - if err != nil { - return totalWritten, probe.NewError(err) + gid, e := strconv.Atoi(attr["gid"]) + if e != nil { + return totalWritten, probe.NewError(e) } + // Change the owner - if err := os.Chown(objectPath, uid, gid); err != nil { - return totalWritten, probe.NewError(err) + if e := os.Chown(objectPath, uid, gid); e != nil { + if !os.IsPermission(e) { + return totalWritten, probe.NewError(e) + } } - atime, err := strconv.ParseInt(attr["atime"], 10, 64) - if err != nil { - return totalWritten, probe.NewError(err) + atime, e := strconv.ParseInt(attr["atime"], 10, 64) + if e != nil { + return totalWritten, probe.NewError(e) } - ctime, err := strconv.ParseInt(attr["ctime"], 10, 64) - if err != nil { - return totalWritten, probe.NewError(err) + ctime, e := strconv.ParseInt(attr["ctime"], 10, 64) + if e != nil { + return totalWritten, probe.NewError(e) } + // Change the access, modify and change time - if err := os.Chtimes(objectPath, time.Unix(atime, 0), time.Unix(ctime, 0)); err != nil { - return totalWritten, probe.NewError(err) + if e := os.Chtimes(objectPath, time.Unix(atime, 0), time.Unix(ctime, 0)); e != nil { + if !os.IsPermission(e) { + return totalWritten, probe.NewError(e) + } } } @@ -505,11 +514,11 @@ func isSysErrNotEmpty(err error) bool { // until it finds one with files in it. Returns nil for a non-empty directory. func deleteFile(deletePath string) error { // Attempt to remove path. - if err := os.Remove((deletePath)); err != nil { - if isSysErrNotEmpty(err) { + if e := os.Remove(deletePath); e != nil { + if isSysErrNotEmpty(e) { return nil } - return err + return e } // Trailing slash is removed when found to ensure @@ -538,15 +547,15 @@ func (f *fsClient) Remove(isIncomplete, isRemoveBucket bool, contentCh <-chan *c if isIncomplete { name += partSuffix } - if err := deleteFile(name); err != nil { - if os.IsPermission(err) { + if e := deleteFile(name); e != nil { + if os.IsPermission(e) { // Ignore permission error. errorCh <- probe.NewError(PathInsufficientPermission{Path: content.URL.Path}) - } else if os.IsNotExist(err) && isRemoveBucket { + } else if os.IsNotExist(e) && isRemoveBucket { // ignore PathNotFound for dir removal. return } else { - errorCh <- probe.NewError(err) + errorCh <- probe.NewError(e) return } } @@ -557,18 +566,18 @@ func (f *fsClient) Remove(isIncomplete, isRemoveBucket bool, contentCh <-chan *c } // List - list files and folders. -func (f *fsClient) List(isRecursive, isIncomplete bool, showDir DirOpt) <-chan *clientContent { +func (f *fsClient) List(isRecursive, isIncomplete, isMetadata bool, showDir DirOpt) <-chan *clientContent { contentCh := make(chan *clientContent) filteredCh := make(chan *clientContent) if isRecursive { if showDir == DirNone { - go f.listRecursiveInRoutine(contentCh) + go f.listRecursiveInRoutine(contentCh, isMetadata) } else { - go f.listDirOpt(contentCh, isIncomplete, showDir) + go f.listDirOpt(contentCh, isIncomplete, isMetadata, showDir) } } else { - go f.listInRoutine(contentCh) + go f.listInRoutine(contentCh, isMetadata) } // This function filters entries from any listing go routine @@ -701,7 +710,7 @@ func (f *fsClient) listPrefixes(prefix string, contentCh chan<- *clientContent) } } -func (f *fsClient) listInRoutine(contentCh chan<- *clientContent) { +func (f *fsClient) listInRoutine(contentCh chan<- *clientContent, isMetadata bool) { // close the channel when the function returns. defer close(contentCh) @@ -795,7 +804,7 @@ func (f *fsClient) listInRoutine(contentCh chan<- *clientContent) { } // List files recursively using non-recursive mode. -func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool, dirOpt DirOpt) { +func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool, isMetadata bool, dirOpt DirOpt) { defer close(contentCh) // Trim trailing / or \. @@ -808,14 +817,18 @@ func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool, // Closure function reads currentPath and sends to contentCh. If a directory is found, it lists the directory content recursively. var listDir func(currentPath string) bool listDir = func(currentPath string) (isStop bool) { - files, err := readDir(currentPath) - if err != nil { - if os.IsPermission(err) { - contentCh <- &clientContent{Err: probe.NewError(PathInsufficientPermission{Path: currentPath})} + files, e := readDir(currentPath) + if e != nil { + if os.IsPermission(e) { + contentCh <- &clientContent{ + Err: probe.NewError(PathInsufficientPermission{ + Path: currentPath, + }), + } return false } - contentCh <- &clientContent{Err: probe.NewError(err)} + contentCh <- &clientContent{Err: probe.NewError(e)} return true } @@ -861,7 +874,7 @@ func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool, } } -func (f *fsClient) listRecursiveInRoutine(contentCh chan *clientContent) { +func (f *fsClient) listRecursiveInRoutine(contentCh chan *clientContent, isMetadata bool) { // close channels upon return. defer close(contentCh) var dirName string diff --git a/cmd/client-fs_test.go b/cmd/client-fs_test.go index cafee57dfa..bcda786e25 100644 --- a/cmd/client-fs_test.go +++ b/cmd/client-fs_test.go @@ -65,7 +65,7 @@ func (s *TestSuite) TestList(c *C) { // Verify previously create files and list them. var contents []*clientContent - for content := range fsClient.List(false, false, DirNone) { + for content := range fsClient.List(false, false, false, DirNone) { if content.Err != nil { err = content.Err break @@ -93,7 +93,7 @@ func (s *TestSuite) TestList(c *C) { contents = nil // List non recursive to list only top level files. - for content := range fsClient.List(false, false, DirNone) { + for content := range fsClient.List(false, false, false, DirNone) { if content.Err != nil { err = content.Err break @@ -109,7 +109,7 @@ func (s *TestSuite) TestList(c *C) { contents = nil // List recursively all files and verify. - for content := range fsClient.List(true, false, DirNone) { + for content := range fsClient.List(true, false, false, DirNone) { if content.Err != nil { err = content.Err break @@ -153,7 +153,7 @@ func (s *TestSuite) TestList(c *C) { contents = nil // List recursively all files and verify. - for content := range fsClient.List(true, false, DirNone) { + for content := range fsClient.List(true, false, false, DirNone) { if content.Err != nil { err = content.Err break diff --git a/cmd/client-s3.go b/cmd/client-s3.go index 7c4fe42261..1df964adb4 100644 --- a/cmd/client-s3.go +++ b/cmd/client-s3.go @@ -1012,22 +1012,37 @@ func (c *s3Client) MakeBucket(region string, ignoreExisting, withLock bool) *pro return probe.NewError(BucketNameEmpty{}) } if object != "" { - if strings.HasSuffix(object, "/") { - retry: - if _, e := c.api.PutObject(bucket, object, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}); e != nil { - switch minio.ToErrorResponse(e).Code { - case "NoSuchBucket": + if !strings.HasSuffix(object, string(c.targetURL.Separator)) { + object = path.Dir(object) + } + if !strings.HasSuffix(object, string(c.targetURL.Separator)) { + return probe.NewError(BucketNameTopLevel{}) + } + var retried bool + for { + _, e := c.api.PutObject(bucket, object, + bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) + if e == nil { + return nil + } + if retried { + return probe.NewError(e) + } + switch minio.ToErrorResponse(e).Code { + case "NoSuchBucket": + if withLock { + e = c.api.MakeBucketWithObjectLock(bucket, region) + } else { e = c.api.MakeBucket(bucket, region) - if e != nil { - return probe.NewError(e) - } - goto retry } - return probe.NewError(e) + if e != nil { + return probe.NewError(e) + } + retried = true + continue } - return nil + return probe.NewError(e) } - return probe.NewError(BucketNameTopLevel{}) } var e error @@ -1141,11 +1156,11 @@ func (c *s3Client) SetAccess(bucketPolicy string, isJSON bool) *probe.Error { } // listObjectWrapper - select ObjectList version depending on the target hostname -func (c *s3Client) listObjectWrapper(bucket, object string, isRecursive bool, doneCh chan struct{}) <-chan minio.ObjectInfo { - if isAmazon(c.targetURL.Host) || isAmazonAccelerated(c.targetURL.Host) { - return c.api.ListObjectsV2(bucket, object, isRecursive, doneCh) +func (c *s3Client) listObjectWrapper(bucket, object string, isRecursive bool, doneCh chan struct{}, metadata bool) <-chan minio.ObjectInfo { + if metadata { + return c.api.ListObjectsV2WithMetadata(bucket, object, isRecursive, doneCh) } - return c.api.ListObjects(bucket, object, isRecursive, doneCh) + return c.api.ListObjectsV2(bucket, object, isRecursive, doneCh) } // Stat - send a 'HEAD' on a bucket or object to fetch its metadata. @@ -1212,14 +1227,13 @@ func (c *s3Client) Stat(isIncomplete, isFetchMeta, isPreserve bool, sse encrypt. opts := minio.StatObjectOptions{} opts.ServerSideEncryption = sse - for objectStat := range c.listObjectWrapper(bucket, prefix, nonRecursive, nil) { + for objectStat := range c.listObjectWrapper(bucket, prefix, nonRecursive, nil, false) { if objectStat.Err != nil { return nil, probe.NewError(objectStat.Err) } if strings.HasSuffix(objectStat.Key, string(c.targetURL.Separator)) { objectMetadata.URL = *c.targetURL objectMetadata.Type = os.ModeDir - if isFetchMeta { stat, err := c.getObjectStat(bucket, object, opts) if err != nil { @@ -1396,7 +1410,7 @@ func (c *s3Client) splitPath(path string) (bucketName, objectName string) { /// Bucket API operations. // List - list at delimited path, if not recursive. -func (c *s3Client) List(isRecursive, isIncomplete bool, showDir DirOpt) <-chan *clientContent { +func (c *s3Client) List(isRecursive, isIncomplete, isMetadata bool, showDir DirOpt) <-chan *clientContent { c.mutex.Lock() defer c.mutex.Unlock() @@ -1414,12 +1428,12 @@ func (c *s3Client) List(isRecursive, isIncomplete bool, showDir DirOpt) <-chan * } else { if isRecursive { if showDir == DirNone { - go c.listRecursiveInRoutine(contentCh) + go c.listRecursiveInRoutine(contentCh, isMetadata) } else { - go c.listRecursiveInRoutineDirOpt(contentCh, showDir) + go c.listRecursiveInRoutineDirOpt(contentCh, showDir, isMetadata) } } else { - go c.listInRoutine(contentCh) + go c.listInRoutine(contentCh, isMetadata) } } @@ -1562,7 +1576,7 @@ func (c *s3Client) objectMultipartInfo2ClientContent(bucket string, entry minio. content.Size = entry.Size content.Time = entry.Initiated - if strings.HasSuffix(entry.Key, "/") { + if strings.HasSuffix(entry.Key, string(c.targetURL.Separator)) { content.Type = os.ModeDir } else { content.Type = os.ModeTemporary @@ -1679,9 +1693,8 @@ func (c *s3Client) joinPath(bucket string, objects ...string) string { } // Convert objectInfo to clientContent -func (c *s3Client) objectInfo2ClientContent(bucket string, entry minio.ObjectInfo) clientContent { - - content := clientContent{} +func (c *s3Client) objectInfo2ClientContent(bucket string, entry minio.ObjectInfo) *clientContent { + content := &clientContent{} url := *c.targetURL // Join bucket and incoming object key. url.Path = c.joinPath(bucket, entry.Key) @@ -1689,9 +1702,14 @@ func (c *s3Client) objectInfo2ClientContent(bucket string, entry minio.ObjectInf content.Size = entry.Size content.ETag = entry.ETag content.Time = entry.LastModified - - if strings.HasSuffix(entry.Key, "/") && entry.Size == 0 && entry.LastModified.IsZero() { + content.Expires = entry.Expires + content.UserMetadata = map[string]string{} + for k, v := range entry.UserMetadata { + content.UserMetadata[k] = v + } + if strings.HasSuffix(entry.Key, string(c.targetURL.Separator)) && entry.Size == 0 && entry.LastModified.IsZero() { content.Type = os.ModeDir + content.Time = time.Now() } else { content.Type = os.FileMode(0664) } @@ -1712,14 +1730,14 @@ func (c *s3Client) bucketStat(bucket string) (*clientContent, *probe.Error) { } // Recursively lists objects. -func (c *s3Client) listRecursiveInRoutineDirOpt(contentCh chan *clientContent, dirOpt DirOpt) { +func (c *s3Client) listRecursiveInRoutineDirOpt(contentCh chan *clientContent, dirOpt DirOpt, metadata bool) { defer close(contentCh) // Closure function reads list objects and sends to contentCh. If a directory is found, it lists // objects of the directory content recursively. var listDir func(bucket, object string) bool listDir = func(bucket, object string) (isStop bool) { isRecursive := false - for entry := range c.listObjectWrapper(bucket, object, isRecursive, nil) { + for entry := range c.listObjectWrapper(bucket, object, isRecursive, nil, metadata) { if entry.Err != nil { url := *c.targetURL url.Path = c.joinPath(bucket, object) @@ -1737,16 +1755,16 @@ func (c *s3Client) listRecursiveInRoutineDirOpt(contentCh chan *clientContent, d // Handle if object.Key is a directory. if content.Type.IsDir() { if dirOpt == DirFirst { - contentCh <- &content + contentCh <- content } if listDir(bucket, entry.Key) { return true } if dirOpt == DirLast { - contentCh <- &content + contentCh <- content } } else { - contentCh <- &content + contentCh <- content } } return false @@ -1809,7 +1827,7 @@ func (c *s3Client) listRecursiveInRoutineDirOpt(contentCh chan *clientContent, d } } -func (c *s3Client) listInRoutine(contentCh chan *clientContent) { +func (c *s3Client) listInRoutine(contentCh chan *clientContent, metadata bool) { defer close(contentCh) // get bucket and object from URL. b, o := c.url2BucketAndObject() @@ -1841,7 +1859,7 @@ func (c *s3Client) listInRoutine(contentCh chan *clientContent) { contentCh <- content default: isRecursive := false - for object := range c.listObjectWrapper(b, o, isRecursive, nil) { + for object := range c.listObjectWrapper(b, o, isRecursive, nil, metadata) { if object.Err != nil { contentCh <- &clientContent{ Err: probe.NewError(object.Err), @@ -1854,24 +1872,7 @@ func (c *s3Client) listInRoutine(contentCh chan *clientContent) { continue } - content := &clientContent{} - url := *c.targetURL - // Join bucket and incoming object key. - url.Path = c.joinPath(b, object.Key) - switch { - case strings.HasSuffix(object.Key, string(c.targetURL.Separator)): - // We need to keep the trailing Separator, do not use filepath.Join(). - content.URL = url - content.Time = time.Now() - content.Type = os.ModeDir - default: - content.URL = url - content.Size = object.Size - content.ETag = object.ETag - content.Time = object.LastModified - content.Type = os.FileMode(0664) - } - contentCh <- content + contentCh <- c.objectInfo2ClientContent(b, object) } } } @@ -1889,7 +1890,7 @@ const ( s3StorageClassGlacier = "GLACIER" ) -func (c *s3Client) listRecursiveInRoutine(contentCh chan *clientContent) { +func (c *s3Client) listRecursiveInRoutine(contentCh chan *clientContent, metadata bool) { defer close(contentCh) // get bucket and object from URL. b, o := c.url2BucketAndObject() @@ -1904,7 +1905,7 @@ func (c *s3Client) listRecursiveInRoutine(contentCh chan *clientContent) { } for _, bucket := range buckets { isRecursive := true - for object := range c.listObjectWrapper(bucket.Name, o, isRecursive, nil) { + for object := range c.listObjectWrapper(bucket.Name, o, isRecursive, nil, metadata) { // Return error if we encountered glacier object and continue. if object.StorageClass == s3StorageClassGlacier { contentCh <- &clientContent{ @@ -1931,7 +1932,7 @@ func (c *s3Client) listRecursiveInRoutine(contentCh chan *clientContent) { } default: isRecursive := true - for object := range c.listObjectWrapper(b, o, isRecursive, nil) { + for object := range c.listObjectWrapper(b, o, isRecursive, nil, metadata) { if object.Err != nil { contentCh <- &clientContent{ Err: probe.NewError(object.Err), @@ -1947,6 +1948,11 @@ func (c *s3Client) listRecursiveInRoutine(contentCh chan *clientContent) { content.ETag = object.ETag content.Time = object.LastModified content.Type = os.FileMode(0664) + content.Expires = object.Expires + content.UserMetadata = map[string]string{} + for k, v := range object.UserMetadata { + content.UserMetadata[k] = v + } contentCh <- content } } diff --git a/cmd/client-s3_test.go b/cmd/client-s3_test.go index acb3f3b966..78b319cc48 100644 --- a/cmd/client-s3_test.go +++ b/cmd/client-s3_test.go @@ -178,7 +178,7 @@ func (s *TestSuite) TestBucketOperations(c *C) { s3c, err = s3New(conf) c.Assert(err, IsNil) - for content := range s3c.List(false, false, DirNone) { + for content := range s3c.List(false, false, false, DirNone) { c.Assert(content.Err, IsNil) c.Assert(content.Type.IsDir(), Equals, true) } @@ -187,7 +187,7 @@ func (s *TestSuite) TestBucketOperations(c *C) { s3c, err = s3New(conf) c.Assert(err, IsNil) - for content := range s3c.List(false, false, DirNone) { + for content := range s3c.List(false, false, false, DirNone) { c.Assert(content.Err, IsNil) c.Assert(content.Type.IsDir(), Equals, true) } @@ -196,7 +196,7 @@ func (s *TestSuite) TestBucketOperations(c *C) { s3c, err = s3New(conf) c.Assert(err, IsNil) - for content := range s3c.List(false, false, DirNone) { + for content := range s3c.List(false, false, false, DirNone) { c.Assert(content.Err, IsNil) c.Assert(content.Type.IsRegular(), Equals, true) } diff --git a/cmd/client-url.go b/cmd/client-url.go index de0a665891..7bb5fcceee 100644 --- a/cmd/client-url.go +++ b/cmd/client-url.go @@ -218,7 +218,8 @@ func isURLPrefixExists(urlPrefix string, incomplete bool) bool { } isRecursive := false isIncomplete := incomplete - for entry := range clnt.List(isRecursive, isIncomplete, DirNone) { + isFetchMeta := false + for entry := range clnt.List(isRecursive, isIncomplete, isFetchMeta, DirNone) { return entry.Err == nil } return false diff --git a/cmd/client.go b/cmd/client.go index c320bac32c..4498a05ba5 100644 --- a/cmd/client.go +++ b/cmd/client.go @@ -46,7 +46,7 @@ const defaultMultipartThreadsNum = 4 type Client interface { // Common operations Stat(isIncomplete, isFetchMeta, isPreserve bool, sse encrypt.ServerSide) (content *clientContent, err *probe.Error) - List(isRecursive, isIncomplete bool, showDir DirOpt) <-chan *clientContent + List(isRecursive, isIncomplete, isFetchMeta bool, showDir DirOpt) <-chan *clientContent // Bucket operations MakeBucket(region string, ignoreExisting, withLock bool) *probe.Error diff --git a/cmd/common-methods.go b/cmd/common-methods.go index 4afd3a7832..98b37d3608 100644 --- a/cmd/common-methods.go +++ b/cmd/common-methods.go @@ -250,6 +250,21 @@ func copySourceToTargetURL(alias string, urlStr string, source string, size int6 return nil } +func filterMetadata(metadata map[string]string) map[string]string { + newMetadata := map[string]string{} + for k, v := range metadata { + if httpguts.ValidHeaderFieldName(k) && httpguts.ValidHeaderFieldValue(v) { + newMetadata[k] = v + } + } + for k := range metadata { + if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Amz-Server-Side-Encryption-") { + delete(newMetadata, k) + } + } + return newMetadata +} + // getAllMetadata - returns a map of user defined function // by combining the usermetadata of object and values passed by attr keyword func getAllMetadata(sourceAlias, sourceURLStr string, srcSSE encrypt.ServerSide, urls URLs) (map[string]string, *probe.Error) { @@ -262,25 +277,16 @@ func getAllMetadata(sourceAlias, sourceURLStr string, srcSSE encrypt.ServerSide, if err != nil { return nil, err.Trace(sourceAlias, sourceURLStr) } + for k, v := range st.Metadata { - if httpguts.ValidHeaderFieldName(k) && httpguts.ValidHeaderFieldValue(v) { - metadata[k] = v - } + metadata[k] = v } for k, v := range urls.TargetContent.UserMetadata { - if httpguts.ValidHeaderFieldName(k) && httpguts.ValidHeaderFieldValue(v) { - metadata[k] = v - } - } - - for k := range metadata { - if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Amz-Server-Side-Encryption-") { - delete(metadata, k) - } + metadata[k] = v } - return metadata, nil + return filterMetadata(metadata), nil } // uploadSourceToTargetURL - uploads to targetURL from source. @@ -300,17 +306,28 @@ func uploadSourceToTargetURL(ctx context.Context, urls URLs, progress io.Reader, tgtSSE := getSSE(targetPath, encKeyDB[targetAlias]) var err *probe.Error - var metadata map[string]string + var metadata = map[string]string{} // Optimize for server side copy if the host is same. if sourceAlias == targetAlias { - metadata, err = getAllMetadata(sourceAlias, sourceURL.String(), srcSSE, urls) - if err != nil { - return urls.WithError(err.Trace(sourceURL.String())) + for k, v := range urls.SourceContent.UserMetadata { + metadata[k] = v + } + for k, v := range urls.SourceContent.Metadata { + metadata[k] = v + } + // If no metadata populated already by the caller + // just do a Stat() to obtain the metadata. + if len(metadata) == 0 { + metadata, err = getAllMetadata(sourceAlias, sourceURL.String(), srcSSE, urls) + if err != nil { + return urls.WithError(err.Trace(sourceURL.String())) + } } sourcePath := filepath.ToSlash(sourceURL.Path) - err = copySourceToTargetURL(targetAlias, targetURL.String(), sourcePath, length, progress, srcSSE, tgtSSE, metadata) + err = copySourceToTargetURL(targetAlias, targetURL.String(), sourcePath, length, + progress, srcSSE, tgtSSE, filterMetadata(metadata)) } else { var reader io.ReadCloser @@ -328,12 +345,8 @@ func uploadSourceToTargetURL(ctx context.Context, urls URLs, progress io.Reader, for k, v := range urls.TargetContent.UserMetadata { metadata[k] = v } - for k := range metadata { - if strings.HasPrefix(http.CanonicalHeaderKey(k), "X-Amz-Server-Side-Encryption-") { - delete(metadata, k) - } - } - _, err = putTargetStream(ctx, targetAlias, targetURL.String(), reader, length, metadata, progress, tgtSSE) + _, err = putTargetStream(ctx, targetAlias, targetURL.String(), reader, length, filterMetadata(metadata), + progress, tgtSSE) } if err != nil { return urls.WithError(err.Trace(sourceURL.String())) diff --git a/cmd/complete.go b/cmd/complete.go index c0eb366741..043d644344 100644 --- a/cmd/complete.go +++ b/cmd/complete.go @@ -50,7 +50,7 @@ func completeS3Path(s3Path string) (prediction []string) { // List dirPath content and only pick elements that corresponds // to the path that we want to complete - for content := range clnt.List(false, false, DirFirst) { + for content := range clnt.List(false, false, false, DirFirst) { completeS3Path := alias + getKey(content) if content.Type.IsDir() { if !strings.HasSuffix(completeS3Path, "/") { diff --git a/cmd/cp-url.go b/cmd/cp-url.go index 68ec3ff56c..ac0a7eb402 100644 --- a/cmd/cp-url.go +++ b/cmd/cp-url.go @@ -165,7 +165,7 @@ func prepareCopyURLsTypeC(sourceURL, targetURL string, isRecursive bool, encKeyD } isIncomplete := false - for sourceContent := range sourceClient.List(isRecursive, isIncomplete, DirNone) { + for sourceContent := range sourceClient.List(isRecursive, isIncomplete, false, DirNone) { if sourceContent.Err != nil { // Listing failed. copyURLsCh <- URLs{Error: sourceContent.Err.Trace(sourceClient.GetURL().String())} diff --git a/cmd/difference.go b/cmd/difference.go index 3a5387b4cd..30b23c9370 100644 --- a/cmd/difference.go +++ b/cmd/difference.go @@ -17,6 +17,7 @@ package cmd import ( + "reflect" "strings" "unicode/utf8" @@ -78,10 +79,10 @@ func difference(sourceClnt, targetClnt Client, sourceURL, targetURL string, isMe // Set default values for listing. isIncomplete := false // we will not compare any incomplete objects. - srcCh := sourceClnt.List(isRecursive, isIncomplete, dirOpt) - tgtCh := targetClnt.List(isRecursive, isIncomplete, dirOpt) + srcCh := sourceClnt.List(isRecursive, isIncomplete, isMetadata, dirOpt) + tgtCh := targetClnt.List(isRecursive, isIncomplete, isMetadata, dirOpt) - diffCh = make(chan diffMessage, 1000) + diffCh = make(chan diffMessage, 10000) go func() { @@ -190,21 +191,24 @@ func difference(sourceClnt, targetClnt Client, sourceURL, targetURL string, isMe firstContent: srcCtnt, secondContent: tgtCtnt, } - } else if srcTime.After(tgtTime) { - // Regular files differing in timestamp. + } else if isMetadata && + !reflect.DeepEqual(srcCtnt.UserMetadata, tgtCtnt.UserMetadata) && + !reflect.DeepEqual(srcCtnt.Metadata, tgtCtnt.Metadata) { + + // Regular files user requesting additional metadata to same file. diffCh <- diffMessage{ FirstURL: srcCtnt.URL.String(), SecondURL: tgtCtnt.URL.String(), - Diff: differInTime, + Diff: differInMetadata, firstContent: srcCtnt, secondContent: tgtCtnt, } - } else if isMetadata { - // Regular files user requesting additional metadata to same file. + } else if srcTime.After(tgtTime) { + // Regular files differing in timestamp. diffCh <- diffMessage{ FirstURL: srcCtnt.URL.String(), SecondURL: tgtCtnt.URL.String(), - Diff: differInMetadata, + Diff: differInTime, firstContent: srcCtnt, secondContent: tgtCtnt, } diff --git a/cmd/du-main.go b/cmd/du-main.go index 1e6a490bc4..03e3542b80 100644 --- a/cmd/du-main.go +++ b/cmd/du-main.go @@ -103,7 +103,7 @@ func du(urlStr string, depth int, encKeyDB map[string][]prefixSSEPair) (int64, e isRecursive := false isIncomplete := false - contentCh := clnt.List(isRecursive, isIncomplete, DirFirst) + contentCh := clnt.List(isRecursive, isIncomplete, false, DirFirst) size := int64(0) for content := range contentCh { if content.Err != nil { diff --git a/cmd/find.go b/cmd/find.go index 8fd3747b42..dc2944ae11 100644 --- a/cmd/find.go +++ b/cmd/find.go @@ -250,7 +250,7 @@ func doFind(ctx *findContext) error { var prevKeyName string // iterate over all content which is within the given directory - for content := range ctx.clnt.List(true, false, DirNone) { + for content := range ctx.clnt.List(true, false, false, DirNone) { if content.Err != nil { switch content.Err.ToGoError().(type) { // handle this specifically for filesystem related errors. diff --git a/cmd/ls.go b/cmd/ls.go index 8a888e1a79..fcf416d546 100644 --- a/cmd/ls.go +++ b/cmd/ls.go @@ -112,7 +112,7 @@ func doList(clnt Client, isRecursive, isIncomplete bool) error { prefixPath = prefixPath[:strings.LastIndex(prefixPath, separator)+1] } var cErr error - for content := range clnt.List(isRecursive, isIncomplete, DirNone) { + for content := range clnt.List(isRecursive, isIncomplete, false, DirNone) { if content.Err != nil { switch content.Err.ToGoError().(type) { // handle this specifically for filesystem related errors. diff --git a/cmd/mirror-main.go b/cmd/mirror-main.go index dab820b3c6..2e61bc72b5 100644 --- a/cmd/mirror-main.go +++ b/cmd/mirror-main.go @@ -331,7 +331,7 @@ func (mj *mirrorJob) monitorMirrorStatus() (errDuringMirror bool) { fmt.Sprintf("Failed to remove `%s`.", sURLs.TargetContent.URL.String())) errDuringMirror = true default: - errorIf(sURLs.Error.Trace(), "Failed to perform mirroring action.") + errorIf(sURLs.Error.Trace(), "Failed to perform mirroring.") errDuringMirror = true } } @@ -509,7 +509,7 @@ func (mj *mirrorJob) startMirror(ctx context.Context, cancelMirror context.Cance mj.parallel.wait() } - isMetadata := len(mj.userMetadata) > 0 + isMetadata := len(mj.userMetadata) > 0 || mj.isPreserve URLsCh := prepareMirrorURLs(mj.sourceURL, mj.targetURL, mj.isFake, mj.isOverwrite, mj.isRemove, isMetadata, mj.excludeOptions, mj.encKeyDB) for { @@ -672,6 +672,26 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr fatalIf(err, "Unable to parse attribute %v", ctx.String("attr")) } + srcClt, err := newClient(srcURL) + fatalIf(err, "Unable to initialize `"+srcURL+"`.") + + dstClt, err := newClient(dstURL) + fatalIf(err, "Unable to initialize `"+dstURL+"`.") + + mirrorAllBuckets := (dstClt.GetURL().Type == objectStorage && + dstClt.GetURL().Path == string(dstClt.GetURL().Separator)) && + (srcClt.GetURL().Type == objectStorage && + srcClt.GetURL().Path == string(srcClt.GetURL().Separator)) + + // Check if we are only trying to mirror one bucket from source. + if dstClt.GetURL().Type == objectStorage && + dstClt.GetURL().Path == string(dstClt.GetURL().Separator) && !mirrorAllBuckets { + dstURL = urlJoinPath(dstURL, srcClt.GetURL().Path) + + dstClt, err = newClient(dstURL) + fatalIf(err, "Unable to initialize `"+dstURL+"`.") + } + // Create a new mirror job and execute it mj := newMirrorJob(srcURL, dstURL, ctx.Bool("fake"), @@ -686,20 +706,11 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr userMetaMap, encKeyDB) - srcClt, err := newClient(srcURL) - fatalIf(err, "Unable to initialize `"+srcURL+"`.") - - dstClt, err := newClient(dstURL) - fatalIf(err, "Unable to initialize `"+srcURL+"`.") - - mirrorAllBuckets := (srcClt.GetURL().Type == objectStorage && srcClt.GetURL().Path == "/") || - (dstClt.GetURL().Type == objectStorage && dstClt.GetURL().Path == "/") - if mirrorAllBuckets { // Synchronize buckets using dirDifference function for d := range dirDifference(srcClt, dstClt, srcURL, dstURL) { if d.Error != nil { - mj.status.fatalIf(d.Error, fmt.Sprintf("Failed to start monitoring.")) + mj.status.fatalIf(d.Error, "Failed to start mirroring.") } if d.Diff == differInSecond { // Ignore buckets that only exist in target instance @@ -721,24 +732,16 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr } // Bucket only exists in the source, create the same bucket in the destination if err := newDstClt.MakeBucket(ctx.String("region"), false, withLock); err != nil { - errorIf(err, "Cannot created bucket in `"+newTgtURL+"`.") + errorIf(err, "Unable to create bucket at `"+newTgtURL+"`.") continue } // object lock configuration set on bucket if mode != nil { - err := newDstClt.SetObjectLockConfig(mode, validity, unit) - if err != nil { - errorIf(err, "Cannot set object lock config in `"+newTgtURL+"`.") - continue - } - } - // Copy policy rules from source to dest if flag is activated - // and all buckets are mirrored. - if ctx.Bool("a") { - if err := copyBucketPolicies(newSrcClt, newDstClt, isOverwrite); err != nil { - errorIf(err, "Cannot copy bucket policies to `"+newDstClt.GetURL().String()+"`.") - } + errorIf(newDstClt.SetObjectLockConfig(mode, validity, unit), + "Unable to set object lock config in `"+newTgtURL+"`.") } + errorIf(copyBucketPolicies(newSrcClt, newDstClt, isOverwrite), + "Unable to copy bucket policies to `"+newDstClt.GetURL().String()+"`.") } if mj.isWatch { @@ -750,19 +753,25 @@ func runMirror(srcURL, dstURL string, ctx *cli.Context, encKeyDB map[string][]pr } } } else { - // Copy policy rules from source to dest if flag is activated - if ctx.Bool("a") { - if err := copyBucketPolicies(srcClt, dstClt, isOverwrite); err != nil { - errorIf(err, "Cannot copy bucket policies to `"+dstClt.GetURL().String()+"`.") - } + withLock := false + mode, validity, unit, err := srcClt.GetObjectLockConfig() + if err == nil { + withLock = true } + + // Create bucket if it doesn't exist at destination. + // ignore if already exists. + fatalIf(dstClt.MakeBucket(ctx.String("region"), true, withLock), + "Unable to create bucket at `"+dstURL+"`.") + // object lock configuration set on bucket - if srcMode, srcValidity, srcUnit, srcErr := srcClt.GetObjectLockConfig(); srcMode != nil && srcErr == nil { - err := dstClt.SetObjectLockConfig(srcMode, srcValidity, srcUnit) - if err != nil { - errorIf(err, "Cannot set object lock config in `"+dstClt.GetURL().String()+"`.") - } + if mode != nil { + errorIf(dstClt.SetObjectLockConfig(mode, validity, unit), + "Unable to set object lock config in `"+dstURL+"`.") } + + errorIf(copyBucketPolicies(srcClt, dstClt, isOverwrite), + "Unable to copy bucket policies to `"+dstClt.GetURL().String()+"`.") } if !mirrorAllBuckets && mj.isWatch { diff --git a/cmd/policy-main.go b/cmd/policy-main.go index 54e5d56fea..c21cc837ee 100644 --- a/cmd/policy-main.go +++ b/cmd/policy-main.go @@ -394,7 +394,7 @@ func runPolicyLinksCmd(args cli.Args, recursive bool) { clnt, err := newClient(newURL) fatalIf(err.Trace(newURL), "Unable to initialize target `"+targetURL+"`.") // Search for public objects - for content := range clnt.List(isRecursive, isIncomplete, DirFirst) { + for content := range clnt.List(isRecursive, isIncomplete, false, DirFirst) { if content.Err != nil { errorIf(content.Err.Trace(clnt.GetURL().String()), "Unable to list folder.") continue diff --git a/cmd/rb-main.go b/cmd/rb-main.go index f57d3e763b..8ebac7f359 100644 --- a/cmd/rb-main.go +++ b/cmd/rb-main.go @@ -124,7 +124,7 @@ func deleteBucket(url string) *probe.Error { contentCh := make(chan *clientContent) errorCh := clnt.Remove(isIncomplete, isRemoveBucket, contentCh) - for content := range clnt.List(true, false, DirLast) { + for content := range clnt.List(true, false, false, DirLast) { if content.Err != nil { switch content.Err.ToGoError().(type) { case PathInsufficientPermission: @@ -227,7 +227,7 @@ func mainRemoveBucket(ctx *cli.Context) error { } } isEmpty := true - for range clnt.List(true, false, DirNone) { + for range clnt.List(true, false, false, DirNone) { isEmpty = false break } diff --git a/cmd/rm-main.go b/cmd/rm-main.go index e0ca3ad280..379d4e7cf7 100644 --- a/cmd/rm-main.go +++ b/cmd/rm-main.go @@ -256,7 +256,7 @@ func removeRecursive(url string, isIncomplete bool, isFake bool, olderThan, newe errorCh := clnt.Remove(isIncomplete, isRemoveBucket, contentCh) isRecursive := true - for content := range clnt.List(isRecursive, isIncomplete, DirLast) { + for content := range clnt.List(isRecursive, isIncomplete, false, DirLast) { if content.Err != nil { errorIf(content.Err.Trace(url), "Failed to remove `"+url+"` recursively.") switch content.Err.ToGoError().(type) { diff --git a/cmd/share-download-main.go b/cmd/share-download-main.go index 230812f2af..69baf2841d 100644 --- a/cmd/share-download-main.go +++ b/cmd/share-download-main.go @@ -147,7 +147,7 @@ func doShareDownloadURL(targetURL string, isRecursive bool, expiry time.Duration // Recursive mode: Share list of objects go func() { defer close(objectsCh) - for content := range clnt.List(isRecursive, isIncomplete, DirNone) { + for content := range clnt.List(isRecursive, isIncomplete, false, DirNone) { objectsCh <- content } }() diff --git a/cmd/sql-main.go b/cmd/sql-main.go index e30462dc27..8891f92546 100644 --- a/cmd/sql-main.go +++ b/cmd/sql-main.go @@ -459,7 +459,7 @@ func mainSQL(ctx *cli.Context) error { continue } - for content := range clnt.List(ctx.Bool("recursive"), false, DirNone) { + for content := range clnt.List(ctx.Bool("recursive"), false, false, DirNone) { if content.Err != nil { errorIf(content.Err.Trace(url), "Unable to list on target `"+url+"`.") continue diff --git a/cmd/stat.go b/cmd/stat.go index 6674c56719..c8be15a329 100644 --- a/cmd/stat.go +++ b/cmd/stat.go @@ -134,7 +134,7 @@ func statURL(targetURL string, isIncomplete, isRecursive bool, encKeyDB map[stri prefixPath = prefixPath[:strings.LastIndex(prefixPath, separator)+1] } var cErr error - for content := range clnt.List(isRecursive, isIncomplete, DirNone) { + for content := range clnt.List(isRecursive, isIncomplete, false, DirNone) { if content.Err != nil { switch content.Err.ToGoError().(type) { // handle this specifically for filesystem related errors. diff --git a/cmd/tree-main.go b/cmd/tree-main.go index aac284937c..4c5b5b8351 100644 --- a/cmd/tree-main.go +++ b/cmd/tree-main.go @@ -215,7 +215,7 @@ func doTree(url string, level int, leaf bool, branchString string, depth int, in return nil } - for content := range clnt.List(false, false, DirNone) { + for content := range clnt.List(false, false, false, DirNone) { if !includeFiles && !content.Type.IsDir() { continue diff --git a/go.mod b/go.mod index ee8a9c73e0..28673b1b0d 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/mattn/go-isatty v0.0.7 github.com/mattn/go-runewidth v0.0.5 // indirect github.com/minio/cli v1.22.0 - github.com/minio/minio v0.0.0-20191121003501-a9b87c0a16cd - github.com/minio/minio-go/v6 v6.0.42 + github.com/minio/minio v0.0.0-20191119214813-7cdb67680e72 + github.com/minio/minio-go/v6 v6.0.43 github.com/minio/sha256-simd v0.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/profile v1.3.0 diff --git a/go.sum b/go.sum index 405e40b0e5..2977fc75bd 100644 --- a/go.sum +++ b/go.sum @@ -440,8 +440,8 @@ github.com/minio/minio v0.0.0-20190510004154-ac3b59645e92/go.mod h1:yFbQSwuA61mB github.com/minio/minio v0.0.0-20190903181048-8a71b0ec5a72/go.mod h1:H8OUaOxwJIALXHlfMpk2ESvk81dW+F7tSnrZi7rln/k= github.com/minio/minio v0.0.0-20190927193314-1c5b05c130fa/go.mod h1:47wx8w7yKTFAPw/hbmYPL4prsPivh6QuUqjTykftGgg= github.com/minio/minio v0.0.0-20191001201215-ff5bf519522f/go.mod h1:+vpJpzImdz4NCQto3r6/vF/24jWeEs5i3lsDfRmEcl4= -github.com/minio/minio v0.0.0-20191121003501-a9b87c0a16cd h1:y++FGJMRcQShNwD55nSrH+aIHVQ7YJ4LViUoKWYYcJY= -github.com/minio/minio v0.0.0-20191121003501-a9b87c0a16cd/go.mod h1:ntnO3zSURYUoyfHsETJzp6Ydz1wmAWfk6uBLbVJijJI= +github.com/minio/minio v0.0.0-20191119214813-7cdb67680e72 h1:3agNxlLLhDtNHIE5JnSE82M05ai2L2MguVx+Sf+c9NM= +github.com/minio/minio v0.0.0-20191119214813-7cdb67680e72/go.mod h1:ntnO3zSURYUoyfHsETJzp6Ydz1wmAWfk6uBLbVJijJI= github.com/minio/minio-go v0.0.0-20190227180923-59af836a7e6d h1:gptD0/Hnam7h4Iq9D/33fscRpHfzOOUqUbH2nPw9HcU= github.com/minio/minio-go v0.0.0-20190227180923-59af836a7e6d/go.mod h1:/haSOWG8hQNx2+JOfLJ9GKp61EAmgPwRVw/Sac0NzaM= github.com/minio/minio-go v0.0.0-20190313212832-5d20267d970d/go.mod h1:/haSOWG8hQNx2+JOfLJ9GKp61EAmgPwRVw/Sac0NzaM= @@ -452,8 +452,8 @@ github.com/minio/minio-go/v6 v6.0.29/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS github.com/minio/minio-go/v6 v6.0.37/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/minio-go/v6 v6.0.39 h1:9qmKCTBpQpMdGlDAbs3mbb4mmL45/lwRUvHL1VLhYUk= github.com/minio/minio-go/v6 v6.0.39/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= -github.com/minio/minio-go/v6 v6.0.42 h1:aIAm+bMIOWCr634eZQdWGxelaVXA8/y2tOBrG9wV8+Y= -github.com/minio/minio-go/v6 v6.0.42/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v6 v6.0.43 h1:D7c6Kx0ZB5U8EXJ6SQVOqPzapaLK/qpxQIktCnPHp/o= +github.com/minio/minio-go/v6 v6.0.43/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679 h1:OMKaN/82sBHUZPvjYNBFituHExa1OGY63eACDGtetKs= github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679/go.mod h1:J+goXSuzlte5imWMqb6cUWC/tbYYysUHctwmKXomYzM= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U=