Skip to content

Commit

Permalink
Add mirror command to compare metadata and copy
Browse files Browse the repository at this point in the history 
mirror now uses the newly introduced
ListObjectsV2WithMetadata API in minio-go
  • Loading branch information
@harshavardhana
harshavardhana committed Nov 21, 2019
1 parent 3dc37f1 commit 5215f0f
Show file tree
Hide file tree
Showing 23 changed files with 251 additions and 205 deletions.
119 changes: 66 additions & 53 deletions cmd/client-fs.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,11 +68,11 @@ func fsNew(path string) (Client, *probe.Error) {
}, nil
}

func isNotSupported(err error) bool {
if err == nil {
func isNotSupported(e error) bool {
if e == nil {
return false
}
errno := err.(*xattr.Error)
errno := e.(*xattr.Error)
if errno == nil {
return false
}
Expand All @@ -83,13 +83,13 @@ func isNotSupported(err error) bool {

// isIgnoredFile returns true if 'filename' is on the exclude list.
func isIgnoredFile(filename string) bool {
matchFile := path.Base(filename)
matchFile := filepath.Base(filename)

// OS specific ignore list.
for _, ignoredFile := range ignoreFiles[runtime.GOOS] {
matched, err := filepath.Match(ignoredFile, matchFile)
if err != nil {
panic(err)
matched, e := filepath.Match(ignoredFile, matchFile)
if e != nil {
panic(e)
}
if matched {
return true
Expand All @@ -98,9 +98,9 @@ func isIgnoredFile(filename string) bool {

// Default ignore list for all OSes.
for _, ignoredFile := range ignoreFiles["default"] {
matched, err := filepath.Match(ignoredFile, matchFile)
if err != nil {
panic(err)
matched, e := filepath.Match(ignoredFile, matchFile)
if e != nil {
panic(e)
}
if matched {
return true
Expand Down Expand Up @@ -343,46 +343,55 @@ func (f *fsClient) put(reader io.Reader, size int64, metadata map[string][]strin
}

if len(metadata["mc-attrs"]) != 0 {
attr, err := parseAttribute(metadata["mc-attrs"][0])
if err != nil {
return totalWritten, probe.NewError(err)
attr, e := parseAttribute(metadata["mc-attrs"][0])
if e != nil {
return totalWritten, probe.NewError(e)
}

mode, err := strconv.ParseUint(attr["mode"], 10, 32)
if err != nil {
return totalWritten, probe.NewError(err)
mode, e := strconv.ParseUint(attr["mode"], 10, 32)
if e != nil {
return totalWritten, probe.NewError(e)
}

// Change the mode of file
if err := os.Chmod(objectPath, os.FileMode(mode)); err != nil {
return totalWritten, probe.NewError(err)
if e := os.Chmod(objectPath, os.FileMode(mode)); e != nil {
if !os.IsPermission(e) {
return totalWritten, probe.NewError(e)
}
}

uid, err := strconv.Atoi(attr["uid"])
if err != nil {
return totalWritten, probe.NewError(err)
uid, e := strconv.Atoi(attr["uid"])
if e != nil {
return totalWritten, probe.NewError(e)
}

gid, err := strconv.Atoi(attr["gid"])
if err != nil {
return totalWritten, probe.NewError(err)
gid, e := strconv.Atoi(attr["gid"])
if e != nil {
return totalWritten, probe.NewError(e)
}

// Change the owner
if err := os.Chown(objectPath, uid, gid); err != nil {
return totalWritten, probe.NewError(err)
if e := os.Chown(objectPath, uid, gid); e != nil {
if !os.IsPermission(e) {
return totalWritten, probe.NewError(e)
}
}

atime, err := strconv.ParseInt(attr["atime"], 10, 64)
if err != nil {
return totalWritten, probe.NewError(err)
atime, e := strconv.ParseInt(attr["atime"], 10, 64)
if e != nil {
return totalWritten, probe.NewError(e)
}

ctime, err := strconv.ParseInt(attr["ctime"], 10, 64)
if err != nil {
return totalWritten, probe.NewError(err)
ctime, e := strconv.ParseInt(attr["ctime"], 10, 64)
if e != nil {
return totalWritten, probe.NewError(e)
}

// Change the access, modify and change time
if err := os.Chtimes(objectPath, time.Unix(atime, 0), time.Unix(ctime, 0)); err != nil {
return totalWritten, probe.NewError(err)
if e := os.Chtimes(objectPath, time.Unix(atime, 0), time.Unix(ctime, 0)); e != nil {
if !os.IsPermission(e) {
return totalWritten, probe.NewError(e)
}
}

}
Expand Down Expand Up @@ -505,11 +514,11 @@ func isSysErrNotEmpty(err error) bool {
// until it finds one with files in it. Returns nil for a non-empty directory.
func deleteFile(deletePath string) error {
// Attempt to remove path.
if err := os.Remove((deletePath)); err != nil {
if isSysErrNotEmpty(err) {
if e := os.Remove(deletePath); e != nil {
if isSysErrNotEmpty(e) {
return nil
}
return err
return e
}

// Trailing slash is removed when found to ensure
Expand Down Expand Up @@ -538,15 +547,15 @@ func (f *fsClient) Remove(isIncomplete, isRemoveBucket bool, contentCh <-chan *c
if isIncomplete {
name += partSuffix
}
if err := deleteFile(name); err != nil {
if os.IsPermission(err) {
if e := deleteFile(name); e != nil {
if os.IsPermission(e) {
// Ignore permission error.
errorCh <- probe.NewError(PathInsufficientPermission{Path: content.URL.Path})
} else if os.IsNotExist(err) && isRemoveBucket {
} else if os.IsNotExist(e) && isRemoveBucket {
// ignore PathNotFound for dir removal.
return
} else {
errorCh <- probe.NewError(err)
errorCh <- probe.NewError(e)
return
}
}
Expand All @@ -557,18 +566,18 @@ func (f *fsClient) Remove(isIncomplete, isRemoveBucket bool, contentCh <-chan *c
}

// List - list files and folders.
func (f *fsClient) List(isRecursive, isIncomplete bool, showDir DirOpt) <-chan *clientContent {
func (f *fsClient) List(isRecursive, isIncomplete, isMetadata bool, showDir DirOpt) <-chan *clientContent {
contentCh := make(chan *clientContent)
filteredCh := make(chan *clientContent)

if isRecursive {
if showDir == DirNone {
go f.listRecursiveInRoutine(contentCh)
go f.listRecursiveInRoutine(contentCh, isMetadata)
} else {
go f.listDirOpt(contentCh, isIncomplete, showDir)
go f.listDirOpt(contentCh, isIncomplete, isMetadata, showDir)
}
} else {
go f.listInRoutine(contentCh)
go f.listInRoutine(contentCh, isMetadata)
}

// This function filters entries from any listing go routine
Expand Down Expand Up @@ -701,7 +710,7 @@ func (f *fsClient) listPrefixes(prefix string, contentCh chan<- *clientContent)
}
}

func (f *fsClient) listInRoutine(contentCh chan<- *clientContent) {
func (f *fsClient) listInRoutine(contentCh chan<- *clientContent, isMetadata bool) {
// close the channel when the function returns.
defer close(contentCh)

Expand Down Expand Up @@ -795,7 +804,7 @@ func (f *fsClient) listInRoutine(contentCh chan<- *clientContent) {
}

// List files recursively using non-recursive mode.
func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool, dirOpt DirOpt) {
func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool, isMetadata bool, dirOpt DirOpt) {
defer close(contentCh)

// Trim trailing / or \.
Expand All @@ -808,14 +817,18 @@ func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool,
// Closure function reads currentPath and sends to contentCh. If a directory is found, it lists the directory content recursively.
var listDir func(currentPath string) bool
listDir = func(currentPath string) (isStop bool) {
files, err := readDir(currentPath)
if err != nil {
if os.IsPermission(err) {
contentCh <- &clientContent{Err: probe.NewError(PathInsufficientPermission{Path: currentPath})}
files, e := readDir(currentPath)
if e != nil {
if os.IsPermission(e) {
contentCh <- &clientContent{
Err: probe.NewError(PathInsufficientPermission{
Path: currentPath,
}),
}
return false
}

contentCh <- &clientContent{Err: probe.NewError(err)}
contentCh <- &clientContent{Err: probe.NewError(e)}
return true
}

Expand Down Expand Up @@ -861,7 +874,7 @@ func (f *fsClient) listDirOpt(contentCh chan *clientContent, isIncomplete bool,
}
}

func (f *fsClient) listRecursiveInRoutine(contentCh chan *clientContent) {
func (f *fsClient) listRecursiveInRoutine(contentCh chan *clientContent, isMetadata bool) {
// close channels upon return.
defer close(contentCh)
var dirName string
Expand Down
8 changes: 4 additions & 4 deletions cmd/client-fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ func (s *TestSuite) TestList(c *C) {

// Verify previously create files and list them.
var contents []*clientContent
for content := range fsClient.List(false, false, DirNone) {
for content := range fsClient.List(false, false, false, DirNone) {
if content.Err != nil {
err = content.Err
break
Expand Down Expand Up @@ -93,7 +93,7 @@ func (s *TestSuite) TestList(c *C) {

contents = nil
// List non recursive to list only top level files.
for content := range fsClient.List(false, false, DirNone) {
for content := range fsClient.List(false, false, false, DirNone) {
if content.Err != nil {
err = content.Err
break
Expand All @@ -109,7 +109,7 @@ func (s *TestSuite) TestList(c *C) {

contents = nil
// List recursively all files and verify.
for content := range fsClient.List(true, false, DirNone) {
for content := range fsClient.List(true, false, false, DirNone) {
if content.Err != nil {
err = content.Err
break
Expand Down Expand Up @@ -153,7 +153,7 @@ func (s *TestSuite) TestList(c *C) {

contents = nil
// List recursively all files and verify.
for content := range fsClient.List(true, false, DirNone) {
for content := range fsClient.List(true, false, false, DirNone) {
if content.Err != nil {
err = content.Err
break
Expand Down
Loading

0 comments on commit 5215f0f

Please sign in to comment.