Fix: List logic was not able to list more than 1000 files

s3.go

@@ -44,10 +44,17 @@ func (s *s3Provider) ListFiles(prefix string) ([]file, error) {
 		return out, err
 	}
 
+	prefixList := []*string{aws.String(path)}
+
+	for {
+		fmt.Printf("Scanning prefixes (%d left)...\r", len(prefixList))
+		var p *string
+		p, prefixList = prefixList[0], prefixList[1:]
 		in := &s3.ListObjectsInput{
 			Bucket:    aws.String(bucket),
-		Marker: nil,
+			Prefix:    p,
-		Prefix: aws.String(path),
+			MaxKeys:   aws.Long(1000),
+			Delimiter: aws.String("/"),
 		}
 		for {
 			o, err := s.conn.ListObjects(in)
@@ -63,12 +70,32 @@ func (s *s3Provider) ListFiles(prefix string) ([]file, error) {
 				})
 			}
 
-		if o.NextMarker == nil {
+			if len(o.CommonPrefixes) > 0 {
+				for _, cp := range o.CommonPrefixes {
+					found := false
+					for _, v := range prefixList {
+						if v == cp.Prefix {
+							found = true
+						}
+					}
+					if !found {
+						prefixList = append(prefixList, cp.Prefix)
+					}
+				}
+			}
+
+			if !*o.IsTruncated {
 				break
 			}
 			in.Marker = o.NextMarker
 		}
 
+		if len(prefixList) == 0 {
+			fmt.Printf("\n")
+			break
+		}
+	}
+
 	return out, nil
 }