Add progress output to CLI upload

Signed-off-by: Knut Ahlers <knut@ahlers.me>

Gopkg.lock

@@ -49,6 +49,14 @@
   revision = "3dd4f56d3cb9d194293525540562216f81bd3f27"
   version = "v1.15.37"
 
+[[projects]]
+  digest = "1:40a1cbb226e60770882b41e5b8e7f5c38426399c602fe75618de717e70dcb871"
+  name = "github.com/cheggaaa/pb"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "2af8bbdea9e99e83b3ac400d8f6b6d1b8cbbf338"
+  version = "v1.0.25"
+
 [[projects]]
   digest = "1:74d9b0a7b4107b41e0ade759fac64502876f82d29fb23d77b3dd24b194ee3dd5"
   name = "github.com/go-ini/ini"
@@ -64,6 +72,14 @@
   pruneopts = "NUT"
   revision = "0b12d6b5"
 
+[[projects]]
+  digest = "1:bff482b22ebed387378546ba6a7850fdef87fd47f8ee58a7c62124a8e889a56b"
+  name = "github.com/mattn/go-runewidth"
+  packages = ["."]
+  pruneopts = "NUT"
+  revision = "ce7b0b5c7b45a81508558cd1dba6bb1e4ddb51bb"
+  version = "v0.0.3"
+
 [[projects]]
   digest = "1:b2339e83ce9b5c4f79405f949429a7f68a9a904fed903c672aac1e7ceb7f5f02"
   name = "github.com/sirupsen/logrus"
@@ -123,6 +139,7 @@
     "github.com/aws/aws-sdk-go/aws",
     "github.com/aws/aws-sdk-go/aws/session",
     "github.com/aws/aws-sdk-go/service/s3",
+    "github.com/cheggaaa/pb",
     "github.com/sirupsen/logrus",
   ]
   solver-name = "gps-cdcl"

Gopkg.toml

@@ -33,6 +33,10 @@
   name = "github.com/aws/aws-sdk-go"
   version = "1.15.37"
 
+[[constraint]]
+  name = "github.com/cheggaaa/pb"
+  version = "^1.0.0"
+
 [[constraint]]
   name = "github.com/sirupsen/logrus"
   version = "1.0.6"

main.go

@@ -21,6 +21,7 @@ var (
 		Bucket         string `flag:"bucket" env:"BUCKET" default:"" description:"S3 bucket to upload files to" validate:"nonzero"`
 		ContentType    string `flag:"content-type,c" default:"" description:"Force content-type to be set to this value"`
 		Listen         string `flag:"listen" env:"LISTEN" default:"" description:"Enable HTTP server if set to IP/Port (e.g. ':3000')"`
+		Progress       bool   `flag:"progress" default:"false" description:"Show progress bar while uploading"`
 		VersionAndExit bool   `flag:"version" default:"false" description:"Prints current version and exits"`
 	}{}
 

progress.go

@@ -0,0 +1,54 @@
+package main
+
+import (
+	"io"
+	"io/ioutil"
+)
+
+type progressSeeker struct {
+	Size     int64
+	Progress int64
+
+	o io.ReadSeeker
+}
+
+func newProgressSeeker(o io.ReadSeeker) (*progressSeeker, error) {
+	if _, err := o.Seek(0, io.SeekStart); err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadAll(o)
+	if err != nil {
+		return nil, err
+	}
+
+	if _, err := o.Seek(0, io.SeekStart); err != nil {
+		return nil, err
+	}
+
+	return &progressSeeker{
+		o:    o,
+		Size: int64(len(data)),
+	}, nil
+}
+
+func (p *progressSeeker) Read(o []byte) (n int, err error) {
+	i, err := p.o.Read(o)
+
+	p.Progress += int64(i)
+
+	return i, err
+}
+
+func (p *progressSeeker) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		p.Progress = offset
+	case io.SeekCurrent:
+		p.Progress = p.Progress + offset
+	case io.SeekEnd:
+		p.Progress = p.Size + offset
+	}
+
+	return p.o.Seek(offset, whence)
+}

upload.go

@@ -10,10 +10,12 @@ import (
 	"mime"
 	"path"
 	"strings"
+	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/cheggaaa/pb"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -43,12 +45,31 @@ func executeUpload(inFileName string, inFileHandle io.ReadSeeker, useCalculatedF
 	sess := session.Must(session.NewSession())
 	svc := s3.New(sess)
 
-	if _, err := inFileHandle.Seek(0, io.SeekStart); err != nil {
+	ps, err := newProgressSeeker(inFileHandle)
+	if err != nil {
 		return "", err
 	}
 
+	if cfg.Progress {
+		bar := pb.New64(ps.Size).Prefix(inFileName).SetUnits(pb.U_BYTES)
+		bar.Start()
+		barUpdate := true
+
+		go func() {
+			for barUpdate {
+				bar.Set64(ps.Progress)
+				<-time.After(100 * time.Millisecond)
+			}
+		}()
+
+		defer func() {
+			barUpdate = false
+			bar.Finish()
+		}()
+	}
+
 	if _, err := svc.PutObject(&s3.PutObjectInput{
-		Body:        inFileHandle,
+		Body:        ps,
 		Bucket:      aws.String(cfg.Bucket),
 		ContentType: aws.String(mimeType),
 		Key:         aws.String(upFile),

vendor/github.com/cheggaaa/pb/LICENSE

@@ -0,0 +1,12 @@
+Copyright (c) 2012-2015, Sergey Cherepanov
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/cheggaaa/pb/format.go

@@ -0,0 +1,125 @@
+package pb
+
+import (
+	"fmt"
+	"time"
+)
+
+type Units int
+
+const (
+	// U_NO are default units, they represent a simple value and are not formatted at all.
+	U_NO Units = iota
+	// U_BYTES units are formatted in a human readable way (B, KiB, MiB, ...)
+	U_BYTES
+	// U_BYTES_DEC units are like U_BYTES, but base 10 (B, KB, MB, ...)
+	U_BYTES_DEC
+	// U_DURATION units are formatted in a human readable way (3h14m15s)
+	U_DURATION
+)
+
+const (
+	KiB = 1024
+	MiB = 1048576
+	GiB = 1073741824
+	TiB = 1099511627776
+
+	KB = 1e3
+	MB = 1e6
+	GB = 1e9
+	TB = 1e12
+)
+
+func Format(i int64) *formatter {
+	return &formatter{n: i}
+}
+
+type formatter struct {
+	n      int64
+	unit   Units
+	width  int
+	perSec bool
+}
+
+func (f *formatter) To(unit Units) *formatter {
+	f.unit = unit
+	return f
+}
+
+func (f *formatter) Width(width int) *formatter {
+	f.width = width
+	return f
+}
+
+func (f *formatter) PerSec() *formatter {
+	f.perSec = true
+	return f
+}
+
+func (f *formatter) String() (out string) {
+	switch f.unit {
+	case U_BYTES:
+		out = formatBytes(f.n)
+	case U_BYTES_DEC:
+		out = formatBytesDec(f.n)
+	case U_DURATION:
+		out = formatDuration(f.n)
+	default:
+		out = fmt.Sprintf(fmt.Sprintf("%%%dd", f.width), f.n)
+	}
+	if f.perSec {
+		out += "/s"
+	}
+	return
+}
+
+// Convert bytes to human readable string. Like 2 MiB, 64.2 KiB, 52 B
+func formatBytes(i int64) (result string) {
+	switch {
+	case i >= TiB:
+		result = fmt.Sprintf("%.02f TiB", float64(i)/TiB)
+	case i >= GiB:
+		result = fmt.Sprintf("%.02f GiB", float64(i)/GiB)
+	case i >= MiB:
+		result = fmt.Sprintf("%.02f MiB", float64(i)/MiB)
+	case i >= KiB:
+		result = fmt.Sprintf("%.02f KiB", float64(i)/KiB)
+	default:
+		result = fmt.Sprintf("%d B", i)
+	}
+	return
+}
+
+// Convert bytes to base-10 human readable string. Like 2 MB, 64.2 KB, 52 B
+func formatBytesDec(i int64) (result string) {
+	switch {
+	case i >= TB:
+		result = fmt.Sprintf("%.02f TB", float64(i)/TB)
+	case i >= GB:
+		result = fmt.Sprintf("%.02f GB", float64(i)/GB)
+	case i >= MB:
+		result = fmt.Sprintf("%.02f MB", float64(i)/MB)
+	case i >= KB:
+		result = fmt.Sprintf("%.02f KB", float64(i)/KB)
+	default:
+		result = fmt.Sprintf("%d B", i)
+	}
+	return
+}
+
+func formatDuration(n int64) (result string) {
+	d := time.Duration(n)
+	if d > time.Hour*24 {
+		result = fmt.Sprintf("%dd", d/24/time.Hour)
+		d -= (d / time.Hour / 24) * (time.Hour * 24)
+	}
+	if d > time.Hour {
+		result = fmt.Sprintf("%s%dh", result, d/time.Hour)
+		d -= d / time.Hour * time.Hour
+	}
+	m := d / time.Minute
+	d -= m * time.Minute
+	s := d / time.Second
+	result = fmt.Sprintf("%s%02dm%02ds", result, m, s)
+	return
+}

vendor/github.com/cheggaaa/pb/pb.go

@@ -0,0 +1,500 @@
+// Simple console progress bars
+package pb
+
+import (
+	"fmt"
+	"io"
+	"math"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+	"unicode/utf8"
+)
+
+// Current version
+const Version = "1.0.25"
+
+const (
+	// Default refresh rate - 200ms
+	DEFAULT_REFRESH_RATE = time.Millisecond * 200
+	FORMAT               = "[=>-]"
+)
+
+// DEPRECATED
+// variables for backward compatibility, from now do not work
+// use pb.Format and pb.SetRefreshRate
+var (
+	DefaultRefreshRate                         = DEFAULT_REFRESH_RATE
+	BarStart, BarEnd, Empty, Current, CurrentN string
+)
+
+// Create new progress bar object
+func New(total int) *ProgressBar {
+	return New64(int64(total))
+}
+
+// Create new progress bar object using int64 as total
+func New64(total int64) *ProgressBar {
+	pb := &ProgressBar{
+		Total:           total,
+		RefreshRate:     DEFAULT_REFRESH_RATE,
+		ShowPercent:     true,
+		ShowCounters:    true,
+		ShowBar:         true,
+		ShowTimeLeft:    true,
+		ShowElapsedTime: false,
+		ShowFinalTime:   true,
+		Units:           U_NO,
+		ManualUpdate:    false,
+		finish:          make(chan struct{}),
+	}
+	return pb.Format(FORMAT)
+}
+
+// Create new object and start
+func StartNew(total int) *ProgressBar {
+	return New(total).Start()
+}
+
+// Callback for custom output
+// For example:
+// bar.Callback = func(s string) {
+//     mySuperPrint(s)
+// }
+//
+type Callback func(out string)
+
+type ProgressBar struct {
+	current  int64 // current must be first member of struct (https://code.google.com/p/go/issues/detail?id=5278)
+	previous int64
+
+	Total                            int64
+	RefreshRate                      time.Duration
+	ShowPercent, ShowCounters        bool
+	ShowSpeed, ShowTimeLeft, ShowBar bool
+	ShowFinalTime, ShowElapsedTime   bool
+	Output                           io.Writer
+	Callback                         Callback
+	NotPrint                         bool
+	Units                            Units
+	Width                            int
+	ForceWidth                       bool
+	ManualUpdate                     bool
+	AutoStat                         bool
+
+	// Default width for the time box.
+	UnitsWidth   int
+	TimeBoxWidth int
+
+	finishOnce sync.Once //Guards isFinish
+	finish     chan struct{}
+	isFinish   bool
+
+	startTime  time.Time
+	startValue int64
+
+	changeTime time.Time
+
+	prefix, postfix string
+
+	mu        sync.Mutex
+	lastPrint string
+
+	BarStart string
+	BarEnd   string
+	Empty    string
+	Current  string
+	CurrentN string
+
+	AlwaysUpdate bool
+}
+
+// Start print
+func (pb *ProgressBar) Start() *ProgressBar {
+	pb.startTime = time.Now()
+	pb.startValue = atomic.LoadInt64(&pb.current)
+	if atomic.LoadInt64(&pb.Total) == 0 {
+		pb.ShowTimeLeft = false
+		pb.ShowPercent = false
+		pb.AutoStat = false
+	}
+	if !pb.ManualUpdate {
+		pb.Update() // Initial printing of the bar before running the bar refresher.
+		go pb.refresher()
+	}
+	return pb
+}
+
+// Increment current value
+func (pb *ProgressBar) Increment() int {
+	return pb.Add(1)
+}
+
+// Get current value
+func (pb *ProgressBar) Get() int64 {
+	c := atomic.LoadInt64(&pb.current)
+	return c
+}
+
+// Set current value
+func (pb *ProgressBar) Set(current int) *ProgressBar {
+	return pb.Set64(int64(current))
+}
+
+// Set64 sets the current value as int64
+func (pb *ProgressBar) Set64(current int64) *ProgressBar {
+	atomic.StoreInt64(&pb.current, current)
+	return pb
+}
+
+// Add to current value
+func (pb *ProgressBar) Add(add int) int {
+	return int(pb.Add64(int64(add)))
+}
+
+func (pb *ProgressBar) Add64(add int64) int64 {
+	return atomic.AddInt64(&pb.current, add)
+}
+
+// Set prefix string
+func (pb *ProgressBar) Prefix(prefix string) *ProgressBar {
+	pb.prefix = prefix
+	return pb
+}
+
+// Set postfix string
+func (pb *ProgressBar) Postfix(postfix string) *ProgressBar {
+	pb.postfix = postfix
+	return pb
+}
+
+// Set custom format for bar
+// Example: bar.Format("[=>_]")
+// Example: bar.Format("[\x00=\x00>\x00-\x00]") // \x00 is the delimiter
+func (pb *ProgressBar) Format(format string) *ProgressBar {
+	var formatEntries []string
+	if utf8.RuneCountInString(format) == 5 {
+		formatEntries = strings.Split(format, "")
+	} else {
+		formatEntries = strings.Split(format, "\x00")
+	}
+	if len(formatEntries) == 5 {
+		pb.BarStart = formatEntries[0]
+		pb.BarEnd = formatEntries[4]
+		pb.Empty = formatEntries[3]
+		pb.Current = formatEntries[1]
+		pb.CurrentN = formatEntries[2]
+	}
+	return pb
+}
+
+// Set bar refresh rate
+func (pb *ProgressBar) SetRefreshRate(rate time.Duration) *ProgressBar {
+	pb.RefreshRate = rate
+	return pb
+}
+
+// Set units
+// bar.SetUnits(U_NO) - by default
+// bar.SetUnits(U_BYTES) - for Mb, Kb, etc
+func (pb *ProgressBar) SetUnits(units Units) *ProgressBar {
+	pb.Units = units
+	return pb
+}
+
+// Set max width, if width is bigger than terminal width, will be ignored
+func (pb *ProgressBar) SetMaxWidth(width int) *ProgressBar {
+	pb.Width = width
+	pb.ForceWidth = false
+	return pb
+}
+
+// Set bar width
+func (pb *ProgressBar) SetWidth(width int) *ProgressBar {
+	pb.Width = width
+	pb.ForceWidth = true
+	return pb
+}
+
+// End print
+func (pb *ProgressBar) Finish() {
+	//Protect multiple calls
+	pb.finishOnce.Do(func() {
+		close(pb.finish)
+		pb.write(atomic.LoadInt64(&pb.Total), atomic.LoadInt64(&pb.current))
+		pb.mu.Lock()
+		defer pb.mu.Unlock()
+		switch {
+		case pb.Output != nil:
+			fmt.Fprintln(pb.Output)
+		case !pb.NotPrint:
+			fmt.Println()
+		}
+		pb.isFinish = true
+	})
+}
+
+// IsFinished return boolean
+func (pb *ProgressBar) IsFinished() bool {
+	pb.mu.Lock()
+	defer pb.mu.Unlock()
+	return pb.isFinish
+}
+
+// End print and write string 'str'
+func (pb *ProgressBar) FinishPrint(str string) {
+	pb.Finish()
+	if pb.Output != nil {
+		fmt.Fprintln(pb.Output, str)
+	} else {
+		fmt.Println(str)
+	}
+}
+
+// implement io.Writer
+func (pb *ProgressBar) Write(p []byte) (n int, err error) {
+	n = len(p)
+	pb.Add(n)
+	return
+}
+
+// implement io.Reader
+func (pb *ProgressBar) Read(p []byte) (n int, err error) {
+	n = len(p)
+	pb.Add(n)
+	return
+}
+
+// Create new proxy reader over bar
+// Takes io.Reader or io.ReadCloser
+func (pb *ProgressBar) NewProxyReader(r io.Reader) *Reader {
+	return &Reader{r, pb}
+}
+
+func (pb *ProgressBar) write(total, current int64) {
+	width := pb.GetWidth()
+
+	var percentBox, countersBox, timeLeftBox, timeSpentBox, speedBox, barBox, end, out string
+
+	// percents
+	if pb.ShowPercent {
+		var percent float64
+		if total > 0 {
+			percent = float64(current) / (float64(total) / float64(100))
+		} else {
+			percent = float64(current) / float64(100)
+		}
+		percentBox = fmt.Sprintf(" %6.02f%%", percent)
+	}
+
+	// counters
+	if pb.ShowCounters {
+		current := Format(current).To(pb.Units).Width(pb.UnitsWidth)
+		if total > 0 {
+			totalS := Format(total).To(pb.Units).Width(pb.UnitsWidth)
+			countersBox = fmt.Sprintf(" %s / %s ", current, totalS)
+		} else {
+			countersBox = fmt.Sprintf(" %s / ? ", current)
+		}
+	}
+
+	// time left
+	pb.mu.Lock()
+	currentFromStart := current - pb.startValue
+	fromStart := time.Now().Sub(pb.startTime)
+	lastChangeTime := pb.changeTime
+	fromChange := lastChangeTime.Sub(pb.startTime)
+	pb.mu.Unlock()
+
+	if pb.ShowElapsedTime {
+		timeSpentBox = fmt.Sprintf(" %s ", (fromStart/time.Second)*time.Second)
+	}
+
+	select {
+	case <-pb.finish:
+		if pb.ShowFinalTime {
+			var left time.Duration
+			left = (fromStart / time.Second) * time.Second
+			timeLeftBox = fmt.Sprintf(" %s", left.String())
+		}
+	default:
+		if pb.ShowTimeLeft && currentFromStart > 0 {
+			perEntry := fromChange / time.Duration(currentFromStart)
+			var left time.Duration
+			if total > 0 {
+				left = time.Duration(total-currentFromStart) * perEntry
+				left -= time.Since(lastChangeTime)
+				left = (left / time.Second) * time.Second
+			} else {
+				left = time.Duration(currentFromStart) * perEntry
+				left = (left / time.Second) * time.Second
+			}
+			if left > 0 {
+				timeLeft := Format(int64(left)).To(U_DURATION).String()
+				timeLeftBox = fmt.Sprintf(" %s", timeLeft)
+			}
+		}
+	}
+
+	if len(timeLeftBox) < pb.TimeBoxWidth {
+		timeLeftBox = fmt.Sprintf("%s%s", strings.Repeat(" ", pb.TimeBoxWidth-len(timeLeftBox)), timeLeftBox)
+	}
+
+	// speed
+	if pb.ShowSpeed && currentFromStart > 0 {
+		fromStart := time.Now().Sub(pb.startTime)
+		speed := float64(currentFromStart) / (float64(fromStart) / float64(time.Second))
+		speedBox = " " + Format(int64(speed)).To(pb.Units).Width(pb.UnitsWidth).PerSec().String()
+	}
+
+	barWidth := escapeAwareRuneCountInString(countersBox + pb.BarStart + pb.BarEnd + percentBox + timeSpentBox + timeLeftBox + speedBox + pb.prefix + pb.postfix)
+	// bar
+	if pb.ShowBar {
+		size := width - barWidth
+		if size > 0 {
+			if total > 0 {
+				curSize := int(math.Ceil((float64(current) / float64(total)) * float64(size)))
+				emptySize := size - curSize
+				barBox = pb.BarStart
+				if emptySize < 0 {
+					emptySize = 0
+				}
+				if curSize > size {
+					curSize = size
+				}
+
+				cursorLen := escapeAwareRuneCountInString(pb.Current)
+				if emptySize <= 0 {
+					barBox += strings.Repeat(pb.Current, curSize/cursorLen)
+				} else if curSize > 0 {
+					cursorEndLen := escapeAwareRuneCountInString(pb.CurrentN)
+					cursorRepetitions := (curSize - cursorEndLen) / cursorLen
+					barBox += strings.Repeat(pb.Current, cursorRepetitions)
+					barBox += pb.CurrentN
+				}
+
+				emptyLen := escapeAwareRuneCountInString(pb.Empty)
+				barBox += strings.Repeat(pb.Empty, emptySize/emptyLen)
+				barBox += pb.BarEnd
+			} else {
+				pos := size - int(current)%int(size)
+				barBox = pb.BarStart
+				if pos-1 > 0 {
+					barBox += strings.Repeat(pb.Empty, pos-1)
+				}
+				barBox += pb.Current
+				if size-pos-1 > 0 {
+					barBox += strings.Repeat(pb.Empty, size-pos-1)
+				}
+				barBox += pb.BarEnd
+			}
+		}
+	}
+
+	// check len
+	out = pb.prefix + timeSpentBox + countersBox + barBox + percentBox + speedBox + timeLeftBox + pb.postfix
+	if cl := escapeAwareRuneCountInString(out); cl < width {
+		end = strings.Repeat(" ", width-cl)
+	}
+
+	// and print!
+	pb.mu.Lock()
+	defer pb.mu.Unlock()
+	pb.lastPrint = out + end
+	isFinish := pb.isFinish
+
+	switch {
+	case isFinish:
+		return
+	case pb.Output != nil:
+		fmt.Fprint(pb.Output, "\r"+out+end)
+	case pb.Callback != nil:
+		pb.Callback(out + end)
+	case !pb.NotPrint:
+		fmt.Print("\r" + out + end)
+	}
+}
+
+// GetTerminalWidth - returns terminal width for all platforms.
+func GetTerminalWidth() (int, error) {
+	return terminalWidth()
+}
+
+func (pb *ProgressBar) GetWidth() int {
+	if pb.ForceWidth {
+		return pb.Width
+	}
+
+	width := pb.Width
+	termWidth, _ := terminalWidth()
+	if width == 0 || termWidth <= width {
+		width = termWidth
+	}
+
+	return width
+}
+
+// Write the current state of the progressbar
+func (pb *ProgressBar) Update() {
+	c := atomic.LoadInt64(&pb.current)
+	p := atomic.LoadInt64(&pb.previous)
+	t := atomic.LoadInt64(&pb.Total)
+	if p != c {
+		pb.mu.Lock()
+		pb.changeTime = time.Now()
+		pb.mu.Unlock()
+		atomic.StoreInt64(&pb.previous, c)
+	}
+	pb.write(t, c)
+	if pb.AutoStat {
+		if c == 0 {
+			pb.startTime = time.Now()
+			pb.startValue = 0
+		} else if c >= t && pb.isFinish != true {
+			pb.Finish()
+		}
+	}
+}
+
+// String return the last bar print
+func (pb *ProgressBar) String() string {
+	pb.mu.Lock()
+	defer pb.mu.Unlock()
+	return pb.lastPrint
+}
+
+// SetTotal atomically sets new total count
+func (pb *ProgressBar) SetTotal(total int) *ProgressBar {
+	return pb.SetTotal64(int64(total))
+}
+
+// SetTotal64 atomically sets new total count
+func (pb *ProgressBar) SetTotal64(total int64) *ProgressBar {
+	atomic.StoreInt64(&pb.Total, total)
+	return pb
+}
+
+// Reset bar and set new total count
+// Does effect only on finished bar
+func (pb *ProgressBar) Reset(total int) *ProgressBar {
+	pb.mu.Lock()
+	defer pb.mu.Unlock()
+	if pb.isFinish {
+		pb.SetTotal(total).Set(0)
+		atomic.StoreInt64(&pb.previous, 0)
+	}
+	return pb
+}
+
+// Internal loop for refreshing the progressbar
+func (pb *ProgressBar) refresher() {
+	for {
+		select {
+		case <-pb.finish:
+			return
+		case <-time.After(pb.RefreshRate):
+			pb.Update()
+		}
+	}
+}

vendor/github.com/cheggaaa/pb/pb_appengine.go

@@ -0,0 +1,11 @@
+// +build appengine
+
+package pb
+
+import "errors"
+
+// terminalWidth returns width of the terminal, which is not supported
+// and should always failed on appengine classic which is a sandboxed PaaS.
+func terminalWidth() (int, error) {
+	return 0, errors.New("Not supported")
+}

vendor/github.com/cheggaaa/pb/pb_win.go

@@ -0,0 +1,143 @@
+// +build windows
+
+package pb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"sync"
+	"syscall"
+	"unsafe"
+)
+
+var tty = os.Stdin
+
+var (
+	kernel32 = syscall.NewLazyDLL("kernel32.dll")
+
+	// GetConsoleScreenBufferInfo retrieves information about the
+	// specified console screen buffer.
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms683171(v=vs.85).aspx
+	procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo")
+
+	// GetConsoleMode retrieves the current input mode of a console's
+	// input buffer or the current output mode of a console screen buffer.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms683167(v=vs.85).aspx
+	getConsoleMode = kernel32.NewProc("GetConsoleMode")
+
+	// SetConsoleMode sets the input mode of a console's input buffer
+	// or the output mode of a console screen buffer.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms686033(v=vs.85).aspx
+	setConsoleMode = kernel32.NewProc("SetConsoleMode")
+
+	// SetConsoleCursorPosition sets the cursor position in the
+	// specified console screen buffer.
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/ms686025(v=vs.85).aspx
+	setConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition")
+)
+
+type (
+	// Defines the coordinates of the upper left and lower right corners
+	// of a rectangle.
+	// See
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms686311(v=vs.85).aspx
+	smallRect struct {
+		Left, Top, Right, Bottom int16
+	}
+
+	// Defines the coordinates of a character cell in a console screen
+	// buffer. The origin of the coordinate system (0,0) is at the top, left cell
+	// of the buffer.
+	// See
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682119(v=vs.85).aspx
+	coordinates struct {
+		X, Y int16
+	}
+
+	word int16
+
+	// Contains information about a console screen buffer.
+	// http://msdn.microsoft.com/en-us/library/windows/desktop/ms682093(v=vs.85).aspx
+	consoleScreenBufferInfo struct {
+		dwSize              coordinates
+		dwCursorPosition    coordinates
+		wAttributes         word
+		srWindow            smallRect
+		dwMaximumWindowSize coordinates
+	}
+)
+
+// terminalWidth returns width of the terminal.
+func terminalWidth() (width int, err error) {
+	var info consoleScreenBufferInfo
+	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0)
+	if e != 0 {
+		return 0, error(e)
+	}
+	return int(info.dwSize.X) - 1, nil
+}
+
+func getCursorPos() (pos coordinates, err error) {
+	var info consoleScreenBufferInfo
+	_, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&info)), 0)
+	if e != 0 {
+		return info.dwCursorPosition, error(e)
+	}
+	return info.dwCursorPosition, nil
+}
+
+func setCursorPos(pos coordinates) error {
+	_, _, e := syscall.Syscall(setConsoleCursorPosition.Addr(), 2, uintptr(syscall.Stdout), uintptr(uint32(uint16(pos.Y))<<16|uint32(uint16(pos.X))), 0)
+	if e != 0 {
+		return error(e)
+	}
+	return nil
+}
+
+var ErrPoolWasStarted = errors.New("Bar pool was started")
+
+var echoLocked bool
+var echoLockMutex sync.Mutex
+
+var oldState word
+
+func lockEcho() (shutdownCh chan struct{}, err error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if echoLocked {
+		err = ErrPoolWasStarted
+		return
+	}
+	echoLocked = true
+
+	if _, _, e := syscall.Syscall(getConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(unsafe.Pointer(&oldState)), 0); e != 0 {
+		err = fmt.Errorf("Can't get terminal settings: %v", e)
+		return
+	}
+
+	newState := oldState
+	const ENABLE_ECHO_INPUT = 0x0004
+	const ENABLE_LINE_INPUT = 0x0002
+	newState = newState & (^(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT))
+	if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(newState), 0); e != 0 {
+		err = fmt.Errorf("Can't set terminal settings: %v", e)
+		return
+	}
+
+	shutdownCh = make(chan struct{})
+	return
+}
+
+func unlockEcho() (err error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if !echoLocked {
+		return
+	}
+	echoLocked = false
+	if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(oldState), 0); e != 0 {
+		err = fmt.Errorf("Can't set terminal settings")
+	}
+	return
+}

vendor/github.com/cheggaaa/pb/pb_x.go

@@ -0,0 +1,108 @@
+// +build linux darwin freebsd netbsd openbsd solaris dragonfly
+// +build !appengine
+
+package pb
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"os/signal"
+	"sync"
+	"syscall"
+
+	"golang.org/x/sys/unix"
+)
+
+var ErrPoolWasStarted = errors.New("Bar pool was started")
+
+var (
+	echoLockMutex    sync.Mutex
+	origTermStatePtr *unix.Termios
+	tty              *os.File
+)
+
+func init() {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+
+	var err error
+	tty, err = os.Open("/dev/tty")
+	if err != nil {
+		tty = os.Stdin
+	}
+}
+
+// terminalWidth returns width of the terminal.
+func terminalWidth() (int, error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+
+	fd := int(tty.Fd())
+
+	ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
+	if err != nil {
+		return 0, err
+	}
+
+	return int(ws.Col), nil
+}
+
+func lockEcho() (shutdownCh chan struct{}, err error) {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if origTermStatePtr != nil {
+		return shutdownCh, ErrPoolWasStarted
+	}
+
+	fd := int(tty.Fd())
+
+	origTermStatePtr, err = unix.IoctlGetTermios(fd, ioctlReadTermios)
+	if err != nil {
+		return nil, fmt.Errorf("Can't get terminal settings: %v", err)
+	}
+
+	oldTermios := *origTermStatePtr
+	newTermios := oldTermios
+	newTermios.Lflag &^= syscall.ECHO
+	newTermios.Lflag |= syscall.ICANON | syscall.ISIG
+	newTermios.Iflag |= syscall.ICRNL
+	if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, &newTermios); err != nil {
+		return nil, fmt.Errorf("Can't set terminal settings: %v", err)
+	}
+
+	shutdownCh = make(chan struct{})
+	go catchTerminate(shutdownCh)
+	return
+}
+
+func unlockEcho() error {
+	echoLockMutex.Lock()
+	defer echoLockMutex.Unlock()
+	if origTermStatePtr == nil {
+		return nil
+	}
+
+	fd := int(tty.Fd())
+
+	if err := unix.IoctlSetTermios(fd, ioctlWriteTermios, origTermStatePtr); err != nil {
+		return fmt.Errorf("Can't set terminal settings: %v", err)
+	}
+
+	origTermStatePtr = nil
+
+	return nil
+}
+
+// listen exit signals and restore terminal state
+func catchTerminate(shutdownCh chan struct{}) {
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGKILL)
+	defer signal.Stop(sig)
+	select {
+	case <-shutdownCh:
+		unlockEcho()
+	case <-sig:
+		unlockEcho()
+	}
+}

vendor/github.com/cheggaaa/pb/pool.go

@@ -0,0 +1,102 @@
+// +build linux darwin freebsd netbsd openbsd solaris dragonfly windows
+
+package pb
+
+import (
+	"io"
+	"sync"
+	"time"
+)
+
+// Create and start new pool with given bars
+// You need call pool.Stop() after work
+func StartPool(pbs ...*ProgressBar) (pool *Pool, err error) {
+	pool = new(Pool)
+	if err = pool.Start(); err != nil {
+		return
+	}
+	pool.Add(pbs...)
+	return
+}
+
+// NewPool initialises a pool with progress bars, but
+// doesn't start it. You need to call Start manually
+func NewPool(pbs ...*ProgressBar) (pool *Pool) {
+	pool = new(Pool)
+	pool.Add(pbs...)
+	return
+}
+
+type Pool struct {
+	Output        io.Writer
+	RefreshRate   time.Duration
+	bars          []*ProgressBar
+	lastBarsCount int
+	shutdownCh    chan struct{}
+	workerCh      chan struct{}
+	m             sync.Mutex
+	finishOnce    sync.Once
+}
+
+// Add progress bars.
+func (p *Pool) Add(pbs ...*ProgressBar) {
+	p.m.Lock()
+	defer p.m.Unlock()
+	for _, bar := range pbs {
+		bar.ManualUpdate = true
+		bar.NotPrint = true
+		bar.Start()
+		p.bars = append(p.bars, bar)
+	}
+}
+
+func (p *Pool) Start() (err error) {
+	p.RefreshRate = DefaultRefreshRate
+	p.shutdownCh, err = lockEcho()
+	if err != nil {
+		return
+	}
+	p.workerCh = make(chan struct{})
+	go p.writer()
+	return
+}
+
+func (p *Pool) writer() {
+	var first = true
+	defer func() {
+		if first == false {
+			p.print(false)
+		} else {
+			p.print(true)
+			p.print(false)
+		}
+		close(p.workerCh)
+	}()
+
+	for {
+		select {
+		case <-time.After(p.RefreshRate):
+			if p.print(first) {
+				p.print(false)
+				return
+			}
+			first = false
+		case <-p.shutdownCh:
+			return
+		}
+	}
+}
+
+// Restore terminal state and close pool
+func (p *Pool) Stop() error {
+	p.finishOnce.Do(func() {
+		close(p.shutdownCh)
+	})
+
+	// Wait for the worker to complete
+	select {
+	case <-p.workerCh:
+	}
+
+	return unlockEcho()
+}

vendor/github.com/cheggaaa/pb/pool_win.go

@@ -0,0 +1,45 @@
+// +build windows
+
+package pb
+
+import (
+	"fmt"
+	"log"
+)
+
+func (p *Pool) print(first bool) bool {
+	p.m.Lock()
+	defer p.m.Unlock()
+	var out string
+	if !first {
+		coords, err := getCursorPos()
+		if err != nil {
+			log.Panic(err)
+		}
+		coords.Y -= int16(p.lastBarsCount)
+		if coords.Y < 0 {
+			coords.Y = 0
+		}
+		coords.X = 0
+
+		err = setCursorPos(coords)
+		if err != nil {
+			log.Panic(err)
+		}
+	}
+	isFinished := true
+	for _, bar := range p.bars {
+		if !bar.IsFinished() {
+			isFinished = false
+		}
+		bar.Update()
+		out += fmt.Sprintf("\r%s\n", bar.String())
+	}
+	if p.Output != nil {
+		fmt.Fprint(p.Output, out)
+	} else {
+		fmt.Print(out)
+	}
+	p.lastBarsCount = len(p.bars)
+	return isFinished
+}

vendor/github.com/cheggaaa/pb/pool_x.go

@@ -0,0 +1,29 @@
+// +build linux darwin freebsd netbsd openbsd solaris dragonfly
+
+package pb
+
+import "fmt"
+
+func (p *Pool) print(first bool) bool {
+	p.m.Lock()
+	defer p.m.Unlock()
+	var out string
+	if !first {
+		out = fmt.Sprintf("\033[%dA", p.lastBarsCount)
+	}
+	isFinished := true
+	for _, bar := range p.bars {
+		if !bar.IsFinished() {
+			isFinished = false
+		}
+		bar.Update()
+		out += fmt.Sprintf("\r%s\n", bar.String())
+	}
+	if p.Output != nil {
+		fmt.Fprint(p.Output, out)
+	} else {
+		fmt.Print(out)
+	}
+	p.lastBarsCount = len(p.bars)
+	return isFinished
+}

vendor/github.com/cheggaaa/pb/reader.go

@@ -0,0 +1,25 @@
+package pb
+
+import (
+	"io"
+)
+
+// It's proxy reader, implement io.Reader
+type Reader struct {
+	io.Reader
+	bar *ProgressBar
+}
+
+func (r *Reader) Read(p []byte) (n int, err error) {
+	n, err = r.Reader.Read(p)
+	r.bar.Add(n)
+	return
+}
+
+// Close the reader when it implements io.Closer
+func (r *Reader) Close() (err error) {
+	if closer, ok := r.Reader.(io.Closer); ok {
+		return closer.Close()
+	}
+	return
+}

vendor/github.com/cheggaaa/pb/runecount.go

@@ -0,0 +1,17 @@
+package pb
+
+import (
+	"github.com/mattn/go-runewidth"
+	"regexp"
+)
+
+// Finds the control character sequences (like colors)
+var ctrlFinder = regexp.MustCompile("\x1b\x5b[0-9]+\x6d")
+
+func escapeAwareRuneCountInString(s string) int {
+	n := runewidth.StringWidth(s)
+	for _, sm := range ctrlFinder.FindAllString(s, -1) {
+		n -= runewidth.StringWidth(sm)
+	}
+	return n
+}

vendor/github.com/cheggaaa/pb/termios_bsd.go

@@ -0,0 +1,9 @@
+// +build darwin freebsd netbsd openbsd dragonfly
+// +build !appengine
+
+package pb
+
+import "syscall"
+
+const ioctlReadTermios = syscall.TIOCGETA
+const ioctlWriteTermios = syscall.TIOCSETA

vendor/github.com/cheggaaa/pb/termios_sysv.go

@@ -0,0 +1,13 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build linux solaris
+// +build !appengine
+
+package pb
+
+import "golang.org/x/sys/unix"
+
+const ioctlReadTermios = unix.TCGETS
+const ioctlWriteTermios = unix.TCSETS

vendor/github.com/mattn/go-runewidth/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Yasuhiro Matsumoto
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

vendor/github.com/mattn/go-runewidth/runewidth_js.go

@@ -0,0 +1,8 @@
+// +build js
+
+package runewidth
+
+func IsEastAsian() bool {
+	// TODO: Implement this for the web. Detect east asian in a compatible way, and return true.
+	return false
+}

vendor/github.com/mattn/go-runewidth/runewidth_posix.go

@@ -0,0 +1,77 @@
+// +build !windows,!js
+
+package runewidth
+
+import (
+	"os"
+	"regexp"
+	"strings"
+)
+
+var reLoc = regexp.MustCompile(`^[a-z][a-z][a-z]?(?:_[A-Z][A-Z])?\.(.+)`)
+
+var mblenTable = map[string]int{
+	"utf-8":   6,
+	"utf8":    6,
+	"jis":     8,
+	"eucjp":   3,
+	"euckr":   2,
+	"euccn":   2,
+	"sjis":    2,
+	"cp932":   2,
+	"cp51932": 2,
+	"cp936":   2,
+	"cp949":   2,
+	"cp950":   2,
+	"big5":    2,
+	"gbk":     2,
+	"gb2312":  2,
+}
+
+func isEastAsian(locale string) bool {
+	charset := strings.ToLower(locale)
+	r := reLoc.FindStringSubmatch(locale)
+	if len(r) == 2 {
+		charset = strings.ToLower(r[1])
+	}
+
+	if strings.HasSuffix(charset, "@cjk_narrow") {
+		return false
+	}
+
+	for pos, b := range []byte(charset) {
+		if b == '@' {
+			charset = charset[:pos]
+			break
+		}
+	}
+	max := 1
+	if m, ok := mblenTable[charset]; ok {
+		max = m
+	}
+	if max > 1 && (charset[0] != 'u' ||
+		strings.HasPrefix(locale, "ja") ||
+		strings.HasPrefix(locale, "ko") ||
+		strings.HasPrefix(locale, "zh")) {
+		return true
+	}
+	return false
+}
+
+// IsEastAsian return true if the current locale is CJK
+func IsEastAsian() bool {
+	locale := os.Getenv("LC_CTYPE")
+	if locale == "" {
+		locale = os.Getenv("LANG")
+	}
+
+	// ignore C locale
+	if locale == "POSIX" || locale == "C" {
+		return false
+	}
+	if len(locale) > 1 && locale[0] == 'C' && (locale[1] == '.' || locale[1] == '-') {
+		return false
+	}
+
+	return isEastAsian(locale)
+}

vendor/github.com/mattn/go-runewidth/runewidth_windows.go

@@ -0,0 +1,25 @@
+package runewidth
+
+import (
+	"syscall"
+)
+
+var (
+	kernel32               = syscall.NewLazyDLL("kernel32")
+	procGetConsoleOutputCP = kernel32.NewProc("GetConsoleOutputCP")
+)
+
+// IsEastAsian return true if the current locale is CJK
+func IsEastAsian() bool {
+	r1, _, _ := procGetConsoleOutputCP.Call()
+	if r1 == 0 {
+		return false
+	}
+
+	switch int(r1) {
+	case 932, 51932, 936, 949, 950:
+		return true
+	}
+
+	return false
+}