duplicity-backup/configfile.go

package main

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"text/template"

	valid "github.com/asaskevich/govalidator"
	"gopkg.in/yaml.v2"
)

const (
	commandBackup           = "backup"
	commandFullBackup       = "full"
	commandIncrBackup       = "incr"
	commandCleanup          = "cleanup"
	commandList             = "list-current-files"
	commandRestore          = "restore"
	commandStatus           = "status"
	commandVerify           = "verify"
	commandRemove           = "__remove_old"
	commandListChangedFiles = "list-changed-files"
)

var (
	notifyCommands = []string{
		commandBackup,
		commandRemove,
		commandFullBackup,
		commandIncrBackup,
	}
	removeCommands = []string{
		commandBackup,
		commandCleanup,
	}
)

type configFile struct {
	RootPath    string `yaml:"root" valid:"required"`
	Hostname    string `yaml:"hostname"`
	Destination string `yaml:"dest" valid:"required"`
	FTPPassword string `yaml:"ftp_password"`
	AWS         struct {
		AccessKeyID     string `yaml:"access_key_id"`
		SecretAccessKey string `yaml:"secret_access_key"`
		StorageClass    string `yaml:"storage_class"`
	} `yaml:"aws"`
	GoogleCloud struct {
		AccessKeyID     string `yaml:"access_key_id"`
		SecretAccessKey string `yaml:"secret_access_key"`
	} `yaml:"google_cloud"`
	Swift struct {
		Username    string `yaml:"username"`
		Password    string `yaml:"password"`
		AuthURL     string `yaml:"auth_url"`
		AuthVersion int    `yaml:"auth_version"`
	} `yaml:"swift"`
	Include            []string `yaml:"inclist"`
	Exclude            []string `yaml:"exclist"`
	IncExcFile         string   `yaml:"incexcfile" valid:"customFileExistsValidator,required"`
	ExcludeDeviceFiles bool     `yaml:"excdevicefiles"`
	Encryption         struct {
		Enable           bool   `yaml:"enable"`
		Passphrase       string `yaml:"passphrase"`
		GPGEncryptionKey string `yaml:"gpg_encryption_key"`
		GPGSignKey       string `yaml:"gpg_sign_key"`
		HideKeyID        bool   `yaml:"hide_key_id"`
		SecretKeyRing    string `yaml:"secret_keyring"`
	} `yaml:"encryption"`
	StaticBackupOptions []string `yaml:"static_options"`
	Cleanup             struct {
		Type  string `yaml:"type"`
		Value string `yaml:"value"`
	} `yaml:"cleanup"`
	LogDirectory  string `yaml:"logdir" valid:"required"`
	Notifications struct {
		Slack struct {
			HookURL  string `yaml:"hook_url"`
			Channel  string `yaml:"channel"`
			Username string `yaml:"username"`
			Emoji    string `yaml:"emoji"`
		} `yaml:"slack"`
		MonDash struct {
			BoardURL  string `yaml:"board"`
			Token     string `yaml:"token"`
			Freshness int64  `yaml:"freshness"`
		} `yaml:"mondash"`
	} `yaml:"notifications"`
}

func init() {
	valid.CustomTypeTagMap.Set("customFileExistsValidator", valid.CustomTypeValidator(func(i interface{}, context interface{}) bool {
		switch v := i.(type) { // this validates a field against the value in another field, i.e. dependent validation
		case string:
			_, err := os.Stat(v)
			return v == "" || err == nil
		}
		return false
	}))
}

func (c *configFile) validate() error {
	result, err := valid.ValidateStruct(c)
	if !result || err != nil {
		return err
	}

	if c.Encryption.Enable && c.Encryption.GPGSignKey != "" && c.Encryption.Passphrase == "" {
		return errors.New("With gpg_sign_key passphrase is required")
	}

	if c.Encryption.Enable && c.Encryption.GPGEncryptionKey == "" && c.Encryption.Passphrase == "" {
		return errors.New("Encryption is enabled but no encryption key or passphrase is specified")
	}

	if c.Destination[0:2] == "s3" && (c.AWS.AccessKeyID == "" || c.AWS.SecretAccessKey == "") {
		return errors.New("Destination is S3 but AWS credentials are not configured")
	}

	if c.Destination[0:2] == "gs" && (c.GoogleCloud.AccessKeyID == "" || c.GoogleCloud.SecretAccessKey == "") {
		return errors.New("Destination is S3 but AWS credentials are not configured")
	}

	return nil
}

func getTemplateFuncMap() template.FuncMap {
	return template.FuncMap{
		"env": func(name string, v ...string) string {
			defaultValue := ""
			if len(v) > 0 {
				defaultValue = v[0]
			}
			if value, ok := os.LookupEnv(name); ok {
				return value
			}
			return defaultValue
		},
	}
}

func loadConfigFile(in io.Reader) (*configFile, error) {
	fileContent, err := ioutil.ReadAll(in)
	if err != nil {
		return nil, err
	}

	buf := bytes.NewBuffer([]byte{})
	tpl, err := template.New("config file").Funcs(getTemplateFuncMap()).Parse(string(fileContent))
	if err != nil {
		return nil, err
	}
	if err := tpl.Execute(buf, nil); err != nil {
		return nil, err
	}

	hostname, _ := os.Hostname()

	res := &configFile{
		Hostname: hostname,
	}
	if err := yaml.Unmarshal(buf.Bytes(), res); err != nil {
		return nil, err
	}

	return res, res.validate()
}

func (c *configFile) GenerateCommand(argv []string, time string) (commandLine []string, env []string, logfilter *regexp.Regexp, err error) {
	var (
		tmpEnv             []string
		option, root, dest string
		addTime            bool
		command            = argv[0]
	)

	switch command {
	case commandBackup:
		option = ""
		root = c.RootPath
		dest = c.Destination
		commandLine, env, err = c.generateFullCommand(option, time, root, dest, addTime, "")
	case commandListChangedFiles:
		option = ""
		root = c.RootPath
		dest = c.Destination
		commandLine, env, err = c.generateFullCommand(option, time, root, dest, addTime, "")
		commandLine = append([]string{"--dry-run", "--verbosity", "8"}, commandLine...)
		logfilter = regexp.MustCompile(`^[ADM] `)
	case commandFullBackup:
		option = command
		root = c.RootPath
		dest = c.Destination
		commandLine, env, err = c.generateFullCommand(option, time, root, dest, addTime, "")
	case commandIncrBackup:
		option = command
		root = c.RootPath
		dest = c.Destination
		commandLine, env, err = c.generateFullCommand(option, time, root, dest, addTime, "")
	case commandCleanup:
		option = command
		commandLine, env, err = c.generateLiteCommand(option, time, addTime)
	case commandList:
		option = command
		commandLine, env, err = c.generateLiteCommand(option, time, addTime)
	case commandRestore:
		addTime = true
		option = command
		root = c.Destination
		restoreFile := ""

		if len(argv) == 3 {
			restoreFile = argv[1]
			dest = argv[2]
		} else if len(argv) == 2 {
			dest = argv[1]
		} else {
			err = errors.New("You need to specify one ore more parameters. See help message.")
			return
		}

		commandLine, env, err = c.generateFullCommand(option, time, root, dest, addTime, restoreFile)
	case commandStatus:
		option = "collection-status"
		commandLine, env, err = c.generateLiteCommand(option, time, addTime)
	case commandVerify:
		option = command
		root = c.Destination
		dest = c.RootPath
		commandLine, env, err = c.generateFullCommand(option, time, root, dest, addTime, "")
	case commandRemove:
		commandLine, env, err = c.generateRemoveCommand()
	default:
		err = fmt.Errorf("Did not understand command '%s', please see 'help' for details what to do.", command)
		return
	}

	// Add destination credentials
	tmpEnv = c.generateCredentialExport()
	env = append(env, tmpEnv...)

	commandLine = c.cleanSlice(commandLine)
	env = c.cleanSlice(env)

	return
}

func (c *configFile) cleanSlice(in []string) (out []string) {
	for _, i := range in {
		if i != "" {
			out = append(out, i)
		}
	}

	return
}

func (c *configFile) generateCredentialExport() (env []string) {
	if c.AWS.AccessKeyID != "" {
		env = append(env, "AWS_ACCESS_KEY_ID="+c.AWS.AccessKeyID)
		env = append(env, "AWS_SECRET_ACCESS_KEY="+c.AWS.SecretAccessKey)
	}
	if c.GoogleCloud.AccessKeyID != "" {
		env = append(env, "GS_ACCESS_KEY_ID="+c.GoogleCloud.AccessKeyID)
		env = append(env, "GS_SECRET_ACCESS_KEY="+c.GoogleCloud.SecretAccessKey)
	}
	if c.Swift.Username != "" {
		env = append(env, "SWIFT_USERNAME="+c.Swift.Username)
		env = append(env, "SWIFT_PASSWORD="+c.Swift.Password)
		env = append(env, "SWIFT_AUTHURL="+c.Swift.AuthURL)
		env = append(env, "SWIFT_AUTHVERSION="+strconv.Itoa(c.Swift.AuthVersion))
	}
	if c.FTPPassword != "" {
		env = append(env, "FTP_PASSWORD="+c.FTPPassword)
	}

	return
}

func (c *configFile) generateRemoveCommand() (commandLine []string, env []string, err error) {
	var tmpArg, tmpEnv []string
	// Assemble command
	commandLine = append(commandLine, c.Cleanup.Type, c.Cleanup.Value)
	// Static Options
	commandLine = append(commandLine, c.StaticBackupOptions...)
	// Encryption options
	tmpArg, tmpEnv = c.generateEncryption(c.Cleanup.Type)
	commandLine = append(commandLine, tmpArg...)
	env = append(env, tmpEnv...)
	// Enforce cleanup
	commandLine = append(commandLine, "--force")
	// Remote repo
	commandLine = append(commandLine, c.Destination)

	return
}

func (c *configFile) generateLiteCommand(option, time string, addTime bool) (commandLine []string, env []string, err error) {
	var tmpArg, tmpEnv []string
	// Assemble command
	commandLine = append(commandLine, option)
	// Static Options
	commandLine = append(commandLine, c.StaticBackupOptions...)
	if addTime && time != "" {
		commandLine = append(commandLine, "--time", time)
	}
	// Encryption options
	tmpArg, tmpEnv = c.generateEncryption(option)
	commandLine = append(commandLine, tmpArg...)
	env = append(env, tmpEnv...)
	// Remote repo
	commandLine = append(commandLine, c.Destination)

	return
}

func (c *configFile) generateFullCommand(option, time, root, dest string, addTime bool, restoreFile string) (commandLine []string, env []string, err error) {
	var tmpArg, tmpEnv []string
	// Assemble command
	commandLine = append(commandLine, option)
	// Static Options
	commandLine = append(commandLine, c.StaticBackupOptions...)
	if addTime && time != "" {
		commandLine = append(commandLine, "--time", time)
	}
	if restoreFile != "" {
		commandLine = append(commandLine, "--file-to-restore", restoreFile)
	}
	// AWS Storage Class (empty if not used, will get stripped)
	commandLine = append(commandLine, c.AWS.StorageClass)
	// Encryption options
	tmpArg, tmpEnv = c.generateEncryption(option)
	commandLine = append(commandLine, tmpArg...)
	env = append(env, tmpEnv...)
	// Includes / Excludes
	tmpArg, tmpEnv = c.generateIncludeExclude()
	commandLine = append(commandLine, tmpArg...)
	env = append(env, tmpEnv...)
	// Source / Destination
	commandLine = append(commandLine, root, dest)

	return
}

func (c *configFile) generateIncludeExclude() (arguments []string, env []string) {
	if c.ExcludeDeviceFiles {
		arguments = append(arguments, "--exclude-device-files")
	}

	for _, exc := range c.Exclude {
		arguments = append(arguments, "--exclude="+exc)
	}

	for _, inc := range c.Include {
		arguments = append(arguments, "--include="+inc)
	}

	if c.IncExcFile != "" {
		arguments = append(arguments, "--include-globbing-filelist", c.IncExcFile)
	}

	if len(c.Include) > 0 || c.IncExcFile != "" {
		arguments = append(arguments, "--exclude=**")
	}

	return
}

func (c *configFile) generateEncryption(command string) (arguments []string, env []string) {
	if !c.Encryption.Enable {
		arguments = append(arguments, "--no-encryption")
		return
	}

	if c.Encryption.Passphrase != "" {
		env = append(env, "PASSPHRASE="+c.Encryption.Passphrase)
	}

	if c.Encryption.GPGEncryptionKey != "" {
		if c.Encryption.HideKeyID {
			arguments = append(arguments, "--hidden-encrypt-key="+c.Encryption.GPGEncryptionKey)
		} else {
			arguments = append(arguments, "--encrypt-key="+c.Encryption.GPGEncryptionKey)
		}
	}

	if c.Encryption.GPGSignKey != "" && command != "restore" {
		arguments = append(arguments, "--sign-key="+c.Encryption.GPGSignKey)
	}

	if c.Encryption.GPGEncryptionKey != "" && c.Encryption.SecretKeyRing != "" {
		arguments = append(arguments, "--encrypt-secret-keyring="+c.Encryption.SecretKeyRing)
	}

	return
}