FileCompare/main.go

package main

import (
	"bufio"
	"crypto/md5"
	"encoding/gob"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"
)

type DataFile struct {
	Filename string
	Filepath string
	Filesize int64
}

type DataStore struct {
	Files map[string][]DataFile
	Paths []string
}

var (
	ds           *DataStore
	dupe, unique int
)

func reinit() {
	ds = &DataStore{
		Files: make(map[string][]DataFile),
		Paths: make([]string, 0),
	}
	dupe, unique = 0, 0
}

func main() {

	reinit()

	fmt.Println("File Catalog")
	fmt.Println("---------------------")
	stdin := bufio.NewReader(os.Stdin)

	catalogue := choseCatalogue(stdin)
	loadCatalogue(catalogue)

	for {
		fmt.Println("Next path to scan (q to quit)")
		fmt.Print("-> ")

		searchDir, _ := stdin.ReadString('\n')
		searchDir = strings.Replace(searchDir, "\r", "", -1)
		searchDir = strings.Replace(searchDir, "\n", "", -1)

		if len(searchDir) == 0 || strings.ToLower(searchDir) == "q" {
			break
		}

		err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error {

			if f != nil && f.IsDir() {
				return nil
			}

			hashError, hash := hashFile(path)
			if hashError != nil {
				return hashError
			}
			file := DataFile{
				filepath.Base(path),
				filepath.Dir(path),
				f.Size(),
			}

			if ds.Files[hash] == nil {
				ds.Files[hash] = make([]DataFile, 0)
				// log.Printf("+[%s] - %s", hash, path)
				unique++
			} else {
				dupe++
				log.Printf("![%d] %s", len(ds.Files[hash])+1, path)
			}
			ds.Files[hash] = append(ds.Files[hash], file)

			return err

			// end file loop
		})
		if err != nil {
			panic(err)
		}

		update := ""
		for update == "" {
			fmt.Printf("Job finished, %d duplicates found, update catalogue?\n", dupe)
			fmt.Printf("[y/N/q]-> ")
			update, _ = stdin.ReadString('\n')
			update = strings.Replace(update, "\r", "", -1)
			update = strings.Replace(update, "\n", "", -1)
			update = strings.ToLower(update)

			if update == "y" {
				saveCatalogue(catalogue)
			} else if len(update) == 0 || update == "n" {
				fmt.Println("Discarding changes...")
				reinit()
				loadCatalogue(catalogue)
				break
			} else if update == "q" {
				fmt.Println("Proceeding to exit...")
				break
			} else {
				update = ""
			}
		}
		if update == "q" {
			break
		}

		// end main loop
	}

	fmt.Println("Save text summary of duplicate files?")
	fmt.Printf("[y/N]-> ")
	summary, _ := stdin.ReadString('\n')
	summary = strings.Replace(summary, "\r", "", -1)
	summary = strings.Replace(summary, "\n", "", -1)
	summary = strings.ToLower(summary)

	if summary == "y" {
		saveSummary("duplicates.csv")
	} else {
		log.Println("Exiting...")
	}
}

func saveSummary(path string) {
	summaryFile, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	defer summaryFile.Close()

	for hash, files := range ds.Files {
		if len(files) > 1 {
			// _, err := summaryFile.WriteString(fmt.Sprintf("%s - %s Duplicates", hash, len(files)))
			for _, file := range files {
				summaryFile.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", hash, filepath.Join(file.Filepath, file.Filename)))
			}
		}
	}

	log.Printf("Saved summary to: %s\n", path)
}

func hashFile(filePath string) (error, string) {
	file, err := os.Open(filePath)
	if err != nil {
		return err, ""
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)

	return nil, hex.EncodeToString(hash.Sum(nil))
}

func choseCatalogue(stdin *bufio.Reader) string {
	fmt.Println("Path to catalogue file")
	fmt.Print("[./save.gob]-> ")

	catalogue, _ := stdin.ReadString('\n')
	catalogue = strings.Replace(catalogue, "\r", "", -1)
	catalogue = strings.Replace(catalogue, "\n", "", -1)

	if catalogue == "" {
		catalogue = "save.gob"
	}

	return catalogue
}

func saveCatalogue(path string) {
	flags := os.O_TRUNC | os.O_RDWR | os.O_EXCL
	file, err := os.Stat(path)
	if file == nil {
		flags |= os.O_CREATE
	}

	dataFile, err := os.OpenFile(path, flags, 0644)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	defer dataFile.Close()

	dataEncoder := gob.NewEncoder(dataFile)
	err = dataEncoder.Encode(*ds)

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	log.Printf("File Saved: %s\n", path)
}

func loadCatalogue(path string) {
	file, err := os.Stat(path)
	if file == nil {
		saveCatalogue(path)
	}
	dataFile, err := os.OpenFile(path, os.O_RDWR|os.O_EXCL, 0644)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	defer dataFile.Close()

	dataDecoder := gob.NewDecoder(dataFile)
	err = dataDecoder.Decode(ds)

	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	log.Printf("File Loaded: %s\n", path)

}