Initial working version - quick and dirty

main.go

@@ -0,0 +1,238 @@
+package main
+
+import (
+	"bufio"
+	"crypto/md5"
+	"encoding/gob"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+type DataFile struct {
+	Filename string
+	Filepath string
+	Filesize int64
+}
+
+type DataStore struct {
+	Files map[string][]DataFile
+	Paths []string
+}
+
+var (
+	ds           *DataStore
+	dupe, unique int
+)
+
+func reinit() {
+	ds = &DataStore{
+		Files: make(map[string][]DataFile),
+		Paths: make([]string, 0),
+	}
+	dupe, unique = 0, 0
+}
+
+func main() {
+
+	reinit()
+
+	fmt.Println("File Catalog")
+	fmt.Println("---------------------")
+	stdin := bufio.NewReader(os.Stdin)
+
+	catalogue := choseCatalogue(stdin)
+	loadCatalogue(catalogue)
+
+	for {
+		fmt.Println("Next path to scan (q to quit)")
+		fmt.Print("-> ")
+
+		searchDir, _ := stdin.ReadString('\n')
+		searchDir = strings.Replace(searchDir, "\r", "", -1)
+		searchDir = strings.Replace(searchDir, "\n", "", -1)
+
+		if len(searchDir) == 0 || strings.ToLower(searchDir) == "q" {
+			break
+		}
+
+		err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error {
+
+			if f != nil && f.IsDir() {
+				return nil
+			}
+
+			hashError, hash := hashFile(path)
+			if hashError != nil {
+				return hashError
+			}
+			file := DataFile{
+				filepath.Base(path),
+				filepath.Dir(path),
+				f.Size(),
+			}
+
+			if ds.Files[hash] == nil {
+				ds.Files[hash] = make([]DataFile, 0)
+				// log.Printf("+[%s] - %s", hash, path)
+				unique++
+			} else {
+				dupe++
+				log.Printf("![%d] %s", len(ds.Files[hash])+1, path)
+			}
+			ds.Files[hash] = append(ds.Files[hash], file)
+
+			return err
+
+			// end file loop
+		})
+		if err != nil {
+			panic(err)
+		}
+
+		update := ""
+		for update == "" {
+			fmt.Printf("Job finished, %d duplicates found, update catalogue?\n", dupe)
+			fmt.Printf("[y/N/q]-> ")
+			update, _ = stdin.ReadString('\n')
+			update = strings.Replace(update, "\r", "", -1)
+			update = strings.Replace(update, "\n", "", -1)
+			update = strings.ToLower(update)
+
+			if update == "y" {
+				saveCatalogue(catalogue)
+			} else if len(update) == 0 || update == "n" {
+				fmt.Println("Discarding changes...")
+				reinit()
+				loadCatalogue(catalogue)
+				break
+			} else if update == "q" {
+				fmt.Println("Proceeding to exit...")
+				break
+			} else {
+				update = ""
+			}
+		}
+		if update == "q" {
+			break
+		}
+
+		// end main loop
+	}
+
+	fmt.Println("Save text summary of duplicate files?")
+	fmt.Printf("[y/N]-> ")
+	summary, _ := stdin.ReadString('\n')
+	summary = strings.Replace(summary, "\r", "", -1)
+	summary = strings.Replace(summary, "\n", "", -1)
+	summary = strings.ToLower(summary)
+
+	if summary == "y" {
+		saveSummary("duplicates.csv")
+	} else {
+		log.Println("Exiting...")
+	}
+}
+
+func saveSummary(path string) {
+	summaryFile, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	defer summaryFile.Close()
+
+	for hash, files := range ds.Files {
+		if len(files) > 1 {
+			// _, err := summaryFile.WriteString(fmt.Sprintf("%s - %s Duplicates", hash, len(files)))
+			for _, file := range files {
+				summaryFile.WriteString(fmt.Sprintf("\"%s\",\"%s\"\n", hash, filepath.Join(file.Filepath, file.Filename)))
+			}
+		}
+	}
+
+	log.Printf("Saved summary to: %s\n", path)
+}
+
+func hashFile(filePath string) (error, string) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return err, ""
+	}
+	defer file.Close()
+
+	hash := md5.New()
+	_, err = io.Copy(hash, file)
+
+	return nil, hex.EncodeToString(hash.Sum(nil))
+}
+
+func choseCatalogue(stdin *bufio.Reader) string {
+	fmt.Println("Path to catalogue file")
+	fmt.Print("[./save.gob]-> ")
+
+	catalogue, _ := stdin.ReadString('\n')
+	catalogue = strings.Replace(catalogue, "\r", "", -1)
+	catalogue = strings.Replace(catalogue, "\n", "", -1)
+
+	if catalogue == "" {
+		catalogue = "save.gob"
+	}
+
+	return catalogue
+}
+
+func saveCatalogue(path string) {
+	flags := os.O_TRUNC | os.O_RDWR | os.O_EXCL
+	file, err := os.Stat(path)
+	if file == nil {
+		flags |= os.O_CREATE
+	}
+
+	dataFile, err := os.OpenFile(path, flags, 0644)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	defer dataFile.Close()
+
+	dataEncoder := gob.NewEncoder(dataFile)
+	err = dataEncoder.Encode(*ds)
+
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	log.Printf("File Saved: %s\n", path)
+}
+
+func loadCatalogue(path string) {
+	file, err := os.Stat(path)
+	if file == nil {
+		saveCatalogue(path)
+	}
+	dataFile, err := os.OpenFile(path, os.O_RDWR|os.O_EXCL, 0644)
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	defer dataFile.Close()
+
+	dataDecoder := gob.NewDecoder(dataFile)
+	err = dataDecoder.Decode(ds)
+
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+
+	log.Printf("File Loaded: %s\n", path)
+
+}