// // pool-import // // Copyright (C) 2016 Christian Pointner // // This file is part of pool-import. // // pool-import is free software: you can redistribute it and/or modify // it under the terms of the GNU General Public License as published by // the Free Software Foundation, either version 3 of the License, or // any later version. // // pool-import is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU General Public License for more details. // // You should have received a copy of the GNU General Public License // along with pool-import. If not, see . // package main import ( "encoding/base64" "io" "log" "os" "path/filepath" "sync" "golang.org/x/crypto/blake2b" ) type FileMap map[string]string func (m FileMap) Merge(m2 FileMap) { for key, value := range m2 { m[key] = value } } type Hasher struct { numThreads uint stdlog *log.Logger filesMtx sync.Mutex Files FileMap } func computeHash(path string) (string, error) { hash, err := blake2b.New256(nil) if err != nil { return "", err } file, err := os.Open(path) if err != nil { return "", err } defer file.Close() if _, err := io.Copy(hash, file); err != nil { return "", err } return base64.URLEncoding.EncodeToString(hash.Sum(nil)), nil } func (h *Hasher) collectHashes(C <-chan string) (wg *sync.WaitGroup) { wg = &sync.WaitGroup{} for i := uint(0); i < h.numThreads; i++ { wg.Add(1) go func(num uint) { defer wg.Done() for { file, ok := <-C if !ok { return } if hash, err := computeHash(file); err != nil { h.stdlog.Printf(" - skipping (%v): %s", err, filepath.Base(file)) } else { h.stdlog.Printf(" - hashed: %s", filepath.Base(file)) h.filesMtx.Lock() h.Files[hash] = file h.filesMtx.Unlock() } } }(i) } return } func (h *Hasher) ComputeHashes(w Walker) (err error) { C := make(chan string, 10) wg := h.collectHashes(C) err = w.Walk(C) if err != nil { return } wg.Wait() return } func NewHasher(numThreads uint, stdlog *log.Logger) (h *Hasher) { h = &Hasher{numThreads: numThreads, stdlog: stdlog} h.Files = make(map[string]string) return }