summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/file-hasher/dir-hasher.go152
-rw-r--r--src/file-hasher/dir-walker.go70
-rw-r--r--src/file-hasher/hasher.go110
-rw-r--r--src/file-hasher/main.go11
-rw-r--r--src/file-hasher/walker.go51
5 files changed, 236 insertions, 158 deletions
diff --git a/src/file-hasher/dir-hasher.go b/src/file-hasher/dir-hasher.go
deleted file mode 100644
index 7f5e45b..0000000
--- a/src/file-hasher/dir-hasher.go
+++ /dev/null
@@ -1,152 +0,0 @@
-//
-// pool-import
-//
-// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
-//
-// This file is part of pool-import.
-//
-// pool-import is free software: you can redistribute it and/or modify
-// it under the terms of the GNU General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// any later version.
-//
-// pool-import is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU General Public License for more details.
-//
-// You should have received a copy of the GNU General Public License
-// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
-//
-
-package main
-
-import (
- "encoding/base64"
- "io"
- "log"
- "os"
- "path/filepath"
- "strings"
- "sync"
-
- "golang.org/x/crypto/blake2b"
-)
-
-type FileMap map[string]string
-
-func (m FileMap) Merge(m2 FileMap) {
- for key, value := range m2 {
- m[key] = value
- }
-}
-
-type MusicDir struct {
- root string
- stdlog *log.Logger
- filesMtx sync.Mutex
- Files FileMap
-}
-
-func handleEntry(C chan<- string, path string, info os.FileInfo, err error, stdlog *log.Logger) error {
- if err != nil {
- return err
- }
- if info.IsDir() {
- stdlog.Printf("entering directory: %s", path)
- return nil
- }
- if !info.Mode().IsRegular() {
- stdlog.Printf(" - skipping (special file): %s", info.Name())
- return nil
- }
- switch strings.ToLower(filepath.Ext(path)) {
- case ".flac":
- fallthrough
- case ".ogg":
- fallthrough
- case ".wav":
- fallthrough
- case ".mp3":
- fallthrough
- case ".aac":
- fallthrough
- case ".mp4":
- fallthrough
- case ".m4a":
- if absPath, err := filepath.Abs(path); err != nil {
- stdlog.Printf(" - skipping (%v): %s", err, info.Name())
- } else {
- C <- absPath
- }
- default:
- stdlog.Printf(" - skipping (unknown extension): %s", info.Name())
- }
- return nil
-}
-
-func computeHash(path string) (string, error) {
- hash, err := blake2b.New256(nil)
- if err != nil {
- return "", err
- }
- file, err := os.Open(path)
- if err != nil {
- return "", err
- }
- defer file.Close()
- if _, err := io.Copy(hash, file); err != nil {
- return "", err
- }
- return base64.URLEncoding.EncodeToString(hash.Sum(nil)), nil
-}
-
-func (d *MusicDir) collectHashes(C <-chan string, numThreads int) (wg *sync.WaitGroup) {
- wg = &sync.WaitGroup{}
-
- for i := 0; i < numThreads; i++ {
- wg.Add(1)
- go func(num int) {
- defer wg.Done()
- for {
- file, ok := <-C
- if !ok {
- return
- }
- if hash, err := computeHash(file); err != nil {
- d.stdlog.Printf(" - skipping (%v): %s", err, filepath.Base(file))
- } else {
- d.stdlog.Printf(" - hashed: %s", filepath.Base(file))
- d.filesMtx.Lock()
- d.Files[hash] = file
- d.filesMtx.Unlock()
- }
- }
- }(i)
- }
- return
-}
-
-func (d *MusicDir) ComputeHashes() (err error) {
- C := make(chan string, 10)
-
- wg := d.collectHashes(C, 4) // TODO: make number of hashing threads configurable
-
- err = filepath.Walk(d.root, func(path string, info os.FileInfo, err error) error {
- return handleEntry(C, path, info, err, d.stdlog)
- })
- close(C)
-
- if err != nil {
- return
- }
-
- wg.Wait()
- return
-}
-
-func NewMusicDir(root string, stdlog *log.Logger) (dir *MusicDir) {
- dir = &MusicDir{root: root, stdlog: stdlog}
- dir.Files = make(map[string]string)
- return
-}
diff --git a/src/file-hasher/dir-walker.go b/src/file-hasher/dir-walker.go
new file mode 100644
index 0000000..0aa7ba5
--- /dev/null
+++ b/src/file-hasher/dir-walker.go
@@ -0,0 +1,70 @@
+//
+// pool-import
+//
+// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
+//
+// This file is part of pool-import.
+//
+// pool-import is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// pool-import is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package main
+
+import (
+ "log"
+ "os"
+ "path/filepath"
+)
+
+type Dir struct {
+ root string
+ stdlog *log.Logger
+}
+
+func handleEntry(C chan<- string, path string, info os.FileInfo, err error, stdlog *log.Logger) error {
+ if err != nil {
+ return err
+ }
+ if info.IsDir() {
+ stdlog.Printf("entering directory: %s", path)
+ return nil
+ }
+ if !info.Mode().IsRegular() {
+ stdlog.Printf(" - skipping (special file): %s", info.Name())
+ return nil
+ }
+ if checkFileExt(path) {
+ if absPath, err := filepath.Abs(path); err != nil {
+ stdlog.Printf(" - skipping (%v): %s", err, info.Name())
+ } else {
+ C <- absPath
+ }
+ } else {
+ stdlog.Printf(" - skipping (unknown extension): %s", info.Name())
+ }
+ return nil
+}
+
+func (d *Dir) Walk(C chan<- string) (err error) {
+ defer close(C)
+
+ return filepath.Walk(d.root, func(path string, info os.FileInfo, err error) error {
+ return handleEntry(C, path, info, err, d.stdlog)
+ })
+}
+
+func NewDir(root string, stdlog *log.Logger) (dir *Dir) {
+ dir = &Dir{root: root, stdlog: stdlog}
+ return
+}
diff --git a/src/file-hasher/hasher.go b/src/file-hasher/hasher.go
new file mode 100644
index 0000000..afc35fd
--- /dev/null
+++ b/src/file-hasher/hasher.go
@@ -0,0 +1,110 @@
+//
+// pool-import
+//
+// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
+//
+// This file is part of pool-import.
+//
+// pool-import is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// pool-import is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package main
+
+import (
+ "encoding/base64"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "sync"
+
+ "golang.org/x/crypto/blake2b"
+)
+
+type FileMap map[string]string
+
+func (m FileMap) Merge(m2 FileMap) {
+ for key, value := range m2 {
+ m[key] = value
+ }
+}
+
+type Hasher struct {
+ numThreads uint
+ stdlog *log.Logger
+ filesMtx sync.Mutex
+ Files FileMap
+}
+
+func computeHash(path string) (string, error) {
+ hash, err := blake2b.New256(nil)
+ if err != nil {
+ return "", err
+ }
+ file, err := os.Open(path)
+ if err != nil {
+ return "", err
+ }
+ defer file.Close()
+ if _, err := io.Copy(hash, file); err != nil {
+ return "", err
+ }
+ return base64.URLEncoding.EncodeToString(hash.Sum(nil)), nil
+}
+
+func (h *Hasher) collectHashes(C <-chan string) (wg *sync.WaitGroup) {
+ wg = &sync.WaitGroup{}
+
+ for i := uint(0); i < h.numThreads; i++ {
+ wg.Add(1)
+ go func(num uint) {
+ defer wg.Done()
+ for {
+ file, ok := <-C
+ if !ok {
+ return
+ }
+ if hash, err := computeHash(file); err != nil {
+ h.stdlog.Printf(" - skipping (%v): %s", err, filepath.Base(file))
+ } else {
+ h.stdlog.Printf(" - hashed: %s", filepath.Base(file))
+ h.filesMtx.Lock()
+ h.Files[hash] = file
+ h.filesMtx.Unlock()
+ }
+ }
+ }(i)
+ }
+ return
+}
+
+func (h *Hasher) ComputeHashes(w Walker) (err error) {
+ C := make(chan string, 10)
+
+ wg := h.collectHashes(C)
+
+ err = w.Walk(C)
+ if err != nil {
+ return
+ }
+
+ wg.Wait()
+ return
+}
+
+func NewHasher(numThreads uint, stdlog *log.Logger) (h *Hasher) {
+ h = &Hasher{numThreads: numThreads, stdlog: stdlog}
+ h.Files = make(map[string]string)
+ return
+}
diff --git a/src/file-hasher/main.go b/src/file-hasher/main.go
index 1bb0132..051bb33 100644
--- a/src/file-hasher/main.go
+++ b/src/file-hasher/main.go
@@ -56,25 +56,24 @@ func main() {
}()
stdlog.Println("********************************************")
- allFiles := make(FileMap)
+ h := NewHasher(4, stdlog) // TODO: make number of threads configurable
for _, root := range directories {
+ d := NewDir(root, stdlog)
stdlog.Printf("*** hashing all the files in '%s'", root)
stdlog.Println("")
- md := NewMusicDir(root, stdlog)
- if err := md.ComputeHashes(); err != nil {
+ if err := h.ComputeHashes(d); err != nil {
return
}
- allFiles.Merge(md.Files)
stdlog.Println("")
stdlog.Println("******************************")
}
stdlog.Println("")
stdlog.Println("***************************************************************")
- stdlog.Printf("*** hashed %d files in %v", len(allFiles), time.Since(starttime))
+ stdlog.Printf("*** hashed %d files in %v", len(h.Files), time.Since(starttime))
stdlog.Println("***************************************************************")
enc := json.NewEncoder(os.Stdout)
- if err := enc.Encode(allFiles); err != nil {
+ if err := enc.Encode(h.Files); err != nil {
stdlog.Printf("Error encoding JSON: %v", err)
}
}()
diff --git a/src/file-hasher/walker.go b/src/file-hasher/walker.go
new file mode 100644
index 0000000..cc543a2
--- /dev/null
+++ b/src/file-hasher/walker.go
@@ -0,0 +1,51 @@
+//
+// pool-import
+//
+// Copyright (C) 2016 Christian Pointner <equinox@helsinki.at>
+//
+// This file is part of pool-import.
+//
+// pool-import is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// any later version.
+//
+// pool-import is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with pool-import. If not, see <http://www.gnu.org/licenses/>.
+//
+
+package main
+
+import (
+ "path/filepath"
+ "strings"
+)
+
+func checkFileExt(path string) bool {
+ switch strings.ToLower(filepath.Ext(path)) {
+ case ".flac":
+ return true
+ case ".ogg":
+ return true
+ case ".wav":
+ return true
+ case ".mp3":
+ return true
+ case ".aac":
+ return true
+ case ".mp4":
+ return true
+ case ".m4a":
+ return true
+ }
+ return false
+}
+
+type Walker interface {
+ Walk(C chan<- string) error
+}
generated by cgit v0.10.2 at 2024-11-22 01:01:57 (GMT)