From b330403d040162efeca890ae207fb24cfa89f44c Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:14:16 +0100 Subject: [PATCH 01/10] re organize project layout --- src/gosync/build.go | 114 -- src/gosync/common.go | 277 ----- src/gosync/diff.go | 145 --- src/gosync/fileformat.md | 21 - src/gosync/main.go | 65 - src/gosync/patch.go | 105 -- vendor/manifest | 24 - .../src/github.com/Redundancy/go-sync/LICENSE | 21 - .../github.com/Redundancy/go-sync/README.md | 119 -- .../go-sync/acceptancetests/nulltest.sh | 1 - .../Redundancy/go-sync/acceptancetests/run.sh | 21 - .../go-sync/acceptancetests/truncated_test.sh | 14 - .../acceptancetests/truncation_test.sh | 14 - .../go-sync/blocksources/blocksourcebase.go | 271 ----- .../blocksources/blocksourcebase_test.go | 303 ----- .../blocksources/fixed_size_block_resolver.go | 73 -- .../fixed_size_block_resolver_test.go | 127 -- .../go-sync/blocksources/helpers.go | 124 -- .../go-sync/blocksources/httpblocksource.go | 110 -- .../blocksources/httpblocksource_test.go | 251 ---- .../blocksources/readseeker_blocksource.go | 54 - .../readseeker_blocksource_test.go | 42 - .../blocksources/sortableBlockResponse.go | 19 - .../Redundancy/go-sync/chunks/chunks.go | 87 -- .../go-sync/circularbuffer/noalloc.go | 170 --- .../go-sync/circularbuffer/noalloc_test.go | 182 --- .../Redundancy/go-sync/comparer/comparer.go | 219 ---- .../go-sync/comparer/comparer_bench_test.go | 76 -- .../go-sync/comparer/comparer_test.go | 445 ------- .../Redundancy/go-sync/comparer/merger.go | 309 ----- .../go-sync/comparer/merger_test.go | 401 ------- .../go-sync/filechecksum/filechecksum.go | 237 ---- .../go-sync/filechecksum/filechecksum_test.go | 309 ----- .../go-sync/filechecksum/verifier.go | 48 - .../go-sync/filechecksum/verifier_test.go | 72 -- .../github.com/Redundancy/go-sync/gosync.go | 16 - .../Redundancy/go-sync/gosync/build.go | 114 -- .../Redundancy/go-sync/gosync/common.go | 277 ----- .../Redundancy/go-sync/gosync/diff.go | 145 --- .../Redundancy/go-sync/gosync/fileformat.md | 21 - .../Redundancy/go-sync/gosync/main.go | 65 - .../Redundancy/go-sync/gosync/patch.go | 105 -- .../Redundancy/go-sync/gosync_test.go | 111 -- .../Redundancy/go-sync/http_test.go | 174 --- .../Redundancy/go-sync/index/index.go | 183 --- .../go-sync/index/index_bench_test.go | 148 --- .../Redundancy/go-sync/index/index_test.go | 206 ---- .../go-sync/indexbuilder/indexbuilder.go | 55 - .../Redundancy/go-sync/patcher/blocksource.go | 58 - .../go-sync/patcher/sequential/sequential.go | 145 --- .../patcher/sequential/sequential_test.go | 188 --- .../Redundancy/go-sync/rollsum/rollsum_32.go | 85 -- .../go-sync/rollsum/rollsum_32_base.go | 86 -- .../go-sync/rollsum/rollsum_32_test.go | 298 ----- .../github.com/Redundancy/go-sync/rsync.go | 347 ------ .../go-sync/util/readers/injectedreader.go | 18 - .../go-sync/util/readers/nonrepeating.go | 35 - .../go-sync/util/readers/nonrepeating_test.go | 58 - .../go-sync/util/readers/readers.go | 6 - .../go-sync/util/readers/sequence_test.go | 1 - .../go-sync/util/readers/sequencelimit.go | 12 - .../go-sync/util/readers/uniform_test.go | 90 -- .../go-sync/util/readers/uniformreader.go | 55 - vendor/src/github.com/codegangsta/cli/LICENSE | 21 - .../src/github.com/codegangsta/cli/README.md | 394 ------- .../github.com/codegangsta/cli/altsrc/flag.go | 439 ------- .../codegangsta/cli/altsrc/flag_test.go | 336 ------ .../codegangsta/cli/altsrc/helpers_test.go | 18 - .../cli/altsrc/input_source_context.go | 21 - .../cli/altsrc/map_input_source.go | 152 --- .../cli/altsrc/yaml_command_test.go | 172 --- .../cli/altsrc/yaml_file_loader.go | 84 -- vendor/src/github.com/codegangsta/cli/app.go | 349 ------ .../github.com/codegangsta/cli/app_test.go | 1047 ----------------- .../github.com/codegangsta/cli/appveyor.yml | 16 - .../cli/autocomplete/bash_autocomplete | 14 - .../cli/autocomplete/zsh_autocomplete | 5 - vendor/src/github.com/codegangsta/cli/cli.go | 40 - .../src/github.com/codegangsta/cli/command.go | 250 ---- .../codegangsta/cli/command_test.go | 97 -- .../src/github.com/codegangsta/cli/context.go | 393 ------- .../codegangsta/cli/context_test.go | 121 -- vendor/src/github.com/codegangsta/cli/flag.go | 546 --------- .../github.com/codegangsta/cli/flag_test.go | 859 -------------- vendor/src/github.com/codegangsta/cli/help.go | 248 ---- .../github.com/codegangsta/cli/help_test.go | 94 -- .../codegangsta/cli/helpers_test.go | 19 - .../github.com/petar/GoLLRB/llrb/avgvar.go | 39 - .../github.com/petar/GoLLRB/llrb/iterator.go | 93 -- .../petar/GoLLRB/llrb/iterator_test.go | 76 -- .../petar/GoLLRB/llrb/llrb-stats.go | 46 - .../src/github.com/petar/GoLLRB/llrb/llrb.go | 456 ------- .../github.com/petar/GoLLRB/llrb/llrb_test.go | 239 ---- .../src/github.com/petar/GoLLRB/llrb/util.go | 17 - 94 files changed, 14673 deletions(-) delete mode 100644 src/gosync/build.go delete mode 100644 src/gosync/common.go delete mode 100644 src/gosync/diff.go delete mode 100644 src/gosync/fileformat.md delete mode 100644 src/gosync/main.go delete mode 100644 src/gosync/patch.go delete mode 100644 vendor/manifest delete mode 100644 vendor/src/github.com/Redundancy/go-sync/LICENSE delete mode 100644 vendor/src/github.com/Redundancy/go-sync/README.md delete mode 100644 vendor/src/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh delete mode 100644 vendor/src/github.com/Redundancy/go-sync/acceptancetests/run.sh delete mode 100644 vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh delete mode 100644 vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/helpers.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/chunks/chunks.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/comparer/comparer.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/comparer/comparer_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/comparer/merger.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/comparer/merger_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync/build.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync/common.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync/diff.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync/fileformat.md delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync/main.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync/patch.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/gosync_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/http_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/index/index.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/index/index_bench_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/index/index_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/patcher/blocksource.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/rsync.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/injectedreader.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/readers.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/sequence_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/sequencelimit.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/uniform_test.go delete mode 100644 vendor/src/github.com/Redundancy/go-sync/util/readers/uniformreader.go delete mode 100644 vendor/src/github.com/codegangsta/cli/LICENSE delete mode 100644 vendor/src/github.com/codegangsta/cli/README.md delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/flag.go delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/flag_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/helpers_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/input_source_context.go delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/map_input_source.go delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/yaml_command_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/altsrc/yaml_file_loader.go delete mode 100644 vendor/src/github.com/codegangsta/cli/app.go delete mode 100644 vendor/src/github.com/codegangsta/cli/app_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/appveyor.yml delete mode 100644 vendor/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete delete mode 100644 vendor/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete delete mode 100644 vendor/src/github.com/codegangsta/cli/cli.go delete mode 100644 vendor/src/github.com/codegangsta/cli/command.go delete mode 100644 vendor/src/github.com/codegangsta/cli/command_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/context.go delete mode 100644 vendor/src/github.com/codegangsta/cli/context_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/flag.go delete mode 100644 vendor/src/github.com/codegangsta/cli/flag_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/help.go delete mode 100644 vendor/src/github.com/codegangsta/cli/help_test.go delete mode 100644 vendor/src/github.com/codegangsta/cli/helpers_test.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/avgvar.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/iterator.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/iterator_test.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/llrb-stats.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/llrb.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/llrb_test.go delete mode 100644 vendor/src/github.com/petar/GoLLRB/llrb/util.go diff --git a/src/gosync/build.go b/src/gosync/build.go deleted file mode 100644 index 7dfb000..0000000 --- a/src/gosync/build.go +++ /dev/null @@ -1,114 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/codegangsta/cli" -) - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "build", - ShortName: "b", - Usage: "build a .gosync file for a file", - Action: Build, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "blocksize", - Value: DefaultBlockSize, - Usage: "The block size to use for the gosync file", - }, - }, - }, - ) -} - -func Build(c *cli.Context) { - filename := c.Args()[0] - blocksize := uint32(c.Int("blocksize")) - generator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) - inputFile, err := os.Open(filename) - - if err != nil { - absInputPath, err2 := filepath.Abs(filename) - if err2 == nil { - handleFileError(absInputPath, err) - } else { - handleFileError(filename, err) - } - - os.Exit(1) - } - - s, _ := inputFile.Stat() - // TODO: Error? - file_size := s.Size() - - defer inputFile.Close() - - ext := filepath.Ext(filename) - outfilePath := filename[:len(filename)-len(ext)] + ".gosync" - outputFile, err := os.Create(outfilePath) - - if err != nil { - handleFileError(outfilePath, err) - os.Exit(1) - } - - defer outputFile.Close() - - if err = writeHeaders( - outputFile, - magicString, - blocksize, - file_size, - []uint16{majorVersion, minorVersion, patchVersion}, - ); err != nil { - fmt.Fprintf( - os.Stderr, - "Error getting file info: %v\n", - filename, - err, - ) - os.Exit(2) - } - - start := time.Now() - _, err = generator.GenerateChecksums(inputFile, outputFile) - end := time.Now() - - if err != nil { - fmt.Fprintf( - os.Stderr, - "Error generating checksum: %v\n", - filename, - err, - ) - os.Exit(2) - } - - inputFileInfo, err := os.Stat(filename) - if err != nil { - fmt.Fprintf( - os.Stderr, - "Error getting file info: %v\n", - filename, - err, - ) - os.Exit(2) - } - - fmt.Fprintf( - os.Stderr, - "Index for %v file generated in %v (%v bytes/S)\n", - inputFileInfo.Size(), - end.Sub(start), - float64(inputFileInfo.Size())/end.Sub(start).Seconds(), - ) -} diff --git a/src/gosync/common.go b/src/gosync/common.go deleted file mode 100644 index f8091a9..0000000 --- a/src/gosync/common.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "bufio" - "encoding/binary" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/patcher" - "github.com/codegangsta/cli" -) - -const ( - // KB - One Kilobyte - KB = 1024 - // MB - One Megabyte - MB = 1000000 -) - -func errorWrapper(c *cli.Context, f func(*cli.Context) error) { - defer func() { - if p := recover(); p != nil { - fmt.Fprintln(os.Stderr, p) - os.Exit(1) - } - }() - - if err := f(c); err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) - } - - return -} - -func openFileAndHandleError(filename string) (f *os.File) { - var err error - f, err = os.Open(filename) - - if err != nil { - f = nil - handleFileError(filename, err) - } - - return -} - -func formatFileError(filename string, err error) error { - switch { - case os.IsExist(err): - return fmt.Errorf( - "Could not open %v (already exists): %v", - filename, - err, - ) - case os.IsNotExist(err): - return fmt.Errorf( - "Could not find %v: %v\n", - filename, - err, - ) - case os.IsPermission(err): - return fmt.Errorf( - "Could not open %v (permission denied): %v\n", - filename, - err, - ) - default: - return fmt.Errorf( - "Unknown error opening %v: %v\n", - filename, - err, - ) - } -} - -func handleFileError(filename string, err error) { - e := formatFileError(filename, err) - fmt.Fprintln(os.Stderr, e) -} - -func getLocalOrRemoteFile(path string) (io.ReadCloser, error) { - url, err := url.Parse(path) - - switch { - case err != nil: - return os.Open(path) - case url.Scheme == "": - return os.Open(path) - default: - response, err := http.Get(path) - - if err != nil { - return nil, err - } - - if response.StatusCode < 200 || response.StatusCode > 299 { - return nil, fmt.Errorf("Request to %v returned status: %v", path, response.Status) - } - - return response.Body, nil - } -} - -func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} - -func writeHeaders( - f *os.File, - magic string, - blocksize uint32, - filesize int64, - versions []uint16, -) (err error) { - if _, err = f.WriteString(magicString); err != nil { - return - } - - for _, v := range versions { - if err = binary.Write(f, binary.LittleEndian, v); err != nil { - return - } - } - - if err = binary.Write(f, binary.LittleEndian, filesize); err != nil { - return - } - - err = binary.Write(f, binary.LittleEndian, blocksize) - return -} - -// reads the file headers and checks the magic string, then the semantic versioning -func readHeadersAndCheck( - r io.Reader, - magic string, - requiredMajorVersion uint16, -) ( - major, minor, patch uint16, - filesize int64, - blocksize uint32, - err error, -) { - b := make([]byte, len(magicString)) - - if _, err = r.Read(b); err != nil { - return - } else if string(b) != magicString { - err = errors.New( - "file header does not match magic string. Not a valid gosync file", - ) - return - } - - for _, v := range []*uint16{&major, &minor, &patch} { - err = binary.Read(r, binary.LittleEndian, v) - if err != nil { - return - } - } - - if requiredMajorVersion != major { - err = fmt.Errorf( - "The major version of the gosync file (%v.%v.%v) does not match the tool (%v.%v.%v).", - major, minor, patch, - majorVersion, minorVersion, patchVersion, - ) - - return - } - - err = binary.Read(r, binary.LittleEndian, &filesize) - if err != nil { - return - } - - err = binary.Read(r, binary.LittleEndian, &blocksize) - return -} - -func readIndex(r io.Reader, blocksize uint) ( - i *index.ChecksumIndex, - checksumLookup filechecksum.ChecksumLookup, - blockCount uint, - err error, -) { - generator := filechecksum.NewFileChecksumGenerator(blocksize) - - readChunks, e := chunks.LoadChecksumsFromReader( - r, - generator.WeakRollingHash.Size(), - generator.StrongHash.Size(), - ) - - err = e - - if err != nil { - return - } - - checksumLookup = chunks.StrongChecksumGetter(readChunks) - i = index.MakeChecksumIndex(readChunks) - blockCount = uint(len(readChunks)) - - return -} - -func multithreadedMatching( - localFile *os.File, - idx *index.ChecksumIndex, - localFileSize, - matcherCount int64, - blocksize uint, -) (*comparer.MatchMerger, *comparer.Comparer) { - // Note: Since not all sections of the file are equal in work - // it would be better to divide things up into more sections and - // pull work from a queue channel as each finish - sectionSize := localFileSize / matcherCount - sectionSize += int64(blocksize) - (sectionSize % int64(blocksize)) - merger := &comparer.MatchMerger{} - compare := &comparer.Comparer{} - - for i := int64(0); i < matcherCount; i++ { - offset := sectionSize * i - - // Sections must overlap by blocksize (strictly blocksize - 1?) - if i > 0 { - offset -= int64(blocksize) - } - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(localFile, offset, sectionSize), - MB, - ) - - sectionGenerator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, idx) - - merger.StartMergeResultStream(matchStream, int64(blocksize)) - } - - return merger, compare -} - -// better way to do this? diff --git a/src/gosync/diff.go b/src/gosync/diff.go deleted file mode 100644 index 14a4acf..0000000 --- a/src/gosync/diff.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - "time" - - "github.com/codegangsta/cli" -) - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "diff", - ShortName: "d", - Usage: "gosync diff ", - Description: `Compare a file with a reference index, and print statistics on the comparison and performance.`, - Action: Diff, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "p", - Value: runtime.NumCPU(), - Usage: "The number of streams to use concurrently", - }, - }, - }, - ) -} - -func Diff(c *cli.Context) { - localFilename := c.Args()[0] - referenceFilename := c.Args()[1] - startTime := time.Now() - - localFile := openFileAndHandleError(localFilename) - - if localFile == nil { - os.Exit(1) - } - - defer localFile.Close() - - var blocksize uint32 - referenceFile := openFileAndHandleError(referenceFilename) - - if referenceFile == nil { - os.Exit(1) - } - - defer referenceFile.Close() - - _, _, _, _, blocksize, e := readHeadersAndCheck( - referenceFile, - magicString, - majorVersion, - ) - - if e != nil { - fmt.Printf("Error loading index: %v", e) - os.Exit(1) - } - - fmt.Println("Blocksize: ", blocksize) - - index, _, _, err := readIndex(referenceFile, uint(blocksize)) - referenceFile.Close() - - if err != nil { - return - } - - fmt.Println("Weak hash count:", index.WeakCount()) - - fi, err := localFile.Stat() - - if err != nil { - fmt.Println("Could not get info on file:", err) - os.Exit(1) - } - - num_matchers := int64(c.Int("p")) - - localFile_size := fi.Size() - - // Don't split up small files - if localFile_size < 1024*1024 { - num_matchers = 1 - } - - merger, compare := multithreadedMatching( - localFile, - index, - localFile_size, - num_matchers, - uint(blocksize), - ) - - mergedBlocks := merger.GetMergedBlocks() - - fmt.Println("\nMatched:") - totalMatchingSize := uint64(0) - matchedBlockCountAfterMerging := uint(0) - - for _, b := range mergedBlocks { - totalMatchingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) - matchedBlockCountAfterMerging += b.EndBlock - b.StartBlock + 1 - } - - fmt.Println("Comparisons:", compare.Comparisons) - fmt.Println("Weak hash hits:", compare.WeakHashHits) - - if compare.Comparisons > 0 { - fmt.Printf( - "Weak hit rate: %.2f%%\n", - 100.0*float64(compare.WeakHashHits)/float64(compare.Comparisons), - ) - } - - fmt.Println("Strong hash hits:", compare.StrongHashHits) - if compare.WeakHashHits > 0 { - fmt.Printf( - "Weak hash error rate: %.2f%%\n", - 100.0*float64(compare.WeakHashHits-compare.StrongHashHits)/float64(compare.WeakHashHits), - ) - } - - fmt.Println("Total matched bytes:", totalMatchingSize) - fmt.Println("Total matched blocks:", matchedBlockCountAfterMerging) - - // TODO: GetMissingBlocks uses the highest index, not the count, this can be pretty confusing - // Should clean up this interface to avoid that - missing := mergedBlocks.GetMissingBlocks(uint(index.BlockCount) - 1) - fmt.Println("Index blocks:", index.BlockCount) - - totalMissingSize := uint64(0) - for _, b := range missing { - //fmt.Printf("%#v\n", b) - totalMissingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) - } - - fmt.Println("Approximate missing bytes:", totalMissingSize) - fmt.Println("Time taken:", time.Now().Sub(startTime)) -} diff --git a/src/gosync/fileformat.md b/src/gosync/fileformat.md deleted file mode 100644 index 0fd5522..0000000 --- a/src/gosync/fileformat.md +++ /dev/null @@ -1,21 +0,0 @@ -*NB: I'm documenting the current format of the gosync files merely as a point in time -reference of format that is in use in the tool that is meant to serve as a practical reference and acceptance testing tool. The gosync tool is not intended as a production-worthy, well supported, tested tool.* - -*The format used exists entirely in service of being able to test the implementation of the gosync library as a cohesive whole in the real world, and therefore backwards and forwards compatibility (or even efficiency) are not primary concerns.* - -# Version 0.2.0 -### The header -(LE = little endian) -* The string "G0S9NC" in UTF-8 -* versions*3 (eg. 0.1.2), uint16 LE -* filesize, int64 LE -* blocksize uint32 LE - -### The body -Repeating: -* WeakChecksum -* StrongChecksum - -each referring to blocks, starting at 0 (file start) and going upwards. - -In the current implementation of the FileChecksumGenerator used the WeakChecksum is the rolling checksum (4 bytes), and StrongChecksum is MD5 (16 bytes). diff --git a/src/gosync/main.go b/src/gosync/main.go deleted file mode 100644 index 3164a8d..0000000 --- a/src/gosync/main.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -gosync is a command-line implementation of the gosync package functionality, primarily as a demonstration of usage -but supposed to be functional in itself. -*/ -package main - -import ( - "fmt" - "log" - "net/http" - _ "net/http/pprof" - "os" - "runtime" - - "github.com/codegangsta/cli" -) - -const ( - DefaultBlockSize = 8192 - magicString = "G0S9NC" // just to confirm the file type is used correctly - majorVersion = uint16(0) - minorVersion = uint16(2) - patchVersion = uint16(1) -) - -var app = cli.NewApp() - -func main() { - app.Name = "gosync" - app.Usage = "Build indexes, patches, patch files" - app.Flags = []cli.Flag{ - cli.BoolFlag{ - Name: "profile", - Usage: "enable HTTP profiling", - }, - cli.IntFlag{ - Name: "profilePort", - Value: 6060, - Usage: "The number of streams to use concurrently", - }, - } - - app.Version = fmt.Sprintf( - "%v.%v.%v", - majorVersion, - minorVersion, - patchVersion, - ) - - runtime.GOMAXPROCS(runtime.NumCPU()) - - app.Before = func(c *cli.Context) error { - if c.Bool("profile") { - port := fmt.Sprint(c.Int("profilePort")) - - go func() { - log.Println(http.ListenAndServe("localhost:"+port, nil)) - }() - } - - return nil - } - - app.Run(os.Args) -} diff --git a/src/gosync/patch.go b/src/gosync/patch.go deleted file mode 100644 index 16730d3..0000000 --- a/src/gosync/patch.go +++ /dev/null @@ -1,105 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - - gosync_main "github.com/Redundancy/go-sync" - "github.com/codegangsta/cli" -) - -const usage = "gosync patch []" - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "patch", - ShortName: "p", - Usage: usage, - Description: `Recreate the reference source file, using an index and a local file that is believed to be similar. -The index should be produced by "gosync build". - - is a .gosync file and may be a local, unc network path or http/https url - is corresponding target and may be a local, unc network path or http/https url - is optional. If not specified, the local file will be overwritten when done.`, - Action: Patch, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "p", - Value: runtime.NumCPU(), - Usage: "The number of streams to use concurrently", - }, - }, - }, - ) -} - -// Patch a file -func Patch(c *cli.Context) { - errorWrapper(c, func(c *cli.Context) error { - - fmt.Fprintln(os.Stderr, "Starting patching process") - - if l := len(c.Args()); l < 3 || l > 4 { - return fmt.Errorf( - "Usage is \"%v\" (invalid number of arguments)", - usage, - ) - } - - localFilename := c.Args()[0] - summaryFile := c.Args()[1] - referencePath := c.Args()[2] - - outFilename := localFilename - if len(c.Args()) == 4 { - outFilename = c.Args()[3] - } - - indexReader, e := os.Open(summaryFile) - if e != nil { - return e - } - defer indexReader.Close() - - _, _, _, filesize, blocksize, e := readHeadersAndCheck( - indexReader, - magicString, - majorVersion, - ) - - index, checksumLookup, blockCount, err := readIndex( - indexReader, - uint(blocksize), - ) - - fs := &gosync_main.BasicSummary{ - ChecksumIndex: index, - ChecksumLookup: checksumLookup, - BlockCount: blockCount, - BlockSize: uint(blocksize), - FileSize: filesize, - } - - rsync, err := gosync_main.MakeRSync( - localFilename, - referencePath, - outFilename, - fs, - ) - - if err != nil { - return err - } - - err = rsync.Patch() - - if err != nil { - return err - } - - return rsync.Close() - }) -} diff --git a/vendor/manifest b/vendor/manifest deleted file mode 100644 index caf6313..0000000 --- a/vendor/manifest +++ /dev/null @@ -1,24 +0,0 @@ -{ - "version": 0, - "dependencies": [ - { - "importpath": "github.com/Redundancy/go-sync", - "repository": "https://github.com/Redundancy/go-sync", - "revision": "aff5b3b10f5934554a1fcfdc089ca13830ec1f35", - "branch": "master" - }, - { - "importpath": "github.com/codegangsta/cli", - "repository": "https://github.com/codegangsta/cli", - "revision": "aca5b047ed14d17224157c3434ea93bf6cdaadee", - "branch": "master" - }, - { - "importpath": "github.com/petar/GoLLRB/llrb", - "repository": "https://github.com/petar/GoLLRB", - "revision": "53be0d36a84c2a886ca057d34b6aa4468df9ccb4", - "branch": "master", - "path": "/llrb" - } - ] -} \ No newline at end of file diff --git a/vendor/src/github.com/Redundancy/go-sync/LICENSE b/vendor/src/github.com/Redundancy/go-sync/LICENSE deleted file mode 100644 index 1e4b9c6..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Daniel Speed - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/src/github.com/Redundancy/go-sync/README.md b/vendor/src/github.com/Redundancy/go-sync/README.md deleted file mode 100644 index 584e77d..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/README.md +++ /dev/null @@ -1,119 +0,0 @@ -Go-Sync ------- -[![Build Status](https://travis-ci.org/Redundancy/go-sync.svg?branch=master)](https://travis-ci.org/Redundancy/go-sync) -[![GoDoc](https://godoc.org/github.com/Redundancy/go-sync?status.svg)](https://godoc.org/github.com/Redundancy/go-sync) - -gosync is a library inspired by zsync and rsync. -Here are the goals: - -### Fast -Using the concurrency and performance features of Golang, Go-sync is designed to take advantage of multiple processors and multiple HTTP connections to make the most of modern hardware and minimize the impact of the bandwidth latency product. - -### Cross Platform -Works on Windows and Linux, without cygwin or fuss. - -### Easy - -A new high-level interface designed to reduce the work of implementing block transfer in your application: -```golang -fs := &BasicSummary{...} - -rsync, err := MakeRSync( - localFilename, - referencePath, - outFilename, - fs, -) - -if err != nil { - return err -} - -err = rsync.Patch() - -if err != nil { - return err -} - -return rsync.Close() -``` - -### Extensible -All functionality is based on interfaces, allowing customization of behavior: - -```golang -// Here, the input version is a local string -inputFile := bytes.NewReader(localVersionAsBytes) - -// And the output is a buffer -patchedFile := bytes.NewBuffer(nil) - -// This information is meta-data on the file that should be loaded / provided -// You can also provide your own implementation of the FileSummary interface -summary := &BasicSummary{ - ChecksumIndex: referenceFileIndex, - // Disable verification of hashes for downloaded data (not really a good idea!) - ChecksumLookup: nil, - BlockCount: uint(blockCount), - BlockSize: blockSize, - FileSize: int64(len(referenceAsBytes)), -} - -rsync := &RSync{ - Input: inputFile, - Output: patchedFile, - // An in-memory block source - Source: blocksources.NewReadSeekerBlockSource( - bytes.NewReader(referenceAsBytes), - blocksources.MakeNullFixedSizeResolver(uint64(blockSize)), - ), - Index: summary, - Summary: summary, - OnClose: nil, -} -``` - -Reuse low level objects to build a new high level library, or implement a new lower-level object to add a new transfer protocol (for example). - -### Tested -GoSync has been built from the ground up with unit tests. -The GoSync command-line tool has acceptance tests, although not everything is covered. - -## Current State -Go-Sync is still probably not ready for production. - -The most obvious areas that still need improvement are the acceptance tests, the error messages, -compression on the blocks that are retrieved from the source and handling of file flags. - -### TODO -- [ ] gzip source blocks (this involves writing out a version of the file that's compressed in block-increments) -- [ ] Clean up naming consistency and clarity: Block / Chunk etc -- [ ] Flesh out full directory build / sync -- [ ] Implement 'patch' payloads from a known start point to a desired end state -- [ ] Validate full file checksum after patching -- [ ] Provide bandwidth limiting / monitoring as part of http blocksource -- [ ] Think about turning the filechecksum into an interface -- [ ] Avoid marshalling / un-marshalling blocks during checksum generation -- [ ] Sequential patcher to resume after error? - -### Testing - -All tests are run by Travis-CI - -#### Unit tests - - go test github.com/Redundancy/go-sync/... - -#### Acceptance Tests -See the "acceptancetests" folder. This is currently difficult to run locally and relies on several linux utilities. - -#### Commandline & files - - go build github.com/Redundancy/go-sync/gosync - gosync build filenameToPatchTo - gosync patch filenameToPatchFrom filenameToPatchTo.gosync filenameToPatchTo - -Note that normally, patching would rely on a remote http/https file source. - -#### Command line tool reference - gosync --help diff --git a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh b/vendor/src/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh deleted file mode 100644 index 8d8b5a7..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh +++ /dev/null @@ -1 +0,0 @@ -echo "ok!" \ No newline at end of file diff --git a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/run.sh b/vendor/src/github.com/Redundancy/go-sync/acceptancetests/run.sh deleted file mode 100644 index f0e7ca3..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -counter=1 -for f in $TRAVIS_BUILD_DIR/acceptancetests/*test.sh -do - [ ! -f "$f" ] && continue - - echo "Running acceptance test $f" - echo 'travis_fold:start:test_output$counter' - sh $f - rc=$? - echo 'travis_fold:end:test_output$counter' - - if [ $rc != 0 ]; then - echo "Test Failed" - exit $rc - fi - - echo "Test Passed" - counter=`expr $counter + 1` -done - - diff --git a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh b/vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh deleted file mode 100644 index 9f1ce7a..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh +++ /dev/null @@ -1,14 +0,0 @@ -echo "Testing filling in a truncated file" -wget -q https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -O test.exe -cp test.exe compare.exe -gosync b test.exe -truncate -s 5000000 test.exe -gosync p test.exe test.gosync https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -diff -q test.exe compare.exe -rc=$? -if [ $rc != 0 ]; then - gosync -version - ls -l compare.exe - ls -l test.exe - exit $rc -fi diff --git a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh b/vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh deleted file mode 100644 index f1903cb..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh +++ /dev/null @@ -1,14 +0,0 @@ -echo "Testing truncating a file too long" -wget -q https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -O test.exe -cp test.exe compare.exe -gosync b test.exe -truncate -s 10000000 test.exe -gosync p test.exe test.gosync https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -diff -q test.exe compare.exe -rc=$? -if [ $rc != 0 ]; then - gosync -version - ls -l compare.exe - ls -l test.exe - exit $rc -fi \ No newline at end of file diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go deleted file mode 100644 index 95e236a..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go +++ /dev/null @@ -1,271 +0,0 @@ -package blocksources - -import ( - "errors" - "fmt" - "sort" - - "github.com/Redundancy/go-sync/patcher" -) - -// BlockSourceRequester does synchronous requests on a remote source of blocks -// concurrency is handled by the BlockSourceBase. This provides a simple way -// of implementing a particular -type BlockSourceRequester interface { - // This method is called on multiple goroutines, and must - // support simultaneous requests - DoRequest(startOffset int64, endOffset int64) (data []byte, err error) - - // If an error raised by DoRequest should cause BlockSourceBase - // to give up, return true - IsFatal(err error) bool -} - -// A BlockSourceOffsetResolver resolves a blockID to a start offset and and end offset in a file -// it also handles splitting up ranges of blocks into multiple requests, allowing requests to be split down to the -// block size, and handling of compressed blocks (given a resolver that can work out the correct range to query for, -// and a BlockSourceRequester that will decompress the result into a full sized block) -type BlockSourceOffsetResolver interface { - GetBlockStartOffset(blockID uint) int64 - GetBlockEndOffset(blockID uint) int64 - SplitBlockRangeToDesiredSize(startBlockID, endBlockID uint) []QueuedRequest -} - -// Checks blocks against their expected checksum -type BlockVerifier interface { - VerifyBlockRange(startBlockID uint, data []byte) bool -} - -func NewBlockSourceBase( - requester BlockSourceRequester, - resolver BlockSourceOffsetResolver, - verifier BlockVerifier, - concurrentRequestCount int, - concurrentBytes int64, -) *BlockSourceBase { - - b := &BlockSourceBase{ - Requester: requester, - BlockSourceResolver: resolver, - Verifier: verifier, - ConcurrentRequests: concurrentRequestCount, - ConcurrentBytes: concurrentBytes, - exitChannel: make(chan bool), - errorChannel: make(chan error), - responseChannel: make(chan patcher.BlockReponse), - requestChannel: make(chan patcher.MissingBlockSpan), - } - - go b.loop() - - return b -} - -// BlockSourceBase provides an implementation of blocksource -// that takes care of everything except for the actual asyncronous request -// this makes blocksources easier and faster to build reliably -// BlockSourceBase implements patcher.BlockSource, and if it's good enough, -// perhaps nobody else ever will have to. -type BlockSourceBase struct { - Requester BlockSourceRequester - BlockSourceResolver BlockSourceOffsetResolver - Verifier BlockVerifier - - // The number of requests that BlockSourceBase may service at once - ConcurrentRequests int - - // The number of bytes that BlockSourceBase may have in-flight - // (requested + pending delivery) - ConcurrentBytes int64 - - hasQuit bool - exitChannel chan bool - errorChannel chan error - responseChannel chan patcher.BlockReponse - requestChannel chan patcher.MissingBlockSpan - - bytesRequested int64 -} - -const ( - STATE_RUNNING = iota - STATE_EXITING -) - -func (s *BlockSourceBase) ReadBytes() int64 { - return s.bytesRequested -} - -func (s *BlockSourceBase) RequestBlocks(block patcher.MissingBlockSpan) error { - s.requestChannel <- block - return nil -} - -func (s *BlockSourceBase) GetResultChannel() <-chan patcher.BlockReponse { - return s.responseChannel -} - -// If the block source encounters an unsurmountable problem -func (s *BlockSourceBase) EncounteredError() <-chan error { - return s.errorChannel -} - -var BlockSourceAlreadyClosedError = errors.New("Block source was already closed") - -func (s *BlockSourceBase) Close() (err error) { - // if it has already been closed, just recover - // however, let the caller know - defer func() { - if recover() != nil { - err = BlockSourceAlreadyClosedError - } - }() - - if !s.hasQuit { - s.exitChannel <- true - } - - return -} - -func (s *BlockSourceBase) loop() { - defer func() { - s.hasQuit = true - close(s.exitChannel) - close(s.errorChannel) - close(s.requestChannel) - close(s.responseChannel) - }() - - state := STATE_RUNNING - inflightRequests := 0 - //inflightBytes := int64(0) - pendingErrors := &errorWatcher{errorChannel: s.errorChannel} - pendingResponse := &pendingResponseHelper{responseChannel: s.responseChannel} - resultChan := make(chan asyncResult) - defer close(resultChan) - - requestQueue := make(QueuedRequestList, 0, s.ConcurrentRequests*2) - - // enable us to order responses for the active requests, lowest to highest - requestOrdering := make(UintSlice, 0, s.ConcurrentRequests) - responseOrdering := make(PendingResponses, 0, s.ConcurrentRequests) - - for state == STATE_RUNNING || inflightRequests > 0 || pendingErrors.Err() != nil { - - // Start any pending work that we can - for inflightRequests < s.ConcurrentRequests && len(requestQueue) > 0 { - inflightRequests += 1 - - nextRequest := requestQueue[len(requestQueue)-1] - - requestOrdering = append(requestOrdering, nextRequest.StartBlockID) - sort.Sort(sort.Reverse(requestOrdering)) - go func() { - resolver := s.BlockSourceResolver - - startOffset := resolver.GetBlockStartOffset( - nextRequest.StartBlockID, - ) - - endOffset := resolver.GetBlockEndOffset( - nextRequest.EndBlockID, - ) - - result, err := s.Requester.DoRequest( - startOffset, - endOffset, - ) - - resultChan <- asyncResult{ - startBlockID: nextRequest.StartBlockID, - endBlockID: nextRequest.EndBlockID, - data: result, - err: err, - } - }() - - // remove dispatched request - requestQueue = requestQueue[:len(requestQueue)-1] - } - - select { - case newRequest := <-s.requestChannel: - requestQueue = append( - requestQueue, - s.BlockSourceResolver.SplitBlockRangeToDesiredSize( - newRequest.StartBlock, - newRequest.EndBlock, - )..., - ) - - sort.Sort(sort.Reverse(requestQueue)) - - case result := <-resultChan: - inflightRequests -= 1 - - if result.err != nil { - pendingErrors.setError(result.err) - pendingResponse.clear() - state = STATE_EXITING - break - } - - s.bytesRequested += int64(len(result.data)) - - if s.Verifier != nil && !s.Verifier.VerifyBlockRange(result.startBlockID, result.data) { - pendingErrors.setError( - fmt.Errorf( - "The returned block range (%v-%v) did not match the expected checksum for the blocks", - result.startBlockID, result.endBlockID, - ), - ) - pendingResponse.clear() - state = STATE_EXITING - break - } - - responseOrdering = append(responseOrdering, - patcher.BlockReponse{ - StartBlock: result.startBlockID, - Data: result.data, - }, - ) - - // sort high to low - sort.Sort(sort.Reverse(responseOrdering)) - - // if we just got the lowest requested block, we can set - // the response. Otherwise, wait. - lowestRequest := requestOrdering[len(requestOrdering)-1] - - if lowestRequest == result.startBlockID { - lowestResponse := responseOrdering[len(responseOrdering)-1] - pendingResponse.clear() - pendingResponse.setResponse(&lowestResponse) - } - - case pendingResponse.sendIfPending() <- pendingResponse.Response(): - pendingResponse.clear() - responseOrdering = responseOrdering[:len(responseOrdering)-1] - requestOrdering = requestOrdering[:len(requestOrdering)-1] - - // check if there's another response to enqueue - if len(responseOrdering) > 0 { - lowestResponse := responseOrdering[len(responseOrdering)-1] - lowestRequest := requestOrdering[len(requestOrdering)-1] - - if lowestRequest == lowestResponse.StartBlock { - pendingResponse.setResponse(&lowestResponse) - - } - } - - case pendingErrors.sendIfSet() <- pendingErrors.Err(): - pendingErrors.clear() - - case <-s.exitChannel: - state = STATE_EXITING - } - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go deleted file mode 100644 index 1aa91b9..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go +++ /dev/null @@ -1,303 +0,0 @@ -package blocksources - -import ( - "bytes" - "github.com/Redundancy/go-sync/patcher" - - //"runtime" - "testing" - "time" -) - -//----------------------------------------------------------------------------- -type erroringRequester struct{} -type testError struct{} - -func (e *testError) Error() string { - return "test" -} - -func (e *erroringRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - return nil, &testError{} -} - -func (e *erroringRequester) IsFatal(err error) bool { - return true -} - -//----------------------------------------------------------------------------- -type FunctionRequester func(a, b int64) ([]byte, error) - -func (f FunctionRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - return f(startOffset, endOffset) -} - -func (f FunctionRequester) IsFatal(err error) bool { - return true -} - -//----------------------------------------------------------------------------- - -func init() { - //if runtime.GOMAXPROCS(0) == 1 { - //runtime.GOMAXPROCS(4) - //} -} - -func TestRangeSlice(t *testing.T) { - a := []int{0, 1, 2, 3, 4} - b := a[:len(a)-1] - - if len(b) != len(a)-1 { - t.Errorf("b is wrong length, only supposed to remove one item: %v %v", a, b) - } -} - -func TestCreateAndCloseBlockSourceBase(t *testing.T) { - b := NewBlockSourceBase(nil, nil, nil, 1, 1024) - b.Close() - - // TODO: Race condition here. Can Close() block? - if !b.hasQuit { - t.Fatal("Block source base did not exit") - } -} - -func TestErrorWatcher(t *testing.T) { - e := errorWatcher{errorChannel: make(chan error)} - - if e.sendIfSet() != nil { - t.Errorf("Channel should be nil when created") - } - - e.setError(&testError{}) - - if e.sendIfSet() == nil { - t.Errorf("Channel should be non-nil when error is set") - } - if e.Err() == nil { - t.Errorf("Error should not be nil when set") - } -} - -func TestBlockSourceBaseError(t *testing.T) { - b := NewBlockSourceBase( - &erroringRequester{}, - MakeNullFixedSizeResolver(4), - nil, - 1, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 1, - }) - - select { - case <-time.After(time.Second): - t.Fatal("Timed out waiting for error") - case <-b.EncounteredError(): - } - -} - -func TestBlockSourceRequest(t *testing.T) { - expected := []byte("test") - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - return expected, nil - }), - MakeNullFixedSizeResolver(4), - nil, - 1, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 1, - }) - - result := <-b.GetResultChannel() - - if result.StartBlock != 1 { - t.Errorf("Unexpected start block in result: %v", result.StartBlock) - } - if bytes.Compare(result.Data, expected) != 0 { - t.Errorf("Unexpected data in result: %v", result.Data) - } -} - -func TestConcurrentBlockRequests(t *testing.T) { - content := []byte("test") - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - return content[start:end], nil - }), - MakeNullFixedSizeResolver(2), - nil, - 2, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 2, - StartBlock: 0, - EndBlock: 0, - }) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 2, - StartBlock: 1, - EndBlock: 1, - }) - - for i := uint(0); i < 2; i++ { - select { - case r := <-b.GetResultChannel(): - if r.StartBlock != i { - t.Errorf("Wrong start block: %v", r.StartBlock) - } - if bytes.Compare(r.Data, content[i*2:(i+1)*2]) != 0 { - t.Errorf("Unexpected result content for result %v: %v", i+1, string(r.Data)) - } - case <-time.After(time.Second): - t.Fatal("Timed out on request", i+1) - } - } -} - -func TestOutOfOrderRequestCompletion(t *testing.T) { - content := []byte("test") - - channeler := []chan bool{ - make(chan bool), - make(chan bool), - } - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - // read from the channel based on the start - <-(channeler[start]) - return content[start:end], nil - }), - MakeNullFixedSizeResolver(1), - nil, - 2, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 1, - StartBlock: 0, - EndBlock: 0, - }) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 1, - StartBlock: 1, - EndBlock: 1, - }) - - // finish the second request - channeler[1] <- true - - select { - case <-b.GetResultChannel(): - t.Error("Should not deliver any blocks yet") - case <-time.After(time.Second): - } - - // once the first block completes, we're ready to send both - channeler[0] <- true - - for i := uint(0); i < 2; i++ { - select { - case r := <-b.GetResultChannel(): - if r.StartBlock != i { - t.Errorf( - "Wrong start block: %v on result %v", - r.StartBlock, - i+1, - ) - } - case <-time.After(time.Second): - t.Fatal("Timed out on request", i+1) - } - } -} - -func TestRequestCountLimiting(t *testing.T) { - counter := make(chan int) - waiter := make(chan bool) - const ( - MAX_CONCURRENCY = 2 - REQUESTS = 4 - ) - call_counter := 0 - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - counter <- 1 - call_counter += 1 - <-waiter - counter <- -1 - return []byte{0, 0}, nil - }), - MakeNullFixedSizeResolver(1), - nil, - MAX_CONCURRENCY, - 1024, - ) - defer b.Close() - - count := 0 - max := 0 - - go func() { - for { - change, ok := <-counter - - if !ok { - break - } - - count += change - - if count > max { - max = count - } - } - }() - - for i := 0; i < REQUESTS; i++ { - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 1, - StartBlock: uint(i), - EndBlock: uint(i), - }) - } - - for i := 0; i < REQUESTS; i++ { - waiter <- true - } - - close(counter) - close(waiter) - - if max > MAX_CONCURRENCY { - t.Errorf("Maximum requests in flight was greater than the requested concurrency: %v", max) - } - if call_counter != REQUESTS { - t.Errorf("Total number of requests is not expected: %v", call_counter) - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go deleted file mode 100644 index d81573e..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go +++ /dev/null @@ -1,73 +0,0 @@ -package blocksources - -type FixedSizeBlockResolver struct { - BlockSize uint64 - FileSize int64 - MaxDesiredRequestSize uint64 -} - -func (r *FixedSizeBlockResolver) GetBlockStartOffset(blockID uint) int64 { - if off := int64(uint64(blockID) * r.BlockSize); r.FileSize != 0 && off > r.FileSize { - return r.FileSize - } else { - return off - } -} - -func (r *FixedSizeBlockResolver) GetBlockEndOffset(blockID uint) int64 { - if off := int64(uint64(blockID+1) * r.BlockSize); r.FileSize != 0 && off > r.FileSize { - return r.FileSize - } else { - return off - } -} - -// Split blocks into chunks of the desired size, or less. This implementation assumes a fixed block size at the source. -func (r *FixedSizeBlockResolver) SplitBlockRangeToDesiredSize(startBlockID, endBlockID uint) []QueuedRequest { - - if r.MaxDesiredRequestSize == 0 { - return []QueuedRequest{ - QueuedRequest{ - StartBlockID: startBlockID, - EndBlockID: endBlockID, - }, - } - } - - maxSize := r.MaxDesiredRequestSize - if r.MaxDesiredRequestSize < r.BlockSize { - maxSize = r.BlockSize - } - - // how many blocks is the desired size? - blockCountPerRequest := uint(maxSize / r.BlockSize) - - requests := make([]QueuedRequest, 0, (endBlockID-startBlockID)/blockCountPerRequest+1) - currentBlockID := startBlockID - - for { - maxEndBlock := currentBlockID + blockCountPerRequest - - if maxEndBlock > endBlockID { - requests = append( - requests, - QueuedRequest{ - StartBlockID: currentBlockID, - EndBlockID: endBlockID, - }, - ) - - return requests - } else { - requests = append( - requests, - QueuedRequest{ - StartBlockID: currentBlockID, - EndBlockID: maxEndBlock - 1, - }, - ) - - currentBlockID = maxEndBlock - } - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go deleted file mode 100644 index f484bc5..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package blocksources - -import ( - "testing" -) - -func TestNullResolverGivesBackTheSameBlocks(t *testing.T) { - n := MakeNullFixedSizeResolver(5) - result := n.SplitBlockRangeToDesiredSize(0, 10000) - - if len(result) != 1 { - t.Fatalf("Unexpected result length (expected 1): %v", result) - } - - r := result[0] - - if r.StartBlockID != 0 { - t.Errorf("Unexpected start block ID: %v", r) - } - - if r.EndBlockID != 10000 { - t.Errorf("Unexpected end block ID: %v", r) - } -} - -func TestFixedSizeResolverSplitsBlocksOfDesiredSize(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 5, - FileSize: 20000, - } - - // Should split two blocks, each of the desired request size - // into two requests - result := res.SplitBlockRangeToDesiredSize(0, 1) - - if len(result) != 2 { - t.Fatalf("Unexpected result length (expected 2): %v", result) - } - - if result[0].StartBlockID != 0 { - t.Errorf("Unexpected start blockID: %v", result[0]) - } - if result[0].EndBlockID != 0 { - t.Errorf("Unexpected end blockID: %v", result[0]) - } - - if result[1].StartBlockID != 1 { - t.Errorf("Unexpected start blockID: %v", result[1]) - } - if result[1].EndBlockID != 1 { - t.Errorf("Unexpected end blockID: %v", result[1]) - } -} - -func TestThatMultipleBlocksAreSplitByRoundingDown(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 12, - FileSize: 20000, - } - - // 0,1 (10) - 2-3 (10) - result := res.SplitBlockRangeToDesiredSize(0, 3) - - if len(result) != 2 { - t.Fatalf("Unexpected result length (expected 2): %v", result) - } - - if result[0].StartBlockID != 0 { - t.Errorf("Unexpected start blockID: %v", result[0]) - } - if result[0].EndBlockID != 1 { - t.Errorf("Unexpected end blockID: %v", result[0]) - } - - if result[1].StartBlockID != 2 { - t.Errorf("Unexpected start blockID: %v", result[1]) - } - if result[1].EndBlockID != 3 { - t.Errorf("Unexpected end blockID: %v", result[1]) - } -} - -func TestThatADesiredSizeSmallerThanABlockResultsInSingleBlocks(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 4, - FileSize: 20000, - } - - // Should split two blocks - result := res.SplitBlockRangeToDesiredSize(0, 1) - - if len(result) != 2 { - t.Fatalf("Unexpected result length (expected 2): %v", result) - } - - if result[0].StartBlockID != 0 { - t.Errorf("Unexpected start blockID: %v", result[0]) - } - if result[0].EndBlockID != 0 { - t.Errorf("Unexpected end blockID: %v", result[0]) - } - - if result[1].StartBlockID != 1 { - t.Errorf("Unexpected start blockID: %v", result[1]) - } - if result[1].EndBlockID != 1 { - t.Errorf("Unexpected end blockID: %v", result[1]) - } -} - -func TestThatFileSizeTruncatesBlockEnds(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 100, - FileSize: 13, - } - - // Should split two blocks - result := res.GetBlockEndOffset(3) - - if result != 13 { - t.Errorf("Unexpected BlockEnd Offset:", result) - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/helpers.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/helpers.go deleted file mode 100644 index 204c484..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/helpers.go +++ /dev/null @@ -1,124 +0,0 @@ -package blocksources - -import ( - "fmt" - "github.com/Redundancy/go-sync/patcher" -) - -// errorWatcher is a small helper object -// sendIfSet will only return a channel if there is an error set -// so w.sendIfSet() <- w.Err() is always safe in a select statement -// even if there is no error set -type errorWatcher struct { - errorChannel chan error - lastError error -} - -func (w *errorWatcher) setError(e error) { - if w.lastError != nil { - panic("cannot set a new error when one is already set!") - } - w.lastError = e -} - -func (w *errorWatcher) clear() { - w.lastError = nil -} - -func (w *errorWatcher) Err() error { - return w.lastError -} - -func (w *errorWatcher) sendIfSet() chan<- error { - if w.lastError != nil { - return w.errorChannel - } else { - return nil - } -} - -type pendingResponseHelper struct { - responseChannel chan patcher.BlockReponse - pendingResponse *patcher.BlockReponse -} - -func (w *pendingResponseHelper) setResponse(r *patcher.BlockReponse) { - if w.pendingResponse != nil { - p := fmt.Sprintf("Setting a response when one is already set! Had startblock %v, got %v", r.StartBlock, w.pendingResponse.StartBlock) - panic(p) - } - w.pendingResponse = r -} - -func (w *pendingResponseHelper) clear() { - w.pendingResponse = nil -} - -func (w *pendingResponseHelper) Response() patcher.BlockReponse { - if w.pendingResponse == nil { - return patcher.BlockReponse{} - } - return *w.pendingResponse -} - -func (w *pendingResponseHelper) sendIfPending() chan<- patcher.BlockReponse { - if w.pendingResponse != nil { - return w.responseChannel - } else { - return nil - } - -} - -type UintSlice []uint - -func (r UintSlice) Len() int { - return len(r) -} - -func (r UintSlice) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r UintSlice) Less(i, j int) bool { - return r[i] < r[j] -} - -type asyncResult struct { - startBlockID uint - endBlockID uint - data []byte - err error -} - -type QueuedRequest struct { - StartBlockID uint - EndBlockID uint -} - -type QueuedRequestList []QueuedRequest - -func (r QueuedRequestList) Len() int { - return len(r) -} - -func (r QueuedRequestList) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r QueuedRequestList) Less(i, j int) bool { - return r[i].StartBlockID < r[j].StartBlockID -} - -func MakeNullFixedSizeResolver(blockSize uint64) BlockSourceOffsetResolver { - return &FixedSizeBlockResolver{ - BlockSize: blockSize, - } -} - -func MakeFileSizedBlockResolver(blockSize uint64, filesize int64) BlockSourceOffsetResolver { - return &FixedSizeBlockResolver{ - BlockSize: blockSize, - FileSize: filesize, - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource.go deleted file mode 100644 index 0e93278..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource.go +++ /dev/null @@ -1,110 +0,0 @@ -package blocksources - -import ( - "bytes" - "errors" - "fmt" - "net/http" - "strings" -) - -const MB = 1024 * 1024 - -var RangedRequestNotSupportedError = errors.New("Ranged request not supported (Server did not respond with 206 Status)") -var ResponseFromServerWasGZiped = errors.New("HTTP response was gzip encoded. Ranges may not match those requested.") - -var ClientNoCompression = &http.Client{ - Transport: &http.Transport{}, -} - -func NewHttpBlockSource( - url string, - concurrentRequests int, - resolver BlockSourceOffsetResolver, - verifier BlockVerifier, -) *BlockSourceBase { - return NewBlockSourceBase( - &HttpRequester{ - url: url, - client: http.DefaultClient, - }, - resolver, - verifier, - concurrentRequests, - 4*MB, - ) -} - -type URLNotFoundError string - -func (url URLNotFoundError) Error() string { - return "404 Error on URL: " + string(url) -} - -// This class provides the implementation of BlockSourceRequester for BlockSourceBase -// this simplifies creating new BlockSources that satisfy the requirements down to -// writing a request function -type HttpRequester struct { - client *http.Client - url string -} - -func (r *HttpRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - rangedRequest, err := http.NewRequest("GET", r.url, nil) - - if err != nil { - return nil, fmt.Errorf("Error creating request for \"%v\": %v", r.url, err) - } - - rangeSpecifier := fmt.Sprintf("bytes=%v-%v", startOffset, endOffset-1) - rangedRequest.ProtoAtLeast(1, 1) - rangedRequest.Header.Add("Range", rangeSpecifier) - rangedRequest.Header.Add("Accept-Encoding", "identity") - rangedResponse, err := r.client.Do(rangedRequest) - - if err != nil { - return nil, fmt.Errorf("Error executing request for \"%v\": %v", r.url, err) - } - - defer rangedResponse.Body.Close() - - if rangedResponse.StatusCode == 404 { - return nil, URLNotFoundError(r.url) - } else if rangedResponse.StatusCode != 206 { - return nil, RangedRequestNotSupportedError - } else if strings.Contains( - rangedResponse.Header.Get("Content-Encoding"), - "gzip", - ) { - return nil, ResponseFromServerWasGZiped - } else { - buf := bytes.NewBuffer(make([]byte, 0, endOffset-startOffset)) - _, err = buf.ReadFrom(rangedResponse.Body) - - if err != nil { - err = fmt.Errorf( - "Failed to read response body for %v (%v-%v): %v", - r.url, - startOffset, endOffset-1, - err, - ) - } - - data = buf.Bytes() - - if int64(len(data)) != endOffset-startOffset { - err = fmt.Errorf( - "Unexpected response length %v (%v): %v", - r.url, - endOffset-startOffset+1, - len(data), - ) - } - - return - } -} - -func (r *HttpRequester) IsFatal(err error) bool { - return true -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go deleted file mode 100644 index d7a411c..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package blocksources - -import ( - "bytes" - "crypto/md5" - "fmt" - "net" - "net/http" - "testing" - "time" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/patcher" -) - -var PORT = 8000 - -var TEST_CONTENT = []byte("This is test content used for evaluation of the unit tests") -var content = bytes.NewReader(TEST_CONTENT) -var LOCAL_URL = "" - -func handler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), content) -} - -var PARTIAL_CONTENT = []byte("abcdef") -var partialContent = bytes.NewReader(PARTIAL_CONTENT) - -func partialContentHandler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), partialContent) -} - -var CORRUPT_CONTENT = []byte("sfdfsfhhrtertert sffsfsdfsdfsdf") -var corruptContent = bytes.NewReader(CORRUPT_CONTENT) - -func corruptContentHandler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), corruptContent) -} - -// set up a http server locally that will respond predictably to ranged requests -// NB: Doing this will prevent deadlocks from being caught! -func init() { - s := http.NewServeMux() - s.HandleFunc("/", handler) - s.HandleFunc("/partial", partialContentHandler) - s.HandleFunc("/corrupt", corruptContentHandler) - s.Handle("/404", http.NotFoundHandler()) - - portChan := make(chan int) - - go func() { - var listener net.Listener - var err error - - for { - PORT++ - p := fmt.Sprintf(":%v", PORT) - listener, err = net.Listen("tcp", p) - - if err == nil { - break - } - } - portChan <- PORT - http.Serve(listener, s) - }() - - p := fmt.Sprintf(":%v", <-portChan) - LOCAL_URL = "http://localhost" + p - -} - -func TestHandler(t *testing.T) { - resp, err := http.Get(LOCAL_URL) - - if err != nil { - t.Fatal(err) - } - - if resp.StatusCode != 200 { - t.Fatal(resp.Status) - } -} - -func TestHttpBlockSource(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/", - 2, - MakeNullFixedSizeResolver(4), - nil, - ) - - err := b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 0, - EndBlock: 0, - }) - - if err != nil { - t.Fatal(err) - } - - results := b.GetResultChannel() - - select { - case r := <-results: - if bytes.Compare(r.Data, TEST_CONTENT[:4]) != 0 { - t.Errorf("Data differed from expected content: \"%v\"", string(r.Data)) - } - case e := <-b.EncounteredError(): - t.Fatal(e) - case <-time.After(time.Second): - t.Fatal("Waited a second for the response, timeout.") - } -} - -func TestHttpBlockSource404(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/404", - 2, - MakeNullFixedSizeResolver(4), - nil, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 0, - EndBlock: 0, - }) - - results := b.GetResultChannel() - - select { - case <-results: - t.Fatal("Should not have gotten a result") - case e := <-b.EncounteredError(): - if e == nil { - t.Fatal("Error was nil!") - } else if _, ok := e.(URLNotFoundError); !ok { - t.Errorf("Unexpected error type: %v", e) - } - case <-time.After(time.Second): - t.Fatal("Waited a second for the response, timeout.") - } -} - -func TestHttpBlockSourceOffsetBlockRequest(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/", - 2, - MakeNullFixedSizeResolver(4), - nil, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 3, - }) - - select { - case result := <-b.GetResultChannel(): - if result.StartBlock != 1 { - t.Errorf( - "Unexpected result start block: %v", - result.StartBlock, - ) - } - case <-time.After(time.Second): - t.Fatalf("Timeout waiting for result") - } -} - -func TestHttpBlockSourcePartialContentRequest(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/partial", - 2, - MakeFileSizedBlockResolver(4, int64(len(PARTIAL_CONTENT))), - nil, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 1, - }) - - select { - case result := <-b.GetResultChannel(): - if result.StartBlock != 1 { - t.Errorf( - "Unexpected result start block: %v", - result.StartBlock, - ) - } - if len(result.Data) != 2 { - t.Errorf( - "Unexpected data length: \"%v\"", - string(result.Data), - ) - } - if string(result.Data) != "ef" { - t.Errorf( - "Unexpected result \"%v\"", - string(result.Data), - ) - } - case err := <-b.EncounteredError(): - t.Fatal(err) - case <-time.After(time.Second): - t.Fatalf("Timeout waiting for result") - } -} - -type SingleBlockSource []byte - -func (d SingleBlockSource) GetStrongChecksumForBlock(blockID int) []byte { - m := md5.New() - return m.Sum(d) -} - -func TestHttpBlockSourceVerification(t *testing.T) { - const BLOCK_SIZE = 4 - - b := NewHttpBlockSource( - LOCAL_URL+"/corrupt", - 2, - MakeNullFixedSizeResolver(BLOCK_SIZE), - &filechecksum.HashVerifier{ - Hash: md5.New(), - BlockSize: BLOCK_SIZE, - BlockChecksumGetter: SingleBlockSource(TEST_CONTENT[0:BLOCK_SIZE]), - }, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: BLOCK_SIZE, - StartBlock: 0, - EndBlock: 0, - }) - - select { - case result := <-b.GetResultChannel(): - t.Fatalf("Should have thrown an error, got %v", result) - case e := <-b.EncounteredError(): - t.Logf("Encountered expected error: %v", e) - return - case <-time.After(time.Second): - t.Fatalf("Timeout waiting for result") - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go deleted file mode 100644 index 67958f6..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go +++ /dev/null @@ -1,54 +0,0 @@ -package blocksources - -import ( - "io" -) - -const ( - from_start = 0 -) - -type ReadSeeker interface { - Read(b []byte) (n int, err error) - Seek(offset int64, whence int) (int64, error) -} - -func NewReadSeekerBlockSource( - r ReadSeeker, - resolver BlockSourceOffsetResolver, -) *BlockSourceBase { - return NewBlockSourceBase( - &ReadSeekerRequester{ - rs: r, - }, - resolver, - nil, // TODO: No verifier! - 1, - 8*MB, - ) -} - -type ReadSeekerRequester struct { - rs ReadSeeker -} - -func (r *ReadSeekerRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - read_length := endOffset - startOffset - buffer := make([]byte, read_length) - - if _, err = r.rs.Seek(startOffset, from_start); err != nil { - return - } - - n, err := io.ReadFull(r.rs, buffer) - - if err != nil && err != io.ErrUnexpectedEOF { - return - } - - return buffer[:n], nil -} - -func (r *ReadSeekerRequester) IsFatal(err error) bool { - return true -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go deleted file mode 100644 index 37d1970..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package blocksources - -import ( - "bytes" - "github.com/Redundancy/go-sync/patcher" - "testing" -) - -const STRING_DATA = "abcdefghijklmnopqrst" - -var BYTE_BLOCK_SOURCE = NewReadSeekerBlockSource( - bytes.NewReader( - []byte(STRING_DATA), - ), - MakeNullFixedSizeResolver(4), -) - -func TestReadFirstBlock(t *testing.T) { - BYTE_BLOCK_SOURCE.RequestBlocks( - patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 0, - EndBlock: 0, - }, - ) - - result := <-BYTE_BLOCK_SOURCE.GetResultChannel() - - if result.StartBlock != 0 { - t.Errorf("Wrong start block: %v", result.StartBlock) - } - - EXPECTED := STRING_DATA[:4] - - if bytes.Compare(result.Data, []byte(EXPECTED)) != 0 { - t.Errorf( - "Unexpected result data: \"%v\" expected: \"%v\"", - string(result.Data), - EXPECTED, - ) - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go b/vendor/src/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go deleted file mode 100644 index 551d75f..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go +++ /dev/null @@ -1,19 +0,0 @@ -package blocksources - -import ( - "github.com/Redundancy/go-sync/patcher" -) - -type PendingResponses []patcher.BlockReponse - -func (r PendingResponses) Len() int { - return len(r) -} - -func (r PendingResponses) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r PendingResponses) Less(i, j int) bool { - return r[i].StartBlock < r[j].StartBlock -} diff --git a/vendor/src/github.com/Redundancy/go-sync/chunks/chunks.go b/vendor/src/github.com/Redundancy/go-sync/chunks/chunks.go deleted file mode 100644 index aac4143..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/chunks/chunks.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Package chunks provides the basic structure for a pair of the weak and strong checksums. -Since this is fairly widely used, splitting this out breaks a number of possible circular dependencies -*/ -package chunks - -import ( - "bytes" - "errors" - "io" -) - -// For a given Block, the Weak & Strong hashes, and the offset -// This structure is only used to generate the index of reference files, since -// computing the strong checksum is not done when comparing unless the weak checksum matches -type ChunkChecksum struct { - // an offset in terms of chunk count - ChunkOffset uint - // the size of the block - Size int64 - WeakChecksum []byte - StrongChecksum []byte -} - -// compares a checksum to another based on the checksums, not the offset -func (chunk ChunkChecksum) Match(other ChunkChecksum) bool { - weakEqual := bytes.Compare(chunk.WeakChecksum, other.WeakChecksum) == 0 - strongEqual := false - if weakEqual { - strongEqual = bytes.Compare(chunk.StrongChecksum, other.StrongChecksum) == 0 - } - return weakEqual && strongEqual -} - -var ErrPartialChecksum = errors.New("Reader length was not a multiple of the checksums") - -// Loads chunks from a reader, assuming alternating weak then strong hashes -func LoadChecksumsFromReader( - r io.Reader, - weakHashSize int, - strongHashSize int, -) ([]ChunkChecksum, error) { - - result := make([]ChunkChecksum, 0, 20) - offset := uint(0) - - temp := ChunkChecksum{} - - for { - weakBuffer := make([]byte, weakHashSize) - n, err := io.ReadFull(r, weakBuffer) - - if n == weakHashSize { - temp.ChunkOffset = offset - temp.WeakChecksum = weakBuffer - } else if n == 0 && err == io.EOF { - break - } else { - return nil, ErrPartialChecksum - } - - strongBuffer := make([]byte, strongHashSize) - n, err = io.ReadFull(r, strongBuffer) - - if n == strongHashSize { - temp.StrongChecksum = strongBuffer - result = append(result, temp) - - if err == io.EOF { - break - } - } else { - return nil, ErrPartialChecksum - } - - offset += 1 - } - - return result, nil -} - -// satisfies filechecksum.ChecksumLookup -type StrongChecksumGetter []ChunkChecksum - -func (s StrongChecksumGetter) GetStrongChecksumForBlock(blockID int) []byte { - return s[blockID].StrongChecksum -} diff --git a/vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc.go b/vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc.go deleted file mode 100644 index 0449739..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc.go +++ /dev/null @@ -1,170 +0,0 @@ -package circularbuffer - -/* -C2 is a circular buffer intended to allow you to write a block of data of up to 'blocksize', and retrieve the -data evicted by that operation, without allocating any extra slice storage - -This requires that it keep at least blocksize*2 data around. In fact, it doubles that again in order to -guarantee that both of these bits of information can always be obtained in a single contiguous block of memory. - -Other than the cost of the extra memory (4xblocksize), this means that it requires 2 writes for every byte stored. -*/ -type C2 struct { - // used to know how much was evicted - lastWritten int - - // total number of written bytes - // used to track if the buffer has been filled, but goes above blocksize - totalWritten int - - // quick access to the circular buffer size - blocksize int - - // double internal buffer storage - a, b doubleSizeBuffer -} - -type doubleSizeBuffer struct { - // used to reset the head pointer - baseOffset int - - // index of the next byte to be written - head int - - // buffer - buffer []byte -} - -func MakeC2Buffer(blockSize int) *C2 { - return &C2{ - blocksize: blockSize, - a: doubleSizeBuffer{ - baseOffset: 0, - buffer: make([]byte, blockSize*2), - }, - b: doubleSizeBuffer{ - baseOffset: blockSize, - head: blockSize, - buffer: make([]byte, blockSize*2), - }, - } -} - -func (c *C2) Reset() { - c.a.Reset() - c.b.Reset() - c.lastWritten = 0 - c.totalWritten = 0 -} - -// Write new data -func (c *C2) Write(b []byte) { - c.a.Write(b) - c.b.Write(b) - c.lastWritten = len(b) - c.totalWritten += c.lastWritten -} - -func (c *C2) getBlockBuffer() *doubleSizeBuffer { - bufferToRead := &c.a - if c.b.head > c.a.head { - bufferToRead = &c.b - } - - return bufferToRead -} - -// the total written, up to the blocksize -func (c *C2) maxWritten() int { - if c.totalWritten < c.blocksize { - return c.totalWritten - } - - return c.blocksize -} - -func (c *C2) Len() int { - return c.maxWritten() -} - -func (c *C2) Empty() bool { - return c.totalWritten == 0 -} - -// Shortens the content of the circular buffer -// and returns the content removed -func (c *C2) Truncate(byteCount int) (evicted []byte) { - max := c.maxWritten() - - if byteCount > max { - byteCount = max - } - - bufferToRead := c.getBlockBuffer() - start := bufferToRead.head - max - - c.totalWritten = c.maxWritten() - byteCount - return bufferToRead.buffer[start : start+byteCount] -} - -// get the current buffer contents of block -func (c *C2) GetBlock() []byte { - // figure out which buffer has it stored contiguously - bufferToRead := c.getBlockBuffer() - start := bufferToRead.head - c.maxWritten() - - return bufferToRead.buffer[start:bufferToRead.head] -} - -// get the data that was evicted by the last write -func (c *C2) Evicted() []byte { - if c.totalWritten <= c.blocksize { - return nil - } - - bufferToRead := c.a - if c.b.head < c.a.head { - bufferToRead = c.b - } - - bufferStart := bufferToRead.head + c.blocksize - readLength := c.lastWritten - - // if the buffer wasn't full, we don't read the full length - if c.totalWritten-c.lastWritten < c.blocksize { - readLength -= c.lastWritten - c.totalWritten + c.blocksize - } - - return bufferToRead.buffer[bufferStart-readLength : bufferStart] -} - -func (buff *doubleSizeBuffer) Reset() { - buff.head = buff.baseOffset -} - -func (buff *doubleSizeBuffer) Write(by []byte) { - remaining := by - - for len(remaining) > 0 { - remaining_len := len(remaining) - availableSpace := len(buff.buffer) - buff.head - writeThisTime := remaining_len - - if writeThisTime > availableSpace { - writeThisTime = availableSpace - } - - copy( - buff.buffer[buff.head:buff.head+writeThisTime], // to - by, - ) - - buff.head += writeThisTime - - if buff.head == len(buff.buffer) { - buff.head = 0 - } - - remaining = remaining[writeThisTime:] - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go b/vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go deleted file mode 100644 index 6315989..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package circularbuffer - -import ( - "bytes" - "testing" -) - -const BLOCK_SIZE = 10 - -var incrementBlock = make([]byte, BLOCK_SIZE) -var incrementBlock2 = make([]byte, BLOCK_SIZE) - -func init() { - for i, _ := range incrementBlock { - incrementBlock[i] = byte(i) - incrementBlock2[i] = byte(i + BLOCK_SIZE) - } -} - -func TestCreateC2Buffer(t *testing.T) { - MakeC2Buffer(BLOCK_SIZE) -} - -func TestWriteBlock(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) -} - -func TestWritingUnderCapacityGivesEmptyEvicted(t *testing.T) { - b := MakeC2Buffer(2) - b.Write([]byte{1, 2}) - - if len(b.Evicted()) != 0 { - t.Fatal("Evicted should have been empty:", b.Evicted()) - } -} - -func TestWritingMultipleBytesWhenBufferIsNotFull(t *testing.T) { - b := MakeC2Buffer(3) - b.Write([]byte{1, 2}) - b.Write([]byte{3, 4}) - - ev := b.Evicted() - - if len(ev) != 1 || ev[0] != 1 { - t.Fatal("Evicted should have been [1,]:", ev) - } -} - -func TestEvictedRegession1(t *testing.T) { - b := MakeC2Buffer(4) - - b.Write([]byte{7, 6}) - b.Write([]byte{5, 1, 2}) - b.Write([]byte{3, 4}) - - ev := b.Evicted() - if len(ev) != 2 || ev[0] != 6 || ev[1] != 5 { - t.Fatalf("Unexpected evicted [6,5]: %v", ev) - } -} - -func TestGetBlock(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - - block := b.GetBlock() - - if len(block) != BLOCK_SIZE { - t.Fatal("Wrong block size returned") - } - - for i, by := range block { - if byte(i) != by { - t.Errorf("byte %v does not match", i) - } - } -} - -func TestWriteTwoBlocksGet(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - b.Write(incrementBlock2) - - if bytes.Compare(b.GetBlock(), incrementBlock2) != 0 { - t.Errorf("Get block did not return the right value: %s", b.GetBlock()) - } -} - -func TestWriteSingleByteGetSingleByte(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - singleByte := []byte{0} - b.Write(singleByte) - - if bytes.Compare(b.GetBlock(), singleByte) != 0 { - t.Errorf("Get block did not return the right value: %s", b.GetBlock()) - } -} - -func TestWriteTwoBlocksGetEvicted(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - b.Write(incrementBlock2) - - if bytes.Compare(b.Evicted(), incrementBlock) != 0 { - t.Errorf("Evicted did not return the right value: %s", b.Evicted()) - } -} - -func TestWriteSingleByteReturnsSingleEvictedByte(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock2) - singleByte := []byte{0} - - b.Write(singleByte) - e := b.Evicted() - - if len(e) != 1 { - t.Fatalf("Evicted length is not correct: %s", e) - } - - if e[0] != byte(10) { - t.Errorf("Evicted content is not correct: %s", e) - } -} - -func TestTruncatingAfterWriting(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - - evicted := b.Truncate(2) - - if len(evicted) != 2 { - t.Fatalf("Truncate did not return expected evicted length: %v", evicted) - } - - if evicted[0] != 0 || evicted[1] != 1 { - t.Errorf("Unexpected content in evicted: %v", evicted) - } -} - -func TestWritingAfterTruncating(t *testing.T) { - // test that after we truncate some content, the next operations - // on the buffer give us the expected results - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - b.Truncate(4) - - b.Write([]byte{34, 46}) - - block := b.GetBlock() - - if len(block) != BLOCK_SIZE-2 { - t.Fatalf( - "Unexpected block length after truncation: %v (%v)", - block, - len(block), - ) - } - - if bytes.Compare(block, []byte{4, 5, 6, 7, 8, 9, 34, 46}) != 0 { - t.Errorf( - "Unexpected block content after truncation: %v (%v)", - block, - len(block)) - } -} - -// This should have no allocations! -func BenchmarkSingleWrites(b *testing.B) { - buffer := MakeC2Buffer(BLOCK_SIZE) - buffer.Write(incrementBlock) - b.ReportAllocs() - - singleByte := []byte{0} - b.StartTimer() - for i := 0; i < b.N; i++ { - buffer.Write(singleByte) - buffer.Evicted() - } - b.StopTimer() -} diff --git a/vendor/src/github.com/Redundancy/go-sync/comparer/comparer.go b/vendor/src/github.com/Redundancy/go-sync/comparer/comparer.go deleted file mode 100644 index b793818..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/comparer/comparer.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -package comparer is responsible for using a FileChecksumGenerator (filechecksum) and an index -to move through a file and compare it to the index, producing a FileDiffSummary -*/ -package comparer - -import ( - "fmt" - "io" - "sync/atomic" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/circularbuffer" - "github.com/Redundancy/go-sync/filechecksum" -) - -const ( - READ_NEXT_BYTE = iota - READ_NEXT_BLOCK - READ_NONE -) - -// If the weak Hash object satisfies this interface, then -// StartFindMatchingBlocks will not allocate a circular buffer -type BlockBuffer interface { - Write([]byte) (int, error) - // the last set of bytes of the size of the circular buffer - // oldest to newest - GetLastBlock() []byte -} - -type BlockMatchResult struct { - // In case of error - Err error - - // The offset the comparison + baseOffset - ComparisonOffset int64 - - // The block from the index that it matched - BlockIdx uint -} - -type Index interface { - FindWeakChecksum2(chk []byte) interface{} - FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum -} - -/* -Iterates though comparison looking for blocks that match ones from the index -it emits each block to be read from the returned channel. Callers should check for -.Err != nil on the results, in which case reading will end immediately. - -StartFindMatchingBlocks is capable of running asyncronously -on sub-sections of a larger file. When doing this, you must overlap -by the block size, and use seperate checksum generators. -*/ - -type Comparer struct { - Comparisons int64 - WeakHashHits int64 - StrongHashHits int64 -} - -func (c *Comparer) StartFindMatchingBlocks( - comparison io.Reader, - baseOffset int64, - generator *filechecksum.FileChecksumGenerator, - referenceIndex Index, -) <-chan BlockMatchResult { - - resultStream := make(chan BlockMatchResult) - - go c.startFindMatchingBlocks_int( - resultStream, - comparison, - baseOffset, - generator, - referenceIndex, - ) - - return resultStream -} - -/* -TODO: When matching duplicated blocks, a channel of BlockMatchResult slices would be more efficient -*/ -func (c *Comparer) startFindMatchingBlocks_int( - results chan<- BlockMatchResult, - comparison io.Reader, - baseOffset int64, - generator *filechecksum.FileChecksumGenerator, - reference Index, -) { - defer close(results) - - block := make([]byte, generator.BlockSize) - var err error - - ReportErr := func(err error) { - results <- BlockMatchResult{ - Err: err, - } - } - - _, err = io.ReadFull(comparison, block) - - if err != nil { - ReportErr( - fmt.Errorf("Error reading first block in comparison: %v", err), - ) - return - } - - generator.WeakRollingHash.SetBlock(block) - singleByte := make([]byte, 1) - weaksum := make([]byte, generator.WeakRollingHash.Size()) - strongSum := make([]byte, 0, generator.GetStrongHash().Size()) - - blockMemory := circularbuffer.MakeC2Buffer(int(generator.BlockSize)) - blockMemory.Write(block) - - strong := generator.GetStrongHash() - // All the bytes - i := int64(0) - next := READ_NEXT_BYTE - - //ReadLoop: - for { - - atomic.AddInt64(&c.Comparisons, 1) - - // look for a weak match - generator.WeakRollingHash.GetSum(weaksum) - if weakMatchList := reference.FindWeakChecksum2(weaksum); weakMatchList != nil { - atomic.AddInt64(&c.WeakHashHits, 1) - - block = blockMemory.GetBlock() - - strong.Reset() - strong.Write(block) - strongSum = strong.Sum(strongSum) - strongList := reference.FindStrongChecksum2(strongSum, weakMatchList) - - // clear the slice - strongSum = strongSum[:0] - - // If there are many matches, it means that this block is - // duplicated in the reference. - // since we care about finding all the blocks in the reference, - // we must report all of them - off := i + baseOffset - for _, strongMatch := range strongList { - results <- BlockMatchResult{ - ComparisonOffset: off, - BlockIdx: strongMatch.ChunkOffset, - } - } - - if len(strongList) > 0 { - atomic.AddInt64(&c.StrongHashHits, 1) - if next == READ_NONE { - // found the match at the end, so exit - break - } - // No point looking for a match that overlaps this block - next = READ_NEXT_BLOCK - } - } - - var n int - var readBytes []byte - - switch next { - case READ_NEXT_BYTE: - n, err = comparison.Read(singleByte) - readBytes = singleByte - case READ_NEXT_BLOCK: - n, err = io.ReadFull(comparison, block) - readBytes = block[:n] - next = READ_NEXT_BYTE - } - - if uint(n) == generator.BlockSize { - generator.WeakRollingHash.SetBlock(block) - blockMemory.Write(block) - i += int64(n) - } else if n > 0 { - b_len := blockMemory.Len() - blockMemory.Write(readBytes) - generator.WeakRollingHash.AddAndRemoveBytes( - readBytes, - blockMemory.Evicted(), - b_len, - ) - i += int64(n) - } - - if next != READ_NONE && (err == io.EOF || err == io.ErrUnexpectedEOF) { - err = io.EOF - next = READ_NONE - } - - if next == READ_NONE { - if blockMemory.Empty() { - break - } - - b_len := blockMemory.Len() - removedByte := blockMemory.Truncate(1) - generator.WeakRollingHash.RemoveBytes(removedByte, b_len) - i += 1 - } - } - - if err != io.EOF { - ReportErr(err) - return - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go b/vendor/src/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go deleted file mode 100644 index cd3b7b5..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package comparer - -import ( - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/util/readers" - "testing" -) - -var test = []byte{0, 1, 2, 3} - -type NegativeWeakIndex struct { -} - -func (i *NegativeWeakIndex) FindWeakChecksum2(chk []byte) interface{} { - return nil -} - -func (i *NegativeWeakIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { - return nil -} - -type NegativeStrongIndex struct { -} - -func (i *NegativeStrongIndex) FindWeakChecksum2(chk []byte) interface{} { - return i -} - -func (i *NegativeStrongIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { - return nil -} - -func BenchmarkWeakComparison(b *testing.B) { - b.ReportAllocs() - b.SetBytes(1) - - const BLOCK_SIZE = 8 - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - - b.StartTimer() - - results := (&Comparer{}).StartFindMatchingBlocks( - readers.OneReader(b.N+BLOCK_SIZE), - 0, - generator, - &NegativeWeakIndex{}, - ) - - for _, ok := <-results; ok; { - } - - b.StopTimer() -} - -func BenchmarkStrongComparison(b *testing.B) { - b.ReportAllocs() - b.SetBytes(1) - - const BLOCK_SIZE = 8 - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - - b.StartTimer() - - results := (&Comparer{}).StartFindMatchingBlocks( - readers.OneReader(b.N+BLOCK_SIZE), - 0, - generator, - &NegativeStrongIndex{}, - ) - - for _, ok := <-results; ok; { - } - - b.StopTimer() -} diff --git a/vendor/src/github.com/Redundancy/go-sync/comparer/comparer_test.go b/vendor/src/github.com/Redundancy/go-sync/comparer/comparer_test.go deleted file mode 100644 index 9337a5e..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/comparer/comparer_test.go +++ /dev/null @@ -1,445 +0,0 @@ -package comparer - -import ( - "bufio" - "bytes" - "io" - "reflect" - "testing" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/indexbuilder" -) - -func CheckResults( - t *testing.T, - original, modified string, - results <-chan BlockMatchResult, - block_size uint, - expectedResults []string, -) { - var resultStrings []string - - for i := range results { - - if i.Err != nil { - t.Fatal(i.Err) - } - - os := i.BlockIdx * block_size - original_max := os + block_size - if original_max > uint(len(original)) { - original_max = uint(len(original)) - } - - orignal := original[os:original_max] - - compare_max := i.ComparisonOffset + int64(block_size) - if compare_max > int64(len(modified)) { - compare_max = int64(len(modified)) - } - - compare := modified[i.ComparisonOffset:compare_max] - - if orignal != compare { - t.Errorf( - "Bad match: \"%v\" to \"%v\" : %#v", - orignal, compare, - i, - ) - } - - resultStrings = append(resultStrings, orignal) - } - - if len(resultStrings) != len(expectedResults) { - t.Fatalf( - "%#v blocks should have matched, got: %#v", - len(expectedResults), - resultStrings, - ) - } - - for i, v := range expectedResults { - if resultStrings[i] != v { - t.Errorf("%#v != %#v", resultStrings[i], v) - } - } -} - -func compare( - original string, - modified string, - block_size uint, -) (results <-chan BlockMatchResult, err error) { - - originalFileContent := bytes.NewBufferString(original) - generator := filechecksum.NewFileChecksumGenerator(block_size) - - _, reference, _, err := indexbuilder.BuildChecksumIndex( - generator, - originalFileContent, - ) - - if err != nil { - return - } - - modifiedContent := bytes.NewBufferString(modified) - - results = (&Comparer{}).StartFindMatchingBlocks( - modifiedContent, - 0, - generator, - reference, - ) - - return -} - -// Splits successive strings into blocks of size n -// 2, ABCD -> AB, CD -// 2, ABCD, E, FG -> AB, CD, E, FG -func split(n int, ss ...string) (result []string) { - totalLength := 0 - for _, s := range ss { - totalLength += len(s)/n + 1 - } - result = make([]string, 0, totalLength) - - for _, x := range ss { - i := int(0) - for i+n < len(x) { - result = append( - result, - x[i:i+n], - ) - - i += n - } - - if i < len(x)-1 { - result = append( - result, - x[i:], - ) - } - } - - return -} - -func TestSplit(t *testing.T) { - INPUT := "abcdef" - EXPECTED := []string{"ab", "cd", "ef"} - result := split(2, INPUT) - - if !reflect.DeepEqual(result, EXPECTED) { - t.Errorf( - "Lists differ: %v vs %v", - result, - EXPECTED, - ) - } -} - -func TestSplitWithPartial(t *testing.T) { - INPUT := "abcdef" - EXPECTED := []string{"abcd", "ef"} - result := split(4, INPUT) - - if !reflect.DeepEqual(result, EXPECTED) { - t.Errorf( - "Lists differ: %v vs %v", - result, - EXPECTED, - ) - } -} - -func TestMultiSplit(t *testing.T) { - INPUT := []string{"abcdef", "ghij"} - EXPECTED := []string{"abcd", "ef", "ghij"} - result := split(4, INPUT...) - - if !reflect.DeepEqual(result, EXPECTED) { - t.Errorf( - "Lists differ: %v vs %v", - result, - EXPECTED, - ) - } -} - -func TestDetectsPrependedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - - const ORIGINAL_STRING = "abcdefghijklmnop" - const PREPENDED_STRING = "12" + ORIGINAL_STRING - - results, err := compare(ORIGINAL_STRING, PREPENDED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - PREPENDED_STRING, - results, - BLOCK_SIZE, - split(4, ORIGINAL_STRING), - ) -} - -func TestDetectsInjectedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefgh" - const B = "ijklmnop" - const ORIGINAL_STRING = A + B - const MODIFIED_STRING = A + "23" + B - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A, B), - ) -} - -func TestDetectsAppendedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - - const ORIGINAL_STRING = "abcdefghijklmnop" - const MODIFIED_STRING = ORIGINAL_STRING + "23" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, ORIGINAL_STRING), - ) -} - -func TestDetectsModifiedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefgh" - const B = "ijkl" - const C = "mnop" - const ORIGINAL_STRING = A + B + C - const MODIFIED_STRING = A + "i2kl" + C - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A, C), - ) -} - -func TestDetectsPartialBlockAtEnd(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefghijklmnopqrstuvwxyz" - const ORIGINAL_STRING = A - const MODIFIED_STRING = A - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A), - ) -} - -func TestDetectsModifiedPartialBlockAtEnd(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefghijklmnopqrstuvwx" - const ORIGINAL_STRING = A + "yz" - const MODIFIED_STRING = A + "23" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A), - ) -} - -func TestDetectsUnmodifiedPartialBlockAtEnd(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefghijklmnopqrst" - const ORIGINAL_STRING = A + "uvwx" + "yz" - const MODIFIED_STRING = A + "us6x" + "yz" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A, "yz"), - ) -} - -func TestMultipleResultsForDuplicatedBlocks(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcd" - const ORIGINAL_STRING = A + A - const MODIFIED_STRING = A - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - []string{A, A}, - ) -} - -func TestRegression1(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" - const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - []string{"The ", "k br", "own ", "fox ", "jump", "the ", "lazy"}, - ) -} - -func TestTwoComparisons(t *testing.T) { - const BLOCK_SIZE = 4 - const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" - const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" - - numMatchers := int64(4) - sectionSize := int64(len(ORIGINAL_STRING)) / numMatchers - sectionSize += int64(BLOCK_SIZE) - (sectionSize % int64(BLOCK_SIZE)) - - merger := &MatchMerger{} - - originalFile := bytes.NewReader([]byte(ORIGINAL_STRING)) - modifiedFile := bytes.NewReader([]byte(MODIFIED_STRING)) - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - - _, reference, _, _ := indexbuilder.BuildChecksumIndex( - generator, - originalFile, - ) - - for i := int64(0); i < numMatchers; i++ { - compare := &Comparer{} - offset := sectionSize * i - - t.Logf("Section %v: %v-%v", i, offset, offset+sectionSize) - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(modifiedFile, offset, sectionSize+BLOCK_SIZE), - 100000, // 1 MB buffer - ) - - // Bakes in the assumption about how to generate checksums (extract) - sectionGenerator := filechecksum.NewFileChecksumGenerator( - uint(BLOCK_SIZE), - ) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, reference, - ) - - merger.StartMergeResultStream(matchStream, int64(BLOCK_SIZE)) - } - - merged := merger.GetMergedBlocks() - missing := merged.GetMissingBlocks(uint(len(ORIGINAL_STRING) / BLOCK_SIZE)) - - expected := []string{ - "quic", "ed over ", " dog", - } - - t.Logf("Missing blocks: %v", len(missing)) - - for x, v := range missing { - start := v.StartBlock * BLOCK_SIZE - end := (v.EndBlock + 1) * BLOCK_SIZE - if end > uint(len(ORIGINAL_STRING)) { - end = uint(len(ORIGINAL_STRING)) - } - s := ORIGINAL_STRING[start:end] - - if s != expected[x] { - t.Errorf( - "Wrong block %v (%v-%v): %#v (expected %#v)", - x, v.StartBlock, v.EndBlock, s, expected[x], - ) - } else { - t.Logf( - "Correct block %v (%v-%v): %#v (expected %#v)", - x, v.StartBlock, v.EndBlock, s, expected[x], - ) - } - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/comparer/merger.go b/vendor/src/github.com/Redundancy/go-sync/comparer/merger.go deleted file mode 100644 index 59f108b..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/comparer/merger.go +++ /dev/null @@ -1,309 +0,0 @@ -package comparer - -import ( - "sort" - "sync" - - "github.com/petar/GoLLRB/llrb" -) - -// The result merger takes many BlockMatchResult -// and combines adjoining results into spans of blocks -type MatchMerger struct { - sync.Mutex - - wait sync.WaitGroup - - // Porting from map to llrb, to enable finding of the next - // largest blockID (What's the first blockID after 5?) - startEndBlockMap *llrb.LLRB - - blockCount uint -} - -// a span of multiple blocks, from start to end, which match the blocks -// starting at an offset of ComparisonStartOffset -type BlockSpan struct { - StartBlock uint - EndBlock uint - - // byte offset in the comparison for the match - ComparisonStartOffset int64 -} - -type BlockSpanIndex interface { - Position() uint -} - -// Wraps a blockspan so that it may easily be used -// in llrb. Corresponds to the start block of the blockspan. -type BlockSpanStart BlockSpan - -func (s BlockSpanStart) Position() uint { - return s.StartBlock -} - -func (s BlockSpanStart) Less(than llrb.Item) bool { - return s.StartBlock < than.(BlockSpanIndex).Position() -} - -// Wraps a blockspan so that it may easily be used -// in llrb. Corresponds to the end of the blockspan. -type BlockSpanEnd BlockSpan - -func (s BlockSpanEnd) Position() uint { - return s.EndBlock -} - -func (s BlockSpanEnd) Less(than llrb.Item) bool { - return s.EndBlock < than.(BlockSpanIndex).Position() -} - -// Wraps a block index, allowing easy use of llrb.Get() -type BlockSpanKey uint - -func (s BlockSpanKey) Position() uint { - return uint(s) -} - -func (k BlockSpanKey) Less(than llrb.Item) bool { - return uint(k) < than.(BlockSpanIndex).Position() -} - -func (b BlockSpan) EndOffset(blockSize int64) int64 { - return b.ComparisonStartOffset + blockSize*int64(b.EndBlock-b.StartBlock+1) -} - -func toBlockSpan(b BlockMatchResult) *BlockSpan { - return &BlockSpan{ - StartBlock: b.BlockIdx, - EndBlock: b.BlockIdx, - ComparisonStartOffset: b.ComparisonOffset, - } -} - -func isBordering(a, b *BlockSpan, blockSize int64) bool { - if a.EndBlock == b.StartBlock-1 && a.EndOffset(blockSize) == b.ComparisonStartOffset { - return true - } else if b.EndBlock == a.StartBlock-1 && b.EndOffset(blockSize) == a.ComparisonStartOffset { - return true - } - - return false -} - -func itemToBlockSpan(in llrb.Item) BlockSpan { - switch i := in.(type) { - case BlockSpanStart: - return BlockSpan(i) - case BlockSpanEnd: - return BlockSpan(i) - } - return BlockSpan{} -} - -// if merged, the block span remaining is the one with the lower start block -func (merger *MatchMerger) merge(block1, block2 *BlockSpan, blockSize int64) { - var a, b *BlockSpan = block1, block2 - - if block1.StartBlock > block2.StartBlock { - a, b = block2, block1 - } - - if isBordering(a, b, blockSize) { - // bordering, merge - // A ------ A B ------ B > A ---------------- A - merger.startEndBlockMap.Delete(BlockSpanKey(a.EndBlock)) - merger.startEndBlockMap.Delete(BlockSpanKey(b.StartBlock)) - a.EndBlock = b.EndBlock - - merger.startEndBlockMap.ReplaceOrInsert(BlockSpanStart(*a)) - merger.startEndBlockMap.ReplaceOrInsert(BlockSpanEnd(*a)) - } -} - -// Can be used on multiple streams of results simultaneously -// starts working asyncronously, call from the initiating goroutine -func (merger *MatchMerger) StartMergeResultStream( - resultStream <-chan BlockMatchResult, - blockSize int64, -) { - // Add should be called on the main goroutine - // to ensure that it has happened before wait is called - merger.wait.Add(1) - - if merger.startEndBlockMap == nil { - merger.startEndBlockMap = llrb.New() - } - - // used by the llrb iterator to signal that it found/didn't find - // an existing key on or spanning the given block - //foundExisting := make(chan bool) - - go func() { - defer merger.wait.Done() - - for result := range resultStream { - if result.Err != nil { - return - } - - merger.Lock() - merger.blockCount += 1 - - blockID := result.BlockIdx - preceeding := merger.startEndBlockMap.Get(BlockSpanKey(blockID - 1)) - following := merger.startEndBlockMap.Get(BlockSpanKey(blockID + 1)) - - asBlockSpan := toBlockSpan(result) - - var foundExisting bool - // Exists, or within an existing span - merger.startEndBlockMap.AscendGreaterOrEqual( - BlockSpanKey(blockID), - // iterator - func(i llrb.Item) bool { - j, ok := i.(BlockSpanIndex) - - if !ok { - foundExisting = true - return false - } - - switch k := j.(type) { - case BlockSpanStart: - // it's only overlapping if its the same blockID - foundExisting = k.StartBlock == blockID - return false - - case BlockSpanEnd: - // we didn't find a start, so there's an end that overlaps - foundExisting = true - return false - default: - foundExisting = true - return false - } - - }, - ) - - if foundExisting { - merger.Unlock() - continue - } - - merger.startEndBlockMap.ReplaceOrInsert( - BlockSpanStart(*toBlockSpan(result)), - ) - - if preceeding != nil && following != nil { - a := itemToBlockSpan(preceeding) - merger.merge( - asBlockSpan, - &a, - blockSize, - ) - - b := itemToBlockSpan(following) - merger.merge( - &a, - &b, - blockSize, - ) - - } else if preceeding != nil { - a := itemToBlockSpan(preceeding) - merger.merge( - asBlockSpan, - &a, - blockSize, - ) - } else if following != nil { - b := itemToBlockSpan(following) - merger.merge( - asBlockSpan, - &b, - blockSize, - ) - } - - merger.Unlock() - } - }() -} - -type BlockSpanList []BlockSpan - -func (l BlockSpanList) Len() int { - return len(l) -} - -func (l BlockSpanList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] -} - -func (l BlockSpanList) Less(i, j int) bool { - return l[i].StartBlock < l[j].StartBlock -} - -// Sorted list of blocks, based on StartBlock -func (merger *MatchMerger) GetMergedBlocks() (sorted BlockSpanList) { - merger.wait.Wait() - var smallestKey uint = 0 - m := merger.startEndBlockMap - - m.AscendGreaterOrEqual(m.Min(), func(item llrb.Item) bool { - switch block := item.(type) { - case BlockSpanStart: - sorted = append(sorted, BlockSpan(block)) - smallestKey = block.StartBlock + 1 - } - return true - }) - - sort.Sort(sorted) - return -} - -// Creates a list of spans that are missing. -// note that maxBlock is blockCount-1 -func (l BlockSpanList) GetMissingBlocks(maxBlock uint) (sorted BlockSpanList) { - // it's difficult to know how many spans we will need - sorted = make(BlockSpanList, 0) - - lastBlockSpanIndex := -1 - for _, blockSpan := range l { - if int(blockSpan.StartBlock) > lastBlockSpanIndex+1 { - sorted = append( - sorted, - BlockSpan{ - StartBlock: uint(lastBlockSpanIndex + 1), - EndBlock: blockSpan.StartBlock - 1, - }, - ) - } - - lastBlockSpanIndex = int(blockSpan.EndBlock) - } - - if lastBlockSpanIndex == -1 { - sorted = append( - sorted, - BlockSpan{ - StartBlock: 0, - EndBlock: maxBlock, - }, - ) - } else if uint(lastBlockSpanIndex) < maxBlock { - sorted = append( - sorted, - BlockSpan{ - StartBlock: uint(lastBlockSpanIndex + 1), - EndBlock: maxBlock, - }, - ) - } - - return sorted -} diff --git a/vendor/src/github.com/Redundancy/go-sync/comparer/merger_test.go b/vendor/src/github.com/Redundancy/go-sync/comparer/merger_test.go deleted file mode 100644 index 01ca053..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/comparer/merger_test.go +++ /dev/null @@ -1,401 +0,0 @@ -package comparer - -import ( - "testing" -) - -func TestMergeAdjacentBlocksAfter(t *testing.T) { - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 1 { - t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0]) - } -} - -func TestMergeAdjacentBlocksBefore(t *testing.T) { - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 1 { - t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0]) - } - - // start and end - if merger.startEndBlockMap.Len() != 2 { - t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) - } -} - -func TestMergeAdjacentBlocksBetween(t *testing.T) { - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 2 * BLOCK_SIZE, - BlockIdx: 2, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - // match in the center - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 2 { - t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0]) - } - if merged[0].StartBlock != 0 { - t.Errorf("Wrong StartBlock, expected 0, got %#v", merged[0]) - } - if merger.startEndBlockMap.Len() != 2 { - t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) - } -} - -func TestMissingBlocksOffsetStart(t *testing.T) { - b := BlockSpanList{ - { - StartBlock: 2, - EndBlock: 3, - }, - } - - m := b.GetMissingBlocks(3) - - if len(m) != 1 { - t.Fatalf("Wrong number of missing blocks: %v", len(m)) - } - - if m[0].StartBlock != 0 { - t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) - } - if m[0].EndBlock != 1 { - t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) - } -} - -func TestMissingCenterBlock(t *testing.T) { - b := BlockSpanList{ - { - StartBlock: 0, - EndBlock: 0, - }, - { - StartBlock: 2, - EndBlock: 3, - }, - } - - m := b.GetMissingBlocks(3) - - if len(m) != 1 { - t.Fatalf("Wrong number of missing blocks: %v", len(m)) - } - - if m[0].StartBlock != 1 { - t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) - } - if m[0].EndBlock != 1 { - t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) - } -} - -func TestMissingEndBlock(t *testing.T) { - b := BlockSpanList{ - { - StartBlock: 0, - EndBlock: 1, - }, - } - - m := b.GetMissingBlocks(3) - - if len(m) != 1 { - t.Fatalf("Wrong number of missing blocks: %v", len(m)) - } - - if m[0].StartBlock != 2 { - t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) - } - if m[0].EndBlock != 3 { - t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) - } -} - -func TestDuplicatedReferenceBlocks(t *testing.T) { - // Reference = AA - // Local = A - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - // When we find multiple strong matches, we send each of them - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 2 { - t.Errorf("Duplicated blocks cannot be merged: %#v", merged) - } - - missing := merged.GetMissingBlocks(1) - - if len(missing) > 0 { - t.Errorf("There were no missing blocks: %#v", missing) - } -} - -func TestDuplicatedLocalBlocks(t *testing.T) { - // Reference = A - // Local = AA - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - // When we find multiple strong matches, we send each of them - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 0, - } - - close(mergeChan) - - // We only need one of the matches in the resulting file - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Errorf("Duplicated blocks cannot be merged: %#v", merged) - } - - missing := merged.GetMissingBlocks(0) - - if len(missing) > 0 { - t.Errorf("There were no missing blocks: %#v", missing) - } -} - -func TestDoublyDuplicatedBlocks(t *testing.T) { - // Reference = AA - // Local = AA - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - // When we find multiple strong matches, we send each of them - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 1, - } - - // Second local match - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 2 { - t.Errorf("Duplicated blocks cannot be merged: %#v", merged) - } - - missing := merged.GetMissingBlocks(1) - - if len(missing) > 0 { - t.Errorf("There were no missing blocks: %#v", missing) - } -} - -func TestBlockWithinSpan(t *testing.T) { - // catch the case where we're informed about a block, - // after we've merged blocks around it, so that the start and end - // are within a span, not bordering one - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 2 * BLOCK_SIZE, - BlockIdx: 2, - } - - // This one is a duplicate of an earlier one - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 2 { - t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0]) - } - - // start and end - if merger.startEndBlockMap.Len() != 2 { - t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) - } -} - -func TestNilBlockSpanList(t *testing.T) { - s := BlockSpanList(nil) - - missing := s.GetMissingBlocks(1) - - if missing == nil { - t.Fail() - } - - if len(missing) == 0 { - t.Fatal("missing should not be empty") - } - - missingItem := missing[0] - - if missingItem.StartBlock != 0 { - t.Errorf("Wrong startblock: %v", missingItem.StartBlock) - } - if missingItem.EndBlock != 1 { - t.Errorf("Wrong endblock: %v", missingItem.EndBlock) - } -} - -func TestRegression1Merger(t *testing.T) { - const BLOCK_SIZE = 4 - const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" - const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" - - results, _ := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - merger := &MatchMerger{} - merger.StartMergeResultStream(results, BLOCK_SIZE) - - merged := merger.GetMergedBlocks() - missing := merged.GetMissingBlocks(uint(len(ORIGINAL_STRING) / BLOCK_SIZE)) - - expected := []string{ - "quic", "ed over ", " dog", - } - - for i, v := range missing { - start := v.StartBlock * BLOCK_SIZE - end := (v.EndBlock + 1) * BLOCK_SIZE - if end > uint(len(ORIGINAL_STRING)) { - end = uint(len(ORIGINAL_STRING)) - } - s := ORIGINAL_STRING[start:end] - - if s != expected[i] { - t.Errorf("Wrong block %v: %v (expected %v)", i, expected[i]) - } - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum.go b/vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum.go deleted file mode 100644 index 4f69701..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum.go +++ /dev/null @@ -1,237 +0,0 @@ -/* -package filechecksum provides the FileChecksumGenerator, whose main responsibility is to read a file, -and generate both weak and strong checksums for every block. It is also used by the comparer, which -will generate weak checksums for potential byte ranges that could match the index, and strong checksums -if needed. -*/ -package filechecksum - -import ( - "crypto/md5" - "hash" - "io" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/rollsum" -) - -// Rsync swapped to this after version 30 -// this is a factory function, because we don't actually want to share hash state -var DefaultStrongHashGenerator = func() hash.Hash { - return md5.New() -} - -// We provide an overall hash of individual files -var DefaultFileHashGenerator = func() hash.Hash { - return md5.New() -} - -// Uses all default hashes (MD5 & rollsum16) -func NewFileChecksumGenerator(blocksize uint) *FileChecksumGenerator { - return &FileChecksumGenerator{ - BlockSize: blocksize, - WeakRollingHash: rollsum.NewRollsum32Base(blocksize), - //WeakRollingHash: rollsum.NewRollsum16Base(blocksize), - StrongHash: DefaultStrongHashGenerator(), - FileChecksumHash: DefaultFileHashGenerator(), - } -} - -type RollingHash interface { - // the size of the hash output - Size() int - - AddByte(b byte) - RemoveByte(b byte, length int) - - AddBytes(bs []byte) - RemoveBytes(bs []byte, length int) - - // pairs up bytes to do remove/add in the right order - AddAndRemoveBytes(add []byte, remove []byte, length int) - - SetBlock(block []byte) - - GetSum(b []byte) - Reset() -} - -/* -FileChecksumGenerator provides a description of what hashing functions to use to -evaluate a file. Since the hashes store state, it is NOT safe to use a generator concurrently -for different things. -*/ -type FileChecksumGenerator struct { - // See BlockBuffer - WeakRollingHash RollingHash - StrongHash hash.Hash - FileChecksumHash hash.Hash - BlockSize uint -} - -// Reset all hashes to initial state -func (check *FileChecksumGenerator) Reset() { - check.WeakRollingHash.Reset() - check.StrongHash.Reset() - check.FileChecksumHash.Reset() -} - -func (check *FileChecksumGenerator) ChecksumSize() int { - return check.WeakRollingHash.Size() + check.GetStrongHash().Size() -} - -func (check *FileChecksumGenerator) GetChecksumSizes() (int, int) { - return check.WeakRollingHash.Size(), check.GetStrongHash().Size() -} - -// Gets the Hash function for the overall file used on each block -// defaults to md5 -func (check *FileChecksumGenerator) GetFileHash() hash.Hash { - return check.FileChecksumHash -} - -// Gets the Hash function for the strong hash used on each block -// defaults to md5, but can be overriden by the generator -func (check *FileChecksumGenerator) GetStrongHash() hash.Hash { - return check.StrongHash -} - -// GenerateChecksums reads each block of the input file, and outputs first the weak, then the strong checksum -// to the output writer. It will return a checksum for the whole file. -// Potentially speaking, this might be better producing a channel of blocks, which would remove the need for io from -// a number of other places. -func (check *FileChecksumGenerator) GenerateChecksums(inputFile io.Reader, output io.Writer) (fileChecksum []byte, err error) { - for chunkResult := range check.StartChecksumGeneration(inputFile, 64, nil) { - if chunkResult.Err != nil { - return nil, chunkResult.Err - } else if chunkResult.Filechecksum != nil { - return chunkResult.Filechecksum, nil - } - - for _, chunk := range chunkResult.Checksums { - output.Write(chunk.WeakChecksum) - output.Write(chunk.StrongChecksum) - } - } - - return nil, nil -} - -type ChecksumResults struct { - // Return multiple chunks at once for performance - Checksums []chunks.ChunkChecksum - // only used for the last item - Filechecksum []byte - // signals that this is the last item - Err error -} - -// A function or object that can compress blocks -// the compression function must also write out the compressed blocks somewhere! -// Compressed blocks should be independently inflatable -type CompressionFunction func([]byte) (compressedSize int64, err error) - -func (check *FileChecksumGenerator) StartChecksumGeneration( - inputFile io.Reader, - blocksPerResult uint, - compressionFunction CompressionFunction, -) <-chan ChecksumResults { - resultChan := make(chan ChecksumResults) - go check.generate(resultChan, blocksPerResult, compressionFunction, inputFile) - return resultChan -} - -func (check *FileChecksumGenerator) generate( - resultChan chan ChecksumResults, - blocksPerResult uint, - compressionFunction CompressionFunction, - inputFile io.Reader, -) { - defer close(resultChan) - - fullChecksum := check.GetFileHash() - strongHash := check.GetStrongHash() - - buffer := make([]byte, check.BlockSize) - - // ensure that the hashes are clean - strongHash.Reset() - fullChecksum.Reset() - - // We reset the hashes when done do we can reuse the generator - defer check.WeakRollingHash.Reset() - defer strongHash.Reset() - defer fullChecksum.Reset() - - results := make([]chunks.ChunkChecksum, 0, blocksPerResult) - - i := uint(0) - for { - n, err := io.ReadFull(inputFile, buffer) - section := buffer[:n] - - if n == 0 { - break - } - - // As hashes, the assumption is that they never error - // additionally, we assume that the only reason not - // to write a full block would be reaching the end of the file - fullChecksum.Write(section) - check.WeakRollingHash.SetBlock(section) - strongHash.Write(section) - - strongChecksumValue := make([]byte, 0, strongHash.Size()) - weakChecksumValue := make([]byte, check.WeakRollingHash.Size()) - - check.WeakRollingHash.GetSum(weakChecksumValue) - strongChecksumValue = strongHash.Sum(strongChecksumValue) - - blockSize := int64(check.BlockSize) - - if compressionFunction != nil { - blockSize, err = compressionFunction(section) - } - - results = append( - results, - chunks.ChunkChecksum{ - ChunkOffset: i, - Size: blockSize, - WeakChecksum: weakChecksumValue, - StrongChecksum: strongChecksumValue, - }, - ) - - i++ - - if len(results) == cap(results) { - resultChan <- ChecksumResults{ - Checksums: results, - } - results = make([]chunks.ChunkChecksum, 0, blocksPerResult) - } - - // clear it again - strongChecksumValue = strongChecksumValue[:0] - - // Reset the strong - strongHash.Reset() - - if n != len(buffer) || err == io.EOF { - break - } - } - - if len(results) > 0 { - resultChan <- ChecksumResults{ - Checksums: results, - } - } - - resultChan <- ChecksumResults{ - Filechecksum: fullChecksum.Sum(nil), - } - - return -} diff --git a/vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go b/vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go deleted file mode 100644 index 76d1e74..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go +++ /dev/null @@ -1,309 +0,0 @@ -package filechecksum - -import ( - "bytes" - "io" - "os" - "testing" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/util/readers" -) - -func TestChecksumGenerationEndsWithFilechecksum(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - lastResult := ChecksumResults{} - - for lastResult = range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { - } - - if lastResult.Checksums != nil { - t.Errorf("Last result had checksums: %#v", lastResult) - } - - if lastResult.Filechecksum == nil { - t.Errorf("Last result did not contain the filechecksum: %#v", lastResult) - } -} - -func TestChecksumGenerationReturnsCorrectChecksumCount(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - resultCount := 0 - - for r := range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { - resultCount += len(r.Checksums) - } - - if resultCount != BLOCK_COUNT { - t.Errorf("Unexpected block count returned: %v", resultCount) - } -} - -func TestChecksumGenerationContainsHashes(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - for r := range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { - for _, r2 := range r.Checksums { - if len(r2.WeakChecksum) != checksum.WeakRollingHash.Size() { - t.Fatalf( - "Wrong length weak checksum: %v vs %v", - len(r2.WeakChecksum), - checksum.WeakRollingHash.Size(), - ) - } - - if len(r2.StrongChecksum) != checksum.StrongHash.Size() { - t.Fatalf( - "Wrong length strong checksum: %v vs %v", - len(r2.StrongChecksum), - checksum.StrongHash.Size(), - ) - } - } - } -} - -func TestRollsumLength(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - output := bytes.NewBuffer(nil) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - // output length is expected to be 20 blocks - expectedLength := (BLOCK_COUNT * checksum.GetStrongHash().Size()) + - (BLOCK_COUNT * checksum.WeakRollingHash.Size()) - - _, err := checksum.GenerateChecksums(emptybuffer, output) - - if err != nil { - t.Fatal(err) - } - - if output.Len() != expectedLength { - t.Errorf( - "output length (%v) did not match expected length (%v)", - output.Len(), - expectedLength, - ) - } -} - -func TestRollsumLengthWithPartialBlockAtEnd(t *testing.T) { - const BLOCKSIZE = 100 - const FULL_BLOCK_COUNT = 20 - const BLOCK_COUNT = FULL_BLOCK_COUNT + 1 - - emptybuffer := bytes.NewBuffer(make([]byte, FULL_BLOCK_COUNT*BLOCKSIZE+50)) - output := bytes.NewBuffer(nil) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - // output length is expected to be 20 blocks - expectedLength := (BLOCK_COUNT * checksum.GetStrongHash().Size()) + - (BLOCK_COUNT * checksum.WeakRollingHash.Size()) - - _, err := checksum.GenerateChecksums(emptybuffer, output) - - if err != nil { - t.Fatal(err) - } - - if output.Len() != expectedLength { - t.Errorf( - "output length (%v) did not match expected length (%v)", - output.Len(), - expectedLength, - ) - } -} - -// Each of the data blocks is the same, so the checksums for the blocks should be the same -func TestChecksumBlocksTheSame(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - output := bytes.NewBuffer(nil) - - _, err := checksum.GenerateChecksums( - readers.OneReader(BLOCKSIZE*BLOCK_COUNT), - output, - ) - - if err != nil { - t.Fatal(err) - } - - weakSize, strongSize := checksum.GetChecksumSizes() - - if output.Len() != BLOCK_COUNT*(strongSize+weakSize) { - t.Errorf( - "Unexpected output length: %v, expected %v", - output.Len(), - BLOCK_COUNT*(strongSize+weakSize), - ) - } - - results, err := chunks.LoadChecksumsFromReader(output, weakSize, strongSize) - - if err != nil { - t.Fatal(err) - } - - if len(results) != BLOCK_COUNT { - t.Fatalf("Results too short! %v", len(results)) - } - - first := results[0] - - for i, chk := range results { - if chk.ChunkOffset != uint(i) { - t.Errorf("Unexpected offset %v on chunk %v", chk.ChunkOffset, i) - } - if !first.Match(chk) { - t.Fatalf("Chunks have different checksums on %v", i) - } - } -} - -func TestPrependedBlocks(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - file1 := io.LimitReader( - readers.NewNonRepeatingSequence(0), - BLOCKSIZE*BLOCK_COUNT, - ) - - file2 := io.LimitReader( - io.MultiReader( - readers.OneReader(BLOCKSIZE), // Off by one block - readers.NewNonRepeatingSequence(0), - ), - BLOCKSIZE*BLOCK_COUNT, - ) - - output1 := bytes.NewBuffer(nil) - chksum1, _ := checksum.GenerateChecksums(file1, output1) - - output2 := bytes.NewBuffer(nil) - chksum2, _ := checksum.GenerateChecksums(file2, output2) - - if bytes.Compare(chksum1, chksum2) == 0 { - t.Fatal("Checksums should be different") - } - - weaksize, strongSize := checksum.GetChecksumSizes() - sums1, _ := chunks.LoadChecksumsFromReader(output1, weaksize, strongSize) - sums2, _ := chunks.LoadChecksumsFromReader(output2, weaksize, strongSize) - - if len(sums1) != len(sums2) { - t.Fatalf("Checksum lengths differ %v vs %v", len(sums1), len(sums2)) - } - - if sums1[0].Match(sums2[0]) { - t.Error("Chunk sums1[0] should differ from sums2[0]") - } - - for i, _ := range sums2 { - if i == 0 { - continue - } - - if !sums1[i-1].Match(sums2[i]) { - t.Errorf("Chunk sums1[%v] equal sums2[%v]", i-1, i) - } - - } -} - -// Test that partial content that ends in the middle of a weak -// hash is caught correctly -func TestInvalidReaderLength_Weak(t *testing.T) { - const BLOCKSIZE = 100 - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - file1 := io.LimitReader( - readers.NewNonRepeatingSequence(0), - int64(checksum.ChecksumSize())+2, - ) - - ws, ss := checksum.GetChecksumSizes() - r, err := chunks.LoadChecksumsFromReader(file1, ws, ss) - - if r != nil || err != chunks.ErrPartialChecksum { - t.Error("Expected partial checksum error") - } -} - -// Test that partial content that ends in the middle of a strong -// hash is caught correctly -func TestInvalidReaderLength_Strong(t *testing.T) { - const BLOCKSIZE = 100 - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - file1 := io.LimitReader( - readers.NewNonRepeatingSequence(0), - int64(checksum.ChecksumSize())+int64(checksum.WeakRollingHash.Size())+2, - ) - - ws, ss := checksum.GetChecksumSizes() - r, err := chunks.LoadChecksumsFromReader(file1, ws, ss) - - if r != nil || err != chunks.ErrPartialChecksum { - t.Error("Expected partial checksum error") - } -} - -func ExampleFileChecksumGenerator_LoadChecksumsFromReader() { - const BLOCKSIZE = 8096 - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - // This could be any source that conforms to io.Reader - // sections of a file, or the body of an http response - file1, err := os.Open("fileChecksums.chk") - - if err != nil { - return - } - - defer file1.Close() - - ws, ss := checksum.GetChecksumSizes() - checksums, err := chunks.LoadChecksumsFromReader(file1, ws, ss) - - if err != nil { - return - } - - // Make an index that we can use against our local - // checksums - i := index.MakeChecksumIndex(checksums) - - // example checksum from a local file - // look for the chunk in the index - i.FindWeakChecksumInIndex([]byte("a")) - -} diff --git a/vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier.go b/vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier.go deleted file mode 100644 index 9d604a5..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier.go +++ /dev/null @@ -1,48 +0,0 @@ -package filechecksum - -import ( - "bytes" - "hash" -) - -type ChecksumLookup interface { - GetStrongChecksumForBlock(blockID int) []byte -} - -type HashVerifier struct { - BlockSize uint - Hash hash.Hash - BlockChecksumGetter ChecksumLookup -} - -func (v *HashVerifier) VerifyBlockRange(startBlockID uint, data []byte) bool { - for i := 0; i*int(v.BlockSize) < len(data); i++ { - start := i * int(v.BlockSize) - end := start + int(v.BlockSize) - - if end > len(data) { - end = len(data) - } - - blockData := data[start:end] - - expectedChecksum := v.BlockChecksumGetter.GetStrongChecksumForBlock( - int(startBlockID) + i, - ) - - if expectedChecksum == nil { - return true - } - - v.Hash.Write(blockData) - hashedData := v.Hash.Sum(nil) - - if bytes.Compare(expectedChecksum, hashedData) != 0 { - return false - } - - v.Hash.Reset() - } - - return true -} diff --git a/vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier_test.go b/vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier_test.go deleted file mode 100644 index ce3938f..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/filechecksum/verifier_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package filechecksum - -import ( - "crypto/md5" - "testing" -) - -type SingleBlockSource []byte - -func (d SingleBlockSource) GetStrongChecksumForBlock(blockID int) []byte { - m := md5.New() - m.Write(d) - return m.Sum(nil) -} - -func TestBlockEqualsItself(t *testing.T) { - data := []byte("fooooo") - - h := HashVerifier{ - Hash: md5.New(), - BlockSize: uint(len(data)), - BlockChecksumGetter: SingleBlockSource(data), - } - - if !h.VerifyBlockRange(0, data) { - t.Error("data did not verify") - } -} - -type FourByteBlockSource []byte - -func (d FourByteBlockSource) GetStrongChecksumForBlock(blockID int) []byte { - m := md5.New() - - start := blockID * 4 - end := start + 4 - - if end >= len(d) { - end = len(d) - } - - m.Write(d[start:end]) - return m.Sum(nil) -} - -func TestSplitBlocksEqualThemselves(t *testing.T) { - data := []byte("foooBaar") - - h := HashVerifier{ - Hash: md5.New(), - BlockSize: uint(4), - BlockChecksumGetter: FourByteBlockSource(data), - } - - if !h.VerifyBlockRange(0, data) { - t.Error("data did not verify") - } -} - -func TestPartialBlock(t *testing.T) { - data := []byte("fo") - - h := HashVerifier{ - Hash: md5.New(), - BlockSize: uint(4), - BlockChecksumGetter: SingleBlockSource(data), - } - - if !h.VerifyBlockRange(0, data) { - t.Error("data did not verify") - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync.go b/vendor/src/github.com/Redundancy/go-sync/gosync.go deleted file mode 100644 index ea07157..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync.go +++ /dev/null @@ -1,16 +0,0 @@ -/* -Package gosync is inspired by zsync, and rsync. It aims to take the fundamentals and create a very flexible library that can be adapted -to work in many ways. - -We rely heavily on built in Go abstractions like io.Reader, hash.Hash and our own interfaces - this makes the code easier to change, and to test. -In particular, no part of the core library should know anything about the transport or layout of the reference data. If you want -to do rsync and do http/https range requests, that's just as good as zsync client-server over an SSH tunnel. The goal is also to allow -support for multiple concurrent connections, so that you can make the best use of your line in the face of the bandwidth latency product -(or other concerns that require concurrency to solve). - -The following optimizations are possible: -* Generate hashes with multiple threads (both during reference generation and local file interrogation) -* Multiple ranged requests (can even be used to get the hashes) - -*/ -package gosync diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync/build.go b/vendor/src/github.com/Redundancy/go-sync/gosync/build.go deleted file mode 100644 index 7dfb000..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync/build.go +++ /dev/null @@ -1,114 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/codegangsta/cli" -) - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "build", - ShortName: "b", - Usage: "build a .gosync file for a file", - Action: Build, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "blocksize", - Value: DefaultBlockSize, - Usage: "The block size to use for the gosync file", - }, - }, - }, - ) -} - -func Build(c *cli.Context) { - filename := c.Args()[0] - blocksize := uint32(c.Int("blocksize")) - generator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) - inputFile, err := os.Open(filename) - - if err != nil { - absInputPath, err2 := filepath.Abs(filename) - if err2 == nil { - handleFileError(absInputPath, err) - } else { - handleFileError(filename, err) - } - - os.Exit(1) - } - - s, _ := inputFile.Stat() - // TODO: Error? - file_size := s.Size() - - defer inputFile.Close() - - ext := filepath.Ext(filename) - outfilePath := filename[:len(filename)-len(ext)] + ".gosync" - outputFile, err := os.Create(outfilePath) - - if err != nil { - handleFileError(outfilePath, err) - os.Exit(1) - } - - defer outputFile.Close() - - if err = writeHeaders( - outputFile, - magicString, - blocksize, - file_size, - []uint16{majorVersion, minorVersion, patchVersion}, - ); err != nil { - fmt.Fprintf( - os.Stderr, - "Error getting file info: %v\n", - filename, - err, - ) - os.Exit(2) - } - - start := time.Now() - _, err = generator.GenerateChecksums(inputFile, outputFile) - end := time.Now() - - if err != nil { - fmt.Fprintf( - os.Stderr, - "Error generating checksum: %v\n", - filename, - err, - ) - os.Exit(2) - } - - inputFileInfo, err := os.Stat(filename) - if err != nil { - fmt.Fprintf( - os.Stderr, - "Error getting file info: %v\n", - filename, - err, - ) - os.Exit(2) - } - - fmt.Fprintf( - os.Stderr, - "Index for %v file generated in %v (%v bytes/S)\n", - inputFileInfo.Size(), - end.Sub(start), - float64(inputFileInfo.Size())/end.Sub(start).Seconds(), - ) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync/common.go b/vendor/src/github.com/Redundancy/go-sync/gosync/common.go deleted file mode 100644 index f8091a9..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync/common.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "bufio" - "encoding/binary" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/patcher" - "github.com/codegangsta/cli" -) - -const ( - // KB - One Kilobyte - KB = 1024 - // MB - One Megabyte - MB = 1000000 -) - -func errorWrapper(c *cli.Context, f func(*cli.Context) error) { - defer func() { - if p := recover(); p != nil { - fmt.Fprintln(os.Stderr, p) - os.Exit(1) - } - }() - - if err := f(c); err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) - } - - return -} - -func openFileAndHandleError(filename string) (f *os.File) { - var err error - f, err = os.Open(filename) - - if err != nil { - f = nil - handleFileError(filename, err) - } - - return -} - -func formatFileError(filename string, err error) error { - switch { - case os.IsExist(err): - return fmt.Errorf( - "Could not open %v (already exists): %v", - filename, - err, - ) - case os.IsNotExist(err): - return fmt.Errorf( - "Could not find %v: %v\n", - filename, - err, - ) - case os.IsPermission(err): - return fmt.Errorf( - "Could not open %v (permission denied): %v\n", - filename, - err, - ) - default: - return fmt.Errorf( - "Unknown error opening %v: %v\n", - filename, - err, - ) - } -} - -func handleFileError(filename string, err error) { - e := formatFileError(filename, err) - fmt.Fprintln(os.Stderr, e) -} - -func getLocalOrRemoteFile(path string) (io.ReadCloser, error) { - url, err := url.Parse(path) - - switch { - case err != nil: - return os.Open(path) - case url.Scheme == "": - return os.Open(path) - default: - response, err := http.Get(path) - - if err != nil { - return nil, err - } - - if response.StatusCode < 200 || response.StatusCode > 299 { - return nil, fmt.Errorf("Request to %v returned status: %v", path, response.Status) - } - - return response.Body, nil - } -} - -func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} - -func writeHeaders( - f *os.File, - magic string, - blocksize uint32, - filesize int64, - versions []uint16, -) (err error) { - if _, err = f.WriteString(magicString); err != nil { - return - } - - for _, v := range versions { - if err = binary.Write(f, binary.LittleEndian, v); err != nil { - return - } - } - - if err = binary.Write(f, binary.LittleEndian, filesize); err != nil { - return - } - - err = binary.Write(f, binary.LittleEndian, blocksize) - return -} - -// reads the file headers and checks the magic string, then the semantic versioning -func readHeadersAndCheck( - r io.Reader, - magic string, - requiredMajorVersion uint16, -) ( - major, minor, patch uint16, - filesize int64, - blocksize uint32, - err error, -) { - b := make([]byte, len(magicString)) - - if _, err = r.Read(b); err != nil { - return - } else if string(b) != magicString { - err = errors.New( - "file header does not match magic string. Not a valid gosync file", - ) - return - } - - for _, v := range []*uint16{&major, &minor, &patch} { - err = binary.Read(r, binary.LittleEndian, v) - if err != nil { - return - } - } - - if requiredMajorVersion != major { - err = fmt.Errorf( - "The major version of the gosync file (%v.%v.%v) does not match the tool (%v.%v.%v).", - major, minor, patch, - majorVersion, minorVersion, patchVersion, - ) - - return - } - - err = binary.Read(r, binary.LittleEndian, &filesize) - if err != nil { - return - } - - err = binary.Read(r, binary.LittleEndian, &blocksize) - return -} - -func readIndex(r io.Reader, blocksize uint) ( - i *index.ChecksumIndex, - checksumLookup filechecksum.ChecksumLookup, - blockCount uint, - err error, -) { - generator := filechecksum.NewFileChecksumGenerator(blocksize) - - readChunks, e := chunks.LoadChecksumsFromReader( - r, - generator.WeakRollingHash.Size(), - generator.StrongHash.Size(), - ) - - err = e - - if err != nil { - return - } - - checksumLookup = chunks.StrongChecksumGetter(readChunks) - i = index.MakeChecksumIndex(readChunks) - blockCount = uint(len(readChunks)) - - return -} - -func multithreadedMatching( - localFile *os.File, - idx *index.ChecksumIndex, - localFileSize, - matcherCount int64, - blocksize uint, -) (*comparer.MatchMerger, *comparer.Comparer) { - // Note: Since not all sections of the file are equal in work - // it would be better to divide things up into more sections and - // pull work from a queue channel as each finish - sectionSize := localFileSize / matcherCount - sectionSize += int64(blocksize) - (sectionSize % int64(blocksize)) - merger := &comparer.MatchMerger{} - compare := &comparer.Comparer{} - - for i := int64(0); i < matcherCount; i++ { - offset := sectionSize * i - - // Sections must overlap by blocksize (strictly blocksize - 1?) - if i > 0 { - offset -= int64(blocksize) - } - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(localFile, offset, sectionSize), - MB, - ) - - sectionGenerator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, idx) - - merger.StartMergeResultStream(matchStream, int64(blocksize)) - } - - return merger, compare -} - -// better way to do this? diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync/diff.go b/vendor/src/github.com/Redundancy/go-sync/gosync/diff.go deleted file mode 100644 index 14a4acf..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync/diff.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - "time" - - "github.com/codegangsta/cli" -) - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "diff", - ShortName: "d", - Usage: "gosync diff ", - Description: `Compare a file with a reference index, and print statistics on the comparison and performance.`, - Action: Diff, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "p", - Value: runtime.NumCPU(), - Usage: "The number of streams to use concurrently", - }, - }, - }, - ) -} - -func Diff(c *cli.Context) { - localFilename := c.Args()[0] - referenceFilename := c.Args()[1] - startTime := time.Now() - - localFile := openFileAndHandleError(localFilename) - - if localFile == nil { - os.Exit(1) - } - - defer localFile.Close() - - var blocksize uint32 - referenceFile := openFileAndHandleError(referenceFilename) - - if referenceFile == nil { - os.Exit(1) - } - - defer referenceFile.Close() - - _, _, _, _, blocksize, e := readHeadersAndCheck( - referenceFile, - magicString, - majorVersion, - ) - - if e != nil { - fmt.Printf("Error loading index: %v", e) - os.Exit(1) - } - - fmt.Println("Blocksize: ", blocksize) - - index, _, _, err := readIndex(referenceFile, uint(blocksize)) - referenceFile.Close() - - if err != nil { - return - } - - fmt.Println("Weak hash count:", index.WeakCount()) - - fi, err := localFile.Stat() - - if err != nil { - fmt.Println("Could not get info on file:", err) - os.Exit(1) - } - - num_matchers := int64(c.Int("p")) - - localFile_size := fi.Size() - - // Don't split up small files - if localFile_size < 1024*1024 { - num_matchers = 1 - } - - merger, compare := multithreadedMatching( - localFile, - index, - localFile_size, - num_matchers, - uint(blocksize), - ) - - mergedBlocks := merger.GetMergedBlocks() - - fmt.Println("\nMatched:") - totalMatchingSize := uint64(0) - matchedBlockCountAfterMerging := uint(0) - - for _, b := range mergedBlocks { - totalMatchingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) - matchedBlockCountAfterMerging += b.EndBlock - b.StartBlock + 1 - } - - fmt.Println("Comparisons:", compare.Comparisons) - fmt.Println("Weak hash hits:", compare.WeakHashHits) - - if compare.Comparisons > 0 { - fmt.Printf( - "Weak hit rate: %.2f%%\n", - 100.0*float64(compare.WeakHashHits)/float64(compare.Comparisons), - ) - } - - fmt.Println("Strong hash hits:", compare.StrongHashHits) - if compare.WeakHashHits > 0 { - fmt.Printf( - "Weak hash error rate: %.2f%%\n", - 100.0*float64(compare.WeakHashHits-compare.StrongHashHits)/float64(compare.WeakHashHits), - ) - } - - fmt.Println("Total matched bytes:", totalMatchingSize) - fmt.Println("Total matched blocks:", matchedBlockCountAfterMerging) - - // TODO: GetMissingBlocks uses the highest index, not the count, this can be pretty confusing - // Should clean up this interface to avoid that - missing := mergedBlocks.GetMissingBlocks(uint(index.BlockCount) - 1) - fmt.Println("Index blocks:", index.BlockCount) - - totalMissingSize := uint64(0) - for _, b := range missing { - //fmt.Printf("%#v\n", b) - totalMissingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) - } - - fmt.Println("Approximate missing bytes:", totalMissingSize) - fmt.Println("Time taken:", time.Now().Sub(startTime)) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync/fileformat.md b/vendor/src/github.com/Redundancy/go-sync/gosync/fileformat.md deleted file mode 100644 index 0fd5522..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync/fileformat.md +++ /dev/null @@ -1,21 +0,0 @@ -*NB: I'm documenting the current format of the gosync files merely as a point in time -reference of format that is in use in the tool that is meant to serve as a practical reference and acceptance testing tool. The gosync tool is not intended as a production-worthy, well supported, tested tool.* - -*The format used exists entirely in service of being able to test the implementation of the gosync library as a cohesive whole in the real world, and therefore backwards and forwards compatibility (or even efficiency) are not primary concerns.* - -# Version 0.2.0 -### The header -(LE = little endian) -* The string "G0S9NC" in UTF-8 -* versions*3 (eg. 0.1.2), uint16 LE -* filesize, int64 LE -* blocksize uint32 LE - -### The body -Repeating: -* WeakChecksum -* StrongChecksum - -each referring to blocks, starting at 0 (file start) and going upwards. - -In the current implementation of the FileChecksumGenerator used the WeakChecksum is the rolling checksum (4 bytes), and StrongChecksum is MD5 (16 bytes). diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync/main.go b/vendor/src/github.com/Redundancy/go-sync/gosync/main.go deleted file mode 100644 index 3164a8d..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync/main.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -gosync is a command-line implementation of the gosync package functionality, primarily as a demonstration of usage -but supposed to be functional in itself. -*/ -package main - -import ( - "fmt" - "log" - "net/http" - _ "net/http/pprof" - "os" - "runtime" - - "github.com/codegangsta/cli" -) - -const ( - DefaultBlockSize = 8192 - magicString = "G0S9NC" // just to confirm the file type is used correctly - majorVersion = uint16(0) - minorVersion = uint16(2) - patchVersion = uint16(1) -) - -var app = cli.NewApp() - -func main() { - app.Name = "gosync" - app.Usage = "Build indexes, patches, patch files" - app.Flags = []cli.Flag{ - cli.BoolFlag{ - Name: "profile", - Usage: "enable HTTP profiling", - }, - cli.IntFlag{ - Name: "profilePort", - Value: 6060, - Usage: "The number of streams to use concurrently", - }, - } - - app.Version = fmt.Sprintf( - "%v.%v.%v", - majorVersion, - minorVersion, - patchVersion, - ) - - runtime.GOMAXPROCS(runtime.NumCPU()) - - app.Before = func(c *cli.Context) error { - if c.Bool("profile") { - port := fmt.Sprint(c.Int("profilePort")) - - go func() { - log.Println(http.ListenAndServe("localhost:"+port, nil)) - }() - } - - return nil - } - - app.Run(os.Args) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync/patch.go b/vendor/src/github.com/Redundancy/go-sync/gosync/patch.go deleted file mode 100644 index 16730d3..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync/patch.go +++ /dev/null @@ -1,105 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - - gosync_main "github.com/Redundancy/go-sync" - "github.com/codegangsta/cli" -) - -const usage = "gosync patch []" - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "patch", - ShortName: "p", - Usage: usage, - Description: `Recreate the reference source file, using an index and a local file that is believed to be similar. -The index should be produced by "gosync build". - - is a .gosync file and may be a local, unc network path or http/https url - is corresponding target and may be a local, unc network path or http/https url - is optional. If not specified, the local file will be overwritten when done.`, - Action: Patch, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "p", - Value: runtime.NumCPU(), - Usage: "The number of streams to use concurrently", - }, - }, - }, - ) -} - -// Patch a file -func Patch(c *cli.Context) { - errorWrapper(c, func(c *cli.Context) error { - - fmt.Fprintln(os.Stderr, "Starting patching process") - - if l := len(c.Args()); l < 3 || l > 4 { - return fmt.Errorf( - "Usage is \"%v\" (invalid number of arguments)", - usage, - ) - } - - localFilename := c.Args()[0] - summaryFile := c.Args()[1] - referencePath := c.Args()[2] - - outFilename := localFilename - if len(c.Args()) == 4 { - outFilename = c.Args()[3] - } - - indexReader, e := os.Open(summaryFile) - if e != nil { - return e - } - defer indexReader.Close() - - _, _, _, filesize, blocksize, e := readHeadersAndCheck( - indexReader, - magicString, - majorVersion, - ) - - index, checksumLookup, blockCount, err := readIndex( - indexReader, - uint(blocksize), - ) - - fs := &gosync_main.BasicSummary{ - ChecksumIndex: index, - ChecksumLookup: checksumLookup, - BlockCount: blockCount, - BlockSize: uint(blocksize), - FileSize: filesize, - } - - rsync, err := gosync_main.MakeRSync( - localFilename, - referencePath, - outFilename, - fs, - ) - - if err != nil { - return err - } - - err = rsync.Patch() - - if err != nil { - return err - } - - return rsync.Close() - }) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/gosync_test.go b/vendor/src/github.com/Redundancy/go-sync/gosync_test.go deleted file mode 100644 index 72195db..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/gosync_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package gosync - -import ( - "fmt" - "testing" - - "bytes" - - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/indexbuilder" - "github.com/Redundancy/go-sync/util/readers" -) - -func Example() { - // due to short example strings, use a very small block size - // using one this small in practice would increase your file transfer! - const blockSize = 4 - - // This is the "file" as described by the authoritive version - const reference = "The quick brown fox jumped over the lazy dog" - - // This is what we have locally. Not too far off, but not correct. - const localVersion = "The qwik brown fox jumped 0v3r the lazy" - - generator := filechecksum.NewFileChecksumGenerator(blockSize) - _, referenceFileIndex, _, err := indexbuilder.BuildIndexFromString( - generator, - reference, - ) - - if err != nil { - return - } - - referenceAsBytes := []byte(reference) - localVersionAsBytes := []byte(localVersion) - - blockCount := len(referenceAsBytes) / blockSize - if len(referenceAsBytes)%blockSize != 0 { - blockCount++ - } - - inputFile := bytes.NewReader(localVersionAsBytes) - patchedFile := bytes.NewBuffer(nil) - - // This is more complicated than usual, because we're using in-memory - // "files" and sources. Normally you would use MakeRSync - summary := &BasicSummary{ - ChecksumIndex: referenceFileIndex, - ChecksumLookup: nil, - BlockCount: uint(blockCount), - BlockSize: blockSize, - FileSize: int64(len(referenceAsBytes)), - } - - rsync := &RSync{ - Input: inputFile, - Output: patchedFile, - Source: blocksources.NewReadSeekerBlockSource( - bytes.NewReader(referenceAsBytes), - blocksources.MakeNullFixedSizeResolver(uint64(blockSize)), - ), - Summary: summary, - OnClose: nil, - } - - if err := rsync.Patch(); err != nil { - fmt.Printf("Error: %v", err) - return - } - - fmt.Printf("Patched result: \"%s\"\n", patchedFile.Bytes()) - // Output: - // Patched result: "The quick brown fox jumped over the lazy dog" -} - -const ( - BYTE = 1 - KB = 1024 * BYTE - MB = 1024 * KB -) - -func BenchmarkIndexComparisons(b *testing.B) { - b.ReportAllocs() - - const SIZE = 200 * KB - b.SetBytes(SIZE) - - file := readers.NewSizedNonRepeatingSequence(6, SIZE) - generator := filechecksum.NewFileChecksumGenerator(8 * KB) - _, index, _, err := indexbuilder.BuildChecksumIndex(generator, file) - - if err != nil { - b.Fatal(err) - } - - b.StartTimer() - for i := 0; i < b.N; i++ { - // must reinitialize the file for each comparison - otherFile := readers.NewSizedNonRepeatingSequence(745656, SIZE) - compare := &comparer.Comparer{} - m := compare.StartFindMatchingBlocks(otherFile, 0, generator, index) - - for _, ok := <-m; ok; { - } - } - - b.StopTimer() -} diff --git a/vendor/src/github.com/Redundancy/go-sync/http_test.go b/vendor/src/github.com/Redundancy/go-sync/http_test.go deleted file mode 100644 index 6d7c047..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/http_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package gosync - -import ( - "bytes" - "crypto/md5" - "fmt" - "net" - "net/http" - "time" - - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/indexbuilder" - "github.com/Redundancy/go-sync/patcher" -) - -// due to short example strings, use a very small block size -// using one this small in practice would increase your file transfer! -const BLOCK_SIZE = 4 - -// This is the "file" as described by the authoritive version -const REFERENCE = "The quick brown fox jumped over the lazy dog" - -// This is what we have locally. Not too far off, but not correct. -const LOCAL_VERSION = "The qwik brown fox jumped 0v3r the lazy" - -var content = bytes.NewReader([]byte(REFERENCE)) - -func handler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), content) -} - -// set up a http server locally that will respond predictably to ranged requests -func setupServer() <-chan int { - var PORT = 8000 - s := http.NewServeMux() - s.HandleFunc("/content", handler) - - portChan := make(chan int) - - go func() { - var listener net.Listener - var err error - - for { - PORT++ - p := fmt.Sprintf(":%v", PORT) - listener, err = net.Listen("tcp", p) - - if err == nil { - break - } - } - portChan <- PORT - http.Serve(listener, s) - }() - - return portChan -} - -// This is exceedingly similar to the module Example, but uses the http blocksource and a local http server -func Example_httpBlockSource() { - PORT := <-setupServer() - LOCAL_URL := fmt.Sprintf("http://localhost:%v/content", PORT) - - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - _, referenceFileIndex, checksumLookup, err := indexbuilder.BuildIndexFromString(generator, REFERENCE) - - if err != nil { - return - } - - fileSize := int64(len([]byte(REFERENCE))) - - // This would normally be saved in a file - - blockCount := fileSize / BLOCK_SIZE - if fileSize%BLOCK_SIZE != 0 { - blockCount++ - } - - fs := &BasicSummary{ - ChecksumIndex: referenceFileIndex, - ChecksumLookup: checksumLookup, - BlockCount: uint(blockCount), - BlockSize: uint(BLOCK_SIZE), - FileSize: fileSize, - } - - /* - // Normally, this would be: - rsync, err := MakeRSync( - "toPatch.file", - "http://localhost/content", - "out.file", - fs, - ) - */ - // Need to replace the output and the input - inputFile := bytes.NewReader([]byte(LOCAL_VERSION)) - patchedFile := bytes.NewBuffer(nil) - - resolver := blocksources.MakeFileSizedBlockResolver( - uint64(fs.GetBlockSize()), - fs.GetFileSize(), - ) - - rsync := &RSync{ - Input: inputFile, - Output: patchedFile, - Source: blocksources.NewHttpBlockSource( - LOCAL_URL, - 1, - resolver, - &filechecksum.HashVerifier{ - Hash: md5.New(), - BlockSize: fs.GetBlockSize(), - BlockChecksumGetter: fs, - }, - ), - Summary: fs, - OnClose: nil, - } - - err = rsync.Patch() - - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - err = rsync.Close() - - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - fmt.Printf("Patched content: \"%v\"\n", patchedFile.String()) - - // Just for inspection - remoteReferenceSource := rsync.Source.(*blocksources.BlockSourceBase) - fmt.Printf("Downloaded Bytes: %v\n", remoteReferenceSource.ReadBytes()) - - // Output: - // Patched content: "The quick brown fox jumped over the lazy dog" - // Downloaded Bytes: 16 -} - -func ToPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func ToPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} diff --git a/vendor/src/github.com/Redundancy/go-sync/index/index.go b/vendor/src/github.com/Redundancy/go-sync/index/index.go deleted file mode 100644 index 6711024..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/index/index.go +++ /dev/null @@ -1,183 +0,0 @@ -/* -Package index provides the functionality to describe a reference 'file' and its contents in terms of -the weak and strong checksums, in such a way that you can check if a weak checksum is present, -then check if there is a strong checksum that matches. - -It also allows lookups in terms of block offsets, so that upon finding a match, you can more efficiently -check if the next block follows it. - -The index structure does not lend itself to being an interface - the pattern of taking the result of looking for -the weak checksum and looking up the strong checksum in that requires us to return an object matching an interface which -both packages must know about. - -Here's the interface: - -type Index interface { - FindWeakChecksum(chk []byte) interface{} - FindStrongChecksum(chk []byte, weak interface{}) []chunks.ChunkChecksum -} - -This allows the implementation to rely on a previously generated value, without the users knowing what it is. -This breaks the dependency that requires so many packages to import index. -*/ -package index - -import ( - "bytes" - "encoding/binary" - "github.com/Redundancy/go-sync/chunks" - "sort" -) - -type ChecksumIndex struct { - BlockCount int - - /* - This datastructure is based on some benchmarking that indicates that it outperforms - a basic map 70ns vs 130ns for ~8192 checksums (which is reasonably large - - say 64 MB with no weak collisions @8192 bytes per block). - - We use a 256 element slice, and the value of the least significant byte to determine - which map to look up into. - */ - weakChecksumLookup []map[uint32]StrongChecksumList - - MaxStrongLength int - AverageStrongLength float32 - Count int -} - -// Builds an index in which chunks can be found, with their corresponding offsets -// We use this for the -func MakeChecksumIndex(checksums []chunks.ChunkChecksum) *ChecksumIndex { - n := &ChecksumIndex{ - BlockCount: len(checksums), - weakChecksumLookup: make([]map[uint32]StrongChecksumList, 256), - } - - for _, chunk := range checksums { - weakChecksumAsInt := binary.LittleEndian.Uint32(chunk.WeakChecksum) - arrayOffset := weakChecksumAsInt & 255 - - if n.weakChecksumLookup[arrayOffset] == nil { - n.weakChecksumLookup[arrayOffset] = make(map[uint32]StrongChecksumList) - } - - n.weakChecksumLookup[arrayOffset][weakChecksumAsInt] = append( - n.weakChecksumLookup[arrayOffset][weakChecksumAsInt], - chunk, - ) - - } - - sum := 0 - count := 0 - - for _, a := range n.weakChecksumLookup { - for _, c := range a { - sort.Sort(c) - if len(c) > n.MaxStrongLength { - n.MaxStrongLength = len(c) - } - sum += len(c) - count += 1 - n.Count += len(c) - } - } - - n.AverageStrongLength = float32(sum) / float32(count) - - return n -} - -func (index *ChecksumIndex) WeakCount() int { - return index.Count -} - -func (index *ChecksumIndex) FindWeakChecksumInIndex(weak []byte) StrongChecksumList { - x := binary.LittleEndian.Uint32(weak) - if index.weakChecksumLookup[x&255] != nil { - if v, ok := index.weakChecksumLookup[x&255][x]; ok { - return v - } - } - return nil -} - -func (index *ChecksumIndex) FindWeakChecksum2(chk []byte) interface{} { - w := index.FindWeakChecksumInIndex(chk) - - if len(w) == 0 { - return nil - } else { - return w - } -} - -func (index *ChecksumIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { - if strongList, ok := weak.(StrongChecksumList); ok { - return strongList.FindStrongChecksum(chk) - } else { - return nil - } -} - -type StrongChecksumList []chunks.ChunkChecksum - -// Sortable interface -func (s StrongChecksumList) Len() int { - return len(s) -} - -// Sortable interface -func (s StrongChecksumList) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -// Sortable interface -func (s StrongChecksumList) Less(i, j int) bool { - return bytes.Compare(s[i].StrongChecksum, s[j].StrongChecksum) == -1 -} - -func (s StrongChecksumList) FindStrongChecksum(strong []byte) (result []chunks.ChunkChecksum) { - n := len(s) - - // average length is 1, so fast path comparison - if n == 1 { - if bytes.Compare(s[0].StrongChecksum, strong) == 0 { - return s - } else { - return nil - } - } - - // find the first possible occurance - first_gte_checksum := sort.Search( - n, - func(i int) bool { - return bytes.Compare(s[i].StrongChecksum, strong) >= 0 - }, - ) - - // out of bounds - if first_gte_checksum == -1 || first_gte_checksum == n { - return nil - } - - // Somewhere in the middle, but the next one didn't match - if bytes.Compare(s[first_gte_checksum].StrongChecksum, strong) != 0 { - return nil - } - - end := first_gte_checksum + 1 - for end < n { - if bytes.Compare(s[end].StrongChecksum, strong) == 0 { - end += 1 - } else { - break - } - - } - - return s[first_gte_checksum:end] -} diff --git a/vendor/src/github.com/Redundancy/go-sync/index/index_bench_test.go b/vendor/src/github.com/Redundancy/go-sync/index/index_bench_test.go deleted file mode 100644 index aace8a1..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/index/index_bench_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package index - -import ( - "github.com/Redundancy/go-sync/chunks" - "math/rand" - "sort" - "testing" -) - -var T = []byte{1, 2, 3, 4} - -func BenchmarkIndex1024(b *testing.B) { - i := ChecksumIndex{} - i.weakChecksumLookup = make([]map[uint32]StrongChecksumList, 256) - - for x := 0; x < 1024; x++ { - w := rand.Uint32() - - if i.weakChecksumLookup[w&255] == nil { - i.weakChecksumLookup[w&255] = make(map[uint32]StrongChecksumList) - } - - i.weakChecksumLookup[w&255][w] = append( - i.weakChecksumLookup[w&255][w], - chunks.ChunkChecksum{}, - ) - } - - b.SetBytes(1) - b.StartTimer() - for x := 0; x < b.N; x++ { - i.FindWeakChecksum2(T) - } - b.StopTimer() - -} - -func BenchmarkIndex8192(b *testing.B) { - i := ChecksumIndex{} - i.weakChecksumLookup = make([]map[uint32]StrongChecksumList, 256) - - for x := 0; x < 8192; x++ { - w := rand.Uint32() - - if i.weakChecksumLookup[w&255] == nil { - i.weakChecksumLookup[w&255] = make(map[uint32]StrongChecksumList) - } - - i.weakChecksumLookup[w&255][w] = append( - i.weakChecksumLookup[w&255][w], - chunks.ChunkChecksum{}, - ) - } - - b.SetBytes(1) - b.StartTimer() - for x := 0; x < b.N; x++ { - i.FindWeakChecksum2(T) - } - b.StopTimer() -} - -// Check how fast a sorted list of 8192 items would be -func BenchmarkIndexAsListBinarySearch8192(b *testing.B) { - b.SkipNow() - - s := make([]int, 8192) - for x := 0; x < 8192; x++ { - s[x] = rand.Int() - } - - sort.Ints(s) - - b.StartTimer() - for x := 0; x < b.N; x++ { - sort.SearchInts(s, rand.Int()) - } - b.StopTimer() -} - -// Check how fast a sorted list of 8192 items would be -// Checking for cache coherency gains -func BenchmarkIndexAsListLinearSearch8192(b *testing.B) { - s := make([]int, 8192) - for x := 0; x < 8192; x++ { - s[x] = rand.Int() - } - - sort.Ints(s) - - l := len(s) - b.StartTimer() - for x := 0; x < b.N; x++ { - v := rand.Int() - for i := 0; i < l; i++ { - if v < s[i] { - break - } - } - } - b.StopTimer() -} - -func Benchmark_256SplitBinarySearch(b *testing.B) { - a := make([][]int, 256) - for x := 0; x < 8192; x++ { - i := rand.Int() - a[i&255] = append( - a[i&255], - i, - ) - } - - for x := 0; x < 256; x++ { - sort.Ints(a[x]) - } - - b.StartTimer() - for x := 0; x < b.N; x++ { - v := rand.Int() - sort.SearchInts(a[v&255], v) - } - b.StopTimer() -} - -/* -This is currently the best performing contender for the index data structure for -weak checksum lookups. -*/ -func Benchmark_256Split_Map(b *testing.B) { - a := make([]map[int]interface{}, 256) - for x := 0; x < 8192; x++ { - i := rand.Int() - if a[i&255] == nil { - a[i&255] = make(map[int]interface{}) - } - a[i&255][i] = nil - } - - b.StartTimer() - for x := 0; x < b.N; x++ { - v := rand.Int() - if _, ok := a[v&255][v]; ok { - - } - } - b.StopTimer() -} diff --git a/vendor/src/github.com/Redundancy/go-sync/index/index_test.go b/vendor/src/github.com/Redundancy/go-sync/index/index_test.go deleted file mode 100644 index 3336134..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/index/index_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package index - -import ( - "testing" - - "github.com/Redundancy/go-sync/chunks" -) - -// Weak checksums must be 4 bytes -var WEAK_A = []byte("aaaa") -var WEAK_B = []byte("bbbb") - -/* -ChunkOffset uint -// the size of the block -Size int64 -WeakChecksum []byte -StrongChecksum []byte -*/ - -func TestMakeIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - }, - ) - - if i.Count != 2 { - t.Fatalf("Wrong count on index %v", i.Count) - } -} - -func TestFindWeakInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - result := i.FindWeakChecksumInIndex(WEAK_B) - - if result == nil { - t.Error("Did not find lookfor in the index") - } else if len(result) != 2 { - t.Errorf("Wrong number of possible matches found: %v", len(result)) - } else if result[0].ChunkOffset != 1 { - t.Errorf("Found chunk had offset %v expected 1", result[0].ChunkOffset) - } -} - -func TestWeakNotInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - result := i.FindWeakChecksumInIndex([]byte("afgh")) - - if result != nil { - t.Error("Result from FindWeakChecksumInIndex should be nil") - } - - result2 := i.FindWeakChecksum2([]byte("afgh")) - - if result2 != nil { - t.Errorf("Result from FindWeakChecksum2 should be nil: %#v", result2) - } -} - -func TestWeakNotInIndex2(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - result := i.FindWeakChecksumInIndex([]byte("llll")) - - if result != nil { - t.Error("Result should be nil") - } -} - -func TestFindStrongInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("c")) - - if len(strongs) != 1 { - t.Errorf("Incorrect number of strong checksums found: %v", len(strongs)) - } else if strongs[0].ChunkOffset != 1 { - t.Errorf("Wrong chunk found, had offset %v", strongs[0].ChunkOffset) - } -} - -func TestNotFoundStrongInIndexAtEnd(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("e")) - - if len(strongs) != 0 { - t.Errorf("Incorrect number of strong checksums found: %v", strongs) - } -} - -func TestNotFoundStrongInIndexInCenter(t *testing.T) { - // The strong checksum we're looking for is not found - // but is < another checksum in the strong list - - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - {ChunkOffset: 3, WeakChecksum: WEAK_B, StrongChecksum: []byte("f")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("e")) - - if len(strongs) != 0 { - t.Errorf("Incorrect number of strong checksums found: %v", strongs) - } -} - -func TestFindDuplicatedBlocksInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 3, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("c")) - - if len(strongs) != 2 { - t.Fatalf("Incorrect number of strong checksums found: %v", strongs) - } - - first := strongs[0] - if first.ChunkOffset != 1 { - t.Errorf("Wrong chunk found, had offset %v", first.ChunkOffset) - } - - second := strongs[1] - if second.ChunkOffset != 3 { - t.Errorf("Wrong chunk found, had offset %v", second.ChunkOffset) - } -} - -func TestFindTwoDuplicatedBlocksInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("c")) - - if len(strongs) != 2 { - t.Fatalf("Incorrect number of strong checksums found: %v", strongs) - } - - first := strongs[0] - if first.ChunkOffset != 1 { - t.Errorf("Wrong chunk found, had offset %v", first.ChunkOffset) - } - - second := strongs[1] - if second.ChunkOffset != 2 { - t.Errorf("Wrong chunk found, had offset %v", second.ChunkOffset) - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go b/vendor/src/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go deleted file mode 100644 index 8d2ebef..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Package indexbuilder provides a few shortbuts to building a checksum index by generating and then loading -the checksums, and building an index from that. It's potentially a sign that the responsibilities here need refactoring. -*/ -package indexbuilder - -import ( - "bytes" - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "io" -) - -// Generates an index from a reader -// This is mostly a utility function to avoid being overly verbose in tests that need -// an index to work, but don't want to construct one by hand in order to avoid the dependencies -// obviously this means that those tests are likely to fail if there are issues with any of the other -// modules, which is not ideal. -// TODO: move to util? -func BuildChecksumIndex(check *filechecksum.FileChecksumGenerator, r io.Reader) ( - fcheck []byte, - i *index.ChecksumIndex, - lookup filechecksum.ChecksumLookup, - err error, -) { - b := bytes.NewBuffer(nil) - fcheck, err = check.GenerateChecksums(r, b) - - if err != nil { - return - } - - weakSize := check.WeakRollingHash.Size() - strongSize := check.GetStrongHash().Size() - readChunks, err := chunks.LoadChecksumsFromReader(b, weakSize, strongSize) - - if err != nil { - return - } - - i = index.MakeChecksumIndex(readChunks) - lookup = chunks.StrongChecksumGetter(readChunks) - - return -} - -func BuildIndexFromString(generator *filechecksum.FileChecksumGenerator, reference string) ( - fileCheckSum []byte, - referenceIndex *index.ChecksumIndex, - lookup filechecksum.ChecksumLookup, - err error, -) { - return BuildChecksumIndex(generator, bytes.NewBufferString(reference)) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/patcher/blocksource.go b/vendor/src/github.com/Redundancy/go-sync/patcher/blocksource.go deleted file mode 100644 index c50241d..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/patcher/blocksource.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Package patcher follows a pattern established by hash, which defines the interface in the top level package, and then provides implementations -below it. -*/ -package patcher - -import ( - "hash" -) - -/* -BlockSource is an interface used by the patchers to obtain blocks from the reference -It does not stipulate where the reference data might be (it could be local, in a pre-built patch file, on S3 or somewhere else) - -It is assumed that the BlockSource may be slow, and may benefit from request pipelining & concurrency. -Therefore patchers should feel free to request as many block spans as they can handle. - -A BlockSource may be a view onto a larger transport concept, so that multiple files can be handled with wider -knowledge of the number of simultaneous requests allowed, etc. The BlockSource may decide to split BlockSpans -into smaller sizes if it wants. - -It is up to the patcher to receive blocks in a timely manner, and decide what to do with them, rather than -bother the BlockSource with more memory management and buffering logic. - -Since these interfaces require specific structs to satisfy, it's expected that implementers will import this module. - -*/ -type BlockSource interface { - RequestBlocks(MissingBlockSpan) error - - GetResultChannel() <-chan BlockReponse - - // If the block source encounters an unsurmountable problem - EncounteredError() <-chan error -} - -type FoundBlockSpan struct { - StartBlock uint - EndBlock uint - BlockSize int64 - MatchOffset int64 -} - -type MissingBlockSpan struct { - StartBlock uint - EndBlock uint - - BlockSize int64 - // a hasher to use to ensure that the block response matches - Hasher hash.Hash - // the hash values that the blocks should have - ExpectedSums [][]byte -} - -type BlockReponse struct { - StartBlock uint - Data []byte -} diff --git a/vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential.go b/vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential.go deleted file mode 100644 index b8a3018..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Sequential Patcher will stream the patched version of the file to output, -since it works strictly in order, it cannot patch the local file directly -(since it might overwrite a block needed later), -so there would have to be a final copy once the patching was done. -*/ -package sequential - -import ( - "fmt" - "github.com/Redundancy/go-sync/patcher" - "io" -) - -const ( - ABSOLUTE_POSITION = 0 -) - -/* -This simple example currently doesn't do any pipelining of needed blocks, nor does it deal with -blocks being delivered out of order. -*/ -func SequentialPatcher( - localFile io.ReadSeeker, - reference patcher.BlockSource, - requiredRemoteBlocks []patcher.MissingBlockSpan, - locallyAvailableBlocks []patcher.FoundBlockSpan, - maxBlockStorage uint64, // the amount of memory we're allowed to use for temporary data storage - output io.Writer, -) error { - - maxBlockMissing := uint(0) - if len(requiredRemoteBlocks) > 0 { - maxBlockMissing = requiredRemoteBlocks[len(requiredRemoteBlocks)-1].EndBlock - } - - maxBlockFound := uint(0) - if len(locallyAvailableBlocks) > 0 { - maxBlockFound = locallyAvailableBlocks[len(locallyAvailableBlocks)-1].EndBlock - } - - if reference == nil { - return fmt.Errorf("No BlockSource set for obtaining reference blocks") - } - - maxBlock := maxBlockMissing - if maxBlockFound > maxBlock { - maxBlock = maxBlockFound - } - - currentBlock := uint(0) - - // TODO: find a way to test this, since it seemed to be the cause of an issue - for currentBlock <= maxBlock { - // where is the next block supposed to come from? - if withinFirstBlockOfLocalBlocks(currentBlock, locallyAvailableBlocks) { - firstMatched := locallyAvailableBlocks[0] - - // we have the current block range in the local file - localFile.Seek(firstMatched.MatchOffset, ABSOLUTE_POSITION) - blockSizeToRead := int64(firstMatched.EndBlock-firstMatched.StartBlock+1) * firstMatched.BlockSize - - if _, err := io.Copy(output, io.LimitReader(localFile, blockSizeToRead)); err != nil { - return fmt.Errorf("Could not copy %v bytes to output: %v", blockSizeToRead, err) - } - - currentBlock = firstMatched.EndBlock + 1 - locallyAvailableBlocks = locallyAvailableBlocks[1:] - - } else if withinFirstBlockOfRemoteBlocks(currentBlock, requiredRemoteBlocks) { - firstMissing := requiredRemoteBlocks[0] - reference.RequestBlocks(firstMissing) - - select { - case result := <-reference.GetResultChannel(): - if result.StartBlock == currentBlock { - if _, err := output.Write(result.Data); err != nil { - return fmt.Errorf( - "Could not write data to output: %v", - err, - ) - } else { - - completed := calculateNumberOfCompletedBlocks( - len(result.Data), - firstMissing.BlockSize, - ) - - if completed != (firstMissing.EndBlock-firstMissing.StartBlock) + 1 { - return fmt.Errorf( - "Unexpected reponse length from remote source: blocks %v-%v (got %v blocks)", - firstMissing.StartBlock, - firstMissing.EndBlock, - completed, - ) - } - - currentBlock += completed - requiredRemoteBlocks = requiredRemoteBlocks[1:] - } - } else { - return fmt.Errorf( - "Received unexpected block: %v", - result.StartBlock, - ) - } - case err := <-reference.EncounteredError(): - return fmt.Errorf( - "Failed to read from reference file: %v", - err, - ) - } - - } else { - return fmt.Errorf( - "Could not find block in missing or matched list: %v\nRemote: %v\nLocal: %v\n", - currentBlock, - requiredRemoteBlocks, - locallyAvailableBlocks, - ) - } - } - - return nil -} - -func withinFirstBlockOfRemoteBlocks(currentBlock uint, remoteBlocks []patcher.MissingBlockSpan) bool { - return len(remoteBlocks) > 0 && remoteBlocks[0].StartBlock <= currentBlock && remoteBlocks[0].EndBlock >= currentBlock -} - -func withinFirstBlockOfLocalBlocks(currentBlock uint, localBlocks []patcher.FoundBlockSpan) bool { - return len(localBlocks) > 0 && localBlocks[0].StartBlock <= currentBlock && localBlocks[0].EndBlock >= currentBlock -} - -func calculateNumberOfCompletedBlocks(resultLength int, blockSize int64) (completedBlockCount uint) { - // TODO: lots of casting to uint here, is it safe? - completedBlockCount = uint(resultLength) / uint(blockSize) - - // round up in the case of a partial block (last block may not be full sized) - if uint(resultLength)%uint(blockSize) != 0 { - completedBlockCount += 1 - } - - return -} diff --git a/vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go b/vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go deleted file mode 100644 index 7608f8b..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package sequential - -import ( - "bytes" - "crypto/md5" - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/patcher" - "io" - "io/ioutil" - "strings" - "testing" -) - -const ( - BLOCKSIZE = 4 - REFERENCE_STRING = "The quick brown fox jumped over the lazy dog" -) - -var ( - REFERENCE_BUFFER = bytes.NewBufferString(REFERENCE_STRING) - REFERENCE_BLOCKS []string - BLOCK_COUNT int - REFERENCE_HASHES [][]byte -) - -func init() { - maxLen := len(REFERENCE_STRING) - m := md5.New() - for i := 0; i < maxLen; i += BLOCKSIZE { - last := i + 4 - - if last >= maxLen { - last = maxLen - 1 - } - - block := REFERENCE_STRING[i:last] - - REFERENCE_BLOCKS = append(REFERENCE_BLOCKS, block) - m.Write([]byte(block)) - REFERENCE_HASHES = append(REFERENCE_HASHES, m.Sum(nil)) - m.Reset() - } - - BLOCK_COUNT = len(REFERENCE_BLOCKS) -} - -func stringToReadSeeker(input string) io.ReadSeeker { - return bytes.NewReader([]byte(input)) -} - -func TestPatchingStart(t *testing.T) { - LOCAL := bytes.NewReader([]byte("48 brown fox jumped over the lazy dog")) - out := bytes.NewBuffer(nil) - - missing := []patcher.MissingBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 0, - EndBlock: 2, - Hasher: md5.New(), - ExpectedSums: REFERENCE_HASHES[0:3], - }, - } - - matched := []patcher.FoundBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 3, - EndBlock: 11, - MatchOffset: 5, - }, - } - - err := SequentialPatcher( - LOCAL, - blocksources.NewReadSeekerBlockSource( - stringToReadSeeker(REFERENCE_STRING), - blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), - ), - missing, - matched, - 1024, - out, - ) - - if err != nil { - t.Fatal(err) - } - - if result, err := ioutil.ReadAll(out); err == nil { - t.Logf("String split is: \"%v\"", strings.Join(REFERENCE_BLOCKS, "\", \"")) - if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { - t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) - } - } else { - t.Fatal(err) - } - // -} - -func TestPatchingEnd(t *testing.T) { - LOCAL := bytes.NewReader([]byte("The quick brown fox jumped over the l4zy d0g")) - out := bytes.NewBuffer(nil) - - missing := []patcher.MissingBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 9, - EndBlock: 10, - Hasher: md5.New(), - ExpectedSums: REFERENCE_HASHES[0:3], - }, - } - - matched := []patcher.FoundBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 0, - EndBlock: 8, - MatchOffset: 0, - }, - } - - err := SequentialPatcher( - LOCAL, - blocksources.NewReadSeekerBlockSource( - stringToReadSeeker(REFERENCE_STRING), - blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), - ), - missing, - matched, - 1024, - out, - ) - - if err != nil { - t.Fatal(err) - } - - if result, err := ioutil.ReadAll(out); err == nil { - if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { - t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) - } - } else { - t.Fatal(err) - } -} - -func TestPatchingEntirelyMissing(t *testing.T) { - LOCAL := bytes.NewReader([]byte("")) - out := bytes.NewBuffer(nil) - - missing := []patcher.MissingBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 0, - EndBlock: 10, - Hasher: md5.New(), - ExpectedSums: REFERENCE_HASHES[0:10], - }, - } - - matched := []patcher.FoundBlockSpan{} - - err := SequentialPatcher( - LOCAL, - blocksources.NewReadSeekerBlockSource( - stringToReadSeeker(REFERENCE_STRING), - blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), - ), - missing, - matched, - 1024, - out, - ) - - if err != nil { - t.Fatal(err) - } - - if result, err := ioutil.ReadAll(out); err == nil { - if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { - t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) - } - } else { - t.Fatal(err) - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32.go b/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32.go deleted file mode 100644 index 1fbe7b8..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -rollsum provides an implementation of a rolling checksum - a checksum that's efficient to advance a byte -or more at a time. It is inspired by the rollsum in rsync, but differs in that the internal values used -are 32bit integers - to make a conformant implementation, a find a replace on "32" should be almost sufficient -(although it would be highly recommended to test against known values from the original implementation). - -Rollsum32 supports the hash.Hash implementation, but is not used much in go-sync, mostly in order to -share and access the underlying circular buffer storage, and use the implementation as efficiently as possible. -*/ -package rollsum - -import ( - "github.com/Redundancy/go-sync/circularbuffer" -) - -func NewRollsum32(blocksize uint) *Rollsum32 { - return &Rollsum32{ - Rollsum32Base: Rollsum32Base{ - blockSize: blocksize, - }, - buffer: circularbuffer.MakeC2Buffer(int(blocksize)), - } -} - -// Rollsum32 is a rolling checksum implemenation -// inspired by rsync, but with 32bit internal values -// Create one using NewRollsum32 -type Rollsum32 struct { - Rollsum32Base - buffer *circularbuffer.C2 -} - -// cannot be called concurrently -func (r *Rollsum32) Write(p []byte) (n int, err error) { - ulen_p := uint(len(p)) - - if ulen_p >= r.blockSize { - // if it's really long, we can just ignore a load of it - remaining := p[ulen_p-r.blockSize:] - r.buffer.Write(remaining) - r.Rollsum32Base.SetBlock(remaining) - } else { - b_len := r.buffer.Len() - r.buffer.Write(p) - evicted := r.buffer.Evicted() - r.Rollsum32Base.AddAndRemoveBytes(p, evicted, b_len) - } - - return len(p), nil -} - -// The most efficient byte length to call Write with -func (r *Rollsum32) BlockSize() int { - return int(r.blockSize) -} - -// the number of bytes -func (r *Rollsum32) Size() int { - return 4 -} - -func (r *Rollsum32) Reset() { - r.Rollsum32Base.Reset() - r.buffer.Reset() -} - -// Sum appends the current hash to b and returns the resulting slice. -// It does not change the underlying hash state. -// Note that this is to allow Sum() to reuse a preallocated buffer -func (r *Rollsum32) Sum(b []byte) []byte { - if b != nil && cap(b)-len(b) >= 4 { - p := len(b) - b = b[:len(b)+4] - r.Rollsum32Base.GetSum(b[p:]) - return b - } else { - result := []byte{0, 0, 0, 0} - r.Rollsum32Base.GetSum(result) - return append(b, result...) - } -} - -func (r *Rollsum32) GetLastBlock() []byte { - return r.buffer.GetBlock() -} diff --git a/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go b/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go deleted file mode 100644 index 8153e72..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go +++ /dev/null @@ -1,86 +0,0 @@ -package rollsum - -import ( - "encoding/binary" -) - -const FULL_BYTES_16 = (1 << 16) - 1 - -// Rollsum32Base decouples the rollsum algorithm from the implementation of -// hash.Hash and the storage the rolling checksum window -// this allows us to write different versions of the storage for the distinctly different -// use-cases and optimize the storage with the usage pattern. -func NewRollsum32Base(blockSize uint) *Rollsum32Base { - return &Rollsum32Base{blockSize: blockSize} -} - -// The specification of hash.Hash is such that it cannot be implemented without implementing storage -// but the most optimal storage scheme depends on usage of the circular buffer & hash -type Rollsum32Base struct { - blockSize uint - a, b uint32 -} - -// Add a single byte into the rollsum -func (r *Rollsum32Base) AddByte(b byte) { - r.a += uint32(b) - r.b += r.a -} - -func (r *Rollsum32Base) AddBytes(bs []byte) { - for _, b := range bs { - r.a += uint32(b) - r.b += r.a - } -} - -// Remove a byte from the end of the rollsum -// Use the previous length (before removal) -func (r *Rollsum32Base) RemoveByte(b byte, length int) { - r.a -= uint32(b) - r.b -= uint32(uint(length) * uint(b)) -} - -func (r *Rollsum32Base) RemoveBytes(bs []byte, length int) { - for _, b := range bs { - r.a -= uint32(b) - r.b -= uint32(uint(length) * uint(b)) - length -= 1 - } -} - -func (r *Rollsum32Base) AddAndRemoveBytes(add []byte, remove []byte, length int) { - len_added := len(add) - len_removed := len(remove) - - startEvicted := len_added - len_removed - r.AddBytes(add[:startEvicted]) - length += startEvicted - - for i := startEvicted; i < len_added; i++ { - r.RemoveByte(remove[i-startEvicted], length) - r.AddByte(add[i]) - } -} - -// Set a whole block of blockSize -func (r *Rollsum32Base) SetBlock(block []byte) { - r.Reset() - r.AddBytes(block) -} - -// Reset the hash to the initial state -func (r *Rollsum32Base) Reset() { - r.a, r.b = 0, 0 -} - -// size of the hash in bytes -func (r *Rollsum32Base) Size() int { - return 4 -} - -// Puts the sum into b. Avoids allocation. b must have length >= 4 -func (r *Rollsum32Base) GetSum(b []byte) { - value := uint32((r.a & FULL_BYTES_16) + ((r.b & FULL_BYTES_16) << 16)) - binary.LittleEndian.PutUint32(b, value) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go b/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go deleted file mode 100644 index cbf4c93..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go +++ /dev/null @@ -1,298 +0,0 @@ -package rollsum - -import ( - "bytes" - "github.com/Redundancy/go-sync/circularbuffer" - "hash" - "io" - "testing" -) - -func TestThatRollsum32SatisfiesHashInterface(t *testing.T) { - var i hash.Hash = NewRollsum32(10) - i.Reset() -} - -func TestThatRollsum32SatisfiedWriterInterface(t *testing.T) { - var i io.Writer = NewRollsum32(10) - n, err := i.Write([]byte{1, 2, 3, 4}) - - if n != 4 { - t.Error("Did not report writing 4 bytes") - } - - if err != nil { - t.Error(err) - } -} - -func TestThatRollsum32IsTheSameAfterBlockSizeBytes(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3, 4}) - - r2.Write([]byte{7, 6}) - r2.Write([]byte{5, 1, 2}) - r2.Write([]byte{3, 4}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums are different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThatRollsum32IsTheSameAfterBlockSizeBytesWithPartialEviction(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3, 4}) - - r2.Write([]byte{7, 5}) - r2.Write([]byte{1, 2, 3, 4}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums are different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestRegression2(t *testing.T) { - const A = "The quick br" - const B = "The qwik br" - - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte(A[:4])) - r1.Reset() - r1.Write([]byte(A[4:8])) - r1.Reset() - r1.Write([]byte(A[8:12])) - - r2.Write([]byte(B[:4])) - r2.Write([]byte(B[4:8])) - for _, c := range B[8:] { - r2.Write([]byte{byte(c)}) - } - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums are different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThatRollsum32RemovesBytesCorrectly(t *testing.T) { - r1 := NewRollsum32Base(2) - - r1.AddByte(255) - r1.AddByte(10) - r1.RemoveByte(255, 2) - r1.AddByte(0) - r1.RemoveByte(10, 2) - r1.AddByte(0) - - if r1.a != 0 || r1.b != 0 { - t.Errorf("Values are not reset: %v %v", r1.a, r1.b) - } -} - -func TestThatRollsum32IsDifferentForDifferentInput(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3, 4}) - r2.Write([]byte{7, 6, 5, 1}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) == 0 { - t.Errorf( - "Rollsums should be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestResettingTheRollsum32(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3}) - - r2.Write([]byte{7, 6}) - r2.Reset() - r2.Write([]byte{1, 2, 3}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums should not be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestTruncatingPartiallyFilledBufferResultsInSameState(t *testing.T) { - r1 := NewRollsum32Base(4) - r2 := NewRollsum32Base(4) - - r1.AddByte(2) - sum1 := make([]byte, 4) - r1.GetSum(sum1) - - r2.AddByte(1) - r2.AddByte(2) - // Removal works from the left - r2.RemoveByte(1, 2) - sum2 := make([]byte, 4) - r2.GetSum(sum2) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums should not be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThat32SumDoesNotChangeTheHashState(t *testing.T) { - r1 := NewRollsum32(4) - - sum1 := r1.Sum([]byte{1, 2, 3}) - sum2 := r1.Sum([]byte{3, 4, 5}) - - if bytes.Compare(sum1[3:], sum2[3:]) != 0 { - t.Errorf( - "Rollsums should not be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThat32OutputLengthMatchesSize(t *testing.T) { - r1 := NewRollsum32(4) - sumLength := len(r1.Sum(nil)) - - if sumLength != r1.Size() { - t.Errorf("Unexpected length: %v vs expected %v", sumLength, r1.Size()) - } -} - -func BenchmarkRollsum32(b *testing.B) { - r := NewRollsum32(100) - buffer := make([]byte, 100) - b.ReportAllocs() - b.SetBytes(int64(len(buffer))) - checksum := make([]byte, 16) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.Write(buffer) - r.Sum(checksum) - checksum = checksum[:0] - } - b.StopTimer() -} - -func BenchmarkRollsum32_8096(b *testing.B) { - r := NewRollsum32(8096) - buffer := make([]byte, 8096) - b.ReportAllocs() - b.SetBytes(int64(len(buffer))) - checksum := make([]byte, 16) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.Write(buffer) - r.Sum(checksum) - checksum = checksum[:0] - } - b.StopTimer() -} - -func BenchmarkRollsum32Base(b *testing.B) { - r := Rollsum32Base{blockSize: 100} - buffer := make([]byte, 100) - checksum := make([]byte, 16) - b.ReportAllocs() - b.SetBytes(int64(len(buffer))) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.SetBlock(buffer) - r.GetSum(checksum) - } - b.StopTimer() - -} - -// This is the benchmark where Rollsum should beat a full MD5 for each blocksize -func BenchmarkIncrementalRollsum32(b *testing.B) { - r := NewRollsum32(100) - buffer := make([]byte, 100) - r.Write(buffer) - b.SetBytes(1) - - b.ReportAllocs() - checksum := make([]byte, 16) - increment := make([]byte, 1) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.Write(increment) - r.Sum(checksum) - checksum = checksum[:0] - } - b.StopTimer() -} - -// The C2 veersion should avoid all allocations in the main loop, and beat the pants off the -// other versions -func BenchmarkIncrementalRollsum32WithC2(b *testing.B) { - const BLOCK_SIZE = 100 - r := NewRollsum32Base(BLOCK_SIZE) - buffer := make([]byte, BLOCK_SIZE) - b.SetBytes(1) - cbuffer := circularbuffer.MakeC2Buffer(BLOCK_SIZE) - - r.AddBytes(buffer) - cbuffer.Write(buffer) - - b.ReportAllocs() - checksum := make([]byte, 16) - increment := make([]byte, 1) - - b.StartTimer() - for i := 0; i < b.N; i++ { - cbuffer.Write(increment) - r.AddAndRemoveBytes(increment, cbuffer.Evicted(), BLOCK_SIZE) - r.GetSum(checksum) - } - b.StopTimer() -} diff --git a/vendor/src/github.com/Redundancy/go-sync/rsync.go b/vendor/src/github.com/Redundancy/go-sync/rsync.go deleted file mode 100644 index 59db7c1..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/rsync.go +++ /dev/null @@ -1,347 +0,0 @@ -package gosync - -import ( - "bufio" - "crypto/md5" - "fmt" - "io" - "io/ioutil" - "os" - "runtime" - - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/patcher" - "github.com/Redundancy/go-sync/patcher/sequential" -) - -const ( - megabyte = 1000000 -) - -var ( - // DefaultConcurrency is the default concurrency level used by patching and downloading - DefaultConcurrency = runtime.NumCPU() -) - -// ReadSeekerAt is the combinaton of ReadSeeker and ReaderAt interfaces -type ReadSeekerAt interface { - io.ReadSeeker - io.ReaderAt -} - -/* -RSync is an object designed to make the standard use-case for gosync as -easy as possible. - -To this end, it hides away many low level choices by default, and makes some -assumptions. -*/ -type RSync struct { - Input ReadSeekerAt - Source patcher.BlockSource - Output io.Writer - - Summary FileSummary - - OnClose []closer -} - -type closer interface { - Close() error -} - -// FileSummary combines many of the interfaces that are needed -// It is expected that you might implement it by embedding existing structs -type FileSummary interface { - GetBlockSize() uint - GetBlockCount() uint - GetFileSize() int64 - FindWeakChecksum2(bytes []byte) interface{} - FindStrongChecksum2(bytes []byte, weak interface{}) []chunks.ChunkChecksum - GetStrongChecksumForBlock(blockID int) []byte -} - -// BasicSummary implements a version of the FileSummary interface -type BasicSummary struct { - BlockSize uint - BlockCount uint - FileSize int64 - *index.ChecksumIndex - filechecksum.ChecksumLookup -} - -// GetBlockSize gets the size of each block -func (fs *BasicSummary) GetBlockSize() uint { - return fs.BlockSize -} - -// GetBlockCount gets the number of blocks -func (fs *BasicSummary) GetBlockCount() uint { - return fs.BlockCount -} - -// GetFileSize gets the file size of the file -func (fs *BasicSummary) GetFileSize() int64 { - return fs.FileSize -} - -// MakeRSync creates an RSync object using string paths, -// inferring most of the configuration -func MakeRSync( - InputFile, - Source, - OutFile string, - Summary FileSummary, -) (r *RSync, err error) { - useTempFile := false - if useTempFile, err = IsSameFile(InputFile, OutFile); err != nil { - return nil, err - } - - inputFile, err := os.Open(InputFile) - - if err != nil { - return - } - - var out io.WriteCloser - var outFilename = OutFile - var copier closer - - if useTempFile { - out, outFilename, err = getTempFile() - - if err != nil { - return - } - - copier = &fileCopyCloser{ - from: outFilename, - to: OutFile, - } - } else { - out, err = getOutFile(OutFile) - - if err != nil { - return - } - - copier = nullCloser{} - } - - // blocksource - var source *blocksources.BlockSourceBase - - resolver := blocksources.MakeFileSizedBlockResolver( - uint64(Summary.GetBlockSize()), - Summary.GetFileSize(), - ) - - source = blocksources.NewHttpBlockSource( - Source, - DefaultConcurrency, - resolver, - &filechecksum.HashVerifier{ - Hash: md5.New(), - BlockSize: Summary.GetBlockSize(), - BlockChecksumGetter: Summary, - }, - ) - - r = &RSync{ - Input: inputFile, - Output: out, - Source: source, - Summary: Summary, - OnClose: []closer{ - &fileCloser{inputFile, InputFile}, - &fileCloser{out, outFilename}, - copier, - }, - } - - return -} - -// Patch the files -func (rsync *RSync) Patch() (err error) { - numMatchers := int64(DefaultConcurrency) - blockSize := rsync.Summary.GetBlockSize() - sectionSize := rsync.Summary.GetFileSize() / numMatchers - sectionSize += int64(blockSize) - (sectionSize % int64(blockSize)) - - merger := &comparer.MatchMerger{} - - for i := int64(0); i < numMatchers; i++ { - compare := &comparer.Comparer{} - offset := sectionSize * i - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(rsync.Input, offset, sectionSize+int64(blockSize)), - megabyte, // 1 MB buffer - ) - - // Bakes in the assumption about how to generate checksums (extract) - sectionGenerator := filechecksum.NewFileChecksumGenerator( - uint(blockSize), - ) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, rsync.Summary, - ) - - merger.StartMergeResultStream(matchStream, int64(blockSize)) - } - - mergedBlocks := merger.GetMergedBlocks() - missing := mergedBlocks.GetMissingBlocks(rsync.Summary.GetBlockCount() - 1) - - return sequential.SequentialPatcher( - rsync.Input, - rsync.Source, - toPatcherMissingSpan(missing, int64(blockSize)), - toPatcherFoundSpan(mergedBlocks, int64(blockSize)), - 20*megabyte, - rsync.Output, - ) -} - -func getOutFile(filename string) (f io.WriteCloser, err error) { - if _, err = os.Stat(filename); os.IsNotExist(err) { - return os.Create(filename) - } - - return os.OpenFile(filename, os.O_WRONLY, 0) -} - -func getTempFile() (f io.WriteCloser, filename string, err error) { - ft, err := ioutil.TempFile(".", "tmp_") - filename = ft.Name() - f = ft - return -} - -// IsSameFile checks if two file paths are the same file -func IsSameFile(path1, path2 string) (same bool, err error) { - - fi1, err := os.Stat(path1) - - switch { - case os.IsNotExist(err): - return false, nil - case err != nil: - return - } - - fi2, err := os.Stat(path2) - - switch { - case os.IsNotExist(err): - return false, nil - case err != nil: - return - } - - return os.SameFile(fi1, fi2), nil -} - -// Close - close open files, copy to the final location from -// a temporary one if neede -func (rsync *RSync) Close() error { - for _, f := range rsync.OnClose { - if err := f.Close(); err != nil { - return err - } - } - return nil -} - -type fileCloser struct { - f io.Closer - path string -} - -// Close - add file path information to closing a file -func (f *fileCloser) Close() error { - err := f.f.Close() - if err != nil { - return fmt.Errorf( - "Could not close file %v: %v", - f.path, - err, - ) - } - return nil -} - -type nullCloser struct{} - -func (n nullCloser) Close() error { - return nil -} - -type fileCopyCloser struct { - from string - to string -} - -func (f *fileCopyCloser) Close() (err error) { - from, err := os.OpenFile(f.from, os.O_RDONLY, 0) - - if err != nil { - return err - } - - defer func() { - e := from.Close() - if err != nil { - err = e - } - }() - - to, err := os.OpenFile(f.to, os.O_TRUNC|os.O_WRONLY, 0) - - if err != nil { - return err - } - - defer func() { - e := to.Close() - if err != nil { - err = e - } - }() - - bufferedReader := bufio.NewReaderSize(from, megabyte) - _, err = io.Copy(to, bufferedReader) - return -} - -func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/injectedreader.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/injectedreader.go deleted file mode 100644 index 5da00af..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/injectedreader.go +++ /dev/null @@ -1,18 +0,0 @@ -package readers - -import ( - "io" -) - -// Injects the second reader into the first at an offset -func InjectedReader( - offsetFromStart int64, - base io.Reader, - inject io.Reader, -) io.Reader { - return io.MultiReader( - io.LimitReader(base, offsetFromStart), - inject, - base, - ) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating.go deleted file mode 100644 index f1edc0d..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating.go +++ /dev/null @@ -1,35 +0,0 @@ -package readers - -import ( - "encoding/binary" - "io" -) - -const nonRepeatingModulo = 87178291199 -const nonRepeatingIncrement = 17180131327 - -// *should* produce a non-repeating sequence of bytes in a deterministic fashion -// use io.LimitReader to limit it to a specific length -type nonRepeatingSequenceReader struct { - value int -} - -func NewNonRepeatingSequence(i int) io.Reader { - return &nonRepeatingSequenceReader{i} -} - -func NewSizedNonRepeatingSequence(i int, s int64) io.Reader { - return io.LimitReader(NewNonRepeatingSequence(i), s) -} - -func (r *nonRepeatingSequenceReader) Read(p []byte) (n int, err error) { - lenp := len(p) - b := []byte{1, 2, 3, 4} - - for i := 0; i < lenp; i++ { - binary.LittleEndian.PutUint32(b, uint32(r.value)) - p[i] = b[0] - r.value = (r.value + nonRepeatingIncrement) % nonRepeatingModulo - } - return lenp, nil -} diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go deleted file mode 100644 index 3c65b1c..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package readers - -import ( - "io" - "io/ioutil" - "testing" -) - -// This is only a very basic test -func TestNonRepeatingSequenceReader(t *testing.T) { - i := NewNonRepeatingSequence(0) - a := []byte{0} - b := []byte{0} - - i.Read(a) - i.Read(b) - - if a[0] == b[0] { - t.Fatalf("Bytes should not be the same! %s vs %s", a, b) - } -} - -func TestNonRepeatingSequenceIsDifferent(t *testing.T) { - i := NewNonRepeatingSequence(0) - i2 := NewNonRepeatingSequence(5) - - a := []byte{0} - b := []byte{0} - - commonalities := 0 - - for x := 0; x < 100; x++ { - i.Read(a) - i2.Read(b) - - if a[0] == b[0] { - commonalities += 1 - } - } - - if commonalities > 5 { - t.Fatal("Sequences are too similar") - } -} - -func BenchmarkNonRepeatingSequence(b *testing.B) { - b.SetBytes(1) - - s := NewSizedNonRepeatingSequence(0, int64(b.N)) - - b.StartTimer() - _, err := io.Copy(ioutil.Discard, s) - b.StopTimer() - - if err != nil { - b.Fatal(err) - } -} diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/readers.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/readers.go deleted file mode 100644 index c9db457..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/readers.go +++ /dev/null @@ -1,6 +0,0 @@ -/* -util/readers exists to provide convenient and composable io.Reader compatible streams to allow testing -without having to check in large binary files. - -*/ -package readers diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/sequence_test.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/sequence_test.go deleted file mode 100644 index 523084e..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/sequence_test.go +++ /dev/null @@ -1 +0,0 @@ -package readers diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/sequencelimit.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/sequencelimit.go deleted file mode 100644 index 96ba1bd..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/sequencelimit.go +++ /dev/null @@ -1,12 +0,0 @@ -package readers - -import ( - "io" -) - -// read from 'readers' in sequence up to a limit of 'size' -func SequenceLimit(size int64, readers ...io.Reader) io.Reader { - return io.LimitReader( - io.MultiReader(readers...), - size) -} diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/uniform_test.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/uniform_test.go deleted file mode 100644 index e24e69b..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/uniform_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package readers - -import ( - "io" - "io/ioutil" - "testing" -) - -func TestUniformReaderLength(t *testing.T) { - r, err := ioutil.ReadAll(OneReader(100)) - - if err != nil { - t.Fatal(err) - } - - if len(r) != 100 { - t.Errorf("Unexpected length: %v", len(r)) - } - - for i, b := range r { - if b != 1 { - t.Errorf("Byte at position %v is not 1: %v", i, b) - } - } -} - -func TestReadIntoLargerBuffer(t *testing.T) { - b := make([]byte, 100) - r := OneReader(10) - - n, err := r.Read(b) - - if n != 10 { - t.Errorf("Wrong read length: %v", n) - } - - if err != io.EOF { - t.Errorf("Did not raise EOF after reading: %v", err) - } -} - -func TestMultiUniformReader(t *testing.T) { - r := io.MultiReader( - OneReader(12), - NewSizedNonRepeatingSequence(0, 88), - ) - - b := make([]byte, 100) - - n, err := r.Read(b) - - if n != 12 { - t.Errorf("Wrong read length: %v", n) - } - - if err == io.EOF { - t.Errorf("Raised EOF after reading! %v", err) - } - - n, err = r.Read(b) - - if n != 88 { - t.Errorf("Wrong read length: %v", n) - } - - n, err = r.Read(b) - - if err != io.EOF { - t.Errorf("Really expected EOF by now: %v %v", err, n) - } -} - -func TestFillBuffer(t *testing.T) { - r := io.MultiReader( - OneReader(12), - NewSizedNonRepeatingSequence(0, 88), - ) - - b := make([]byte, 100) - _, err := io.ReadFull(r, b) - - if err != nil && err != io.EOF { - t.Error(err) - } - - if len(b) != cap(b) { - t.Errorf("Expected to fill b: %v", len(b)) - } - -} diff --git a/vendor/src/github.com/Redundancy/go-sync/util/readers/uniformreader.go b/vendor/src/github.com/Redundancy/go-sync/util/readers/uniformreader.go deleted file mode 100644 index e6045da..0000000 --- a/vendor/src/github.com/Redundancy/go-sync/util/readers/uniformreader.go +++ /dev/null @@ -1,55 +0,0 @@ -package readers - -import ( - "io" -) - -// Reads a continuous stream of bytes with the same value, up to length -type uniformReader struct { - value byte - length int - read int -} - -func (r *uniformReader) Read(p []byte) (n int, err error) { - destinationLength := len(p) - readable := r.length - r.read - read := destinationLength - - if readable < destinationLength { - read = readable - } - - if read == 0 { - return 0, io.EOF - } - - for i := 0; i < read; i++ { - p[i] = r.value - } - - var result error = nil - if read == readable { - result = io.EOF - } - - r.read += read - - return read, result -} - -func ZeroReader(length int) io.Reader { - return &uniformReader{ - value: 0, - length: length, - read: 0, - } -} - -func OneReader(length int) io.Reader { - return &uniformReader{ - value: 1, - length: length, - read: 0, - } -} diff --git a/vendor/src/github.com/codegangsta/cli/LICENSE b/vendor/src/github.com/codegangsta/cli/LICENSE deleted file mode 100644 index 5515ccf..0000000 --- a/vendor/src/github.com/codegangsta/cli/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (C) 2013 Jeremy Saenz -All Rights Reserved. - -MIT LICENSE - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/src/github.com/codegangsta/cli/README.md b/vendor/src/github.com/codegangsta/cli/README.md deleted file mode 100644 index 364c964..0000000 --- a/vendor/src/github.com/codegangsta/cli/README.md +++ /dev/null @@ -1,394 +0,0 @@ -[![Coverage](http://gocover.io/_badge/github.com/codegangsta/cli?0)](http://gocover.io/github.com/codegangsta/cli) -[![Build Status](https://travis-ci.org/codegangsta/cli.svg?branch=master)](https://travis-ci.org/codegangsta/cli) -[![GoDoc](https://godoc.org/github.com/codegangsta/cli?status.svg)](https://godoc.org/github.com/codegangsta/cli) - -# cli.go - -`cli.go` is simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way. - -## Overview - -Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app. - -**This is where `cli.go` comes into play.** `cli.go` makes command line programming fun, organized, and expressive! - -## Installation - -Make sure you have a working Go environment (go 1.1+ is *required*). [See the install instructions](http://golang.org/doc/install.html). - -To install `cli.go`, simply run: -``` -$ go get github.com/codegangsta/cli -``` - -Make sure your `PATH` includes to the `$GOPATH/bin` directory so your commands can be easily used: -``` -export PATH=$PATH:$GOPATH/bin -``` - -## Getting Started - -One of the philosophies behind `cli.go` is that an API should be playful and full of discovery. So a `cli.go` app can be as little as one line of code in `main()`. - -``` go -package main - -import ( - "os" - "github.com/codegangsta/cli" -) - -func main() { - cli.NewApp().Run(os.Args) -} -``` - -This app will run and show help text, but is not very useful. Let's give an action to execute and some help documentation: - -``` go -package main - -import ( - "os" - "github.com/codegangsta/cli" -) - -func main() { - app := cli.NewApp() - app.Name = "boom" - app.Usage = "make an explosive entrance" - app.Action = func(c *cli.Context) { - println("boom! I say!") - } - - app.Run(os.Args) -} -``` - -Running this already gives you a ton of functionality, plus support for things like subcommands and flags, which are covered below. - -## Example - -Being a programmer can be a lonely job. Thankfully by the power of automation that is not the case! Let's create a greeter app to fend off our demons of loneliness! - -Start by creating a directory named `greet`, and within it, add a file, `greet.go` with the following code in it: - -``` go -package main - -import ( - "os" - "github.com/codegangsta/cli" -) - -func main() { - app := cli.NewApp() - app.Name = "greet" - app.Usage = "fight the loneliness!" - app.Action = func(c *cli.Context) { - println("Hello friend!") - } - - app.Run(os.Args) -} -``` - -Install our command to the `$GOPATH/bin` directory: - -``` -$ go install -``` - -Finally run our new command: - -``` -$ greet -Hello friend! -``` - -`cli.go` also generates neat help text: - -``` -$ greet help -NAME: - greet - fight the loneliness! - -USAGE: - greet [global options] command [command options] [arguments...] - -VERSION: - 0.0.0 - -COMMANDS: - help, h Shows a list of commands or help for one command - -GLOBAL OPTIONS - --version Shows version information -``` - -### Arguments - -You can lookup arguments by calling the `Args` function on `cli.Context`. - -``` go -... -app.Action = func(c *cli.Context) { - println("Hello", c.Args()[0]) -} -... -``` - -### Flags - -Setting and querying flags is simple. - -``` go -... -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang", - Value: "english", - Usage: "language for the greeting", - }, -} -app.Action = func(c *cli.Context) { - name := "someone" - if c.NArg() > 0 { - name = c.Args()[0] - } - if c.String("lang") == "spanish" { - println("Hola", name) - } else { - println("Hello", name) - } -} -... -``` - -You can also set a destination variable for a flag, to which the content will be scanned. - -``` go -... -var language string -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang", - Value: "english", - Usage: "language for the greeting", - Destination: &language, - }, -} -app.Action = func(c *cli.Context) { - name := "someone" - if c.NArg() > 0 { - name = c.Args()[0] - } - if language == "spanish" { - println("Hola", name) - } else { - println("Hello", name) - } -} -... -``` - -See full list of flags at http://godoc.org/github.com/codegangsta/cli - -#### Alternate Names - -You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g. - -``` go -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang, l", - Value: "english", - Usage: "language for the greeting", - }, -} -``` - -That flag can then be set with `--lang spanish` or `-l spanish`. Note that giving two different forms of the same flag in the same command invocation is an error. - -#### Values from the Environment - -You can also have the default value set from the environment via `EnvVar`. e.g. - -``` go -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang, l", - Value: "english", - Usage: "language for the greeting", - EnvVar: "APP_LANG", - }, -} -``` - -The `EnvVar` may also be given as a comma-delimited "cascade", where the first environment variable that resolves is used as the default. - -``` go -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang, l", - Value: "english", - Usage: "language for the greeting", - EnvVar: "LEGACY_COMPAT_LANG,APP_LANG,LANG", - }, -} -``` - -#### Values from alternate input sources (YAML and others) - -There is a separate package altsrc that adds support for getting flag values from other input sources like YAML. - -In order to get values for a flag from an alternate input source the following code would be added to wrap an existing cli.Flag like below: - -``` go - altsrc.NewIntFlag(cli.IntFlag{Name: "test"}) -``` - -Initialization must also occur for these flags. Below is an example initializing getting data from a yaml file below. - -``` go - command.Before = altsrc.InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) -``` - -The code above will use the "load" string as a flag name to get the file name of a yaml file from the cli.Context. -It will then use that file name to initialize the yaml input source for any flags that are defined on that command. -As a note the "load" flag used would also have to be defined on the command flags in order for this code snipped to work. - -Currently only YAML files are supported but developers can add support for other input sources by implementing the -altsrc.InputSourceContext for their given sources. - -Here is a more complete sample of a command using YAML support: - -``` go - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - // Action to run - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - err := command.Run(c) -``` - -### Subcommands - -Subcommands can be defined for a more git-like command line app. - -```go -... -app.Commands = []cli.Command{ - { - Name: "add", - Aliases: []string{"a"}, - Usage: "add a task to the list", - Action: func(c *cli.Context) { - println("added task: ", c.Args().First()) - }, - }, - { - Name: "complete", - Aliases: []string{"c"}, - Usage: "complete a task on the list", - Action: func(c *cli.Context) { - println("completed task: ", c.Args().First()) - }, - }, - { - Name: "template", - Aliases: []string{"r"}, - Usage: "options for task templates", - Subcommands: []cli.Command{ - { - Name: "add", - Usage: "add a new template", - Action: func(c *cli.Context) { - println("new task template: ", c.Args().First()) - }, - }, - { - Name: "remove", - Usage: "remove an existing template", - Action: func(c *cli.Context) { - println("removed task template: ", c.Args().First()) - }, - }, - }, - }, -} -... -``` - -### Bash Completion - -You can enable completion commands by setting the `EnableBashCompletion` -flag on the `App` object. By default, this setting will only auto-complete to -show an app's subcommands, but you can write your own completion methods for -the App or its subcommands. - -```go -... -var tasks = []string{"cook", "clean", "laundry", "eat", "sleep", "code"} -app := cli.NewApp() -app.EnableBashCompletion = true -app.Commands = []cli.Command{ - { - Name: "complete", - Aliases: []string{"c"}, - Usage: "complete a task on the list", - Action: func(c *cli.Context) { - println("completed task: ", c.Args().First()) - }, - BashComplete: func(c *cli.Context) { - // This will complete if no args are passed - if c.NArg() > 0 { - return - } - for _, t := range tasks { - fmt.Println(t) - } - }, - } -} -... -``` - -#### To Enable - -Source the `autocomplete/bash_autocomplete` file in your `.bashrc` file while -setting the `PROG` variable to the name of your program: - -`PROG=myprogram source /.../cli/autocomplete/bash_autocomplete` - -#### To Distribute - -Copy `autocomplete/bash_autocomplete` into `/etc/bash_completion.d/` and rename -it to the name of the program you wish to add autocomplete support for (or -automatically install it there if you are distributing a package). Don't forget -to source the file to make it active in the current shell. - -``` -sudo cp src/bash_autocomplete /etc/bash_completion.d/ -source /etc/bash_completion.d/ -``` - -Alternatively, you can just document that users should source the generic -`autocomplete/bash_autocomplete` in their bash configuration with `$PROG` set -to the name of their program (as above). - -## Contribution Guidelines - -Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch. - -If you have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together. - -If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out. diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/flag.go b/vendor/src/github.com/codegangsta/cli/altsrc/flag.go deleted file mode 100644 index f13ffb4..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/flag.go +++ /dev/null @@ -1,439 +0,0 @@ -package altsrc - -import ( - "flag" - "fmt" - "os" - "strconv" - "strings" - - "github.com/codegangsta/cli" -) - -// FlagInputSourceExtension is an extension interface of cli.Flag that -// allows a value to be set on the existing parsed flags. -type FlagInputSourceExtension interface { - cli.Flag - ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error -} - -// ApplyInputSourceValues iterates over all provided flags and -// executes ApplyInputSourceValue on flags implementing the -// FlagInputSourceExtension interface to initialize these flags -// to an alternate input source. -func ApplyInputSourceValues(context *cli.Context, inputSourceContext InputSourceContext, flags []cli.Flag) error { - for _, f := range flags { - inputSourceExtendedFlag, isType := f.(FlagInputSourceExtension) - if isType { - err := inputSourceExtendedFlag.ApplyInputSourceValue(context, inputSourceContext) - if err != nil { - return err - } - } - } - - return nil -} - -// InitInputSource is used to to setup an InputSourceContext on a cli.Command Before method. It will create a new -// input source based on the func provided. If there is no error it will then apply the new input source to any flags -// that are supported by the input source -func InitInputSource(flags []cli.Flag, createInputSource func() (InputSourceContext, error)) func(context *cli.Context) error { - return func(context *cli.Context) error { - inputSource, err := createInputSource() - if err != nil { - return fmt.Errorf("Unable to create input source: inner error: \n'%v'", err.Error()) - } - - return ApplyInputSourceValues(context, inputSource, flags) - } -} - -// InitInputSourceWithContext is used to to setup an InputSourceContext on a cli.Command Before method. It will create a new -// input source based on the func provided with potentially using existing cli.Context values to initialize itself. If there is -// no error it will then apply the new input source to any flags that are supported by the input source -func InitInputSourceWithContext(flags []cli.Flag, createInputSource func(context *cli.Context) (InputSourceContext, error)) func(context *cli.Context) error { - return func(context *cli.Context) error { - inputSource, err := createInputSource(context) - if err != nil { - return fmt.Errorf("Unable to create input source with context: inner error: \n'%v'", err.Error()) - } - - return ApplyInputSourceValues(context, inputSource, flags) - } -} - -// GenericFlag is the flag type that wraps cli.GenericFlag to allow -// for other values to be specified -type GenericFlag struct { - cli.GenericFlag - set *flag.FlagSet -} - -// NewGenericFlag creates a new GenericFlag -func NewGenericFlag(flag cli.GenericFlag) *GenericFlag { - return &GenericFlag{GenericFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a generic value to the flagSet if required -func (f *GenericFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.Generic(f.GenericFlag.Name) - if err != nil { - return err - } - if value != nil { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, value.String()) - }) - } - } - } - - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped GenericFlag.Apply -func (f *GenericFlag) Apply(set *flag.FlagSet) { - f.set = set - f.GenericFlag.Apply(set) -} - -// StringSliceFlag is the flag type that wraps cli.StringSliceFlag to allow -// for other values to be specified -type StringSliceFlag struct { - cli.StringSliceFlag - set *flag.FlagSet -} - -// NewStringSliceFlag creates a new StringSliceFlag -func NewStringSliceFlag(flag cli.StringSliceFlag) *StringSliceFlag { - return &StringSliceFlag{StringSliceFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a StringSlice value to the flagSet if required -func (f *StringSliceFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.StringSlice(f.StringSliceFlag.Name) - if err != nil { - return err - } - if value != nil { - var sliceValue cli.StringSlice = value - eachName(f.Name, func(name string) { - underlyingFlag := f.set.Lookup(f.Name) - if underlyingFlag != nil { - underlyingFlag.Value = &sliceValue - } - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped StringSliceFlag.Apply -func (f *StringSliceFlag) Apply(set *flag.FlagSet) { - f.set = set - f.StringSliceFlag.Apply(set) -} - -// IntSliceFlag is the flag type that wraps cli.IntSliceFlag to allow -// for other values to be specified -type IntSliceFlag struct { - cli.IntSliceFlag - set *flag.FlagSet -} - -// NewIntSliceFlag creates a new IntSliceFlag -func NewIntSliceFlag(flag cli.IntSliceFlag) *IntSliceFlag { - return &IntSliceFlag{IntSliceFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a IntSlice value if required -func (f *IntSliceFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.IntSlice(f.IntSliceFlag.Name) - if err != nil { - return err - } - if value != nil { - var sliceValue cli.IntSlice = value - eachName(f.Name, func(name string) { - underlyingFlag := f.set.Lookup(f.Name) - if underlyingFlag != nil { - underlyingFlag.Value = &sliceValue - } - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped IntSliceFlag.Apply -func (f *IntSliceFlag) Apply(set *flag.FlagSet) { - f.set = set - f.IntSliceFlag.Apply(set) -} - -// BoolFlag is the flag type that wraps cli.BoolFlag to allow -// for other values to be specified -type BoolFlag struct { - cli.BoolFlag - set *flag.FlagSet -} - -// NewBoolFlag creates a new BoolFlag -func NewBoolFlag(flag cli.BoolFlag) *BoolFlag { - return &BoolFlag{BoolFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a Bool value to the flagSet if required -func (f *BoolFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.Bool(f.BoolFlag.Name) - if err != nil { - return err - } - if value { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, strconv.FormatBool(value)) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped BoolFlag.Apply -func (f *BoolFlag) Apply(set *flag.FlagSet) { - f.set = set - f.BoolFlag.Apply(set) -} - -// BoolTFlag is the flag type that wraps cli.BoolTFlag to allow -// for other values to be specified -type BoolTFlag struct { - cli.BoolTFlag - set *flag.FlagSet -} - -// NewBoolTFlag creates a new BoolTFlag -func NewBoolTFlag(flag cli.BoolTFlag) *BoolTFlag { - return &BoolTFlag{BoolTFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a BoolT value to the flagSet if required -func (f *BoolTFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.BoolT(f.BoolTFlag.Name) - if err != nil { - return err - } - if !value { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, strconv.FormatBool(value)) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped BoolTFlag.Apply -func (f *BoolTFlag) Apply(set *flag.FlagSet) { - f.set = set - - f.BoolTFlag.Apply(set) -} - -// StringFlag is the flag type that wraps cli.StringFlag to allow -// for other values to be specified -type StringFlag struct { - cli.StringFlag - set *flag.FlagSet -} - -// NewStringFlag creates a new StringFlag -func NewStringFlag(flag cli.StringFlag) *StringFlag { - return &StringFlag{StringFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a String value to the flagSet if required -func (f *StringFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.String(f.StringFlag.Name) - if err != nil { - return err - } - if value != "" { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, value) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped StringFlag.Apply -func (f *StringFlag) Apply(set *flag.FlagSet) { - f.set = set - - f.StringFlag.Apply(set) -} - -// IntFlag is the flag type that wraps cli.IntFlag to allow -// for other values to be specified -type IntFlag struct { - cli.IntFlag - set *flag.FlagSet -} - -// NewIntFlag creates a new IntFlag -func NewIntFlag(flag cli.IntFlag) *IntFlag { - return &IntFlag{IntFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a int value to the flagSet if required -func (f *IntFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.Int(f.IntFlag.Name) - if err != nil { - return err - } - if value > 0 { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, strconv.FormatInt(int64(value), 10)) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped IntFlag.Apply -func (f *IntFlag) Apply(set *flag.FlagSet) { - f.set = set - f.IntFlag.Apply(set) -} - -// DurationFlag is the flag type that wraps cli.DurationFlag to allow -// for other values to be specified -type DurationFlag struct { - cli.DurationFlag - set *flag.FlagSet -} - -// NewDurationFlag creates a new DurationFlag -func NewDurationFlag(flag cli.DurationFlag) *DurationFlag { - return &DurationFlag{DurationFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a Duration value to the flagSet if required -func (f *DurationFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.Duration(f.DurationFlag.Name) - if err != nil { - return err - } - if value > 0 { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, value.String()) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped DurationFlag.Apply -func (f *DurationFlag) Apply(set *flag.FlagSet) { - f.set = set - - f.DurationFlag.Apply(set) -} - -// Float64Flag is the flag type that wraps cli.Float64Flag to allow -// for other values to be specified -type Float64Flag struct { - cli.Float64Flag - set *flag.FlagSet -} - -// NewFloat64Flag creates a new Float64Flag -func NewFloat64Flag(flag cli.Float64Flag) *Float64Flag { - return &Float64Flag{Float64Flag: flag, set: nil} -} - -// ApplyInputSourceValue applies a Float64 value to the flagSet if required -func (f *Float64Flag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.Float64(f.Float64Flag.Name) - if err != nil { - return err - } - if value > 0 { - floatStr := float64ToString(value) - eachName(f.Name, func(name string) { - f.set.Set(f.Name, floatStr) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped Float64Flag.Apply -func (f *Float64Flag) Apply(set *flag.FlagSet) { - f.set = set - - f.Float64Flag.Apply(set) -} - -func isEnvVarSet(envVars string) bool { - for _, envVar := range strings.Split(envVars, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - // TODO: Can't use this for bools as - // set means that it was true or false based on - // Bool flag type, should work for other types - if len(envVal) > 0 { - return true - } - } - } - - return false -} - -func float64ToString(f float64) string { - return fmt.Sprintf("%v", f) -} - -func eachName(longName string, fn func(string)) { - parts := strings.Split(longName, ",") - for _, name := range parts { - name = strings.Trim(name, " ") - fn(name) - } -} diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/flag_test.go b/vendor/src/github.com/codegangsta/cli/altsrc/flag_test.go deleted file mode 100644 index ac4d1f5..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/flag_test.go +++ /dev/null @@ -1,336 +0,0 @@ -package altsrc - -import ( - "flag" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/codegangsta/cli" -) - -type testApplyInputSource struct { - Flag FlagInputSourceExtension - FlagName string - FlagSetName string - Expected string - ContextValueString string - ContextValue flag.Value - EnvVarValue string - EnvVarName string - MapValue interface{} -} - -func TestGenericApplyInputSourceValue(t *testing.T) { - v := &Parser{"abc", "def"} - c := runTest(t, testApplyInputSource{ - Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}}), - FlagName: "test", - MapValue: v, - }) - expect(t, v, c.Generic("test")) -} - -func TestGenericApplyInputSourceMethodContextSet(t *testing.T) { - p := &Parser{"abc", "def"} - c := runTest(t, testApplyInputSource{ - Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}}), - FlagName: "test", - MapValue: &Parser{"efg", "hig"}, - ContextValueString: p.String(), - }) - expect(t, p, c.Generic("test")) -} - -func TestGenericApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}, EnvVar: "TEST"}), - FlagName: "test", - MapValue: &Parser{"efg", "hij"}, - EnvVarName: "TEST", - EnvVarValue: "abc,def", - }) - expect(t, &Parser{"abc", "def"}, c.Generic("test")) -} - -func TestStringSliceApplyInputSourceValue(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []string{"hello", "world"}, - }) - expect(t, c.StringSlice("test"), []string{"hello", "world"}) -} - -func TestStringSliceApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []string{"hello", "world"}, - ContextValueString: "ohno", - }) - expect(t, c.StringSlice("test"), []string{"ohno"}) -} - -func TestStringSliceApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: []string{"hello", "world"}, - EnvVarName: "TEST", - EnvVarValue: "oh,no", - }) - expect(t, c.StringSlice("test"), []string{"oh", "no"}) -} - -func TestIntSliceApplyInputSourceValue(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []int{1, 2}, - }) - expect(t, c.IntSlice("test"), []int{1, 2}) -} - -func TestIntSliceApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []int{1, 2}, - ContextValueString: "3", - }) - expect(t, c.IntSlice("test"), []int{3}) -} - -func TestIntSliceApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: []int{1, 2}, - EnvVarName: "TEST", - EnvVarValue: "3,4", - }) - expect(t, c.IntSlice("test"), []int{3, 4}) -} - -func TestBoolApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolFlag(cli.BoolFlag{Name: "test"}), - FlagName: "test", - MapValue: true, - }) - expect(t, true, c.Bool("test")) -} - -func TestBoolApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolFlag(cli.BoolFlag{Name: "test"}), - FlagName: "test", - MapValue: false, - ContextValueString: "true", - }) - expect(t, true, c.Bool("test")) -} - -func TestBoolApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolFlag(cli.BoolFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: false, - EnvVarName: "TEST", - EnvVarValue: "true", - }) - expect(t, true, c.Bool("test")) -} - -func TestBoolTApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test"}), - FlagName: "test", - MapValue: false, - }) - expect(t, false, c.BoolT("test")) -} - -func TestBoolTApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test"}), - FlagName: "test", - MapValue: true, - ContextValueString: "false", - }) - expect(t, false, c.BoolT("test")) -} - -func TestBoolTApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: true, - EnvVarName: "TEST", - EnvVarValue: "false", - }) - expect(t, false, c.BoolT("test")) -} - -func TestStringApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringFlag(cli.StringFlag{Name: "test"}), - FlagName: "test", - MapValue: "hello", - }) - expect(t, "hello", c.String("test")) -} - -func TestStringApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringFlag(cli.StringFlag{Name: "test"}), - FlagName: "test", - MapValue: "hello", - ContextValueString: "goodbye", - }) - expect(t, "goodbye", c.String("test")) -} - -func TestStringApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringFlag(cli.StringFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: "hello", - EnvVarName: "TEST", - EnvVarValue: "goodbye", - }) - expect(t, "goodbye", c.String("test")) -} - -func TestIntApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntFlag(cli.IntFlag{Name: "test"}), - FlagName: "test", - MapValue: 15, - }) - expect(t, 15, c.Int("test")) -} - -func TestIntApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntFlag(cli.IntFlag{Name: "test"}), - FlagName: "test", - MapValue: 15, - ContextValueString: "7", - }) - expect(t, 7, c.Int("test")) -} - -func TestIntApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntFlag(cli.IntFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: 15, - EnvVarName: "TEST", - EnvVarValue: "12", - }) - expect(t, 12, c.Int("test")) -} - -func TestDurationApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewDurationFlag(cli.DurationFlag{Name: "test"}), - FlagName: "test", - MapValue: time.Duration(30 * time.Second), - }) - expect(t, time.Duration(30*time.Second), c.Duration("test")) -} - -func TestDurationApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewDurationFlag(cli.DurationFlag{Name: "test"}), - FlagName: "test", - MapValue: time.Duration(30 * time.Second), - ContextValueString: time.Duration(15 * time.Second).String(), - }) - expect(t, time.Duration(15*time.Second), c.Duration("test")) -} - -func TestDurationApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewDurationFlag(cli.DurationFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: time.Duration(30 * time.Second), - EnvVarName: "TEST", - EnvVarValue: time.Duration(15 * time.Second).String(), - }) - expect(t, time.Duration(15*time.Second), c.Duration("test")) -} - -func TestFloat64ApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewFloat64Flag(cli.Float64Flag{Name: "test"}), - FlagName: "test", - MapValue: 1.3, - }) - expect(t, 1.3, c.Float64("test")) -} - -func TestFloat64ApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewFloat64Flag(cli.Float64Flag{Name: "test"}), - FlagName: "test", - MapValue: 1.3, - ContextValueString: fmt.Sprintf("%v", 1.4), - }) - expect(t, 1.4, c.Float64("test")) -} - -func TestFloat64ApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewFloat64Flag(cli.Float64Flag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: 1.3, - EnvVarName: "TEST", - EnvVarValue: fmt.Sprintf("%v", 1.4), - }) - expect(t, 1.4, c.Float64("test")) -} - -func runTest(t *testing.T, test testApplyInputSource) *cli.Context { - inputSource := &MapInputSource{valueMap: map[string]interface{}{test.FlagName: test.MapValue}} - set := flag.NewFlagSet(test.FlagSetName, flag.ContinueOnError) - c := cli.NewContext(nil, set, nil) - if test.EnvVarName != "" && test.EnvVarValue != "" { - os.Setenv(test.EnvVarName, test.EnvVarValue) - defer os.Setenv(test.EnvVarName, "") - } - - test.Flag.Apply(set) - if test.ContextValue != nil { - flag := set.Lookup(test.FlagName) - flag.Value = test.ContextValue - } - if test.ContextValueString != "" { - set.Set(test.FlagName, test.ContextValueString) - } - test.Flag.ApplyInputSourceValue(c, inputSource) - - return c -} - -type Parser [2]string - -func (p *Parser) Set(value string) error { - parts := strings.Split(value, ",") - if len(parts) != 2 { - return fmt.Errorf("invalid format") - } - - (*p)[0] = parts[0] - (*p)[1] = parts[1] - - return nil -} - -func (p *Parser) String() string { - return fmt.Sprintf("%s,%s", p[0], p[1]) -} diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/helpers_test.go b/vendor/src/github.com/codegangsta/cli/altsrc/helpers_test.go deleted file mode 100644 index 3b7f7e9..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/helpers_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package altsrc - -import ( - "reflect" - "testing" -) - -func expect(t *testing.T, a interface{}, b interface{}) { - if !reflect.DeepEqual(b, a) { - t.Errorf("Expected %#v (type %v) - Got %#v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} - -func refute(t *testing.T, a interface{}, b interface{}) { - if a == b { - t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/input_source_context.go b/vendor/src/github.com/codegangsta/cli/altsrc/input_source_context.go deleted file mode 100644 index 6d695ff..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/input_source_context.go +++ /dev/null @@ -1,21 +0,0 @@ -package altsrc - -import ( - "time" - - "github.com/codegangsta/cli" -) - -// InputSourceContext is an interface used to allow -// other input sources to be implemented as needed. -type InputSourceContext interface { - Int(name string) (int, error) - Duration(name string) (time.Duration, error) - Float64(name string) (float64, error) - String(name string) (string, error) - StringSlice(name string) ([]string, error) - IntSlice(name string) ([]int, error) - Generic(name string) (cli.Generic, error) - Bool(name string) (bool, error) - BoolT(name string) (bool, error) -} diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/map_input_source.go b/vendor/src/github.com/codegangsta/cli/altsrc/map_input_source.go deleted file mode 100644 index f1670fb..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/map_input_source.go +++ /dev/null @@ -1,152 +0,0 @@ -package altsrc - -import ( - "fmt" - "reflect" - "time" - - "github.com/codegangsta/cli" -) - -// MapInputSource implements InputSourceContext to return -// data from the map that is loaded. -type MapInputSource struct { - valueMap map[string]interface{} -} - -// Int returns an int from the map if it exists otherwise returns 0 -func (fsm *MapInputSource) Int(name string) (int, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(int) - if !isType { - return 0, incorrectTypeForFlagError(name, "int", otherGenericValue) - } - - return otherValue, nil - } - - return 0, nil -} - -// Duration returns a duration from the map if it exists otherwise returns 0 -func (fsm *MapInputSource) Duration(name string) (time.Duration, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(time.Duration) - if !isType { - return 0, incorrectTypeForFlagError(name, "duration", otherGenericValue) - } - return otherValue, nil - } - - return 0, nil -} - -// Float64 returns an float64 from the map if it exists otherwise returns 0 -func (fsm *MapInputSource) Float64(name string) (float64, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(float64) - if !isType { - return 0, incorrectTypeForFlagError(name, "float64", otherGenericValue) - } - return otherValue, nil - } - - return 0, nil -} - -// String returns a string from the map if it exists otherwise returns an empty string -func (fsm *MapInputSource) String(name string) (string, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(string) - if !isType { - return "", incorrectTypeForFlagError(name, "string", otherGenericValue) - } - return otherValue, nil - } - - return "", nil -} - -// StringSlice returns an []string from the map if it exists otherwise returns nil -func (fsm *MapInputSource) StringSlice(name string) ([]string, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.([]string) - if !isType { - return nil, incorrectTypeForFlagError(name, "[]string", otherGenericValue) - } - return otherValue, nil - } - - return nil, nil -} - -// IntSlice returns an []int from the map if it exists otherwise returns nil -func (fsm *MapInputSource) IntSlice(name string) ([]int, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.([]int) - if !isType { - return nil, incorrectTypeForFlagError(name, "[]int", otherGenericValue) - } - return otherValue, nil - } - - return nil, nil -} - -// Generic returns an cli.Generic from the map if it exists otherwise returns nil -func (fsm *MapInputSource) Generic(name string) (cli.Generic, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(cli.Generic) - if !isType { - return nil, incorrectTypeForFlagError(name, "cli.Generic", otherGenericValue) - } - return otherValue, nil - } - - return nil, nil -} - -// Bool returns an bool from the map otherwise returns false -func (fsm *MapInputSource) Bool(name string) (bool, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(bool) - if !isType { - return false, incorrectTypeForFlagError(name, "bool", otherGenericValue) - } - return otherValue, nil - } - - return false, nil -} - -// BoolT returns an bool from the map otherwise returns true -func (fsm *MapInputSource) BoolT(name string) (bool, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(bool) - if !isType { - return true, incorrectTypeForFlagError(name, "bool", otherGenericValue) - } - return otherValue, nil - } - - return true, nil -} - -func incorrectTypeForFlagError(name, expectedTypeName string, value interface{}) error { - valueType := reflect.TypeOf(value) - valueTypeName := "" - if valueType != nil { - valueTypeName = valueType.Name() - } - - return fmt.Errorf("Mismatched type for flag '%s'. Expected '%s' but actual is '%s'", name, expectedTypeName, valueTypeName) -} diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/yaml_command_test.go b/vendor/src/github.com/codegangsta/cli/altsrc/yaml_command_test.go deleted file mode 100644 index c7ccbf7..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/yaml_command_test.go +++ /dev/null @@ -1,172 +0,0 @@ -// Disabling building of yaml support in cases where golang is 1.0 or 1.1 -// as the encoding library is not implemented or supported. - -// +build !go1,!go1.1 - -package altsrc - -import ( - "flag" - "io/ioutil" - "os" - "testing" - - "github.com/codegangsta/cli" -) - -func TestCommandYamlFileTest(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 15) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileTestGlobalEnvVarWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - os.Setenv("THE_TEST", "10") - defer os.Setenv("THE_TEST", "") - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 10) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test", EnvVar: "THE_TEST"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileTestSpecifiedFlagWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - test := []string{"test-cmd", "--load", "current.yaml", "--test", "7"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 7) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileTestDefaultValueFileWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 15) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test", Value: 7}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileFlagHasDefaultGlobalEnvYamlSetGlobalEnvWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - os.Setenv("THE_TEST", "11") - defer os.Setenv("THE_TEST", "") - - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 11) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test", Value: 7, EnvVar: "THE_TEST"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - err := command.Run(c) - - expect(t, err, nil) -} diff --git a/vendor/src/github.com/codegangsta/cli/altsrc/yaml_file_loader.go b/vendor/src/github.com/codegangsta/cli/altsrc/yaml_file_loader.go deleted file mode 100644 index 1251aeb..0000000 --- a/vendor/src/github.com/codegangsta/cli/altsrc/yaml_file_loader.go +++ /dev/null @@ -1,84 +0,0 @@ -// Disabling building of yaml support in cases where golang is 1.0 or 1.1 -// as the encoding library is not implemented or supported. - -// +build !go1,!go1.1 - -package altsrc - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - - "github.com/codegangsta/cli" - - "gopkg.in/yaml.v2" -) - -type yamlSourceContext struct { - FilePath string -} - -// NewYamlSourceFromFile creates a new Yaml InputSourceContext from a filepath. -func NewYamlSourceFromFile(file string) (InputSourceContext, error) { - ymlLoader := &yamlSourceLoader{FilePath: file} - var results map[string]interface{} - err := readCommandYaml(ysl.FilePath, &results) - if err != nil { - return fmt.Errorf("Unable to load Yaml file '%s': inner error: \n'%v'", filePath, err.Error()) - } - - return &MapInputSource{valueMap: results}, nil -} - -// NewYamlSourceFromFlagFunc creates a new Yaml InputSourceContext from a provided flag name and source context. -func NewYamlSourceFromFlagFunc(flagFileName string) func(InputSourceContext, error) { - return func(context cli.Context) { - filePath := context.String(flagFileName) - return NewYamlSourceFromFile(filePath) - } -} - -func readCommandYaml(filePath string, container interface{}) (err error) { - b, err := loadDataFrom(filePath) - if err != nil { - return err - } - - err = yaml.Unmarshal(b, container) - if err != nil { - return err - } - - err = nil - return -} - -func loadDataFrom(filePath string) ([]byte, error) { - u, err := url.Parse(filePath) - if err != nil { - return nil, err - } - - if u.Host != "" { // i have a host, now do i support the scheme? - switch u.Scheme { - case "http", "https": - res, err := http.Get(filePath) - if err != nil { - return nil, err - } - return ioutil.ReadAll(res.Body) - default: - return nil, fmt.Errorf("scheme of %s is unsupported", filePath) - } - } else if u.Path != "" { // i dont have a host, but I have a path. I am a local file. - if _, notFoundFileErr := os.Stat(filePath); notFoundFileErr != nil { - return nil, fmt.Errorf("Cannot read from file: '%s' because it does not exist.", filePath) - } - return ioutil.ReadFile(filePath) - } else { - return nil, fmt.Errorf("unable to determine how to load from path %s", filePath) - } -} diff --git a/vendor/src/github.com/codegangsta/cli/app.go b/vendor/src/github.com/codegangsta/cli/app.go deleted file mode 100644 index 1ea3fd0..0000000 --- a/vendor/src/github.com/codegangsta/cli/app.go +++ /dev/null @@ -1,349 +0,0 @@ -package cli - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path" - "time" -) - -// App is the main structure of a cli application. It is recommended that -// an app be created with the cli.NewApp() function -type App struct { - // The name of the program. Defaults to path.Base(os.Args[0]) - Name string - // Full name of command for help, defaults to Name - HelpName string - // Description of the program. - Usage string - // Text to override the USAGE section of help - UsageText string - // Description of the program argument format. - ArgsUsage string - // Version of the program - Version string - // List of commands to execute - Commands []Command - // List of flags to parse - Flags []Flag - // Boolean to enable bash completion commands - EnableBashCompletion bool - // Boolean to hide built-in help command - HideHelp bool - // Boolean to hide built-in version flag - HideVersion bool - // An action to execute when the bash-completion flag is set - BashComplete func(context *Context) - // An action to execute before any subcommands are run, but after the context is ready - // If a non-nil error is returned, no subcommands are run - Before func(context *Context) error - // An action to execute after any subcommands are run, but after the subcommand has finished - // It is run even if Action() panics - After func(context *Context) error - // The action to execute when no subcommands are specified - Action func(context *Context) - // Execute this function if the proper command cannot be found - CommandNotFound func(context *Context, command string) - // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. - // This function is able to replace the original error messages. - // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. - OnUsageError func(context *Context, err error, isSubcommand bool) error - // Compilation date - Compiled time.Time - // List of all authors who contributed - Authors []Author - // Copyright of the binary if any - Copyright string - // Name of Author (Note: Use App.Authors, this is deprecated) - Author string - // Email of Author (Note: Use App.Authors, this is deprecated) - Email string - // Writer writer to write output to - Writer io.Writer -} - -// Tries to find out when this binary was compiled. -// Returns the current time if it fails to find it. -func compileTime() time.Time { - info, err := os.Stat(os.Args[0]) - if err != nil { - return time.Now() - } - return info.ModTime() -} - -// Creates a new cli Application with some reasonable defaults for Name, Usage, Version and Action. -func NewApp() *App { - return &App{ - Name: path.Base(os.Args[0]), - HelpName: path.Base(os.Args[0]), - Usage: "A new cli application", - UsageText: "", - Version: "0.0.0", - BashComplete: DefaultAppComplete, - Action: helpCommand.Action, - Compiled: compileTime(), - Writer: os.Stdout, - } -} - -// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination -func (a *App) Run(arguments []string) (err error) { - if a.Author != "" || a.Email != "" { - a.Authors = append(a.Authors, Author{Name: a.Author, Email: a.Email}) - } - - newCmds := []Command{} - for _, c := range a.Commands { - if c.HelpName == "" { - c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name) - } - newCmds = append(newCmds, c) - } - a.Commands = newCmds - - // append help to commands - if a.Command(helpCommand.Name) == nil && !a.HideHelp { - a.Commands = append(a.Commands, helpCommand) - if (HelpFlag != BoolFlag{}) { - a.appendFlag(HelpFlag) - } - } - - //append version/help flags - if a.EnableBashCompletion { - a.appendFlag(BashCompletionFlag) - } - - if !a.HideVersion { - a.appendFlag(VersionFlag) - } - - // parse flags - set := flagSet(a.Name, a.Flags) - set.SetOutput(ioutil.Discard) - err = set.Parse(arguments[1:]) - nerr := normalizeFlags(a.Flags, set) - context := NewContext(a, set, nil) - if nerr != nil { - fmt.Fprintln(a.Writer, nerr) - ShowAppHelp(context) - return nerr - } - - if checkCompletions(context) { - return nil - } - - if err != nil { - if a.OnUsageError != nil { - err := a.OnUsageError(context, err, false) - return err - } else { - fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") - ShowAppHelp(context) - return err - } - } - - if !a.HideHelp && checkHelp(context) { - ShowAppHelp(context) - return nil - } - - if !a.HideVersion && checkVersion(context) { - ShowVersion(context) - return nil - } - - if a.After != nil { - defer func() { - if afterErr := a.After(context); afterErr != nil { - if err != nil { - err = NewMultiError(err, afterErr) - } else { - err = afterErr - } - } - }() - } - - if a.Before != nil { - err = a.Before(context) - if err != nil { - fmt.Fprintf(a.Writer, "%v\n\n", err) - ShowAppHelp(context) - return err - } - } - - args := context.Args() - if args.Present() { - name := args.First() - c := a.Command(name) - if c != nil { - return c.Run(context) - } - } - - // Run default Action - a.Action(context) - return nil -} - -// Another entry point to the cli app, takes care of passing arguments and error handling -func (a *App) RunAndExitOnError() { - if err := a.Run(os.Args); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags -func (a *App) RunAsSubcommand(ctx *Context) (err error) { - // append help to commands - if len(a.Commands) > 0 { - if a.Command(helpCommand.Name) == nil && !a.HideHelp { - a.Commands = append(a.Commands, helpCommand) - if (HelpFlag != BoolFlag{}) { - a.appendFlag(HelpFlag) - } - } - } - - newCmds := []Command{} - for _, c := range a.Commands { - if c.HelpName == "" { - c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name) - } - newCmds = append(newCmds, c) - } - a.Commands = newCmds - - // append flags - if a.EnableBashCompletion { - a.appendFlag(BashCompletionFlag) - } - - // parse flags - set := flagSet(a.Name, a.Flags) - set.SetOutput(ioutil.Discard) - err = set.Parse(ctx.Args().Tail()) - nerr := normalizeFlags(a.Flags, set) - context := NewContext(a, set, ctx) - - if nerr != nil { - fmt.Fprintln(a.Writer, nerr) - fmt.Fprintln(a.Writer) - if len(a.Commands) > 0 { - ShowSubcommandHelp(context) - } else { - ShowCommandHelp(ctx, context.Args().First()) - } - return nerr - } - - if checkCompletions(context) { - return nil - } - - if err != nil { - if a.OnUsageError != nil { - err = a.OnUsageError(context, err, true) - return err - } else { - fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") - ShowSubcommandHelp(context) - return err - } - } - - if len(a.Commands) > 0 { - if checkSubcommandHelp(context) { - return nil - } - } else { - if checkCommandHelp(ctx, context.Args().First()) { - return nil - } - } - - if a.After != nil { - defer func() { - afterErr := a.After(context) - if afterErr != nil { - if err != nil { - err = NewMultiError(err, afterErr) - } else { - err = afterErr - } - } - }() - } - - if a.Before != nil { - err := a.Before(context) - if err != nil { - return err - } - } - - args := context.Args() - if args.Present() { - name := args.First() - c := a.Command(name) - if c != nil { - return c.Run(context) - } - } - - // Run default Action - a.Action(context) - - return nil -} - -// Returns the named command on App. Returns nil if the command does not exist -func (a *App) Command(name string) *Command { - for _, c := range a.Commands { - if c.HasName(name) { - return &c - } - } - - return nil -} - -func (a *App) hasFlag(flag Flag) bool { - for _, f := range a.Flags { - if flag == f { - return true - } - } - - return false -} - -func (a *App) appendFlag(flag Flag) { - if !a.hasFlag(flag) { - a.Flags = append(a.Flags, flag) - } -} - -// Author represents someone who has contributed to a cli project. -type Author struct { - Name string // The Authors name - Email string // The Authors email -} - -// String makes Author comply to the Stringer interface, to allow an easy print in the templating process -func (a Author) String() string { - e := "" - if a.Email != "" { - e = "<" + a.Email + "> " - } - - return fmt.Sprintf("%v %v", a.Name, e) -} diff --git a/vendor/src/github.com/codegangsta/cli/app_test.go b/vendor/src/github.com/codegangsta/cli/app_test.go deleted file mode 100644 index 7feaf1f..0000000 --- a/vendor/src/github.com/codegangsta/cli/app_test.go +++ /dev/null @@ -1,1047 +0,0 @@ -package cli - -import ( - "bytes" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - "testing" -) - -func ExampleApp_Run() { - // set args for examples sake - os.Args = []string{"greet", "--name", "Jeremy"} - - app := NewApp() - app.Name = "greet" - app.Flags = []Flag{ - StringFlag{Name: "name", Value: "bob", Usage: "a name to say"}, - } - app.Action = func(c *Context) { - fmt.Printf("Hello %v\n", c.String("name")) - } - app.UsageText = "app [first_arg] [second_arg]" - app.Author = "Harrison" - app.Email = "harrison@lolwut.com" - app.Authors = []Author{Author{Name: "Oliver Allen", Email: "oliver@toyshop.com"}} - app.Run(os.Args) - // Output: - // Hello Jeremy -} - -func ExampleApp_Run_subcommand() { - // set args for examples sake - os.Args = []string{"say", "hi", "english", "--name", "Jeremy"} - app := NewApp() - app.Name = "say" - app.Commands = []Command{ - { - Name: "hello", - Aliases: []string{"hi"}, - Usage: "use it to see a description", - Description: "This is how we describe hello the function", - Subcommands: []Command{ - { - Name: "english", - Aliases: []string{"en"}, - Usage: "sends a greeting in english", - Description: "greets someone in english", - Flags: []Flag{ - StringFlag{ - Name: "name", - Value: "Bob", - Usage: "Name of the person to greet", - }, - }, - Action: func(c *Context) { - fmt.Println("Hello,", c.String("name")) - }, - }, - }, - }, - } - - app.Run(os.Args) - // Output: - // Hello, Jeremy -} - -func ExampleApp_Run_help() { - // set args for examples sake - os.Args = []string{"greet", "h", "describeit"} - - app := NewApp() - app.Name = "greet" - app.Flags = []Flag{ - StringFlag{Name: "name", Value: "bob", Usage: "a name to say"}, - } - app.Commands = []Command{ - { - Name: "describeit", - Aliases: []string{"d"}, - Usage: "use it to see a description", - Description: "This is how we describe describeit the function", - Action: func(c *Context) { - fmt.Printf("i like to describe things") - }, - }, - } - app.Run(os.Args) - // Output: - // NAME: - // greet describeit - use it to see a description - // - // USAGE: - // greet describeit [arguments...] - // - // DESCRIPTION: - // This is how we describe describeit the function -} - -func ExampleApp_Run_bashComplete() { - // set args for examples sake - os.Args = []string{"greet", "--generate-bash-completion"} - - app := NewApp() - app.Name = "greet" - app.EnableBashCompletion = true - app.Commands = []Command{ - { - Name: "describeit", - Aliases: []string{"d"}, - Usage: "use it to see a description", - Description: "This is how we describe describeit the function", - Action: func(c *Context) { - fmt.Printf("i like to describe things") - }, - }, { - Name: "next", - Usage: "next example", - Description: "more stuff to see when generating bash completion", - Action: func(c *Context) { - fmt.Printf("the next example") - }, - }, - } - - app.Run(os.Args) - // Output: - // describeit - // d - // next - // help - // h -} - -func TestApp_Run(t *testing.T) { - s := "" - - app := NewApp() - app.Action = func(c *Context) { - s = s + c.Args().First() - } - - err := app.Run([]string{"command", "foo"}) - expect(t, err, nil) - err = app.Run([]string{"command", "bar"}) - expect(t, err, nil) - expect(t, s, "foobar") -} - -var commandAppTests = []struct { - name string - expected bool -}{ - {"foobar", true}, - {"batbaz", true}, - {"b", true}, - {"f", true}, - {"bat", false}, - {"nothing", false}, -} - -func TestApp_Command(t *testing.T) { - app := NewApp() - fooCommand := Command{Name: "foobar", Aliases: []string{"f"}} - batCommand := Command{Name: "batbaz", Aliases: []string{"b"}} - app.Commands = []Command{ - fooCommand, - batCommand, - } - - for _, test := range commandAppTests { - expect(t, app.Command(test.name) != nil, test.expected) - } -} - -func TestApp_CommandWithArgBeforeFlags(t *testing.T) { - var parsedOption, firstArg string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - StringFlag{Name: "option", Value: "", Usage: "some option"}, - }, - Action: func(c *Context) { - parsedOption = c.String("option") - firstArg = c.Args().First() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "--option", "my-option"}) - - expect(t, parsedOption, "my-option") - expect(t, firstArg, "my-arg") -} - -func TestApp_RunAsSubcommandParseFlags(t *testing.T) { - var context *Context - - a := NewApp() - a.Commands = []Command{ - { - Name: "foo", - Action: func(c *Context) { - context = c - }, - Flags: []Flag{ - StringFlag{ - Name: "lang", - Value: "english", - Usage: "language for the greeting", - }, - }, - Before: func(_ *Context) error { return nil }, - }, - } - a.Run([]string{"", "foo", "--lang", "spanish", "abcd"}) - - expect(t, context.Args().Get(0), "abcd") - expect(t, context.String("lang"), "spanish") -} - -func TestApp_CommandWithFlagBeforeTerminator(t *testing.T) { - var parsedOption string - var args []string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - StringFlag{Name: "option", Value: "", Usage: "some option"}, - }, - Action: func(c *Context) { - parsedOption = c.String("option") - args = c.Args() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "--option", "my-option", "--", "--notARealFlag"}) - - expect(t, parsedOption, "my-option") - expect(t, args[0], "my-arg") - expect(t, args[1], "--") - expect(t, args[2], "--notARealFlag") -} - -func TestApp_CommandWithDash(t *testing.T) { - var args []string - - app := NewApp() - command := Command{ - Name: "cmd", - Action: func(c *Context) { - args = c.Args() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "-"}) - - expect(t, args[0], "my-arg") - expect(t, args[1], "-") -} - -func TestApp_CommandWithNoFlagBeforeTerminator(t *testing.T) { - var args []string - - app := NewApp() - command := Command{ - Name: "cmd", - Action: func(c *Context) { - args = c.Args() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "--", "notAFlagAtAll"}) - - expect(t, args[0], "my-arg") - expect(t, args[1], "--") - expect(t, args[2], "notAFlagAtAll") -} - -func TestApp_Float64Flag(t *testing.T) { - var meters float64 - - app := NewApp() - app.Flags = []Flag{ - Float64Flag{Name: "height", Value: 1.5, Usage: "Set the height, in meters"}, - } - app.Action = func(c *Context) { - meters = c.Float64("height") - } - - app.Run([]string{"", "--height", "1.93"}) - expect(t, meters, 1.93) -} - -func TestApp_ParseSliceFlags(t *testing.T) { - var parsedOption, firstArg string - var parsedIntSlice []int - var parsedStringSlice []string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - IntSliceFlag{Name: "p", Value: &IntSlice{}, Usage: "set one or more ip addr"}, - StringSliceFlag{Name: "ip", Value: &StringSlice{}, Usage: "set one or more ports to open"}, - }, - Action: func(c *Context) { - parsedIntSlice = c.IntSlice("p") - parsedStringSlice = c.StringSlice("ip") - parsedOption = c.String("option") - firstArg = c.Args().First() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "-p", "22", "-p", "80", "-ip", "8.8.8.8", "-ip", "8.8.4.4"}) - - IntsEquals := func(a, b []int) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true - } - - StrsEquals := func(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true - } - var expectedIntSlice = []int{22, 80} - var expectedStringSlice = []string{"8.8.8.8", "8.8.4.4"} - - if !IntsEquals(parsedIntSlice, expectedIntSlice) { - t.Errorf("%v does not match %v", parsedIntSlice, expectedIntSlice) - } - - if !StrsEquals(parsedStringSlice, expectedStringSlice) { - t.Errorf("%v does not match %v", parsedStringSlice, expectedStringSlice) - } -} - -func TestApp_ParseSliceFlagsWithMissingValue(t *testing.T) { - var parsedIntSlice []int - var parsedStringSlice []string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - IntSliceFlag{Name: "a", Usage: "set numbers"}, - StringSliceFlag{Name: "str", Usage: "set strings"}, - }, - Action: func(c *Context) { - parsedIntSlice = c.IntSlice("a") - parsedStringSlice = c.StringSlice("str") - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "-a", "2", "-str", "A"}) - - var expectedIntSlice = []int{2} - var expectedStringSlice = []string{"A"} - - if parsedIntSlice[0] != expectedIntSlice[0] { - t.Errorf("%v does not match %v", parsedIntSlice[0], expectedIntSlice[0]) - } - - if parsedStringSlice[0] != expectedStringSlice[0] { - t.Errorf("%v does not match %v", parsedIntSlice[0], expectedIntSlice[0]) - } -} - -func TestApp_DefaultStdout(t *testing.T) { - app := NewApp() - - if app.Writer != os.Stdout { - t.Error("Default output writer not set.") - } -} - -type mockWriter struct { - written []byte -} - -func (fw *mockWriter) Write(p []byte) (n int, err error) { - if fw.written == nil { - fw.written = p - } else { - fw.written = append(fw.written, p...) - } - - return len(p), nil -} - -func (fw *mockWriter) GetWritten() (b []byte) { - return fw.written -} - -func TestApp_SetStdout(t *testing.T) { - w := &mockWriter{} - - app := NewApp() - app.Name = "test" - app.Writer = w - - err := app.Run([]string{"help"}) - - if err != nil { - t.Fatalf("Run error: %s", err) - } - - if len(w.written) == 0 { - t.Error("App did not write output to desired writer.") - } -} - -func TestApp_BeforeFunc(t *testing.T) { - beforeRun, subcommandRun := false, false - beforeError := fmt.Errorf("fail") - var err error - - app := NewApp() - - app.Before = func(c *Context) error { - beforeRun = true - s := c.String("opt") - if s == "fail" { - return beforeError - } - - return nil - } - - app.Commands = []Command{ - Command{ - Name: "sub", - Action: func(c *Context) { - subcommandRun = true - }, - }, - } - - app.Flags = []Flag{ - StringFlag{Name: "opt"}, - } - - // run with the Before() func succeeding - err = app.Run([]string{"command", "--opt", "succeed", "sub"}) - - if err != nil { - t.Fatalf("Run error: %s", err) - } - - if beforeRun == false { - t.Errorf("Before() not executed when expected") - } - - if subcommandRun == false { - t.Errorf("Subcommand not executed when expected") - } - - // reset - beforeRun, subcommandRun = false, false - - // run with the Before() func failing - err = app.Run([]string{"command", "--opt", "fail", "sub"}) - - // should be the same error produced by the Before func - if err != beforeError { - t.Errorf("Run error expected, but not received") - } - - if beforeRun == false { - t.Errorf("Before() not executed when expected") - } - - if subcommandRun == true { - t.Errorf("Subcommand executed when NOT expected") - } - -} - -func TestApp_AfterFunc(t *testing.T) { - afterRun, subcommandRun := false, false - afterError := fmt.Errorf("fail") - var err error - - app := NewApp() - - app.After = func(c *Context) error { - afterRun = true - s := c.String("opt") - if s == "fail" { - return afterError - } - - return nil - } - - app.Commands = []Command{ - Command{ - Name: "sub", - Action: func(c *Context) { - subcommandRun = true - }, - }, - } - - app.Flags = []Flag{ - StringFlag{Name: "opt"}, - } - - // run with the After() func succeeding - err = app.Run([]string{"command", "--opt", "succeed", "sub"}) - - if err != nil { - t.Fatalf("Run error: %s", err) - } - - if afterRun == false { - t.Errorf("After() not executed when expected") - } - - if subcommandRun == false { - t.Errorf("Subcommand not executed when expected") - } - - // reset - afterRun, subcommandRun = false, false - - // run with the Before() func failing - err = app.Run([]string{"command", "--opt", "fail", "sub"}) - - // should be the same error produced by the Before func - if err != afterError { - t.Errorf("Run error expected, but not received") - } - - if afterRun == false { - t.Errorf("After() not executed when expected") - } - - if subcommandRun == false { - t.Errorf("Subcommand not executed when expected") - } -} - -func TestAppNoHelpFlag(t *testing.T) { - oldFlag := HelpFlag - defer func() { - HelpFlag = oldFlag - }() - - HelpFlag = BoolFlag{} - - app := NewApp() - app.Writer = ioutil.Discard - err := app.Run([]string{"test", "-h"}) - - if err != flag.ErrHelp { - t.Errorf("expected error about missing help flag, but got: %s (%T)", err, err) - } -} - -func TestAppHelpPrinter(t *testing.T) { - oldPrinter := HelpPrinter - defer func() { - HelpPrinter = oldPrinter - }() - - var wasCalled = false - HelpPrinter = func(w io.Writer, template string, data interface{}) { - wasCalled = true - } - - app := NewApp() - app.Run([]string{"-h"}) - - if wasCalled == false { - t.Errorf("Help printer expected to be called, but was not") - } -} - -func TestAppVersionPrinter(t *testing.T) { - oldPrinter := VersionPrinter - defer func() { - VersionPrinter = oldPrinter - }() - - var wasCalled = false - VersionPrinter = func(c *Context) { - wasCalled = true - } - - app := NewApp() - ctx := NewContext(app, nil, nil) - ShowVersion(ctx) - - if wasCalled == false { - t.Errorf("Version printer expected to be called, but was not") - } -} - -func TestAppCommandNotFound(t *testing.T) { - beforeRun, subcommandRun := false, false - app := NewApp() - - app.CommandNotFound = func(c *Context, command string) { - beforeRun = true - } - - app.Commands = []Command{ - Command{ - Name: "bar", - Action: func(c *Context) { - subcommandRun = true - }, - }, - } - - app.Run([]string{"command", "foo"}) - - expect(t, beforeRun, true) - expect(t, subcommandRun, false) -} - -func TestGlobalFlag(t *testing.T) { - var globalFlag string - var globalFlagSet bool - app := NewApp() - app.Flags = []Flag{ - StringFlag{Name: "global, g", Usage: "global"}, - } - app.Action = func(c *Context) { - globalFlag = c.GlobalString("global") - globalFlagSet = c.GlobalIsSet("global") - } - app.Run([]string{"command", "-g", "foo"}) - expect(t, globalFlag, "foo") - expect(t, globalFlagSet, true) - -} - -func TestGlobalFlagsInSubcommands(t *testing.T) { - subcommandRun := false - parentFlag := false - app := NewApp() - - app.Flags = []Flag{ - BoolFlag{Name: "debug, d", Usage: "Enable debugging"}, - } - - app.Commands = []Command{ - Command{ - Name: "foo", - Flags: []Flag{ - BoolFlag{Name: "parent, p", Usage: "Parent flag"}, - }, - Subcommands: []Command{ - { - Name: "bar", - Action: func(c *Context) { - if c.GlobalBool("debug") { - subcommandRun = true - } - if c.GlobalBool("parent") { - parentFlag = true - } - }, - }, - }, - }, - } - - app.Run([]string{"command", "-d", "foo", "-p", "bar"}) - - expect(t, subcommandRun, true) - expect(t, parentFlag, true) -} - -func TestApp_Run_CommandWithSubcommandHasHelpTopic(t *testing.T) { - var subcommandHelpTopics = [][]string{ - {"command", "foo", "--help"}, - {"command", "foo", "-h"}, - {"command", "foo", "help"}, - } - - for _, flagSet := range subcommandHelpTopics { - t.Logf("==> checking with flags %v", flagSet) - - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - - subCmdBar := Command{ - Name: "bar", - Usage: "does bar things", - } - subCmdBaz := Command{ - Name: "baz", - Usage: "does baz things", - } - cmd := Command{ - Name: "foo", - Description: "descriptive wall of text about how it does foo things", - Subcommands: []Command{subCmdBar, subCmdBaz}, - } - - app.Commands = []Command{cmd} - err := app.Run(flagSet) - - if err != nil { - t.Error(err) - } - - output := buf.String() - t.Logf("output: %q\n", buf.Bytes()) - - if strings.Contains(output, "No help topic for") { - t.Errorf("expect a help topic, got none: \n%q", output) - } - - for _, shouldContain := range []string{ - cmd.Name, cmd.Description, - subCmdBar.Name, subCmdBar.Usage, - subCmdBaz.Name, subCmdBaz.Usage, - } { - if !strings.Contains(output, shouldContain) { - t.Errorf("want help to contain %q, did not: \n%q", shouldContain, output) - } - } - } -} - -func TestApp_Run_SubcommandFullPath(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "command" - subCmd := Command{ - Name: "bar", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "bar", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "command foo bar - does bar things") { - t.Errorf("expected full path to subcommand: %s", output) - } - if !strings.Contains(output, "command foo bar [arguments...]") { - t.Errorf("expected full path to subcommand: %s", output) - } -} - -func TestApp_Run_SubcommandHelpName(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "command" - subCmd := Command{ - Name: "bar", - HelpName: "custom", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "bar", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "custom - does bar things") { - t.Errorf("expected HelpName for subcommand: %s", output) - } - if !strings.Contains(output, "custom [arguments...]") { - t.Errorf("expected HelpName to subcommand: %s", output) - } -} - -func TestApp_Run_CommandHelpName(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "command" - subCmd := Command{ - Name: "bar", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - HelpName: "custom", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "bar", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "command foo bar - does bar things") { - t.Errorf("expected full path to subcommand: %s", output) - } - if !strings.Contains(output, "command foo bar [arguments...]") { - t.Errorf("expected full path to subcommand: %s", output) - } -} - -func TestApp_Run_CommandSubcommandHelpName(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "base" - subCmd := Command{ - Name: "bar", - HelpName: "custom", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "base foo - foo commands") { - t.Errorf("expected full path to subcommand: %s", output) - } - if !strings.Contains(output, "base foo command [command options] [arguments...]") { - t.Errorf("expected full path to subcommand: %s", output) - } -} - -func TestApp_Run_Help(t *testing.T) { - var helpArguments = [][]string{{"boom", "--help"}, {"boom", "-h"}, {"boom", "help"}} - - for _, args := range helpArguments { - buf := new(bytes.Buffer) - - t.Logf("==> checking with arguments %v", args) - - app := NewApp() - app.Name = "boom" - app.Usage = "make an explosive entrance" - app.Writer = buf - app.Action = func(c *Context) { - buf.WriteString("boom I say!") - } - - err := app.Run(args) - if err != nil { - t.Error(err) - } - - output := buf.String() - t.Logf("output: %q\n", buf.Bytes()) - - if !strings.Contains(output, "boom - make an explosive entrance") { - t.Errorf("want help to contain %q, did not: \n%q", "boom - make an explosive entrance", output) - } - } -} - -func TestApp_Run_Version(t *testing.T) { - var versionArguments = [][]string{{"boom", "--version"}, {"boom", "-v"}} - - for _, args := range versionArguments { - buf := new(bytes.Buffer) - - t.Logf("==> checking with arguments %v", args) - - app := NewApp() - app.Name = "boom" - app.Usage = "make an explosive entrance" - app.Version = "0.1.0" - app.Writer = buf - app.Action = func(c *Context) { - buf.WriteString("boom I say!") - } - - err := app.Run(args) - if err != nil { - t.Error(err) - } - - output := buf.String() - t.Logf("output: %q\n", buf.Bytes()) - - if !strings.Contains(output, "0.1.0") { - t.Errorf("want version to contain %q, did not: \n%q", "0.1.0", output) - } - } -} - -func TestApp_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) { - app := NewApp() - app.Action = func(c *Context) {} - app.Before = func(c *Context) error { return fmt.Errorf("before error") } - app.After = func(c *Context) error { return fmt.Errorf("after error") } - - err := app.Run([]string{"foo"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.Contains(err.Error(), "before error") { - t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) - } - if !strings.Contains(err.Error(), "after error") { - t.Errorf("expected text of error from After method, but got none in \"%v\"", err) - } -} - -func TestApp_Run_SubcommandDoesNotOverwriteErrorFromBefore(t *testing.T) { - app := NewApp() - app.Commands = []Command{ - Command{ - Subcommands: []Command{ - Command{ - Name: "sub", - }, - }, - Name: "bar", - Before: func(c *Context) error { return fmt.Errorf("before error") }, - After: func(c *Context) error { return fmt.Errorf("after error") }, - }, - } - - err := app.Run([]string{"foo", "bar"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.Contains(err.Error(), "before error") { - t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) - } - if !strings.Contains(err.Error(), "after error") { - t.Errorf("expected text of error from After method, but got none in \"%v\"", err) - } -} - -func TestApp_OnUsageError_WithWrongFlagValue(t *testing.T) { - app := NewApp() - app.Flags = []Flag{ - IntFlag{Name: "flag"}, - } - app.OnUsageError = func(c *Context, err error, isSubcommand bool) error { - if isSubcommand { - t.Errorf("Expect no subcommand") - } - if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { - t.Errorf("Expect an invalid value error, but got \"%v\"", err) - } - return errors.New("intercepted: " + err.Error()) - } - app.Commands = []Command{ - Command{ - Name: "bar", - }, - } - - err := app.Run([]string{"foo", "--flag=wrong"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { - t.Errorf("Expect an intercepted error, but got \"%v\"", err) - } -} - -func TestApp_OnUsageError_WithWrongFlagValue_ForSubcommand(t *testing.T) { - app := NewApp() - app.Flags = []Flag{ - IntFlag{Name: "flag"}, - } - app.OnUsageError = func(c *Context, err error, isSubcommand bool) error { - if isSubcommand { - t.Errorf("Expect subcommand") - } - if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { - t.Errorf("Expect an invalid value error, but got \"%v\"", err) - } - return errors.New("intercepted: " + err.Error()) - } - app.Commands = []Command{ - Command{ - Name: "bar", - }, - } - - err := app.Run([]string{"foo", "--flag=wrong", "bar"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { - t.Errorf("Expect an intercepted error, but got \"%v\"", err) - } -} diff --git a/vendor/src/github.com/codegangsta/cli/appveyor.yml b/vendor/src/github.com/codegangsta/cli/appveyor.yml deleted file mode 100644 index 3ca7afa..0000000 --- a/vendor/src/github.com/codegangsta/cli/appveyor.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "{build}" - -os: Windows Server 2012 R2 - -install: - - go version - - go env - -build_script: - - cd %APPVEYOR_BUILD_FOLDER% - - go vet ./... - - go test -v ./... - -test: off - -deploy: off diff --git a/vendor/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete b/vendor/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete deleted file mode 100644 index 21a232f..0000000 --- a/vendor/src/github.com/codegangsta/cli/autocomplete/bash_autocomplete +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -: ${PROG:=$(basename ${BASH_SOURCE})} - -_cli_bash_autocomplete() { - local cur opts base - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 - } - - complete -F _cli_bash_autocomplete $PROG diff --git a/vendor/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete b/vendor/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete deleted file mode 100644 index 5430a18..0000000 --- a/vendor/src/github.com/codegangsta/cli/autocomplete/zsh_autocomplete +++ /dev/null @@ -1,5 +0,0 @@ -autoload -U compinit && compinit -autoload -U bashcompinit && bashcompinit - -script_dir=$(dirname $0) -source ${script_dir}/bash_autocomplete diff --git a/vendor/src/github.com/codegangsta/cli/cli.go b/vendor/src/github.com/codegangsta/cli/cli.go deleted file mode 100644 index 31dc912..0000000 --- a/vendor/src/github.com/codegangsta/cli/cli.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package cli provides a minimal framework for creating and organizing command line -// Go applications. cli is designed to be easy to understand and write, the most simple -// cli application can be written as follows: -// func main() { -// cli.NewApp().Run(os.Args) -// } -// -// Of course this application does not do much, so let's make this an actual application: -// func main() { -// app := cli.NewApp() -// app.Name = "greet" -// app.Usage = "say a greeting" -// app.Action = func(c *cli.Context) { -// println("Greetings") -// } -// -// app.Run(os.Args) -// } -package cli - -import ( - "strings" -) - -type MultiError struct { - Errors []error -} - -func NewMultiError(err ...error) MultiError { - return MultiError{Errors: err} -} - -func (m MultiError) Error() string { - errs := make([]string, len(m.Errors)) - for i, err := range m.Errors { - errs[i] = err.Error() - } - - return strings.Join(errs, "\n") -} diff --git a/vendor/src/github.com/codegangsta/cli/command.go b/vendor/src/github.com/codegangsta/cli/command.go deleted file mode 100644 index 0153713..0000000 --- a/vendor/src/github.com/codegangsta/cli/command.go +++ /dev/null @@ -1,250 +0,0 @@ -package cli - -import ( - "fmt" - "io/ioutil" - "strings" -) - -// Command is a subcommand for a cli.App. -type Command struct { - // The name of the command - Name string - // short name of the command. Typically one character (deprecated, use `Aliases`) - ShortName string - // A list of aliases for the command - Aliases []string - // A short description of the usage of this command - Usage string - // Custom text to show on USAGE section of help - UsageText string - // A longer explanation of how the command works - Description string - // A short description of the arguments of this command - ArgsUsage string - // The function to call when checking for bash command completions - BashComplete func(context *Context) - // An action to execute before any sub-subcommands are run, but after the context is ready - // If a non-nil error is returned, no sub-subcommands are run - Before func(context *Context) error - // An action to execute after any subcommands are run, but before the subcommand has finished - // It is run even if Action() panics - After func(context *Context) error - // The function to call when this command is invoked - Action func(context *Context) - // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. - // This function is able to replace the original error messages. - // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. - OnUsageError func(context *Context, err error) error - // List of child commands - Subcommands []Command - // List of flags to parse - Flags []Flag - // Treat all flags as normal arguments if true - SkipFlagParsing bool - // Boolean to hide built-in help command - HideHelp bool - - // Full name of command for help, defaults to full command name, including parent commands. - HelpName string - commandNamePath []string -} - -// Returns the full name of the command. -// For subcommands this ensures that parent commands are part of the command path -func (c Command) FullName() string { - if c.commandNamePath == nil { - return c.Name - } - return strings.Join(c.commandNamePath, " ") -} - -// Invokes the command given the context, parses ctx.Args() to generate command-specific flags -func (c Command) Run(ctx *Context) (err error) { - if len(c.Subcommands) > 0 { - return c.startApp(ctx) - } - - if !c.HideHelp && (HelpFlag != BoolFlag{}) { - // append help to flags - c.Flags = append( - c.Flags, - HelpFlag, - ) - } - - if ctx.App.EnableBashCompletion { - c.Flags = append(c.Flags, BashCompletionFlag) - } - - set := flagSet(c.Name, c.Flags) - set.SetOutput(ioutil.Discard) - - if !c.SkipFlagParsing { - firstFlagIndex := -1 - terminatorIndex := -1 - for index, arg := range ctx.Args() { - if arg == "--" { - terminatorIndex = index - break - } else if arg == "-" { - // Do nothing. A dash alone is not really a flag. - continue - } else if strings.HasPrefix(arg, "-") && firstFlagIndex == -1 { - firstFlagIndex = index - } - } - - if firstFlagIndex > -1 { - args := ctx.Args() - regularArgs := make([]string, len(args[1:firstFlagIndex])) - copy(regularArgs, args[1:firstFlagIndex]) - - var flagArgs []string - if terminatorIndex > -1 { - flagArgs = args[firstFlagIndex:terminatorIndex] - regularArgs = append(regularArgs, args[terminatorIndex:]...) - } else { - flagArgs = args[firstFlagIndex:] - } - - err = set.Parse(append(flagArgs, regularArgs...)) - } else { - err = set.Parse(ctx.Args().Tail()) - } - } else { - if c.SkipFlagParsing { - err = set.Parse(append([]string{"--"}, ctx.Args().Tail()...)) - } - } - - if err != nil { - if c.OnUsageError != nil { - err := c.OnUsageError(ctx, err) - return err - } else { - fmt.Fprintln(ctx.App.Writer, "Incorrect Usage.") - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) - return err - } - } - - nerr := normalizeFlags(c.Flags, set) - if nerr != nil { - fmt.Fprintln(ctx.App.Writer, nerr) - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) - return nerr - } - context := NewContext(ctx.App, set, ctx) - - if checkCommandCompletions(context, c.Name) { - return nil - } - - if checkCommandHelp(context, c.Name) { - return nil - } - - if c.After != nil { - defer func() { - afterErr := c.After(context) - if afterErr != nil { - if err != nil { - err = NewMultiError(err, afterErr) - } else { - err = afterErr - } - } - }() - } - - if c.Before != nil { - err := c.Before(context) - if err != nil { - fmt.Fprintln(ctx.App.Writer, err) - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) - return err - } - } - - context.Command = c - c.Action(context) - return nil -} - -func (c Command) Names() []string { - names := []string{c.Name} - - if c.ShortName != "" { - names = append(names, c.ShortName) - } - - return append(names, c.Aliases...) -} - -// Returns true if Command.Name or Command.ShortName matches given name -func (c Command) HasName(name string) bool { - for _, n := range c.Names() { - if n == name { - return true - } - } - return false -} - -func (c Command) startApp(ctx *Context) error { - app := NewApp() - - // set the name and usage - app.Name = fmt.Sprintf("%s %s", ctx.App.Name, c.Name) - if c.HelpName == "" { - app.HelpName = c.HelpName - } else { - app.HelpName = app.Name - } - - if c.Description != "" { - app.Usage = c.Description - } else { - app.Usage = c.Usage - } - - // set CommandNotFound - app.CommandNotFound = ctx.App.CommandNotFound - - // set the flags and commands - app.Commands = c.Subcommands - app.Flags = c.Flags - app.HideHelp = c.HideHelp - - app.Version = ctx.App.Version - app.HideVersion = ctx.App.HideVersion - app.Compiled = ctx.App.Compiled - app.Author = ctx.App.Author - app.Email = ctx.App.Email - app.Writer = ctx.App.Writer - - // bash completion - app.EnableBashCompletion = ctx.App.EnableBashCompletion - if c.BashComplete != nil { - app.BashComplete = c.BashComplete - } - - // set the actions - app.Before = c.Before - app.After = c.After - if c.Action != nil { - app.Action = c.Action - } else { - app.Action = helpSubcommand.Action - } - - for index, cc := range app.Commands { - app.Commands[index].commandNamePath = []string{c.Name, cc.Name} - } - - return app.RunAsSubcommand(ctx) -} diff --git a/vendor/src/github.com/codegangsta/cli/command_test.go b/vendor/src/github.com/codegangsta/cli/command_test.go deleted file mode 100644 index 827da1d..0000000 --- a/vendor/src/github.com/codegangsta/cli/command_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package cli - -import ( - "errors" - "flag" - "fmt" - "io/ioutil" - "strings" - "testing" -) - -func TestCommandFlagParsing(t *testing.T) { - cases := []struct { - testArgs []string - skipFlagParsing bool - expectedErr error - }{ - {[]string{"blah", "blah", "-break"}, false, errors.New("flag provided but not defined: -break")}, // Test normal "not ignoring flags" flow - {[]string{"blah", "blah"}, true, nil}, // Test SkipFlagParsing without any args that look like flags - {[]string{"blah", "-break"}, true, nil}, // Test SkipFlagParsing with random flag arg - {[]string{"blah", "-help"}, true, nil}, // Test SkipFlagParsing with "special" help flag arg - } - - for _, c := range cases { - app := NewApp() - app.Writer = ioutil.Discard - set := flag.NewFlagSet("test", 0) - set.Parse(c.testArgs) - - context := NewContext(app, set, nil) - - command := Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(_ *Context) {}, - } - - command.SkipFlagParsing = c.skipFlagParsing - - err := command.Run(context) - - expect(t, err, c.expectedErr) - expect(t, []string(context.Args()), c.testArgs) - } -} - -func TestCommand_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) { - app := NewApp() - app.Commands = []Command{ - Command{ - Name: "bar", - Before: func(c *Context) error { return fmt.Errorf("before error") }, - After: func(c *Context) error { return fmt.Errorf("after error") }, - }, - } - - err := app.Run([]string{"foo", "bar"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.Contains(err.Error(), "before error") { - t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) - } - if !strings.Contains(err.Error(), "after error") { - t.Errorf("expected text of error from After method, but got none in \"%v\"", err) - } -} - -func TestCommand_OnUsageError_WithWrongFlagValue(t *testing.T) { - app := NewApp() - app.Commands = []Command{ - Command{ - Name: "bar", - Flags: []Flag{ - IntFlag{Name: "flag"}, - }, - OnUsageError: func(c *Context, err error) error { - if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { - t.Errorf("Expect an invalid value error, but got \"%v\"", err) - } - return errors.New("intercepted: " + err.Error()) - }, - }, - } - - err := app.Run([]string{"foo", "bar", "--flag=wrong"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { - t.Errorf("Expect an intercepted error, but got \"%v\"", err) - } -} diff --git a/vendor/src/github.com/codegangsta/cli/context.go b/vendor/src/github.com/codegangsta/cli/context.go deleted file mode 100644 index b66d278..0000000 --- a/vendor/src/github.com/codegangsta/cli/context.go +++ /dev/null @@ -1,393 +0,0 @@ -package cli - -import ( - "errors" - "flag" - "strconv" - "strings" - "time" -) - -// Context is a type that is passed through to -// each Handler action in a cli application. Context -// can be used to retrieve context-specific Args and -// parsed command-line options. -type Context struct { - App *App - Command Command - flagSet *flag.FlagSet - setFlags map[string]bool - globalSetFlags map[string]bool - parentContext *Context -} - -// Creates a new context. For use in when invoking an App or Command action. -func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context { - return &Context{App: app, flagSet: set, parentContext: parentCtx} -} - -// Looks up the value of a local int flag, returns 0 if no int flag exists -func (c *Context) Int(name string) int { - return lookupInt(name, c.flagSet) -} - -// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists -func (c *Context) Duration(name string) time.Duration { - return lookupDuration(name, c.flagSet) -} - -// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists -func (c *Context) Float64(name string) float64 { - return lookupFloat64(name, c.flagSet) -} - -// Looks up the value of a local bool flag, returns false if no bool flag exists -func (c *Context) Bool(name string) bool { - return lookupBool(name, c.flagSet) -} - -// Looks up the value of a local boolT flag, returns false if no bool flag exists -func (c *Context) BoolT(name string) bool { - return lookupBoolT(name, c.flagSet) -} - -// Looks up the value of a local string flag, returns "" if no string flag exists -func (c *Context) String(name string) string { - return lookupString(name, c.flagSet) -} - -// Looks up the value of a local string slice flag, returns nil if no string slice flag exists -func (c *Context) StringSlice(name string) []string { - return lookupStringSlice(name, c.flagSet) -} - -// Looks up the value of a local int slice flag, returns nil if no int slice flag exists -func (c *Context) IntSlice(name string) []int { - return lookupIntSlice(name, c.flagSet) -} - -// Looks up the value of a local generic flag, returns nil if no generic flag exists -func (c *Context) Generic(name string) interface{} { - return lookupGeneric(name, c.flagSet) -} - -// Looks up the value of a global int flag, returns 0 if no int flag exists -func (c *Context) GlobalInt(name string) int { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupInt(name, fs) - } - return 0 -} - -// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists -func (c *Context) GlobalDuration(name string) time.Duration { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupDuration(name, fs) - } - return 0 -} - -// Looks up the value of a global bool flag, returns false if no bool flag exists -func (c *Context) GlobalBool(name string) bool { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupBool(name, fs) - } - return false -} - -// Looks up the value of a global string flag, returns "" if no string flag exists -func (c *Context) GlobalString(name string) string { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupString(name, fs) - } - return "" -} - -// Looks up the value of a global string slice flag, returns nil if no string slice flag exists -func (c *Context) GlobalStringSlice(name string) []string { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupStringSlice(name, fs) - } - return nil -} - -// Looks up the value of a global int slice flag, returns nil if no int slice flag exists -func (c *Context) GlobalIntSlice(name string) []int { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupIntSlice(name, fs) - } - return nil -} - -// Looks up the value of a global generic flag, returns nil if no generic flag exists -func (c *Context) GlobalGeneric(name string) interface{} { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupGeneric(name, fs) - } - return nil -} - -// Returns the number of flags set -func (c *Context) NumFlags() int { - return c.flagSet.NFlag() -} - -// Determines if the flag was actually set -func (c *Context) IsSet(name string) bool { - if c.setFlags == nil { - c.setFlags = make(map[string]bool) - c.flagSet.Visit(func(f *flag.Flag) { - c.setFlags[f.Name] = true - }) - } - return c.setFlags[name] == true -} - -// Determines if the global flag was actually set -func (c *Context) GlobalIsSet(name string) bool { - if c.globalSetFlags == nil { - c.globalSetFlags = make(map[string]bool) - ctx := c - if ctx.parentContext != nil { - ctx = ctx.parentContext - } - for ; ctx != nil && c.globalSetFlags[name] == false; ctx = ctx.parentContext { - ctx.flagSet.Visit(func(f *flag.Flag) { - c.globalSetFlags[f.Name] = true - }) - } - } - return c.globalSetFlags[name] -} - -// Returns a slice of flag names used in this context. -func (c *Context) FlagNames() (names []string) { - for _, flag := range c.Command.Flags { - name := strings.Split(flag.GetName(), ",")[0] - if name == "help" { - continue - } - names = append(names, name) - } - return -} - -// Returns a slice of global flag names used by the app. -func (c *Context) GlobalFlagNames() (names []string) { - for _, flag := range c.App.Flags { - name := strings.Split(flag.GetName(), ",")[0] - if name == "help" || name == "version" { - continue - } - names = append(names, name) - } - return -} - -// Returns the parent context, if any -func (c *Context) Parent() *Context { - return c.parentContext -} - -type Args []string - -// Returns the command line arguments associated with the context. -func (c *Context) Args() Args { - args := Args(c.flagSet.Args()) - return args -} - -// Returns the number of the command line arguments. -func (c *Context) NArg() int { - return len(c.Args()) -} - -// Returns the nth argument, or else a blank string -func (a Args) Get(n int) string { - if len(a) > n { - return a[n] - } - return "" -} - -// Returns the first argument, or else a blank string -func (a Args) First() string { - return a.Get(0) -} - -// Return the rest of the arguments (not the first one) -// or else an empty string slice -func (a Args) Tail() []string { - if len(a) >= 2 { - return []string(a)[1:] - } - return []string{} -} - -// Checks if there are any arguments present -func (a Args) Present() bool { - return len(a) != 0 -} - -// Swaps arguments at the given indexes -func (a Args) Swap(from, to int) error { - if from >= len(a) || to >= len(a) { - return errors.New("index out of range") - } - a[from], a[to] = a[to], a[from] - return nil -} - -func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet { - if ctx.parentContext != nil { - ctx = ctx.parentContext - } - for ; ctx != nil; ctx = ctx.parentContext { - if f := ctx.flagSet.Lookup(name); f != nil { - return ctx.flagSet - } - } - return nil -} - -func lookupInt(name string, set *flag.FlagSet) int { - f := set.Lookup(name) - if f != nil { - val, err := strconv.Atoi(f.Value.String()) - if err != nil { - return 0 - } - return val - } - - return 0 -} - -func lookupDuration(name string, set *flag.FlagSet) time.Duration { - f := set.Lookup(name) - if f != nil { - val, err := time.ParseDuration(f.Value.String()) - if err == nil { - return val - } - } - - return 0 -} - -func lookupFloat64(name string, set *flag.FlagSet) float64 { - f := set.Lookup(name) - if f != nil { - val, err := strconv.ParseFloat(f.Value.String(), 64) - if err != nil { - return 0 - } - return val - } - - return 0 -} - -func lookupString(name string, set *flag.FlagSet) string { - f := set.Lookup(name) - if f != nil { - return f.Value.String() - } - - return "" -} - -func lookupStringSlice(name string, set *flag.FlagSet) []string { - f := set.Lookup(name) - if f != nil { - return (f.Value.(*StringSlice)).Value() - - } - - return nil -} - -func lookupIntSlice(name string, set *flag.FlagSet) []int { - f := set.Lookup(name) - if f != nil { - return (f.Value.(*IntSlice)).Value() - - } - - return nil -} - -func lookupGeneric(name string, set *flag.FlagSet) interface{} { - f := set.Lookup(name) - if f != nil { - return f.Value - } - return nil -} - -func lookupBool(name string, set *flag.FlagSet) bool { - f := set.Lookup(name) - if f != nil { - val, err := strconv.ParseBool(f.Value.String()) - if err != nil { - return false - } - return val - } - - return false -} - -func lookupBoolT(name string, set *flag.FlagSet) bool { - f := set.Lookup(name) - if f != nil { - val, err := strconv.ParseBool(f.Value.String()) - if err != nil { - return true - } - return val - } - - return false -} - -func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) { - switch ff.Value.(type) { - case *StringSlice: - default: - set.Set(name, ff.Value.String()) - } -} - -func normalizeFlags(flags []Flag, set *flag.FlagSet) error { - visited := make(map[string]bool) - set.Visit(func(f *flag.Flag) { - visited[f.Name] = true - }) - for _, f := range flags { - parts := strings.Split(f.GetName(), ",") - if len(parts) == 1 { - continue - } - var ff *flag.Flag - for _, name := range parts { - name = strings.Trim(name, " ") - if visited[name] { - if ff != nil { - return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name) - } - ff = set.Lookup(name) - } - } - if ff == nil { - continue - } - for _, name := range parts { - name = strings.Trim(name, " ") - if !visited[name] { - copyFlag(name, ff, set) - } - } - } - return nil -} diff --git a/vendor/src/github.com/codegangsta/cli/context_test.go b/vendor/src/github.com/codegangsta/cli/context_test.go deleted file mode 100644 index b8ab37d..0000000 --- a/vendor/src/github.com/codegangsta/cli/context_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package cli - -import ( - "flag" - "testing" - "time" -) - -func TestNewContext(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Int("myflag", 12, "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Int("myflag", 42, "doc") - globalCtx := NewContext(nil, globalSet, nil) - command := Command{Name: "mycommand"} - c := NewContext(nil, set, globalCtx) - c.Command = command - expect(t, c.Int("myflag"), 12) - expect(t, c.GlobalInt("myflag"), 42) - expect(t, c.Command.Name, "mycommand") -} - -func TestContext_Int(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Int("myflag", 12, "doc") - c := NewContext(nil, set, nil) - expect(t, c.Int("myflag"), 12) -} - -func TestContext_Duration(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Duration("myflag", time.Duration(12*time.Second), "doc") - c := NewContext(nil, set, nil) - expect(t, c.Duration("myflag"), time.Duration(12*time.Second)) -} - -func TestContext_String(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.String("myflag", "hello world", "doc") - c := NewContext(nil, set, nil) - expect(t, c.String("myflag"), "hello world") -} - -func TestContext_Bool(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - c := NewContext(nil, set, nil) - expect(t, c.Bool("myflag"), false) -} - -func TestContext_BoolT(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", true, "doc") - c := NewContext(nil, set, nil) - expect(t, c.BoolT("myflag"), true) -} - -func TestContext_Args(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - c := NewContext(nil, set, nil) - set.Parse([]string{"--myflag", "bat", "baz"}) - expect(t, len(c.Args()), 2) - expect(t, c.Bool("myflag"), true) -} - -func TestContext_NArg(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - c := NewContext(nil, set, nil) - set.Parse([]string{"--myflag", "bat", "baz"}) - expect(t, c.NArg(), 2) -} - -func TestContext_IsSet(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - set.String("otherflag", "hello world", "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Bool("myflagGlobal", true, "doc") - globalCtx := NewContext(nil, globalSet, nil) - c := NewContext(nil, set, globalCtx) - set.Parse([]string{"--myflag", "bat", "baz"}) - globalSet.Parse([]string{"--myflagGlobal", "bat", "baz"}) - expect(t, c.IsSet("myflag"), true) - expect(t, c.IsSet("otherflag"), false) - expect(t, c.IsSet("bogusflag"), false) - expect(t, c.IsSet("myflagGlobal"), false) -} - -func TestContext_GlobalIsSet(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - set.String("otherflag", "hello world", "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Bool("myflagGlobal", true, "doc") - globalSet.Bool("myflagGlobalUnset", true, "doc") - globalCtx := NewContext(nil, globalSet, nil) - c := NewContext(nil, set, globalCtx) - set.Parse([]string{"--myflag", "bat", "baz"}) - globalSet.Parse([]string{"--myflagGlobal", "bat", "baz"}) - expect(t, c.GlobalIsSet("myflag"), false) - expect(t, c.GlobalIsSet("otherflag"), false) - expect(t, c.GlobalIsSet("bogusflag"), false) - expect(t, c.GlobalIsSet("myflagGlobal"), true) - expect(t, c.GlobalIsSet("myflagGlobalUnset"), false) - expect(t, c.GlobalIsSet("bogusGlobal"), false) -} - -func TestContext_NumFlags(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - set.String("otherflag", "hello world", "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Bool("myflagGlobal", true, "doc") - globalCtx := NewContext(nil, globalSet, nil) - c := NewContext(nil, set, globalCtx) - set.Parse([]string{"--myflag", "--otherflag=foo"}) - globalSet.Parse([]string{"--myflagGlobal"}) - expect(t, c.NumFlags(), 2) -} diff --git a/vendor/src/github.com/codegangsta/cli/flag.go b/vendor/src/github.com/codegangsta/cli/flag.go deleted file mode 100644 index e951c2d..0000000 --- a/vendor/src/github.com/codegangsta/cli/flag.go +++ /dev/null @@ -1,546 +0,0 @@ -package cli - -import ( - "flag" - "fmt" - "os" - "runtime" - "strconv" - "strings" - "time" -) - -// This flag enables bash-completion for all commands and subcommands -var BashCompletionFlag = BoolFlag{ - Name: "generate-bash-completion", -} - -// This flag prints the version for the application -var VersionFlag = BoolFlag{ - Name: "version, v", - Usage: "print the version", -} - -// This flag prints the help for all commands and subcommands -// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand -// unless HideHelp is set to true) -var HelpFlag = BoolFlag{ - Name: "help, h", - Usage: "show help", -} - -// Flag is a common interface related to parsing flags in cli. -// For more advanced flag parsing techniques, it is recommended that -// this interface be implemented. -type Flag interface { - fmt.Stringer - // Apply Flag settings to the given flag set - Apply(*flag.FlagSet) - GetName() string -} - -func flagSet(name string, flags []Flag) *flag.FlagSet { - set := flag.NewFlagSet(name, flag.ContinueOnError) - - for _, f := range flags { - f.Apply(set) - } - return set -} - -func eachName(longName string, fn func(string)) { - parts := strings.Split(longName, ",") - for _, name := range parts { - name = strings.Trim(name, " ") - fn(name) - } -} - -// Generic is a generic parseable type identified by a specific flag -type Generic interface { - Set(value string) error - String() string -} - -// GenericFlag is the flag type for types implementing Generic -type GenericFlag struct { - Name string - Value Generic - Usage string - EnvVar string -} - -// String returns the string representation of the generic flag to display the -// help text to the user (uses the String() method of the generic flag to show -// the value) -func (f GenericFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) -} - -func (f GenericFlag) FormatValueHelp() string { - if f.Value == nil { - return "" - } - s := f.Value.String() - if len(s) == 0 { - return "" - } - return fmt.Sprintf("\"%s\"", s) -} - -// Apply takes the flagset and calls Set on the generic flag with the value -// provided by the user for parsing by the flag -func (f GenericFlag) Apply(set *flag.FlagSet) { - val := f.Value - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - val.Set(envVal) - break - } - } - } - - eachName(f.Name, func(name string) { - set.Var(f.Value, name, f.Usage) - }) -} - -func (f GenericFlag) GetName() string { - return f.Name -} - -// StringSlice is an opaque type for []string to satisfy flag.Value -type StringSlice []string - -// Set appends the string value to the list of values -func (f *StringSlice) Set(value string) error { - *f = append(*f, value) - return nil -} - -// String returns a readable representation of this value (for usage defaults) -func (f *StringSlice) String() string { - return fmt.Sprintf("%s", *f) -} - -// Value returns the slice of strings set by this flag -func (f *StringSlice) Value() []string { - return *f -} - -// StringSlice is a string flag that can be specified multiple times on the -// command-line -type StringSliceFlag struct { - Name string - Value *StringSlice - Usage string - EnvVar string -} - -// String returns the usage -func (f StringSliceFlag) String() string { - firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") - pref := prefixFor(firstName) - return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f StringSliceFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - newVal := &StringSlice{} - for _, s := range strings.Split(envVal, ",") { - s = strings.TrimSpace(s) - newVal.Set(s) - } - f.Value = newVal - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Value == nil { - f.Value = &StringSlice{} - } - set.Var(f.Value, name, f.Usage) - }) -} - -func (f StringSliceFlag) GetName() string { - return f.Name -} - -// StringSlice is an opaque type for []int to satisfy flag.Value -type IntSlice []int - -// Set parses the value into an integer and appends it to the list of values -func (f *IntSlice) Set(value string) error { - tmp, err := strconv.Atoi(value) - if err != nil { - return err - } else { - *f = append(*f, tmp) - } - return nil -} - -// String returns a readable representation of this value (for usage defaults) -func (f *IntSlice) String() string { - return fmt.Sprintf("%d", *f) -} - -// Value returns the slice of ints set by this flag -func (f *IntSlice) Value() []int { - return *f -} - -// IntSliceFlag is an int flag that can be specified multiple times on the -// command-line -type IntSliceFlag struct { - Name string - Value *IntSlice - Usage string - EnvVar string -} - -// String returns the usage -func (f IntSliceFlag) String() string { - firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") - pref := prefixFor(firstName) - return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f IntSliceFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - newVal := &IntSlice{} - for _, s := range strings.Split(envVal, ",") { - s = strings.TrimSpace(s) - err := newVal.Set(s) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - } - } - f.Value = newVal - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Value == nil { - f.Value = &IntSlice{} - } - set.Var(f.Value, name, f.Usage) - }) -} - -func (f IntSliceFlag) GetName() string { - return f.Name -} - -// BoolFlag is a switch that defaults to false -type BoolFlag struct { - Name string - Usage string - EnvVar string - Destination *bool -} - -// String returns a readable representation of this value (for usage defaults) -func (f BoolFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f BoolFlag) Apply(set *flag.FlagSet) { - val := false - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValBool, err := strconv.ParseBool(envVal) - if err == nil { - val = envValBool - } - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.BoolVar(f.Destination, name, val, f.Usage) - return - } - set.Bool(name, val, f.Usage) - }) -} - -func (f BoolFlag) GetName() string { - return f.Name -} - -// BoolTFlag this represents a boolean flag that is true by default, but can -// still be set to false by --some-flag=false -type BoolTFlag struct { - Name string - Usage string - EnvVar string - Destination *bool -} - -// String returns a readable representation of this value (for usage defaults) -func (f BoolTFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f BoolTFlag) Apply(set *flag.FlagSet) { - val := true - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValBool, err := strconv.ParseBool(envVal) - if err == nil { - val = envValBool - break - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.BoolVar(f.Destination, name, val, f.Usage) - return - } - set.Bool(name, val, f.Usage) - }) -} - -func (f BoolTFlag) GetName() string { - return f.Name -} - -// StringFlag represents a flag that takes as string value -type StringFlag struct { - Name string - Value string - Usage string - EnvVar string - Destination *string -} - -// String returns the usage -func (f StringFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) -} - -func (f StringFlag) FormatValueHelp() string { - s := f.Value - if len(s) == 0 { - return "" - } - return fmt.Sprintf("\"%s\"", s) -} - -// Apply populates the flag given the flag set and environment -func (f StringFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - f.Value = envVal - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.StringVar(f.Destination, name, f.Value, f.Usage) - return - } - set.String(name, f.Value, f.Usage) - }) -} - -func (f StringFlag) GetName() string { - return f.Name -} - -// IntFlag is a flag that takes an integer -// Errors if the value provided cannot be parsed -type IntFlag struct { - Name string - Value int - Usage string - EnvVar string - Destination *int -} - -// String returns the usage -func (f IntFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f IntFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValInt, err := strconv.ParseInt(envVal, 0, 64) - if err == nil { - f.Value = int(envValInt) - break - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.IntVar(f.Destination, name, f.Value, f.Usage) - return - } - set.Int(name, f.Value, f.Usage) - }) -} - -func (f IntFlag) GetName() string { - return f.Name -} - -// DurationFlag is a flag that takes a duration specified in Go's duration -// format: https://golang.org/pkg/time/#ParseDuration -type DurationFlag struct { - Name string - Value time.Duration - Usage string - EnvVar string - Destination *time.Duration -} - -// String returns a readable representation of this value (for usage defaults) -func (f DurationFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f DurationFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValDuration, err := time.ParseDuration(envVal) - if err == nil { - f.Value = envValDuration - break - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.DurationVar(f.Destination, name, f.Value, f.Usage) - return - } - set.Duration(name, f.Value, f.Usage) - }) -} - -func (f DurationFlag) GetName() string { - return f.Name -} - -// Float64Flag is a flag that takes an float value -// Errors if the value provided cannot be parsed -type Float64Flag struct { - Name string - Value float64 - Usage string - EnvVar string - Destination *float64 -} - -// String returns the usage -func (f Float64Flag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f Float64Flag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValFloat, err := strconv.ParseFloat(envVal, 10) - if err == nil { - f.Value = float64(envValFloat) - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.Float64Var(f.Destination, name, f.Value, f.Usage) - return - } - set.Float64(name, f.Value, f.Usage) - }) -} - -func (f Float64Flag) GetName() string { - return f.Name -} - -func prefixFor(name string) (prefix string) { - if len(name) == 1 { - prefix = "-" - } else { - prefix = "--" - } - - return -} - -func prefixedNames(fullName string) (prefixed string) { - parts := strings.Split(fullName, ",") - for i, name := range parts { - name = strings.Trim(name, " ") - prefixed += prefixFor(name) + name - if i < len(parts)-1 { - prefixed += ", " - } - } - return -} - -func withEnvHint(envVar, str string) string { - envText := "" - if envVar != "" { - prefix := "$" - suffix := "" - sep := ", $" - if runtime.GOOS == "windows" { - prefix = "%" - suffix = "%" - sep = "%, %" - } - envText = fmt.Sprintf(" [%s%s%s]", prefix, strings.Join(strings.Split(envVar, ","), sep), suffix) - } - return str + envText -} diff --git a/vendor/src/github.com/codegangsta/cli/flag_test.go b/vendor/src/github.com/codegangsta/cli/flag_test.go deleted file mode 100644 index 3caa70a..0000000 --- a/vendor/src/github.com/codegangsta/cli/flag_test.go +++ /dev/null @@ -1,859 +0,0 @@ -package cli - -import ( - "fmt" - "os" - "reflect" - "strings" - "testing" - "runtime" -) - -var boolFlagTests = []struct { - name string - expected string -}{ - {"help", "--help\t"}, - {"h", "-h\t"}, -} - -func TestBoolFlagHelpOutput(t *testing.T) { - - for _, test := range boolFlagTests { - flag := BoolFlag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -var stringFlagTests = []struct { - name string - value string - expected string -}{ - {"help", "", "--help \t"}, - {"h", "", "-h \t"}, - {"h", "", "-h \t"}, - {"test", "Something", "--test \"Something\"\t"}, -} - -func TestStringFlagHelpOutput(t *testing.T) { - - for _, test := range stringFlagTests { - flag := StringFlag{Name: test.name, Value: test.value} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestStringFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_FOO", "derp") - for _, test := range stringFlagTests { - flag := StringFlag{Name: test.name, Value: test.value, EnvVar: "APP_FOO"} - output := flag.String() - - expectedSuffix := " [$APP_FOO]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_FOO%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var stringSliceFlagTests = []struct { - name string - value *StringSlice - expected string -}{ - {"help", func() *StringSlice { - s := &StringSlice{} - s.Set("") - return s - }(), "--help [--help option --help option]\t"}, - {"h", func() *StringSlice { - s := &StringSlice{} - s.Set("") - return s - }(), "-h [-h option -h option]\t"}, - {"h", func() *StringSlice { - s := &StringSlice{} - s.Set("") - return s - }(), "-h [-h option -h option]\t"}, - {"test", func() *StringSlice { - s := &StringSlice{} - s.Set("Something") - return s - }(), "--test [--test option --test option]\t"}, -} - -func TestStringSliceFlagHelpOutput(t *testing.T) { - - for _, test := range stringSliceFlagTests { - flag := StringSliceFlag{Name: test.name, Value: test.value} - output := flag.String() - - if output != test.expected { - t.Errorf("%q does not match %q", output, test.expected) - } - } -} - -func TestStringSliceFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_QWWX", "11,4") - for _, test := range stringSliceFlagTests { - flag := StringSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_QWWX"} - output := flag.String() - - expectedSuffix := " [$APP_QWWX]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_QWWX%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%q does not end with" + expectedSuffix, output) - } - } -} - -var intFlagTests = []struct { - name string - expected string -}{ - {"help", "--help \"0\"\t"}, - {"h", "-h \"0\"\t"}, -} - -func TestIntFlagHelpOutput(t *testing.T) { - - for _, test := range intFlagTests { - flag := IntFlag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestIntFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_BAR", "2") - for _, test := range intFlagTests { - flag := IntFlag{Name: test.name, EnvVar: "APP_BAR"} - output := flag.String() - - expectedSuffix := " [$APP_BAR]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_BAR%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var durationFlagTests = []struct { - name string - expected string -}{ - {"help", "--help \"0\"\t"}, - {"h", "-h \"0\"\t"}, -} - -func TestDurationFlagHelpOutput(t *testing.T) { - - for _, test := range durationFlagTests { - flag := DurationFlag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestDurationFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_BAR", "2h3m6s") - for _, test := range durationFlagTests { - flag := DurationFlag{Name: test.name, EnvVar: "APP_BAR"} - output := flag.String() - - expectedSuffix := " [$APP_BAR]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_BAR%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var intSliceFlagTests = []struct { - name string - value *IntSlice - expected string -}{ - {"help", &IntSlice{}, "--help [--help option --help option]\t"}, - {"h", &IntSlice{}, "-h [-h option -h option]\t"}, - {"h", &IntSlice{}, "-h [-h option -h option]\t"}, - {"test", func() *IntSlice { - i := &IntSlice{} - i.Set("9") - return i - }(), "--test [--test option --test option]\t"}, -} - -func TestIntSliceFlagHelpOutput(t *testing.T) { - - for _, test := range intSliceFlagTests { - flag := IntSliceFlag{Name: test.name, Value: test.value} - output := flag.String() - - if output != test.expected { - t.Errorf("%q does not match %q", output, test.expected) - } - } -} - -func TestIntSliceFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_SMURF", "42,3") - for _, test := range intSliceFlagTests { - flag := IntSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_SMURF"} - output := flag.String() - - expectedSuffix := " [$APP_SMURF]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_SMURF%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%q does not end with" + expectedSuffix, output) - } - } -} - -var float64FlagTests = []struct { - name string - expected string -}{ - {"help", "--help \"0\"\t"}, - {"h", "-h \"0\"\t"}, -} - -func TestFloat64FlagHelpOutput(t *testing.T) { - - for _, test := range float64FlagTests { - flag := Float64Flag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestFloat64FlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_BAZ", "99.4") - for _, test := range float64FlagTests { - flag := Float64Flag{Name: test.name, EnvVar: "APP_BAZ"} - output := flag.String() - - expectedSuffix := " [$APP_BAZ]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_BAZ%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var genericFlagTests = []struct { - name string - value Generic - expected string -}{ - {"test", &Parser{"abc", "def"}, "--test \"abc,def\"\ttest flag"}, - {"t", &Parser{"abc", "def"}, "-t \"abc,def\"\ttest flag"}, -} - -func TestGenericFlagHelpOutput(t *testing.T) { - - for _, test := range genericFlagTests { - flag := GenericFlag{Name: test.name, Value: test.value, Usage: "test flag"} - output := flag.String() - - if output != test.expected { - t.Errorf("%q does not match %q", output, test.expected) - } - } -} - -func TestGenericFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_ZAP", "3") - for _, test := range genericFlagTests { - flag := GenericFlag{Name: test.name, EnvVar: "APP_ZAP"} - output := flag.String() - - expectedSuffix := " [$APP_ZAP]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_ZAP%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -func TestParseMultiString(t *testing.T) { - (&App{ - Flags: []Flag{ - StringFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.String("serve") != "10" { - t.Errorf("main name not set") - } - if ctx.String("s") != "10" { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run", "-s", "10"}) -} - -func TestParseDestinationString(t *testing.T) { - var dest string - a := App{ - Flags: []Flag{ - StringFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != "10" { - t.Errorf("expected destination String 10") - } - }, - } - a.Run([]string{"run", "--dest", "10"}) -} - -func TestParseMultiStringFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_COUNT", "20") - (&App{ - Flags: []Flag{ - StringFlag{Name: "count, c", EnvVar: "APP_COUNT"}, - }, - Action: func(ctx *Context) { - if ctx.String("count") != "20" { - t.Errorf("main name not set") - } - if ctx.String("c") != "20" { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiStringFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_COUNT", "20") - (&App{ - Flags: []Flag{ - StringFlag{Name: "count, c", EnvVar: "COMPAT_COUNT,APP_COUNT"}, - }, - Action: func(ctx *Context) { - if ctx.String("count") != "20" { - t.Errorf("main name not set") - } - if ctx.String("c") != "20" { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiStringSlice(t *testing.T) { - (&App{ - Flags: []Flag{ - StringSliceFlag{Name: "serve, s", Value: &StringSlice{}}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.StringSlice("serve"), []string{"10", "20"}) { - t.Errorf("main name not set") - } - if !reflect.DeepEqual(ctx.StringSlice("s"), []string{"10", "20"}) { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run", "-s", "10", "-s", "20"}) -} - -func TestParseMultiStringSliceFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - StringSliceFlag{Name: "intervals, i", Value: &StringSlice{}, EnvVar: "APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiStringSliceFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - StringSliceFlag{Name: "intervals, i", Value: &StringSlice{}, EnvVar: "COMPAT_INTERVALS,APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiInt(t *testing.T) { - a := App{ - Flags: []Flag{ - IntFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.Int("serve") != 10 { - t.Errorf("main name not set") - } - if ctx.Int("s") != 10 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "-s", "10"}) -} - -func TestParseDestinationInt(t *testing.T) { - var dest int - a := App{ - Flags: []Flag{ - IntFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != 10 { - t.Errorf("expected destination Int 10") - } - }, - } - a.Run([]string{"run", "--dest", "10"}) -} - -func TestParseMultiIntFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "10") - a := App{ - Flags: []Flag{ - IntFlag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Int("timeout") != 10 { - t.Errorf("main name not set") - } - if ctx.Int("t") != 10 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiIntFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "10") - a := App{ - Flags: []Flag{ - IntFlag{Name: "timeout, t", EnvVar: "COMPAT_TIMEOUT_SECONDS,APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Int("timeout") != 10 { - t.Errorf("main name not set") - } - if ctx.Int("t") != 10 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiIntSlice(t *testing.T) { - (&App{ - Flags: []Flag{ - IntSliceFlag{Name: "serve, s", Value: &IntSlice{}}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.IntSlice("serve"), []int{10, 20}) { - t.Errorf("main name not set") - } - if !reflect.DeepEqual(ctx.IntSlice("s"), []int{10, 20}) { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run", "-s", "10", "-s", "20"}) -} - -func TestParseMultiIntSliceFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - IntSliceFlag{Name: "intervals, i", Value: &IntSlice{}, EnvVar: "APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiIntSliceFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - IntSliceFlag{Name: "intervals, i", Value: &IntSlice{}, EnvVar: "COMPAT_INTERVALS,APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiFloat64(t *testing.T) { - a := App{ - Flags: []Flag{ - Float64Flag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.Float64("serve") != 10.2 { - t.Errorf("main name not set") - } - if ctx.Float64("s") != 10.2 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "-s", "10.2"}) -} - -func TestParseDestinationFloat64(t *testing.T) { - var dest float64 - a := App{ - Flags: []Flag{ - Float64Flag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != 10.2 { - t.Errorf("expected destination Float64 10.2") - } - }, - } - a.Run([]string{"run", "--dest", "10.2"}) -} - -func TestParseMultiFloat64FromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "15.5") - a := App{ - Flags: []Flag{ - Float64Flag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Float64("timeout") != 15.5 { - t.Errorf("main name not set") - } - if ctx.Float64("t") != 15.5 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiFloat64FromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "15.5") - a := App{ - Flags: []Flag{ - Float64Flag{Name: "timeout, t", EnvVar: "COMPAT_TIMEOUT_SECONDS,APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Float64("timeout") != 15.5 { - t.Errorf("main name not set") - } - if ctx.Float64("t") != 15.5 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBool(t *testing.T) { - a := App{ - Flags: []Flag{ - BoolFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("serve") != true { - t.Errorf("main name not set") - } - if ctx.Bool("s") != true { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "--serve"}) -} - -func TestParseDestinationBool(t *testing.T) { - var dest bool - a := App{ - Flags: []Flag{ - BoolFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != true { - t.Errorf("expected destination Bool true") - } - }, - } - a.Run([]string{"run", "--dest"}) -} - -func TestParseMultiBoolFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "1") - a := App{ - Flags: []Flag{ - BoolFlag{Name: "debug, d", EnvVar: "APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("debug") != true { - t.Errorf("main name not set from env") - } - if ctx.Bool("d") != true { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBoolFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "1") - a := App{ - Flags: []Flag{ - BoolFlag{Name: "debug, d", EnvVar: "COMPAT_DEBUG,APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("debug") != true { - t.Errorf("main name not set from env") - } - if ctx.Bool("d") != true { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBoolT(t *testing.T) { - a := App{ - Flags: []Flag{ - BoolTFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.BoolT("serve") != true { - t.Errorf("main name not set") - } - if ctx.BoolT("s") != true { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "--serve"}) -} - -func TestParseDestinationBoolT(t *testing.T) { - var dest bool - a := App{ - Flags: []Flag{ - BoolTFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != true { - t.Errorf("expected destination BoolT true") - } - }, - } - a.Run([]string{"run", "--dest"}) -} - -func TestParseMultiBoolTFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "0") - a := App{ - Flags: []Flag{ - BoolTFlag{Name: "debug, d", EnvVar: "APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.BoolT("debug") != false { - t.Errorf("main name not set from env") - } - if ctx.BoolT("d") != false { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBoolTFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "0") - a := App{ - Flags: []Flag{ - BoolTFlag{Name: "debug, d", EnvVar: "COMPAT_DEBUG,APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.BoolT("debug") != false { - t.Errorf("main name not set from env") - } - if ctx.BoolT("d") != false { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -type Parser [2]string - -func (p *Parser) Set(value string) error { - parts := strings.Split(value, ",") - if len(parts) != 2 { - return fmt.Errorf("invalid format") - } - - (*p)[0] = parts[0] - (*p)[1] = parts[1] - - return nil -} - -func (p *Parser) String() string { - return fmt.Sprintf("%s,%s", p[0], p[1]) -} - -func TestParseGeneric(t *testing.T) { - a := App{ - Flags: []Flag{ - GenericFlag{Name: "serve, s", Value: &Parser{}}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"10", "20"}) { - t.Errorf("main name not set") - } - if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"10", "20"}) { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "-s", "10,20"}) -} - -func TestParseGenericFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_SERVE", "20,30") - a := App{ - Flags: []Flag{ - GenericFlag{Name: "serve, s", Value: &Parser{}, EnvVar: "APP_SERVE"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"20", "30"}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"20", "30"}) { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseGenericFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_FOO", "99,2000") - a := App{ - Flags: []Flag{ - GenericFlag{Name: "foos", Value: &Parser{}, EnvVar: "COMPAT_FOO,APP_FOO"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.Generic("foos"), &Parser{"99", "2000"}) { - t.Errorf("value not set from env") - } - }, - } - a.Run([]string{"run"}) -} diff --git a/vendor/src/github.com/codegangsta/cli/help.go b/vendor/src/github.com/codegangsta/cli/help.go deleted file mode 100644 index 15916f8..0000000 --- a/vendor/src/github.com/codegangsta/cli/help.go +++ /dev/null @@ -1,248 +0,0 @@ -package cli - -import ( - "fmt" - "io" - "strings" - "text/tabwriter" - "text/template" -) - -// The text template for the Default help topic. -// cli.go uses text/template to render templates. You can -// render custom help text by setting this variable. -var AppHelpTemplate = `NAME: - {{.Name}} - {{.Usage}} - -USAGE: - {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .Flags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}} - {{if .Version}} -VERSION: - {{.Version}} - {{end}}{{if len .Authors}} -AUTHOR(S): - {{range .Authors}}{{ . }}{{end}} - {{end}}{{if .Commands}} -COMMANDS: - {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -GLOBAL OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}}{{if .Copyright }} -COPYRIGHT: - {{.Copyright}} - {{end}} -` - -// The text template for the command help topic. -// cli.go uses text/template to render templates. You can -// render custom help text by setting this variable. -var CommandHelpTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}}{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{if .Description}} - -DESCRIPTION: - {{.Description}}{{end}}{{if .Flags}} - -OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{ end }} -` - -// The text template for the subcommand help topic. -// cli.go uses text/template to render templates. You can -// render custom help text by setting this variable. -var SubcommandHelpTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} command{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}} - -COMMANDS: - {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{if .Flags}} -OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}} -` - -var helpCommand = Command{ - Name: "help", - Aliases: []string{"h"}, - Usage: "Shows a list of commands or help for one command", - ArgsUsage: "[command]", - Action: func(c *Context) { - args := c.Args() - if args.Present() { - ShowCommandHelp(c, args.First()) - } else { - ShowAppHelp(c) - } - }, -} - -var helpSubcommand = Command{ - Name: "help", - Aliases: []string{"h"}, - Usage: "Shows a list of commands or help for one command", - ArgsUsage: "[command]", - Action: func(c *Context) { - args := c.Args() - if args.Present() { - ShowCommandHelp(c, args.First()) - } else { - ShowSubcommandHelp(c) - } - }, -} - -// Prints help for the App or Command -type helpPrinter func(w io.Writer, templ string, data interface{}) - -var HelpPrinter helpPrinter = printHelp - -// Prints version for the App -var VersionPrinter = printVersion - -func ShowAppHelp(c *Context) { - HelpPrinter(c.App.Writer, AppHelpTemplate, c.App) -} - -// Prints the list of subcommands as the default app completion method -func DefaultAppComplete(c *Context) { - for _, command := range c.App.Commands { - for _, name := range command.Names() { - fmt.Fprintln(c.App.Writer, name) - } - } -} - -// Prints help for the given command -func ShowCommandHelp(ctx *Context, command string) { - // show the subcommand help for a command with subcommands - if command == "" { - HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App) - return - } - - for _, c := range ctx.App.Commands { - if c.HasName(command) { - HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c) - return - } - } - - if ctx.App.CommandNotFound != nil { - ctx.App.CommandNotFound(ctx, command) - } else { - fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command) - } -} - -// Prints help for the given subcommand -func ShowSubcommandHelp(c *Context) { - ShowCommandHelp(c, c.Command.Name) -} - -// Prints the version number of the App -func ShowVersion(c *Context) { - VersionPrinter(c) -} - -func printVersion(c *Context) { - fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version) -} - -// Prints the lists of commands within a given context -func ShowCompletions(c *Context) { - a := c.App - if a != nil && a.BashComplete != nil { - a.BashComplete(c) - } -} - -// Prints the custom completions for a given command -func ShowCommandCompletions(ctx *Context, command string) { - c := ctx.App.Command(command) - if c != nil && c.BashComplete != nil { - c.BashComplete(ctx) - } -} - -func printHelp(out io.Writer, templ string, data interface{}) { - funcMap := template.FuncMap{ - "join": strings.Join, - } - - w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0) - t := template.Must(template.New("help").Funcs(funcMap).Parse(templ)) - err := t.Execute(w, data) - if err != nil { - // If the writer is closed, t.Execute will fail, and there's nothing - // we can do to recover. We could send this to os.Stderr if we need. - return - } - w.Flush() -} - -func checkVersion(c *Context) bool { - found := false - if VersionFlag.Name != "" { - eachName(VersionFlag.Name, func(name string) { - if c.GlobalBool(name) || c.Bool(name) { - found = true - } - }) - } - return found -} - -func checkHelp(c *Context) bool { - found := false - if HelpFlag.Name != "" { - eachName(HelpFlag.Name, func(name string) { - if c.GlobalBool(name) || c.Bool(name) { - found = true - } - }) - } - return found -} - -func checkCommandHelp(c *Context, name string) bool { - if c.Bool("h") || c.Bool("help") { - ShowCommandHelp(c, name) - return true - } - - return false -} - -func checkSubcommandHelp(c *Context) bool { - if c.GlobalBool("h") || c.GlobalBool("help") { - ShowSubcommandHelp(c) - return true - } - - return false -} - -func checkCompletions(c *Context) bool { - if (c.GlobalBool(BashCompletionFlag.Name) || c.Bool(BashCompletionFlag.Name)) && c.App.EnableBashCompletion { - ShowCompletions(c) - return true - } - - return false -} - -func checkCommandCompletions(c *Context, name string) bool { - if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion { - ShowCommandCompletions(c, name) - return true - } - - return false -} diff --git a/vendor/src/github.com/codegangsta/cli/help_test.go b/vendor/src/github.com/codegangsta/cli/help_test.go deleted file mode 100644 index 350e263..0000000 --- a/vendor/src/github.com/codegangsta/cli/help_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package cli - -import ( - "bytes" - "testing" -) - -func Test_ShowAppHelp_NoAuthor(t *testing.T) { - output := new(bytes.Buffer) - app := NewApp() - app.Writer = output - - c := NewContext(app, nil, nil) - - ShowAppHelp(c) - - if bytes.Index(output.Bytes(), []byte("AUTHOR(S):")) != -1 { - t.Errorf("expected\n%snot to include %s", output.String(), "AUTHOR(S):") - } -} - -func Test_ShowAppHelp_NoVersion(t *testing.T) { - output := new(bytes.Buffer) - app := NewApp() - app.Writer = output - - app.Version = "" - - c := NewContext(app, nil, nil) - - ShowAppHelp(c) - - if bytes.Index(output.Bytes(), []byte("VERSION:")) != -1 { - t.Errorf("expected\n%snot to include %s", output.String(), "VERSION:") - } -} - -func Test_Help_Custom_Flags(t *testing.T) { - oldFlag := HelpFlag - defer func() { - HelpFlag = oldFlag - }() - - HelpFlag = BoolFlag{ - Name: "help, x", - Usage: "show help", - } - - app := App{ - Flags: []Flag{ - BoolFlag{Name: "foo, h"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("h") != true { - t.Errorf("custom help flag not set") - } - }, - } - output := new(bytes.Buffer) - app.Writer = output - app.Run([]string{"test", "-h"}) - if output.Len() > 0 { - t.Errorf("unexpected output: %s", output.String()) - } -} - -func Test_Version_Custom_Flags(t *testing.T) { - oldFlag := VersionFlag - defer func() { - VersionFlag = oldFlag - }() - - VersionFlag = BoolFlag{ - Name: "version, V", - Usage: "show version", - } - - app := App{ - Flags: []Flag{ - BoolFlag{Name: "foo, v"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("v") != true { - t.Errorf("custom version flag not set") - } - }, - } - output := new(bytes.Buffer) - app.Writer = output - app.Run([]string{"test", "-v"}) - if output.Len() > 0 { - t.Errorf("unexpected output: %s", output.String()) - } -} diff --git a/vendor/src/github.com/codegangsta/cli/helpers_test.go b/vendor/src/github.com/codegangsta/cli/helpers_test.go deleted file mode 100644 index b1b7339..0000000 --- a/vendor/src/github.com/codegangsta/cli/helpers_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package cli - -import ( - "reflect" - "testing" -) - -/* Test Helpers */ -func expect(t *testing.T, a interface{}, b interface{}) { - if !reflect.DeepEqual(a, b) { - t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} - -func refute(t *testing.T, a interface{}, b interface{}) { - if reflect.DeepEqual(a, b) { - t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/avgvar.go b/vendor/src/github.com/petar/GoLLRB/llrb/avgvar.go deleted file mode 100644 index 2d7e2a3..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/avgvar.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -import "math" - -// avgVar maintains the average and variance of a stream of numbers -// in a space-efficient manner. -type avgVar struct { - count int64 - sum, sumsq float64 -} - -func (av *avgVar) Init() { - av.count = 0 - av.sum = 0.0 - av.sumsq = 0.0 -} - -func (av *avgVar) Add(sample float64) { - av.count++ - av.sum += sample - av.sumsq += sample * sample -} - -func (av *avgVar) GetCount() int64 { return av.count } - -func (av *avgVar) GetAvg() float64 { return av.sum / float64(av.count) } - -func (av *avgVar) GetTotal() float64 { return av.sum } - -func (av *avgVar) GetVar() float64 { - a := av.GetAvg() - return av.sumsq/float64(av.count) - a*a -} - -func (av *avgVar) GetStdDev() float64 { return math.Sqrt(av.GetVar()) } diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/iterator.go b/vendor/src/github.com/petar/GoLLRB/llrb/iterator.go deleted file mode 100644 index ee7b27f..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/iterator.go +++ /dev/null @@ -1,93 +0,0 @@ -package llrb - -type ItemIterator func(i Item) bool - -//func (t *Tree) Ascend(iterator ItemIterator) { -// t.AscendGreaterOrEqual(Inf(-1), iterator) -//} - -func (t *LLRB) AscendRange(greaterOrEqual, lessThan Item, iterator ItemIterator) { - t.ascendRange(t.root, greaterOrEqual, lessThan, iterator) -} - -func (t *LLRB) ascendRange(h *Node, inf, sup Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if !less(h.Item, sup) { - return t.ascendRange(h.Left, inf, sup, iterator) - } - if less(h.Item, inf) { - return t.ascendRange(h.Right, inf, sup, iterator) - } - - if !t.ascendRange(h.Left, inf, sup, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - return t.ascendRange(h.Right, inf, sup, iterator) -} - -// AscendGreaterOrEqual will call iterator once for each element greater or equal to -// pivot in ascending order. It will stop whenever the iterator returns false. -func (t *LLRB) AscendGreaterOrEqual(pivot Item, iterator ItemIterator) { - t.ascendGreaterOrEqual(t.root, pivot, iterator) -} - -func (t *LLRB) ascendGreaterOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if !less(h.Item, pivot) { - if !t.ascendGreaterOrEqual(h.Left, pivot, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - } - return t.ascendGreaterOrEqual(h.Right, pivot, iterator) -} - -func (t *LLRB) AscendLessThan(pivot Item, iterator ItemIterator) { - t.ascendLessThan(t.root, pivot, iterator) -} - -func (t *LLRB) ascendLessThan(h *Node, pivot Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if !t.ascendLessThan(h.Left, pivot, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - if less(h.Item, pivot) { - return t.ascendLessThan(h.Left, pivot, iterator) - } - return true -} - -// DescendLessOrEqual will call iterator once for each element less than the -// pivot in descending order. It will stop whenever the iterator returns false. -func (t *LLRB) DescendLessOrEqual(pivot Item, iterator ItemIterator) { - t.descendLessOrEqual(t.root, pivot, iterator) -} - -func (t *LLRB) descendLessOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if less(h.Item, pivot) || !less(pivot, h.Item) { - if !t.descendLessOrEqual(h.Right, pivot, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - } - return t.descendLessOrEqual(h.Left, pivot, iterator) -} diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/iterator_test.go b/vendor/src/github.com/petar/GoLLRB/llrb/iterator_test.go deleted file mode 100644 index db5e12c..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/iterator_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package llrb - -import ( - "reflect" - "testing" -) - -func TestAscendGreaterOrEqual(t *testing.T) { - tree := New() - tree.InsertNoReplace(Int(4)) - tree.InsertNoReplace(Int(6)) - tree.InsertNoReplace(Int(1)) - tree.InsertNoReplace(Int(3)) - var ary []Item - tree.AscendGreaterOrEqual(Int(-1), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected := []Item{Int(1), Int(3), Int(4), Int(6)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.AscendGreaterOrEqual(Int(3), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(3), Int(4), Int(6)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.AscendGreaterOrEqual(Int(2), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(3), Int(4), Int(6)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } -} - -func TestDescendLessOrEqual(t *testing.T) { - tree := New() - tree.InsertNoReplace(Int(4)) - tree.InsertNoReplace(Int(6)) - tree.InsertNoReplace(Int(1)) - tree.InsertNoReplace(Int(3)) - var ary []Item - tree.DescendLessOrEqual(Int(10), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected := []Item{Int(6), Int(4), Int(3), Int(1)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.DescendLessOrEqual(Int(4), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(4), Int(3), Int(1)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.DescendLessOrEqual(Int(5), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(4), Int(3), Int(1)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } -} diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/llrb-stats.go b/vendor/src/github.com/petar/GoLLRB/llrb/llrb-stats.go deleted file mode 100644 index 47126a3..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/llrb-stats.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -// GetHeight() returns an item in the tree with key @key, and it's height in the tree -func (t *LLRB) GetHeight(key Item) (result Item, depth int) { - return t.getHeight(t.root, key) -} - -func (t *LLRB) getHeight(h *Node, item Item) (Item, int) { - if h == nil { - return nil, 0 - } - if less(item, h.Item) { - result, depth := t.getHeight(h.Left, item) - return result, depth + 1 - } - if less(h.Item, item) { - result, depth := t.getHeight(h.Right, item) - return result, depth + 1 - } - return h.Item, 0 -} - -// HeightStats() returns the average and standard deviation of the height -// of elements in the tree -func (t *LLRB) HeightStats() (avg, stddev float64) { - av := &avgVar{} - heightStats(t.root, 0, av) - return av.GetAvg(), av.GetStdDev() -} - -func heightStats(h *Node, d int, av *avgVar) { - if h == nil { - return - } - av.Add(float64(d)) - if h.Left != nil { - heightStats(h.Left, d+1, av) - } - if h.Right != nil { - heightStats(h.Right, d+1, av) - } -} diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/llrb.go b/vendor/src/github.com/petar/GoLLRB/llrb/llrb.go deleted file mode 100644 index 81373fb..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/llrb.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// A Left-Leaning Red-Black (LLRB) implementation of 2-3 balanced binary search trees, -// based on the following work: -// -// http://www.cs.princeton.edu/~rs/talks/LLRB/08Penn.pdf -// http://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf -// http://www.cs.princeton.edu/~rs/talks/LLRB/Java/RedBlackBST.java -// -// 2-3 trees (and the run-time equivalent 2-3-4 trees) are the de facto standard BST -// algoritms found in implementations of Python, Java, and other libraries. The LLRB -// implementation of 2-3 trees is a recent improvement on the traditional implementation, -// observed and documented by Robert Sedgewick. -// -package llrb - -// Tree is a Left-Leaning Red-Black (LLRB) implementation of 2-3 trees -type LLRB struct { - count int - root *Node -} - -type Node struct { - Item - Left, Right *Node // Pointers to left and right child nodes - Black bool // If set, the color of the link (incoming from the parent) is black - // In the LLRB, new nodes are always red, hence the zero-value for node -} - -type Item interface { - Less(than Item) bool -} - -// -func less(x, y Item) bool { - if x == pinf { - return false - } - if x == ninf { - return true - } - return x.Less(y) -} - -// Inf returns an Item that is "bigger than" any other item, if sign is positive. -// Otherwise it returns an Item that is "smaller than" any other item. -func Inf(sign int) Item { - if sign == 0 { - panic("sign") - } - if sign > 0 { - return pinf - } - return ninf -} - -var ( - ninf = nInf{} - pinf = pInf{} -) - -type nInf struct{} - -func (nInf) Less(Item) bool { - return true -} - -type pInf struct{} - -func (pInf) Less(Item) bool { - return false -} - -// New() allocates a new tree -func New() *LLRB { - return &LLRB{} -} - -// SetRoot sets the root node of the tree. -// It is intended to be used by functions that deserialize the tree. -func (t *LLRB) SetRoot(r *Node) { - t.root = r -} - -// Root returns the root node of the tree. -// It is intended to be used by functions that serialize the tree. -func (t *LLRB) Root() *Node { - return t.root -} - -// Len returns the number of nodes in the tree. -func (t *LLRB) Len() int { return t.count } - -// Has returns true if the tree contains an element whose order is the same as that of key. -func (t *LLRB) Has(key Item) bool { - return t.Get(key) != nil -} - -// Get retrieves an element from the tree whose order is the same as that of key. -func (t *LLRB) Get(key Item) Item { - h := t.root - for h != nil { - switch { - case less(key, h.Item): - h = h.Left - case less(h.Item, key): - h = h.Right - default: - return h.Item - } - } - return nil -} - -// Min returns the minimum element in the tree. -func (t *LLRB) Min() Item { - h := t.root - if h == nil { - return nil - } - for h.Left != nil { - h = h.Left - } - return h.Item -} - -// Max returns the maximum element in the tree. -func (t *LLRB) Max() Item { - h := t.root - if h == nil { - return nil - } - for h.Right != nil { - h = h.Right - } - return h.Item -} - -func (t *LLRB) ReplaceOrInsertBulk(items ...Item) { - for _, i := range items { - t.ReplaceOrInsert(i) - } -} - -func (t *LLRB) InsertNoReplaceBulk(items ...Item) { - for _, i := range items { - t.InsertNoReplace(i) - } -} - -// ReplaceOrInsert inserts item into the tree. If an existing -// element has the same order, it is removed from the tree and returned. -func (t *LLRB) ReplaceOrInsert(item Item) Item { - if item == nil { - panic("inserting nil item") - } - var replaced Item - t.root, replaced = t.replaceOrInsert(t.root, item) - t.root.Black = true - if replaced == nil { - t.count++ - } - return replaced -} - -func (t *LLRB) replaceOrInsert(h *Node, item Item) (*Node, Item) { - if h == nil { - return newNode(item), nil - } - - h = walkDownRot23(h) - - var replaced Item - if less(item, h.Item) { // BUG - h.Left, replaced = t.replaceOrInsert(h.Left, item) - } else if less(h.Item, item) { - h.Right, replaced = t.replaceOrInsert(h.Right, item) - } else { - replaced, h.Item = h.Item, item - } - - h = walkUpRot23(h) - - return h, replaced -} - -// InsertNoReplace inserts item into the tree. If an existing -// element has the same order, both elements remain in the tree. -func (t *LLRB) InsertNoReplace(item Item) { - if item == nil { - panic("inserting nil item") - } - t.root = t.insertNoReplace(t.root, item) - t.root.Black = true - t.count++ -} - -func (t *LLRB) insertNoReplace(h *Node, item Item) *Node { - if h == nil { - return newNode(item) - } - - h = walkDownRot23(h) - - if less(item, h.Item) { - h.Left = t.insertNoReplace(h.Left, item) - } else { - h.Right = t.insertNoReplace(h.Right, item) - } - - return walkUpRot23(h) -} - -// Rotation driver routines for 2-3 algorithm - -func walkDownRot23(h *Node) *Node { return h } - -func walkUpRot23(h *Node) *Node { - if isRed(h.Right) && !isRed(h.Left) { - h = rotateLeft(h) - } - - if isRed(h.Left) && isRed(h.Left.Left) { - h = rotateRight(h) - } - - if isRed(h.Left) && isRed(h.Right) { - flip(h) - } - - return h -} - -// Rotation driver routines for 2-3-4 algorithm - -func walkDownRot234(h *Node) *Node { - if isRed(h.Left) && isRed(h.Right) { - flip(h) - } - - return h -} - -func walkUpRot234(h *Node) *Node { - if isRed(h.Right) && !isRed(h.Left) { - h = rotateLeft(h) - } - - if isRed(h.Left) && isRed(h.Left.Left) { - h = rotateRight(h) - } - - return h -} - -// DeleteMin deletes the minimum element in the tree and returns the -// deleted item or nil otherwise. -func (t *LLRB) DeleteMin() Item { - var deleted Item - t.root, deleted = deleteMin(t.root) - if t.root != nil { - t.root.Black = true - } - if deleted != nil { - t.count-- - } - return deleted -} - -// deleteMin code for LLRB 2-3 trees -func deleteMin(h *Node) (*Node, Item) { - if h == nil { - return nil, nil - } - if h.Left == nil { - return nil, h.Item - } - - if !isRed(h.Left) && !isRed(h.Left.Left) { - h = moveRedLeft(h) - } - - var deleted Item - h.Left, deleted = deleteMin(h.Left) - - return fixUp(h), deleted -} - -// DeleteMax deletes the maximum element in the tree and returns -// the deleted item or nil otherwise -func (t *LLRB) DeleteMax() Item { - var deleted Item - t.root, deleted = deleteMax(t.root) - if t.root != nil { - t.root.Black = true - } - if deleted != nil { - t.count-- - } - return deleted -} - -func deleteMax(h *Node) (*Node, Item) { - if h == nil { - return nil, nil - } - if isRed(h.Left) { - h = rotateRight(h) - } - if h.Right == nil { - return nil, h.Item - } - if !isRed(h.Right) && !isRed(h.Right.Left) { - h = moveRedRight(h) - } - var deleted Item - h.Right, deleted = deleteMax(h.Right) - - return fixUp(h), deleted -} - -// Delete deletes an item from the tree whose key equals key. -// The deleted item is return, otherwise nil is returned. -func (t *LLRB) Delete(key Item) Item { - var deleted Item - t.root, deleted = t.delete(t.root, key) - if t.root != nil { - t.root.Black = true - } - if deleted != nil { - t.count-- - } - return deleted -} - -func (t *LLRB) delete(h *Node, item Item) (*Node, Item) { - var deleted Item - if h == nil { - return nil, nil - } - if less(item, h.Item) { - if h.Left == nil { // item not present. Nothing to delete - return h, nil - } - if !isRed(h.Left) && !isRed(h.Left.Left) { - h = moveRedLeft(h) - } - h.Left, deleted = t.delete(h.Left, item) - } else { - if isRed(h.Left) { - h = rotateRight(h) - } - // If @item equals @h.Item and no right children at @h - if !less(h.Item, item) && h.Right == nil { - return nil, h.Item - } - // PETAR: Added 'h.Right != nil' below - if h.Right != nil && !isRed(h.Right) && !isRed(h.Right.Left) { - h = moveRedRight(h) - } - // If @item equals @h.Item, and (from above) 'h.Right != nil' - if !less(h.Item, item) { - var subDeleted Item - h.Right, subDeleted = deleteMin(h.Right) - if subDeleted == nil { - panic("logic") - } - deleted, h.Item = h.Item, subDeleted - } else { // Else, @item is bigger than @h.Item - h.Right, deleted = t.delete(h.Right, item) - } - } - - return fixUp(h), deleted -} - -// Internal node manipulation routines - -func newNode(item Item) *Node { return &Node{Item: item} } - -func isRed(h *Node) bool { - if h == nil { - return false - } - return !h.Black -} - -func rotateLeft(h *Node) *Node { - x := h.Right - if x.Black { - panic("rotating a black link") - } - h.Right = x.Left - x.Left = h - x.Black = h.Black - h.Black = false - return x -} - -func rotateRight(h *Node) *Node { - x := h.Left - if x.Black { - panic("rotating a black link") - } - h.Left = x.Right - x.Right = h - x.Black = h.Black - h.Black = false - return x -} - -// REQUIRE: Left and Right children must be present -func flip(h *Node) { - h.Black = !h.Black - h.Left.Black = !h.Left.Black - h.Right.Black = !h.Right.Black -} - -// REQUIRE: Left and Right children must be present -func moveRedLeft(h *Node) *Node { - flip(h) - if isRed(h.Right.Left) { - h.Right = rotateRight(h.Right) - h = rotateLeft(h) - flip(h) - } - return h -} - -// REQUIRE: Left and Right children must be present -func moveRedRight(h *Node) *Node { - flip(h) - if isRed(h.Left.Left) { - h = rotateRight(h) - flip(h) - } - return h -} - -func fixUp(h *Node) *Node { - if isRed(h.Right) { - h = rotateLeft(h) - } - - if isRed(h.Left) && isRed(h.Left.Left) { - h = rotateRight(h) - } - - if isRed(h.Left) && isRed(h.Right) { - flip(h) - } - - return h -} diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/llrb_test.go b/vendor/src/github.com/petar/GoLLRB/llrb/llrb_test.go deleted file mode 100644 index b7bc978..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/llrb_test.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -import ( - "math" - "math/rand" - "testing" -) - -func TestCases(t *testing.T) { - tree := New() - tree.ReplaceOrInsert(Int(1)) - tree.ReplaceOrInsert(Int(1)) - if tree.Len() != 1 { - t.Errorf("expecting len 1") - } - if !tree.Has(Int(1)) { - t.Errorf("expecting to find key=1") - } - - tree.Delete(Int(1)) - if tree.Len() != 0 { - t.Errorf("expecting len 0") - } - if tree.Has(Int(1)) { - t.Errorf("not expecting to find key=1") - } - - tree.Delete(Int(1)) - if tree.Len() != 0 { - t.Errorf("expecting len 0") - } - if tree.Has(Int(1)) { - t.Errorf("not expecting to find key=1") - } -} - -func TestReverseInsertOrder(t *testing.T) { - tree := New() - n := 100 - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(n - i)) - } - i := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - i++ - if item.(Int) != Int(i) { - t.Errorf("bad order: got %d, expect %d", item.(Int), i) - } - return true - }) -} - -func TestRange(t *testing.T) { - tree := New() - order := []String{ - "ab", "aba", "abc", "a", "aa", "aaa", "b", "a-", "a!", - } - for _, i := range order { - tree.ReplaceOrInsert(i) - } - k := 0 - tree.AscendRange(String("ab"), String("ac"), func(item Item) bool { - if k > 3 { - t.Fatalf("returned more items than expected") - } - i1 := order[k] - i2 := item.(String) - if i1 != i2 { - t.Errorf("expecting %s, got %s", i1, i2) - } - k++ - return true - }) -} - -func TestRandomInsertOrder(t *testing.T) { - tree := New() - n := 1000 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - j := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - if item.(Int) != Int(j) { - t.Fatalf("bad order") - } - j++ - return true - }) -} - -func TestRandomReplace(t *testing.T) { - tree := New() - n := 100 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - perm = rand.Perm(n) - for i := 0; i < n; i++ { - if replaced := tree.ReplaceOrInsert(Int(perm[i])); replaced == nil || replaced.(Int) != Int(perm[i]) { - t.Errorf("error replacing") - } - } -} - -func TestRandomInsertSequentialDelete(t *testing.T) { - tree := New() - n := 1000 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - for i := 0; i < n; i++ { - tree.Delete(Int(i)) - } -} - -func TestRandomInsertDeleteNonExistent(t *testing.T) { - tree := New() - n := 100 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - if tree.Delete(Int(200)) != nil { - t.Errorf("deleted non-existent item") - } - if tree.Delete(Int(-2)) != nil { - t.Errorf("deleted non-existent item") - } - for i := 0; i < n; i++ { - if u := tree.Delete(Int(i)); u == nil || u.(Int) != Int(i) { - t.Errorf("delete failed") - } - } - if tree.Delete(Int(200)) != nil { - t.Errorf("deleted non-existent item") - } - if tree.Delete(Int(-2)) != nil { - t.Errorf("deleted non-existent item") - } -} - -func TestRandomInsertPartialDeleteOrder(t *testing.T) { - tree := New() - n := 100 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - for i := 1; i < n-1; i++ { - tree.Delete(Int(i)) - } - j := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - switch j { - case 0: - if item.(Int) != Int(0) { - t.Errorf("expecting 0") - } - case 1: - if item.(Int) != Int(n-1) { - t.Errorf("expecting %d", n-1) - } - } - j++ - return true - }) -} - -func TestRandomInsertStats(t *testing.T) { - tree := New() - n := 100000 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - avg, _ := tree.HeightStats() - expAvg := math.Log2(float64(n)) - 1.5 - if math.Abs(avg-expAvg) >= 2.0 { - t.Errorf("too much deviation from expected average height") - } -} - -func BenchmarkInsert(b *testing.B) { - tree := New() - for i := 0; i < b.N; i++ { - tree.ReplaceOrInsert(Int(b.N - i)) - } -} - -func BenchmarkDelete(b *testing.B) { - b.StopTimer() - tree := New() - for i := 0; i < b.N; i++ { - tree.ReplaceOrInsert(Int(b.N - i)) - } - b.StartTimer() - for i := 0; i < b.N; i++ { - tree.Delete(Int(i)) - } -} - -func BenchmarkDeleteMin(b *testing.B) { - b.StopTimer() - tree := New() - for i := 0; i < b.N; i++ { - tree.ReplaceOrInsert(Int(b.N - i)) - } - b.StartTimer() - for i := 0; i < b.N; i++ { - tree.DeleteMin() - } -} - -func TestInsertNoReplace(t *testing.T) { - tree := New() - n := 1000 - for q := 0; q < 2; q++ { - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.InsertNoReplace(Int(perm[i])) - } - } - j := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - if item.(Int) != Int(j/2) { - t.Fatalf("bad order") - } - j++ - return true - }) -} diff --git a/vendor/src/github.com/petar/GoLLRB/llrb/util.go b/vendor/src/github.com/petar/GoLLRB/llrb/util.go deleted file mode 100644 index 63dbdb2..0000000 --- a/vendor/src/github.com/petar/GoLLRB/llrb/util.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -type Int int - -func (x Int) Less(than Item) bool { - return x < than.(Int) -} - -type String string - -func (x String) Less(than Item) bool { - return x < than.(String) -} From baa06d805769efb86b2d52968f593f9bf1ab7922 Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:14:28 +0100 Subject: [PATCH 02/10] re organize project layout --- build.go | 114 ++ common.go | 277 +++++ diff.go | 145 +++ fileformat.md | 21 + main.go | 65 + patch.go | 105 ++ vendor/github.com/Redundancy/go-sync/LICENSE | 21 + .../github.com/Redundancy/go-sync/README.md | 119 ++ .../go-sync/acceptancetests/nulltest.sh | 1 + .../Redundancy/go-sync/acceptancetests/run.sh | 21 + .../go-sync/acceptancetests/truncated_test.sh | 14 + .../acceptancetests/truncation_test.sh | 14 + .../go-sync/blocksources/blocksourcebase.go | 271 +++++ .../blocksources/blocksourcebase_test.go | 303 +++++ .../blocksources/fixed_size_block_resolver.go | 73 ++ .../fixed_size_block_resolver_test.go | 127 ++ .../go-sync/blocksources/helpers.go | 124 ++ .../go-sync/blocksources/httpblocksource.go | 110 ++ .../blocksources/httpblocksource_test.go | 251 ++++ .../blocksources/readseeker_blocksource.go | 54 + .../readseeker_blocksource_test.go | 42 + .../blocksources/sortableBlockResponse.go | 19 + .../Redundancy/go-sync/chunks/chunks.go | 87 ++ .../go-sync/circularbuffer/noalloc.go | 170 +++ .../go-sync/circularbuffer/noalloc_test.go | 182 +++ .../Redundancy/go-sync/comparer/comparer.go | 219 ++++ .../go-sync/comparer/comparer_bench_test.go | 76 ++ .../go-sync/comparer/comparer_test.go | 445 +++++++ .../Redundancy/go-sync/comparer/merger.go | 309 +++++ .../go-sync/comparer/merger_test.go | 401 +++++++ .../go-sync/filechecksum/filechecksum.go | 237 ++++ .../go-sync/filechecksum/filechecksum_test.go | 309 +++++ .../go-sync/filechecksum/verifier.go | 48 + .../go-sync/filechecksum/verifier_test.go | 72 ++ .../github.com/Redundancy/go-sync/gosync.go | 16 + .../Redundancy/go-sync/gosync/build.go | 114 ++ .../Redundancy/go-sync/gosync/common.go | 277 +++++ .../Redundancy/go-sync/gosync/diff.go | 145 +++ .../Redundancy/go-sync/gosync/fileformat.md | 21 + .../Redundancy/go-sync/gosync/main.go | 65 + .../Redundancy/go-sync/gosync/patch.go | 105 ++ .../Redundancy/go-sync/gosync_test.go | 111 ++ .../Redundancy/go-sync/http_test.go | 174 +++ .../Redundancy/go-sync/index/index.go | 183 +++ .../go-sync/index/index_bench_test.go | 148 +++ .../Redundancy/go-sync/index/index_test.go | 206 ++++ .../go-sync/indexbuilder/indexbuilder.go | 55 + .../Redundancy/go-sync/patcher/blocksource.go | 58 + .../go-sync/patcher/sequential/sequential.go | 145 +++ .../patcher/sequential/sequential_test.go | 188 +++ .../Redundancy/go-sync/rollsum/rollsum_32.go | 85 ++ .../go-sync/rollsum/rollsum_32_base.go | 86 ++ .../go-sync/rollsum/rollsum_32_test.go | 298 +++++ vendor/github.com/Redundancy/go-sync/rsync.go | 347 ++++++ .../go-sync/util/readers/injectedreader.go | 18 + .../go-sync/util/readers/nonrepeating.go | 35 + .../go-sync/util/readers/nonrepeating_test.go | 58 + .../go-sync/util/readers/readers.go | 6 + .../go-sync/util/readers/sequence_test.go | 1 + .../go-sync/util/readers/sequencelimit.go | 12 + .../go-sync/util/readers/uniform_test.go | 90 ++ .../go-sync/util/readers/uniformreader.go | 55 + vendor/github.com/codegangsta/cli/LICENSE | 21 + vendor/github.com/codegangsta/cli/README.md | 394 +++++++ .../github.com/codegangsta/cli/altsrc/flag.go | 439 +++++++ .../codegangsta/cli/altsrc/flag_test.go | 336 ++++++ .../codegangsta/cli/altsrc/helpers_test.go | 18 + .../cli/altsrc/input_source_context.go | 21 + .../cli/altsrc/map_input_source.go | 152 +++ .../cli/altsrc/yaml_command_test.go | 172 +++ .../cli/altsrc/yaml_file_loader.go | 84 ++ vendor/github.com/codegangsta/cli/app.go | 349 ++++++ vendor/github.com/codegangsta/cli/app_test.go | 1047 +++++++++++++++++ .../github.com/codegangsta/cli/appveyor.yml | 16 + .../cli/autocomplete/bash_autocomplete | 14 + .../cli/autocomplete/zsh_autocomplete | 5 + vendor/github.com/codegangsta/cli/cli.go | 40 + vendor/github.com/codegangsta/cli/command.go | 250 ++++ .../codegangsta/cli/command_test.go | 97 ++ vendor/github.com/codegangsta/cli/context.go | 393 +++++++ .../codegangsta/cli/context_test.go | 121 ++ vendor/github.com/codegangsta/cli/flag.go | 546 +++++++++ .../github.com/codegangsta/cli/flag_test.go | 859 ++++++++++++++ vendor/github.com/codegangsta/cli/help.go | 248 ++++ .../github.com/codegangsta/cli/help_test.go | 94 ++ .../codegangsta/cli/helpers_test.go | 19 + vendor/github.com/petar/GoLLRB/llrb/avgvar.go | 39 + .../github.com/petar/GoLLRB/llrb/iterator.go | 93 ++ .../petar/GoLLRB/llrb/iterator_test.go | 76 ++ .../petar/GoLLRB/llrb/llrb-stats.go | 46 + vendor/github.com/petar/GoLLRB/llrb/llrb.go | 456 +++++++ .../github.com/petar/GoLLRB/llrb/llrb_test.go | 239 ++++ vendor/github.com/petar/GoLLRB/llrb/util.go | 17 + 93 files changed, 14649 insertions(+) create mode 100644 build.go create mode 100644 common.go create mode 100644 diff.go create mode 100644 fileformat.md create mode 100644 main.go create mode 100644 patch.go create mode 100644 vendor/github.com/Redundancy/go-sync/LICENSE create mode 100644 vendor/github.com/Redundancy/go-sync/README.md create mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh create mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh create mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh create mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/helpers.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go create mode 100644 vendor/github.com/Redundancy/go-sync/chunks/chunks.go create mode 100644 vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go create mode 100644 vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/comparer/comparer.go create mode 100644 vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/comparer/merger.go create mode 100644 vendor/github.com/Redundancy/go-sync/comparer/merger_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go create mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go create mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync/build.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync/common.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync/diff.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync/fileformat.md create mode 100644 vendor/github.com/Redundancy/go-sync/gosync/main.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync/patch.go create mode 100644 vendor/github.com/Redundancy/go-sync/gosync_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/http_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/index/index.go create mode 100644 vendor/github.com/Redundancy/go-sync/index/index_bench_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/index/index_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go create mode 100644 vendor/github.com/Redundancy/go-sync/patcher/blocksource.go create mode 100644 vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go create mode 100644 vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go create mode 100644 vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go create mode 100644 vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/rsync.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/readers.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go create mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go create mode 100644 vendor/github.com/codegangsta/cli/LICENSE create mode 100644 vendor/github.com/codegangsta/cli/README.md create mode 100644 vendor/github.com/codegangsta/cli/altsrc/flag.go create mode 100644 vendor/github.com/codegangsta/cli/altsrc/flag_test.go create mode 100644 vendor/github.com/codegangsta/cli/altsrc/helpers_test.go create mode 100644 vendor/github.com/codegangsta/cli/altsrc/input_source_context.go create mode 100644 vendor/github.com/codegangsta/cli/altsrc/map_input_source.go create mode 100644 vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go create mode 100644 vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go create mode 100644 vendor/github.com/codegangsta/cli/app.go create mode 100644 vendor/github.com/codegangsta/cli/app_test.go create mode 100644 vendor/github.com/codegangsta/cli/appveyor.yml create mode 100644 vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete create mode 100644 vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete create mode 100644 vendor/github.com/codegangsta/cli/cli.go create mode 100644 vendor/github.com/codegangsta/cli/command.go create mode 100644 vendor/github.com/codegangsta/cli/command_test.go create mode 100644 vendor/github.com/codegangsta/cli/context.go create mode 100644 vendor/github.com/codegangsta/cli/context_test.go create mode 100644 vendor/github.com/codegangsta/cli/flag.go create mode 100644 vendor/github.com/codegangsta/cli/flag_test.go create mode 100644 vendor/github.com/codegangsta/cli/help.go create mode 100644 vendor/github.com/codegangsta/cli/help_test.go create mode 100644 vendor/github.com/codegangsta/cli/helpers_test.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/avgvar.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/iterator.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/iterator_test.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/llrb.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/llrb_test.go create mode 100644 vendor/github.com/petar/GoLLRB/llrb/util.go diff --git a/build.go b/build.go new file mode 100644 index 0000000..7dfb000 --- /dev/null +++ b/build.go @@ -0,0 +1,114 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/Redundancy/go-sync/filechecksum" + "github.com/codegangsta/cli" +) + +func init() { + app.Commands = append( + app.Commands, + cli.Command{ + Name: "build", + ShortName: "b", + Usage: "build a .gosync file for a file", + Action: Build, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "blocksize", + Value: DefaultBlockSize, + Usage: "The block size to use for the gosync file", + }, + }, + }, + ) +} + +func Build(c *cli.Context) { + filename := c.Args()[0] + blocksize := uint32(c.Int("blocksize")) + generator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) + inputFile, err := os.Open(filename) + + if err != nil { + absInputPath, err2 := filepath.Abs(filename) + if err2 == nil { + handleFileError(absInputPath, err) + } else { + handleFileError(filename, err) + } + + os.Exit(1) + } + + s, _ := inputFile.Stat() + // TODO: Error? + file_size := s.Size() + + defer inputFile.Close() + + ext := filepath.Ext(filename) + outfilePath := filename[:len(filename)-len(ext)] + ".gosync" + outputFile, err := os.Create(outfilePath) + + if err != nil { + handleFileError(outfilePath, err) + os.Exit(1) + } + + defer outputFile.Close() + + if err = writeHeaders( + outputFile, + magicString, + blocksize, + file_size, + []uint16{majorVersion, minorVersion, patchVersion}, + ); err != nil { + fmt.Fprintf( + os.Stderr, + "Error getting file info: %v\n", + filename, + err, + ) + os.Exit(2) + } + + start := time.Now() + _, err = generator.GenerateChecksums(inputFile, outputFile) + end := time.Now() + + if err != nil { + fmt.Fprintf( + os.Stderr, + "Error generating checksum: %v\n", + filename, + err, + ) + os.Exit(2) + } + + inputFileInfo, err := os.Stat(filename) + if err != nil { + fmt.Fprintf( + os.Stderr, + "Error getting file info: %v\n", + filename, + err, + ) + os.Exit(2) + } + + fmt.Fprintf( + os.Stderr, + "Index for %v file generated in %v (%v bytes/S)\n", + inputFileInfo.Size(), + end.Sub(start), + float64(inputFileInfo.Size())/end.Sub(start).Seconds(), + ) +} diff --git a/common.go b/common.go new file mode 100644 index 0000000..f8091a9 --- /dev/null +++ b/common.go @@ -0,0 +1,277 @@ +package main + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/comparer" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/index" + "github.com/Redundancy/go-sync/patcher" + "github.com/codegangsta/cli" +) + +const ( + // KB - One Kilobyte + KB = 1024 + // MB - One Megabyte + MB = 1000000 +) + +func errorWrapper(c *cli.Context, f func(*cli.Context) error) { + defer func() { + if p := recover(); p != nil { + fmt.Fprintln(os.Stderr, p) + os.Exit(1) + } + }() + + if err := f(c); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + return +} + +func openFileAndHandleError(filename string) (f *os.File) { + var err error + f, err = os.Open(filename) + + if err != nil { + f = nil + handleFileError(filename, err) + } + + return +} + +func formatFileError(filename string, err error) error { + switch { + case os.IsExist(err): + return fmt.Errorf( + "Could not open %v (already exists): %v", + filename, + err, + ) + case os.IsNotExist(err): + return fmt.Errorf( + "Could not find %v: %v\n", + filename, + err, + ) + case os.IsPermission(err): + return fmt.Errorf( + "Could not open %v (permission denied): %v\n", + filename, + err, + ) + default: + return fmt.Errorf( + "Unknown error opening %v: %v\n", + filename, + err, + ) + } +} + +func handleFileError(filename string, err error) { + e := formatFileError(filename, err) + fmt.Fprintln(os.Stderr, e) +} + +func getLocalOrRemoteFile(path string) (io.ReadCloser, error) { + url, err := url.Parse(path) + + switch { + case err != nil: + return os.Open(path) + case url.Scheme == "": + return os.Open(path) + default: + response, err := http.Get(path) + + if err != nil { + return nil, err + } + + if response.StatusCode < 200 || response.StatusCode > 299 { + return nil, fmt.Errorf("Request to %v returned status: %v", path, response.Status) + } + + return response.Body, nil + } +} + +func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { + result := make([]patcher.FoundBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].MatchOffset = v.ComparisonStartOffset + result[i].BlockSize = blockSize + } + + return result +} + +func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { + result := make([]patcher.MissingBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].BlockSize = blockSize + } + + return result +} + +func writeHeaders( + f *os.File, + magic string, + blocksize uint32, + filesize int64, + versions []uint16, +) (err error) { + if _, err = f.WriteString(magicString); err != nil { + return + } + + for _, v := range versions { + if err = binary.Write(f, binary.LittleEndian, v); err != nil { + return + } + } + + if err = binary.Write(f, binary.LittleEndian, filesize); err != nil { + return + } + + err = binary.Write(f, binary.LittleEndian, blocksize) + return +} + +// reads the file headers and checks the magic string, then the semantic versioning +func readHeadersAndCheck( + r io.Reader, + magic string, + requiredMajorVersion uint16, +) ( + major, minor, patch uint16, + filesize int64, + blocksize uint32, + err error, +) { + b := make([]byte, len(magicString)) + + if _, err = r.Read(b); err != nil { + return + } else if string(b) != magicString { + err = errors.New( + "file header does not match magic string. Not a valid gosync file", + ) + return + } + + for _, v := range []*uint16{&major, &minor, &patch} { + err = binary.Read(r, binary.LittleEndian, v) + if err != nil { + return + } + } + + if requiredMajorVersion != major { + err = fmt.Errorf( + "The major version of the gosync file (%v.%v.%v) does not match the tool (%v.%v.%v).", + major, minor, patch, + majorVersion, minorVersion, patchVersion, + ) + + return + } + + err = binary.Read(r, binary.LittleEndian, &filesize) + if err != nil { + return + } + + err = binary.Read(r, binary.LittleEndian, &blocksize) + return +} + +func readIndex(r io.Reader, blocksize uint) ( + i *index.ChecksumIndex, + checksumLookup filechecksum.ChecksumLookup, + blockCount uint, + err error, +) { + generator := filechecksum.NewFileChecksumGenerator(blocksize) + + readChunks, e := chunks.LoadChecksumsFromReader( + r, + generator.WeakRollingHash.Size(), + generator.StrongHash.Size(), + ) + + err = e + + if err != nil { + return + } + + checksumLookup = chunks.StrongChecksumGetter(readChunks) + i = index.MakeChecksumIndex(readChunks) + blockCount = uint(len(readChunks)) + + return +} + +func multithreadedMatching( + localFile *os.File, + idx *index.ChecksumIndex, + localFileSize, + matcherCount int64, + blocksize uint, +) (*comparer.MatchMerger, *comparer.Comparer) { + // Note: Since not all sections of the file are equal in work + // it would be better to divide things up into more sections and + // pull work from a queue channel as each finish + sectionSize := localFileSize / matcherCount + sectionSize += int64(blocksize) - (sectionSize % int64(blocksize)) + merger := &comparer.MatchMerger{} + compare := &comparer.Comparer{} + + for i := int64(0); i < matcherCount; i++ { + offset := sectionSize * i + + // Sections must overlap by blocksize (strictly blocksize - 1?) + if i > 0 { + offset -= int64(blocksize) + } + + sectionReader := bufio.NewReaderSize( + io.NewSectionReader(localFile, offset, sectionSize), + MB, + ) + + sectionGenerator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) + + matchStream := compare.StartFindMatchingBlocks( + sectionReader, offset, sectionGenerator, idx) + + merger.StartMergeResultStream(matchStream, int64(blocksize)) + } + + return merger, compare +} + +// better way to do this? diff --git a/diff.go b/diff.go new file mode 100644 index 0000000..14a4acf --- /dev/null +++ b/diff.go @@ -0,0 +1,145 @@ +package main + +import ( + "fmt" + "os" + "runtime" + "time" + + "github.com/codegangsta/cli" +) + +func init() { + app.Commands = append( + app.Commands, + cli.Command{ + Name: "diff", + ShortName: "d", + Usage: "gosync diff ", + Description: `Compare a file with a reference index, and print statistics on the comparison and performance.`, + Action: Diff, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "p", + Value: runtime.NumCPU(), + Usage: "The number of streams to use concurrently", + }, + }, + }, + ) +} + +func Diff(c *cli.Context) { + localFilename := c.Args()[0] + referenceFilename := c.Args()[1] + startTime := time.Now() + + localFile := openFileAndHandleError(localFilename) + + if localFile == nil { + os.Exit(1) + } + + defer localFile.Close() + + var blocksize uint32 + referenceFile := openFileAndHandleError(referenceFilename) + + if referenceFile == nil { + os.Exit(1) + } + + defer referenceFile.Close() + + _, _, _, _, blocksize, e := readHeadersAndCheck( + referenceFile, + magicString, + majorVersion, + ) + + if e != nil { + fmt.Printf("Error loading index: %v", e) + os.Exit(1) + } + + fmt.Println("Blocksize: ", blocksize) + + index, _, _, err := readIndex(referenceFile, uint(blocksize)) + referenceFile.Close() + + if err != nil { + return + } + + fmt.Println("Weak hash count:", index.WeakCount()) + + fi, err := localFile.Stat() + + if err != nil { + fmt.Println("Could not get info on file:", err) + os.Exit(1) + } + + num_matchers := int64(c.Int("p")) + + localFile_size := fi.Size() + + // Don't split up small files + if localFile_size < 1024*1024 { + num_matchers = 1 + } + + merger, compare := multithreadedMatching( + localFile, + index, + localFile_size, + num_matchers, + uint(blocksize), + ) + + mergedBlocks := merger.GetMergedBlocks() + + fmt.Println("\nMatched:") + totalMatchingSize := uint64(0) + matchedBlockCountAfterMerging := uint(0) + + for _, b := range mergedBlocks { + totalMatchingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) + matchedBlockCountAfterMerging += b.EndBlock - b.StartBlock + 1 + } + + fmt.Println("Comparisons:", compare.Comparisons) + fmt.Println("Weak hash hits:", compare.WeakHashHits) + + if compare.Comparisons > 0 { + fmt.Printf( + "Weak hit rate: %.2f%%\n", + 100.0*float64(compare.WeakHashHits)/float64(compare.Comparisons), + ) + } + + fmt.Println("Strong hash hits:", compare.StrongHashHits) + if compare.WeakHashHits > 0 { + fmt.Printf( + "Weak hash error rate: %.2f%%\n", + 100.0*float64(compare.WeakHashHits-compare.StrongHashHits)/float64(compare.WeakHashHits), + ) + } + + fmt.Println("Total matched bytes:", totalMatchingSize) + fmt.Println("Total matched blocks:", matchedBlockCountAfterMerging) + + // TODO: GetMissingBlocks uses the highest index, not the count, this can be pretty confusing + // Should clean up this interface to avoid that + missing := mergedBlocks.GetMissingBlocks(uint(index.BlockCount) - 1) + fmt.Println("Index blocks:", index.BlockCount) + + totalMissingSize := uint64(0) + for _, b := range missing { + //fmt.Printf("%#v\n", b) + totalMissingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) + } + + fmt.Println("Approximate missing bytes:", totalMissingSize) + fmt.Println("Time taken:", time.Now().Sub(startTime)) +} diff --git a/fileformat.md b/fileformat.md new file mode 100644 index 0000000..0fd5522 --- /dev/null +++ b/fileformat.md @@ -0,0 +1,21 @@ +*NB: I'm documenting the current format of the gosync files merely as a point in time +reference of format that is in use in the tool that is meant to serve as a practical reference and acceptance testing tool. The gosync tool is not intended as a production-worthy, well supported, tested tool.* + +*The format used exists entirely in service of being able to test the implementation of the gosync library as a cohesive whole in the real world, and therefore backwards and forwards compatibility (or even efficiency) are not primary concerns.* + +# Version 0.2.0 +### The header +(LE = little endian) +* The string "G0S9NC" in UTF-8 +* versions*3 (eg. 0.1.2), uint16 LE +* filesize, int64 LE +* blocksize uint32 LE + +### The body +Repeating: +* WeakChecksum +* StrongChecksum + +each referring to blocks, starting at 0 (file start) and going upwards. + +In the current implementation of the FileChecksumGenerator used the WeakChecksum is the rolling checksum (4 bytes), and StrongChecksum is MD5 (16 bytes). diff --git a/main.go b/main.go new file mode 100644 index 0000000..3164a8d --- /dev/null +++ b/main.go @@ -0,0 +1,65 @@ +/* +gosync is a command-line implementation of the gosync package functionality, primarily as a demonstration of usage +but supposed to be functional in itself. +*/ +package main + +import ( + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "os" + "runtime" + + "github.com/codegangsta/cli" +) + +const ( + DefaultBlockSize = 8192 + magicString = "G0S9NC" // just to confirm the file type is used correctly + majorVersion = uint16(0) + minorVersion = uint16(2) + patchVersion = uint16(1) +) + +var app = cli.NewApp() + +func main() { + app.Name = "gosync" + app.Usage = "Build indexes, patches, patch files" + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "profile", + Usage: "enable HTTP profiling", + }, + cli.IntFlag{ + Name: "profilePort", + Value: 6060, + Usage: "The number of streams to use concurrently", + }, + } + + app.Version = fmt.Sprintf( + "%v.%v.%v", + majorVersion, + minorVersion, + patchVersion, + ) + + runtime.GOMAXPROCS(runtime.NumCPU()) + + app.Before = func(c *cli.Context) error { + if c.Bool("profile") { + port := fmt.Sprint(c.Int("profilePort")) + + go func() { + log.Println(http.ListenAndServe("localhost:"+port, nil)) + }() + } + + return nil + } + + app.Run(os.Args) +} diff --git a/patch.go b/patch.go new file mode 100644 index 0000000..16730d3 --- /dev/null +++ b/patch.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "os" + "runtime" + + gosync_main "github.com/Redundancy/go-sync" + "github.com/codegangsta/cli" +) + +const usage = "gosync patch []" + +func init() { + app.Commands = append( + app.Commands, + cli.Command{ + Name: "patch", + ShortName: "p", + Usage: usage, + Description: `Recreate the reference source file, using an index and a local file that is believed to be similar. +The index should be produced by "gosync build". + + is a .gosync file and may be a local, unc network path or http/https url + is corresponding target and may be a local, unc network path or http/https url + is optional. If not specified, the local file will be overwritten when done.`, + Action: Patch, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "p", + Value: runtime.NumCPU(), + Usage: "The number of streams to use concurrently", + }, + }, + }, + ) +} + +// Patch a file +func Patch(c *cli.Context) { + errorWrapper(c, func(c *cli.Context) error { + + fmt.Fprintln(os.Stderr, "Starting patching process") + + if l := len(c.Args()); l < 3 || l > 4 { + return fmt.Errorf( + "Usage is \"%v\" (invalid number of arguments)", + usage, + ) + } + + localFilename := c.Args()[0] + summaryFile := c.Args()[1] + referencePath := c.Args()[2] + + outFilename := localFilename + if len(c.Args()) == 4 { + outFilename = c.Args()[3] + } + + indexReader, e := os.Open(summaryFile) + if e != nil { + return e + } + defer indexReader.Close() + + _, _, _, filesize, blocksize, e := readHeadersAndCheck( + indexReader, + magicString, + majorVersion, + ) + + index, checksumLookup, blockCount, err := readIndex( + indexReader, + uint(blocksize), + ) + + fs := &gosync_main.BasicSummary{ + ChecksumIndex: index, + ChecksumLookup: checksumLookup, + BlockCount: blockCount, + BlockSize: uint(blocksize), + FileSize: filesize, + } + + rsync, err := gosync_main.MakeRSync( + localFilename, + referencePath, + outFilename, + fs, + ) + + if err != nil { + return err + } + + err = rsync.Patch() + + if err != nil { + return err + } + + return rsync.Close() + }) +} diff --git a/vendor/github.com/Redundancy/go-sync/LICENSE b/vendor/github.com/Redundancy/go-sync/LICENSE new file mode 100644 index 0000000..1e4b9c6 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2014 Daniel Speed + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/Redundancy/go-sync/README.md b/vendor/github.com/Redundancy/go-sync/README.md new file mode 100644 index 0000000..584e77d --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/README.md @@ -0,0 +1,119 @@ +Go-Sync +------ +[![Build Status](https://travis-ci.org/Redundancy/go-sync.svg?branch=master)](https://travis-ci.org/Redundancy/go-sync) +[![GoDoc](https://godoc.org/github.com/Redundancy/go-sync?status.svg)](https://godoc.org/github.com/Redundancy/go-sync) + +gosync is a library inspired by zsync and rsync. +Here are the goals: + +### Fast +Using the concurrency and performance features of Golang, Go-sync is designed to take advantage of multiple processors and multiple HTTP connections to make the most of modern hardware and minimize the impact of the bandwidth latency product. + +### Cross Platform +Works on Windows and Linux, without cygwin or fuss. + +### Easy + +A new high-level interface designed to reduce the work of implementing block transfer in your application: +```golang +fs := &BasicSummary{...} + +rsync, err := MakeRSync( + localFilename, + referencePath, + outFilename, + fs, +) + +if err != nil { + return err +} + +err = rsync.Patch() + +if err != nil { + return err +} + +return rsync.Close() +``` + +### Extensible +All functionality is based on interfaces, allowing customization of behavior: + +```golang +// Here, the input version is a local string +inputFile := bytes.NewReader(localVersionAsBytes) + +// And the output is a buffer +patchedFile := bytes.NewBuffer(nil) + +// This information is meta-data on the file that should be loaded / provided +// You can also provide your own implementation of the FileSummary interface +summary := &BasicSummary{ + ChecksumIndex: referenceFileIndex, + // Disable verification of hashes for downloaded data (not really a good idea!) + ChecksumLookup: nil, + BlockCount: uint(blockCount), + BlockSize: blockSize, + FileSize: int64(len(referenceAsBytes)), +} + +rsync := &RSync{ + Input: inputFile, + Output: patchedFile, + // An in-memory block source + Source: blocksources.NewReadSeekerBlockSource( + bytes.NewReader(referenceAsBytes), + blocksources.MakeNullFixedSizeResolver(uint64(blockSize)), + ), + Index: summary, + Summary: summary, + OnClose: nil, +} +``` + +Reuse low level objects to build a new high level library, or implement a new lower-level object to add a new transfer protocol (for example). + +### Tested +GoSync has been built from the ground up with unit tests. +The GoSync command-line tool has acceptance tests, although not everything is covered. + +## Current State +Go-Sync is still probably not ready for production. + +The most obvious areas that still need improvement are the acceptance tests, the error messages, +compression on the blocks that are retrieved from the source and handling of file flags. + +### TODO +- [ ] gzip source blocks (this involves writing out a version of the file that's compressed in block-increments) +- [ ] Clean up naming consistency and clarity: Block / Chunk etc +- [ ] Flesh out full directory build / sync +- [ ] Implement 'patch' payloads from a known start point to a desired end state +- [ ] Validate full file checksum after patching +- [ ] Provide bandwidth limiting / monitoring as part of http blocksource +- [ ] Think about turning the filechecksum into an interface +- [ ] Avoid marshalling / un-marshalling blocks during checksum generation +- [ ] Sequential patcher to resume after error? + +### Testing + +All tests are run by Travis-CI + +#### Unit tests + + go test github.com/Redundancy/go-sync/... + +#### Acceptance Tests +See the "acceptancetests" folder. This is currently difficult to run locally and relies on several linux utilities. + +#### Commandline & files + + go build github.com/Redundancy/go-sync/gosync + gosync build filenameToPatchTo + gosync patch filenameToPatchFrom filenameToPatchTo.gosync filenameToPatchTo + +Note that normally, patching would rely on a remote http/https file source. + +#### Command line tool reference + gosync --help diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh new file mode 100644 index 0000000..8d8b5a7 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh @@ -0,0 +1 @@ +echo "ok!" \ No newline at end of file diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh new file mode 100644 index 0000000..f0e7ca3 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh @@ -0,0 +1,21 @@ +counter=1 +for f in $TRAVIS_BUILD_DIR/acceptancetests/*test.sh +do + [ ! -f "$f" ] && continue + + echo "Running acceptance test $f" + echo 'travis_fold:start:test_output$counter' + sh $f + rc=$? + echo 'travis_fold:end:test_output$counter' + + if [ $rc != 0 ]; then + echo "Test Failed" + exit $rc + fi + + echo "Test Passed" + counter=`expr $counter + 1` +done + + diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh new file mode 100644 index 0000000..9f1ce7a --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh @@ -0,0 +1,14 @@ +echo "Testing filling in a truncated file" +wget -q https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -O test.exe +cp test.exe compare.exe +gosync b test.exe +truncate -s 5000000 test.exe +gosync p test.exe test.gosync https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe +diff -q test.exe compare.exe +rc=$? +if [ $rc != 0 ]; then + gosync -version + ls -l compare.exe + ls -l test.exe + exit $rc +fi diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh new file mode 100644 index 0000000..f1903cb --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh @@ -0,0 +1,14 @@ +echo "Testing truncating a file too long" +wget -q https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -O test.exe +cp test.exe compare.exe +gosync b test.exe +truncate -s 10000000 test.exe +gosync p test.exe test.gosync https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe +diff -q test.exe compare.exe +rc=$? +if [ $rc != 0 ]; then + gosync -version + ls -l compare.exe + ls -l test.exe + exit $rc +fi \ No newline at end of file diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go b/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go new file mode 100644 index 0000000..95e236a --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go @@ -0,0 +1,271 @@ +package blocksources + +import ( + "errors" + "fmt" + "sort" + + "github.com/Redundancy/go-sync/patcher" +) + +// BlockSourceRequester does synchronous requests on a remote source of blocks +// concurrency is handled by the BlockSourceBase. This provides a simple way +// of implementing a particular +type BlockSourceRequester interface { + // This method is called on multiple goroutines, and must + // support simultaneous requests + DoRequest(startOffset int64, endOffset int64) (data []byte, err error) + + // If an error raised by DoRequest should cause BlockSourceBase + // to give up, return true + IsFatal(err error) bool +} + +// A BlockSourceOffsetResolver resolves a blockID to a start offset and and end offset in a file +// it also handles splitting up ranges of blocks into multiple requests, allowing requests to be split down to the +// block size, and handling of compressed blocks (given a resolver that can work out the correct range to query for, +// and a BlockSourceRequester that will decompress the result into a full sized block) +type BlockSourceOffsetResolver interface { + GetBlockStartOffset(blockID uint) int64 + GetBlockEndOffset(blockID uint) int64 + SplitBlockRangeToDesiredSize(startBlockID, endBlockID uint) []QueuedRequest +} + +// Checks blocks against their expected checksum +type BlockVerifier interface { + VerifyBlockRange(startBlockID uint, data []byte) bool +} + +func NewBlockSourceBase( + requester BlockSourceRequester, + resolver BlockSourceOffsetResolver, + verifier BlockVerifier, + concurrentRequestCount int, + concurrentBytes int64, +) *BlockSourceBase { + + b := &BlockSourceBase{ + Requester: requester, + BlockSourceResolver: resolver, + Verifier: verifier, + ConcurrentRequests: concurrentRequestCount, + ConcurrentBytes: concurrentBytes, + exitChannel: make(chan bool), + errorChannel: make(chan error), + responseChannel: make(chan patcher.BlockReponse), + requestChannel: make(chan patcher.MissingBlockSpan), + } + + go b.loop() + + return b +} + +// BlockSourceBase provides an implementation of blocksource +// that takes care of everything except for the actual asyncronous request +// this makes blocksources easier and faster to build reliably +// BlockSourceBase implements patcher.BlockSource, and if it's good enough, +// perhaps nobody else ever will have to. +type BlockSourceBase struct { + Requester BlockSourceRequester + BlockSourceResolver BlockSourceOffsetResolver + Verifier BlockVerifier + + // The number of requests that BlockSourceBase may service at once + ConcurrentRequests int + + // The number of bytes that BlockSourceBase may have in-flight + // (requested + pending delivery) + ConcurrentBytes int64 + + hasQuit bool + exitChannel chan bool + errorChannel chan error + responseChannel chan patcher.BlockReponse + requestChannel chan patcher.MissingBlockSpan + + bytesRequested int64 +} + +const ( + STATE_RUNNING = iota + STATE_EXITING +) + +func (s *BlockSourceBase) ReadBytes() int64 { + return s.bytesRequested +} + +func (s *BlockSourceBase) RequestBlocks(block patcher.MissingBlockSpan) error { + s.requestChannel <- block + return nil +} + +func (s *BlockSourceBase) GetResultChannel() <-chan patcher.BlockReponse { + return s.responseChannel +} + +// If the block source encounters an unsurmountable problem +func (s *BlockSourceBase) EncounteredError() <-chan error { + return s.errorChannel +} + +var BlockSourceAlreadyClosedError = errors.New("Block source was already closed") + +func (s *BlockSourceBase) Close() (err error) { + // if it has already been closed, just recover + // however, let the caller know + defer func() { + if recover() != nil { + err = BlockSourceAlreadyClosedError + } + }() + + if !s.hasQuit { + s.exitChannel <- true + } + + return +} + +func (s *BlockSourceBase) loop() { + defer func() { + s.hasQuit = true + close(s.exitChannel) + close(s.errorChannel) + close(s.requestChannel) + close(s.responseChannel) + }() + + state := STATE_RUNNING + inflightRequests := 0 + //inflightBytes := int64(0) + pendingErrors := &errorWatcher{errorChannel: s.errorChannel} + pendingResponse := &pendingResponseHelper{responseChannel: s.responseChannel} + resultChan := make(chan asyncResult) + defer close(resultChan) + + requestQueue := make(QueuedRequestList, 0, s.ConcurrentRequests*2) + + // enable us to order responses for the active requests, lowest to highest + requestOrdering := make(UintSlice, 0, s.ConcurrentRequests) + responseOrdering := make(PendingResponses, 0, s.ConcurrentRequests) + + for state == STATE_RUNNING || inflightRequests > 0 || pendingErrors.Err() != nil { + + // Start any pending work that we can + for inflightRequests < s.ConcurrentRequests && len(requestQueue) > 0 { + inflightRequests += 1 + + nextRequest := requestQueue[len(requestQueue)-1] + + requestOrdering = append(requestOrdering, nextRequest.StartBlockID) + sort.Sort(sort.Reverse(requestOrdering)) + go func() { + resolver := s.BlockSourceResolver + + startOffset := resolver.GetBlockStartOffset( + nextRequest.StartBlockID, + ) + + endOffset := resolver.GetBlockEndOffset( + nextRequest.EndBlockID, + ) + + result, err := s.Requester.DoRequest( + startOffset, + endOffset, + ) + + resultChan <- asyncResult{ + startBlockID: nextRequest.StartBlockID, + endBlockID: nextRequest.EndBlockID, + data: result, + err: err, + } + }() + + // remove dispatched request + requestQueue = requestQueue[:len(requestQueue)-1] + } + + select { + case newRequest := <-s.requestChannel: + requestQueue = append( + requestQueue, + s.BlockSourceResolver.SplitBlockRangeToDesiredSize( + newRequest.StartBlock, + newRequest.EndBlock, + )..., + ) + + sort.Sort(sort.Reverse(requestQueue)) + + case result := <-resultChan: + inflightRequests -= 1 + + if result.err != nil { + pendingErrors.setError(result.err) + pendingResponse.clear() + state = STATE_EXITING + break + } + + s.bytesRequested += int64(len(result.data)) + + if s.Verifier != nil && !s.Verifier.VerifyBlockRange(result.startBlockID, result.data) { + pendingErrors.setError( + fmt.Errorf( + "The returned block range (%v-%v) did not match the expected checksum for the blocks", + result.startBlockID, result.endBlockID, + ), + ) + pendingResponse.clear() + state = STATE_EXITING + break + } + + responseOrdering = append(responseOrdering, + patcher.BlockReponse{ + StartBlock: result.startBlockID, + Data: result.data, + }, + ) + + // sort high to low + sort.Sort(sort.Reverse(responseOrdering)) + + // if we just got the lowest requested block, we can set + // the response. Otherwise, wait. + lowestRequest := requestOrdering[len(requestOrdering)-1] + + if lowestRequest == result.startBlockID { + lowestResponse := responseOrdering[len(responseOrdering)-1] + pendingResponse.clear() + pendingResponse.setResponse(&lowestResponse) + } + + case pendingResponse.sendIfPending() <- pendingResponse.Response(): + pendingResponse.clear() + responseOrdering = responseOrdering[:len(responseOrdering)-1] + requestOrdering = requestOrdering[:len(requestOrdering)-1] + + // check if there's another response to enqueue + if len(responseOrdering) > 0 { + lowestResponse := responseOrdering[len(responseOrdering)-1] + lowestRequest := requestOrdering[len(requestOrdering)-1] + + if lowestRequest == lowestResponse.StartBlock { + pendingResponse.setResponse(&lowestResponse) + + } + } + + case pendingErrors.sendIfSet() <- pendingErrors.Err(): + pendingErrors.clear() + + case <-s.exitChannel: + state = STATE_EXITING + } + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go new file mode 100644 index 0000000..1aa91b9 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go @@ -0,0 +1,303 @@ +package blocksources + +import ( + "bytes" + "github.com/Redundancy/go-sync/patcher" + + //"runtime" + "testing" + "time" +) + +//----------------------------------------------------------------------------- +type erroringRequester struct{} +type testError struct{} + +func (e *testError) Error() string { + return "test" +} + +func (e *erroringRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { + return nil, &testError{} +} + +func (e *erroringRequester) IsFatal(err error) bool { + return true +} + +//----------------------------------------------------------------------------- +type FunctionRequester func(a, b int64) ([]byte, error) + +func (f FunctionRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { + return f(startOffset, endOffset) +} + +func (f FunctionRequester) IsFatal(err error) bool { + return true +} + +//----------------------------------------------------------------------------- + +func init() { + //if runtime.GOMAXPROCS(0) == 1 { + //runtime.GOMAXPROCS(4) + //} +} + +func TestRangeSlice(t *testing.T) { + a := []int{0, 1, 2, 3, 4} + b := a[:len(a)-1] + + if len(b) != len(a)-1 { + t.Errorf("b is wrong length, only supposed to remove one item: %v %v", a, b) + } +} + +func TestCreateAndCloseBlockSourceBase(t *testing.T) { + b := NewBlockSourceBase(nil, nil, nil, 1, 1024) + b.Close() + + // TODO: Race condition here. Can Close() block? + if !b.hasQuit { + t.Fatal("Block source base did not exit") + } +} + +func TestErrorWatcher(t *testing.T) { + e := errorWatcher{errorChannel: make(chan error)} + + if e.sendIfSet() != nil { + t.Errorf("Channel should be nil when created") + } + + e.setError(&testError{}) + + if e.sendIfSet() == nil { + t.Errorf("Channel should be non-nil when error is set") + } + if e.Err() == nil { + t.Errorf("Error should not be nil when set") + } +} + +func TestBlockSourceBaseError(t *testing.T) { + b := NewBlockSourceBase( + &erroringRequester{}, + MakeNullFixedSizeResolver(4), + nil, + 1, + 1024, + ) + defer b.Close() + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 1, + EndBlock: 1, + }) + + select { + case <-time.After(time.Second): + t.Fatal("Timed out waiting for error") + case <-b.EncounteredError(): + } + +} + +func TestBlockSourceRequest(t *testing.T) { + expected := []byte("test") + + b := NewBlockSourceBase( + FunctionRequester(func(start, end int64) (data []byte, err error) { + return expected, nil + }), + MakeNullFixedSizeResolver(4), + nil, + 1, + 1024, + ) + defer b.Close() + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 1, + EndBlock: 1, + }) + + result := <-b.GetResultChannel() + + if result.StartBlock != 1 { + t.Errorf("Unexpected start block in result: %v", result.StartBlock) + } + if bytes.Compare(result.Data, expected) != 0 { + t.Errorf("Unexpected data in result: %v", result.Data) + } +} + +func TestConcurrentBlockRequests(t *testing.T) { + content := []byte("test") + + b := NewBlockSourceBase( + FunctionRequester(func(start, end int64) (data []byte, err error) { + return content[start:end], nil + }), + MakeNullFixedSizeResolver(2), + nil, + 2, + 1024, + ) + defer b.Close() + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 2, + StartBlock: 0, + EndBlock: 0, + }) + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 2, + StartBlock: 1, + EndBlock: 1, + }) + + for i := uint(0); i < 2; i++ { + select { + case r := <-b.GetResultChannel(): + if r.StartBlock != i { + t.Errorf("Wrong start block: %v", r.StartBlock) + } + if bytes.Compare(r.Data, content[i*2:(i+1)*2]) != 0 { + t.Errorf("Unexpected result content for result %v: %v", i+1, string(r.Data)) + } + case <-time.After(time.Second): + t.Fatal("Timed out on request", i+1) + } + } +} + +func TestOutOfOrderRequestCompletion(t *testing.T) { + content := []byte("test") + + channeler := []chan bool{ + make(chan bool), + make(chan bool), + } + + b := NewBlockSourceBase( + FunctionRequester(func(start, end int64) (data []byte, err error) { + // read from the channel based on the start + <-(channeler[start]) + return content[start:end], nil + }), + MakeNullFixedSizeResolver(1), + nil, + 2, + 1024, + ) + defer b.Close() + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 1, + StartBlock: 0, + EndBlock: 0, + }) + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 1, + StartBlock: 1, + EndBlock: 1, + }) + + // finish the second request + channeler[1] <- true + + select { + case <-b.GetResultChannel(): + t.Error("Should not deliver any blocks yet") + case <-time.After(time.Second): + } + + // once the first block completes, we're ready to send both + channeler[0] <- true + + for i := uint(0); i < 2; i++ { + select { + case r := <-b.GetResultChannel(): + if r.StartBlock != i { + t.Errorf( + "Wrong start block: %v on result %v", + r.StartBlock, + i+1, + ) + } + case <-time.After(time.Second): + t.Fatal("Timed out on request", i+1) + } + } +} + +func TestRequestCountLimiting(t *testing.T) { + counter := make(chan int) + waiter := make(chan bool) + const ( + MAX_CONCURRENCY = 2 + REQUESTS = 4 + ) + call_counter := 0 + + b := NewBlockSourceBase( + FunctionRequester(func(start, end int64) (data []byte, err error) { + counter <- 1 + call_counter += 1 + <-waiter + counter <- -1 + return []byte{0, 0}, nil + }), + MakeNullFixedSizeResolver(1), + nil, + MAX_CONCURRENCY, + 1024, + ) + defer b.Close() + + count := 0 + max := 0 + + go func() { + for { + change, ok := <-counter + + if !ok { + break + } + + count += change + + if count > max { + max = count + } + } + }() + + for i := 0; i < REQUESTS; i++ { + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 1, + StartBlock: uint(i), + EndBlock: uint(i), + }) + } + + for i := 0; i < REQUESTS; i++ { + waiter <- true + } + + close(counter) + close(waiter) + + if max > MAX_CONCURRENCY { + t.Errorf("Maximum requests in flight was greater than the requested concurrency: %v", max) + } + if call_counter != REQUESTS { + t.Errorf("Total number of requests is not expected: %v", call_counter) + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go b/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go new file mode 100644 index 0000000..d81573e --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go @@ -0,0 +1,73 @@ +package blocksources + +type FixedSizeBlockResolver struct { + BlockSize uint64 + FileSize int64 + MaxDesiredRequestSize uint64 +} + +func (r *FixedSizeBlockResolver) GetBlockStartOffset(blockID uint) int64 { + if off := int64(uint64(blockID) * r.BlockSize); r.FileSize != 0 && off > r.FileSize { + return r.FileSize + } else { + return off + } +} + +func (r *FixedSizeBlockResolver) GetBlockEndOffset(blockID uint) int64 { + if off := int64(uint64(blockID+1) * r.BlockSize); r.FileSize != 0 && off > r.FileSize { + return r.FileSize + } else { + return off + } +} + +// Split blocks into chunks of the desired size, or less. This implementation assumes a fixed block size at the source. +func (r *FixedSizeBlockResolver) SplitBlockRangeToDesiredSize(startBlockID, endBlockID uint) []QueuedRequest { + + if r.MaxDesiredRequestSize == 0 { + return []QueuedRequest{ + QueuedRequest{ + StartBlockID: startBlockID, + EndBlockID: endBlockID, + }, + } + } + + maxSize := r.MaxDesiredRequestSize + if r.MaxDesiredRequestSize < r.BlockSize { + maxSize = r.BlockSize + } + + // how many blocks is the desired size? + blockCountPerRequest := uint(maxSize / r.BlockSize) + + requests := make([]QueuedRequest, 0, (endBlockID-startBlockID)/blockCountPerRequest+1) + currentBlockID := startBlockID + + for { + maxEndBlock := currentBlockID + blockCountPerRequest + + if maxEndBlock > endBlockID { + requests = append( + requests, + QueuedRequest{ + StartBlockID: currentBlockID, + EndBlockID: endBlockID, + }, + ) + + return requests + } else { + requests = append( + requests, + QueuedRequest{ + StartBlockID: currentBlockID, + EndBlockID: maxEndBlock - 1, + }, + ) + + currentBlockID = maxEndBlock + } + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go new file mode 100644 index 0000000..f484bc5 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go @@ -0,0 +1,127 @@ +package blocksources + +import ( + "testing" +) + +func TestNullResolverGivesBackTheSameBlocks(t *testing.T) { + n := MakeNullFixedSizeResolver(5) + result := n.SplitBlockRangeToDesiredSize(0, 10000) + + if len(result) != 1 { + t.Fatalf("Unexpected result length (expected 1): %v", result) + } + + r := result[0] + + if r.StartBlockID != 0 { + t.Errorf("Unexpected start block ID: %v", r) + } + + if r.EndBlockID != 10000 { + t.Errorf("Unexpected end block ID: %v", r) + } +} + +func TestFixedSizeResolverSplitsBlocksOfDesiredSize(t *testing.T) { + res := &FixedSizeBlockResolver{ + BlockSize: 5, + MaxDesiredRequestSize: 5, + FileSize: 20000, + } + + // Should split two blocks, each of the desired request size + // into two requests + result := res.SplitBlockRangeToDesiredSize(0, 1) + + if len(result) != 2 { + t.Fatalf("Unexpected result length (expected 2): %v", result) + } + + if result[0].StartBlockID != 0 { + t.Errorf("Unexpected start blockID: %v", result[0]) + } + if result[0].EndBlockID != 0 { + t.Errorf("Unexpected end blockID: %v", result[0]) + } + + if result[1].StartBlockID != 1 { + t.Errorf("Unexpected start blockID: %v", result[1]) + } + if result[1].EndBlockID != 1 { + t.Errorf("Unexpected end blockID: %v", result[1]) + } +} + +func TestThatMultipleBlocksAreSplitByRoundingDown(t *testing.T) { + res := &FixedSizeBlockResolver{ + BlockSize: 5, + MaxDesiredRequestSize: 12, + FileSize: 20000, + } + + // 0,1 (10) - 2-3 (10) + result := res.SplitBlockRangeToDesiredSize(0, 3) + + if len(result) != 2 { + t.Fatalf("Unexpected result length (expected 2): %v", result) + } + + if result[0].StartBlockID != 0 { + t.Errorf("Unexpected start blockID: %v", result[0]) + } + if result[0].EndBlockID != 1 { + t.Errorf("Unexpected end blockID: %v", result[0]) + } + + if result[1].StartBlockID != 2 { + t.Errorf("Unexpected start blockID: %v", result[1]) + } + if result[1].EndBlockID != 3 { + t.Errorf("Unexpected end blockID: %v", result[1]) + } +} + +func TestThatADesiredSizeSmallerThanABlockResultsInSingleBlocks(t *testing.T) { + res := &FixedSizeBlockResolver{ + BlockSize: 5, + MaxDesiredRequestSize: 4, + FileSize: 20000, + } + + // Should split two blocks + result := res.SplitBlockRangeToDesiredSize(0, 1) + + if len(result) != 2 { + t.Fatalf("Unexpected result length (expected 2): %v", result) + } + + if result[0].StartBlockID != 0 { + t.Errorf("Unexpected start blockID: %v", result[0]) + } + if result[0].EndBlockID != 0 { + t.Errorf("Unexpected end blockID: %v", result[0]) + } + + if result[1].StartBlockID != 1 { + t.Errorf("Unexpected start blockID: %v", result[1]) + } + if result[1].EndBlockID != 1 { + t.Errorf("Unexpected end blockID: %v", result[1]) + } +} + +func TestThatFileSizeTruncatesBlockEnds(t *testing.T) { + res := &FixedSizeBlockResolver{ + BlockSize: 5, + MaxDesiredRequestSize: 100, + FileSize: 13, + } + + // Should split two blocks + result := res.GetBlockEndOffset(3) + + if result != 13 { + t.Errorf("Unexpected BlockEnd Offset:", result) + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/helpers.go b/vendor/github.com/Redundancy/go-sync/blocksources/helpers.go new file mode 100644 index 0000000..204c484 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/helpers.go @@ -0,0 +1,124 @@ +package blocksources + +import ( + "fmt" + "github.com/Redundancy/go-sync/patcher" +) + +// errorWatcher is a small helper object +// sendIfSet will only return a channel if there is an error set +// so w.sendIfSet() <- w.Err() is always safe in a select statement +// even if there is no error set +type errorWatcher struct { + errorChannel chan error + lastError error +} + +func (w *errorWatcher) setError(e error) { + if w.lastError != nil { + panic("cannot set a new error when one is already set!") + } + w.lastError = e +} + +func (w *errorWatcher) clear() { + w.lastError = nil +} + +func (w *errorWatcher) Err() error { + return w.lastError +} + +func (w *errorWatcher) sendIfSet() chan<- error { + if w.lastError != nil { + return w.errorChannel + } else { + return nil + } +} + +type pendingResponseHelper struct { + responseChannel chan patcher.BlockReponse + pendingResponse *patcher.BlockReponse +} + +func (w *pendingResponseHelper) setResponse(r *patcher.BlockReponse) { + if w.pendingResponse != nil { + p := fmt.Sprintf("Setting a response when one is already set! Had startblock %v, got %v", r.StartBlock, w.pendingResponse.StartBlock) + panic(p) + } + w.pendingResponse = r +} + +func (w *pendingResponseHelper) clear() { + w.pendingResponse = nil +} + +func (w *pendingResponseHelper) Response() patcher.BlockReponse { + if w.pendingResponse == nil { + return patcher.BlockReponse{} + } + return *w.pendingResponse +} + +func (w *pendingResponseHelper) sendIfPending() chan<- patcher.BlockReponse { + if w.pendingResponse != nil { + return w.responseChannel + } else { + return nil + } + +} + +type UintSlice []uint + +func (r UintSlice) Len() int { + return len(r) +} + +func (r UintSlice) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} + +func (r UintSlice) Less(i, j int) bool { + return r[i] < r[j] +} + +type asyncResult struct { + startBlockID uint + endBlockID uint + data []byte + err error +} + +type QueuedRequest struct { + StartBlockID uint + EndBlockID uint +} + +type QueuedRequestList []QueuedRequest + +func (r QueuedRequestList) Len() int { + return len(r) +} + +func (r QueuedRequestList) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} + +func (r QueuedRequestList) Less(i, j int) bool { + return r[i].StartBlockID < r[j].StartBlockID +} + +func MakeNullFixedSizeResolver(blockSize uint64) BlockSourceOffsetResolver { + return &FixedSizeBlockResolver{ + BlockSize: blockSize, + } +} + +func MakeFileSizedBlockResolver(blockSize uint64, filesize int64) BlockSourceOffsetResolver { + return &FixedSizeBlockResolver{ + BlockSize: blockSize, + FileSize: filesize, + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go b/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go new file mode 100644 index 0000000..0e93278 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go @@ -0,0 +1,110 @@ +package blocksources + +import ( + "bytes" + "errors" + "fmt" + "net/http" + "strings" +) + +const MB = 1024 * 1024 + +var RangedRequestNotSupportedError = errors.New("Ranged request not supported (Server did not respond with 206 Status)") +var ResponseFromServerWasGZiped = errors.New("HTTP response was gzip encoded. Ranges may not match those requested.") + +var ClientNoCompression = &http.Client{ + Transport: &http.Transport{}, +} + +func NewHttpBlockSource( + url string, + concurrentRequests int, + resolver BlockSourceOffsetResolver, + verifier BlockVerifier, +) *BlockSourceBase { + return NewBlockSourceBase( + &HttpRequester{ + url: url, + client: http.DefaultClient, + }, + resolver, + verifier, + concurrentRequests, + 4*MB, + ) +} + +type URLNotFoundError string + +func (url URLNotFoundError) Error() string { + return "404 Error on URL: " + string(url) +} + +// This class provides the implementation of BlockSourceRequester for BlockSourceBase +// this simplifies creating new BlockSources that satisfy the requirements down to +// writing a request function +type HttpRequester struct { + client *http.Client + url string +} + +func (r *HttpRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { + rangedRequest, err := http.NewRequest("GET", r.url, nil) + + if err != nil { + return nil, fmt.Errorf("Error creating request for \"%v\": %v", r.url, err) + } + + rangeSpecifier := fmt.Sprintf("bytes=%v-%v", startOffset, endOffset-1) + rangedRequest.ProtoAtLeast(1, 1) + rangedRequest.Header.Add("Range", rangeSpecifier) + rangedRequest.Header.Add("Accept-Encoding", "identity") + rangedResponse, err := r.client.Do(rangedRequest) + + if err != nil { + return nil, fmt.Errorf("Error executing request for \"%v\": %v", r.url, err) + } + + defer rangedResponse.Body.Close() + + if rangedResponse.StatusCode == 404 { + return nil, URLNotFoundError(r.url) + } else if rangedResponse.StatusCode != 206 { + return nil, RangedRequestNotSupportedError + } else if strings.Contains( + rangedResponse.Header.Get("Content-Encoding"), + "gzip", + ) { + return nil, ResponseFromServerWasGZiped + } else { + buf := bytes.NewBuffer(make([]byte, 0, endOffset-startOffset)) + _, err = buf.ReadFrom(rangedResponse.Body) + + if err != nil { + err = fmt.Errorf( + "Failed to read response body for %v (%v-%v): %v", + r.url, + startOffset, endOffset-1, + err, + ) + } + + data = buf.Bytes() + + if int64(len(data)) != endOffset-startOffset { + err = fmt.Errorf( + "Unexpected response length %v (%v): %v", + r.url, + endOffset-startOffset+1, + len(data), + ) + } + + return + } +} + +func (r *HttpRequester) IsFatal(err error) bool { + return true +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go new file mode 100644 index 0000000..d7a411c --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go @@ -0,0 +1,251 @@ +package blocksources + +import ( + "bytes" + "crypto/md5" + "fmt" + "net" + "net/http" + "testing" + "time" + + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/patcher" +) + +var PORT = 8000 + +var TEST_CONTENT = []byte("This is test content used for evaluation of the unit tests") +var content = bytes.NewReader(TEST_CONTENT) +var LOCAL_URL = "" + +func handler(w http.ResponseWriter, req *http.Request) { + http.ServeContent(w, req, "", time.Now(), content) +} + +var PARTIAL_CONTENT = []byte("abcdef") +var partialContent = bytes.NewReader(PARTIAL_CONTENT) + +func partialContentHandler(w http.ResponseWriter, req *http.Request) { + http.ServeContent(w, req, "", time.Now(), partialContent) +} + +var CORRUPT_CONTENT = []byte("sfdfsfhhrtertert sffsfsdfsdfsdf") +var corruptContent = bytes.NewReader(CORRUPT_CONTENT) + +func corruptContentHandler(w http.ResponseWriter, req *http.Request) { + http.ServeContent(w, req, "", time.Now(), corruptContent) +} + +// set up a http server locally that will respond predictably to ranged requests +// NB: Doing this will prevent deadlocks from being caught! +func init() { + s := http.NewServeMux() + s.HandleFunc("/", handler) + s.HandleFunc("/partial", partialContentHandler) + s.HandleFunc("/corrupt", corruptContentHandler) + s.Handle("/404", http.NotFoundHandler()) + + portChan := make(chan int) + + go func() { + var listener net.Listener + var err error + + for { + PORT++ + p := fmt.Sprintf(":%v", PORT) + listener, err = net.Listen("tcp", p) + + if err == nil { + break + } + } + portChan <- PORT + http.Serve(listener, s) + }() + + p := fmt.Sprintf(":%v", <-portChan) + LOCAL_URL = "http://localhost" + p + +} + +func TestHandler(t *testing.T) { + resp, err := http.Get(LOCAL_URL) + + if err != nil { + t.Fatal(err) + } + + if resp.StatusCode != 200 { + t.Fatal(resp.Status) + } +} + +func TestHttpBlockSource(t *testing.T) { + b := NewHttpBlockSource( + LOCAL_URL+"/", + 2, + MakeNullFixedSizeResolver(4), + nil, + ) + + err := b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 0, + EndBlock: 0, + }) + + if err != nil { + t.Fatal(err) + } + + results := b.GetResultChannel() + + select { + case r := <-results: + if bytes.Compare(r.Data, TEST_CONTENT[:4]) != 0 { + t.Errorf("Data differed from expected content: \"%v\"", string(r.Data)) + } + case e := <-b.EncounteredError(): + t.Fatal(e) + case <-time.After(time.Second): + t.Fatal("Waited a second for the response, timeout.") + } +} + +func TestHttpBlockSource404(t *testing.T) { + b := NewHttpBlockSource( + LOCAL_URL+"/404", + 2, + MakeNullFixedSizeResolver(4), + nil, + ) + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 0, + EndBlock: 0, + }) + + results := b.GetResultChannel() + + select { + case <-results: + t.Fatal("Should not have gotten a result") + case e := <-b.EncounteredError(): + if e == nil { + t.Fatal("Error was nil!") + } else if _, ok := e.(URLNotFoundError); !ok { + t.Errorf("Unexpected error type: %v", e) + } + case <-time.After(time.Second): + t.Fatal("Waited a second for the response, timeout.") + } +} + +func TestHttpBlockSourceOffsetBlockRequest(t *testing.T) { + b := NewHttpBlockSource( + LOCAL_URL+"/", + 2, + MakeNullFixedSizeResolver(4), + nil, + ) + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 1, + EndBlock: 3, + }) + + select { + case result := <-b.GetResultChannel(): + if result.StartBlock != 1 { + t.Errorf( + "Unexpected result start block: %v", + result.StartBlock, + ) + } + case <-time.After(time.Second): + t.Fatalf("Timeout waiting for result") + } +} + +func TestHttpBlockSourcePartialContentRequest(t *testing.T) { + b := NewHttpBlockSource( + LOCAL_URL+"/partial", + 2, + MakeFileSizedBlockResolver(4, int64(len(PARTIAL_CONTENT))), + nil, + ) + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 1, + EndBlock: 1, + }) + + select { + case result := <-b.GetResultChannel(): + if result.StartBlock != 1 { + t.Errorf( + "Unexpected result start block: %v", + result.StartBlock, + ) + } + if len(result.Data) != 2 { + t.Errorf( + "Unexpected data length: \"%v\"", + string(result.Data), + ) + } + if string(result.Data) != "ef" { + t.Errorf( + "Unexpected result \"%v\"", + string(result.Data), + ) + } + case err := <-b.EncounteredError(): + t.Fatal(err) + case <-time.After(time.Second): + t.Fatalf("Timeout waiting for result") + } +} + +type SingleBlockSource []byte + +func (d SingleBlockSource) GetStrongChecksumForBlock(blockID int) []byte { + m := md5.New() + return m.Sum(d) +} + +func TestHttpBlockSourceVerification(t *testing.T) { + const BLOCK_SIZE = 4 + + b := NewHttpBlockSource( + LOCAL_URL+"/corrupt", + 2, + MakeNullFixedSizeResolver(BLOCK_SIZE), + &filechecksum.HashVerifier{ + Hash: md5.New(), + BlockSize: BLOCK_SIZE, + BlockChecksumGetter: SingleBlockSource(TEST_CONTENT[0:BLOCK_SIZE]), + }, + ) + + b.RequestBlocks(patcher.MissingBlockSpan{ + BlockSize: BLOCK_SIZE, + StartBlock: 0, + EndBlock: 0, + }) + + select { + case result := <-b.GetResultChannel(): + t.Fatalf("Should have thrown an error, got %v", result) + case e := <-b.EncounteredError(): + t.Logf("Encountered expected error: %v", e) + return + case <-time.After(time.Second): + t.Fatalf("Timeout waiting for result") + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go b/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go new file mode 100644 index 0000000..67958f6 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go @@ -0,0 +1,54 @@ +package blocksources + +import ( + "io" +) + +const ( + from_start = 0 +) + +type ReadSeeker interface { + Read(b []byte) (n int, err error) + Seek(offset int64, whence int) (int64, error) +} + +func NewReadSeekerBlockSource( + r ReadSeeker, + resolver BlockSourceOffsetResolver, +) *BlockSourceBase { + return NewBlockSourceBase( + &ReadSeekerRequester{ + rs: r, + }, + resolver, + nil, // TODO: No verifier! + 1, + 8*MB, + ) +} + +type ReadSeekerRequester struct { + rs ReadSeeker +} + +func (r *ReadSeekerRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { + read_length := endOffset - startOffset + buffer := make([]byte, read_length) + + if _, err = r.rs.Seek(startOffset, from_start); err != nil { + return + } + + n, err := io.ReadFull(r.rs, buffer) + + if err != nil && err != io.ErrUnexpectedEOF { + return + } + + return buffer[:n], nil +} + +func (r *ReadSeekerRequester) IsFatal(err error) bool { + return true +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go new file mode 100644 index 0000000..37d1970 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go @@ -0,0 +1,42 @@ +package blocksources + +import ( + "bytes" + "github.com/Redundancy/go-sync/patcher" + "testing" +) + +const STRING_DATA = "abcdefghijklmnopqrst" + +var BYTE_BLOCK_SOURCE = NewReadSeekerBlockSource( + bytes.NewReader( + []byte(STRING_DATA), + ), + MakeNullFixedSizeResolver(4), +) + +func TestReadFirstBlock(t *testing.T) { + BYTE_BLOCK_SOURCE.RequestBlocks( + patcher.MissingBlockSpan{ + BlockSize: 4, + StartBlock: 0, + EndBlock: 0, + }, + ) + + result := <-BYTE_BLOCK_SOURCE.GetResultChannel() + + if result.StartBlock != 0 { + t.Errorf("Wrong start block: %v", result.StartBlock) + } + + EXPECTED := STRING_DATA[:4] + + if bytes.Compare(result.Data, []byte(EXPECTED)) != 0 { + t.Errorf( + "Unexpected result data: \"%v\" expected: \"%v\"", + string(result.Data), + EXPECTED, + ) + } +} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go b/vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go new file mode 100644 index 0000000..551d75f --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go @@ -0,0 +1,19 @@ +package blocksources + +import ( + "github.com/Redundancy/go-sync/patcher" +) + +type PendingResponses []patcher.BlockReponse + +func (r PendingResponses) Len() int { + return len(r) +} + +func (r PendingResponses) Swap(i, j int) { + r[i], r[j] = r[j], r[i] +} + +func (r PendingResponses) Less(i, j int) bool { + return r[i].StartBlock < r[j].StartBlock +} diff --git a/vendor/github.com/Redundancy/go-sync/chunks/chunks.go b/vendor/github.com/Redundancy/go-sync/chunks/chunks.go new file mode 100644 index 0000000..aac4143 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/chunks/chunks.go @@ -0,0 +1,87 @@ +/* +Package chunks provides the basic structure for a pair of the weak and strong checksums. +Since this is fairly widely used, splitting this out breaks a number of possible circular dependencies +*/ +package chunks + +import ( + "bytes" + "errors" + "io" +) + +// For a given Block, the Weak & Strong hashes, and the offset +// This structure is only used to generate the index of reference files, since +// computing the strong checksum is not done when comparing unless the weak checksum matches +type ChunkChecksum struct { + // an offset in terms of chunk count + ChunkOffset uint + // the size of the block + Size int64 + WeakChecksum []byte + StrongChecksum []byte +} + +// compares a checksum to another based on the checksums, not the offset +func (chunk ChunkChecksum) Match(other ChunkChecksum) bool { + weakEqual := bytes.Compare(chunk.WeakChecksum, other.WeakChecksum) == 0 + strongEqual := false + if weakEqual { + strongEqual = bytes.Compare(chunk.StrongChecksum, other.StrongChecksum) == 0 + } + return weakEqual && strongEqual +} + +var ErrPartialChecksum = errors.New("Reader length was not a multiple of the checksums") + +// Loads chunks from a reader, assuming alternating weak then strong hashes +func LoadChecksumsFromReader( + r io.Reader, + weakHashSize int, + strongHashSize int, +) ([]ChunkChecksum, error) { + + result := make([]ChunkChecksum, 0, 20) + offset := uint(0) + + temp := ChunkChecksum{} + + for { + weakBuffer := make([]byte, weakHashSize) + n, err := io.ReadFull(r, weakBuffer) + + if n == weakHashSize { + temp.ChunkOffset = offset + temp.WeakChecksum = weakBuffer + } else if n == 0 && err == io.EOF { + break + } else { + return nil, ErrPartialChecksum + } + + strongBuffer := make([]byte, strongHashSize) + n, err = io.ReadFull(r, strongBuffer) + + if n == strongHashSize { + temp.StrongChecksum = strongBuffer + result = append(result, temp) + + if err == io.EOF { + break + } + } else { + return nil, ErrPartialChecksum + } + + offset += 1 + } + + return result, nil +} + +// satisfies filechecksum.ChecksumLookup +type StrongChecksumGetter []ChunkChecksum + +func (s StrongChecksumGetter) GetStrongChecksumForBlock(blockID int) []byte { + return s[blockID].StrongChecksum +} diff --git a/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go b/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go new file mode 100644 index 0000000..0449739 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go @@ -0,0 +1,170 @@ +package circularbuffer + +/* +C2 is a circular buffer intended to allow you to write a block of data of up to 'blocksize', and retrieve the +data evicted by that operation, without allocating any extra slice storage + +This requires that it keep at least blocksize*2 data around. In fact, it doubles that again in order to +guarantee that both of these bits of information can always be obtained in a single contiguous block of memory. + +Other than the cost of the extra memory (4xblocksize), this means that it requires 2 writes for every byte stored. +*/ +type C2 struct { + // used to know how much was evicted + lastWritten int + + // total number of written bytes + // used to track if the buffer has been filled, but goes above blocksize + totalWritten int + + // quick access to the circular buffer size + blocksize int + + // double internal buffer storage + a, b doubleSizeBuffer +} + +type doubleSizeBuffer struct { + // used to reset the head pointer + baseOffset int + + // index of the next byte to be written + head int + + // buffer + buffer []byte +} + +func MakeC2Buffer(blockSize int) *C2 { + return &C2{ + blocksize: blockSize, + a: doubleSizeBuffer{ + baseOffset: 0, + buffer: make([]byte, blockSize*2), + }, + b: doubleSizeBuffer{ + baseOffset: blockSize, + head: blockSize, + buffer: make([]byte, blockSize*2), + }, + } +} + +func (c *C2) Reset() { + c.a.Reset() + c.b.Reset() + c.lastWritten = 0 + c.totalWritten = 0 +} + +// Write new data +func (c *C2) Write(b []byte) { + c.a.Write(b) + c.b.Write(b) + c.lastWritten = len(b) + c.totalWritten += c.lastWritten +} + +func (c *C2) getBlockBuffer() *doubleSizeBuffer { + bufferToRead := &c.a + if c.b.head > c.a.head { + bufferToRead = &c.b + } + + return bufferToRead +} + +// the total written, up to the blocksize +func (c *C2) maxWritten() int { + if c.totalWritten < c.blocksize { + return c.totalWritten + } + + return c.blocksize +} + +func (c *C2) Len() int { + return c.maxWritten() +} + +func (c *C2) Empty() bool { + return c.totalWritten == 0 +} + +// Shortens the content of the circular buffer +// and returns the content removed +func (c *C2) Truncate(byteCount int) (evicted []byte) { + max := c.maxWritten() + + if byteCount > max { + byteCount = max + } + + bufferToRead := c.getBlockBuffer() + start := bufferToRead.head - max + + c.totalWritten = c.maxWritten() - byteCount + return bufferToRead.buffer[start : start+byteCount] +} + +// get the current buffer contents of block +func (c *C2) GetBlock() []byte { + // figure out which buffer has it stored contiguously + bufferToRead := c.getBlockBuffer() + start := bufferToRead.head - c.maxWritten() + + return bufferToRead.buffer[start:bufferToRead.head] +} + +// get the data that was evicted by the last write +func (c *C2) Evicted() []byte { + if c.totalWritten <= c.blocksize { + return nil + } + + bufferToRead := c.a + if c.b.head < c.a.head { + bufferToRead = c.b + } + + bufferStart := bufferToRead.head + c.blocksize + readLength := c.lastWritten + + // if the buffer wasn't full, we don't read the full length + if c.totalWritten-c.lastWritten < c.blocksize { + readLength -= c.lastWritten - c.totalWritten + c.blocksize + } + + return bufferToRead.buffer[bufferStart-readLength : bufferStart] +} + +func (buff *doubleSizeBuffer) Reset() { + buff.head = buff.baseOffset +} + +func (buff *doubleSizeBuffer) Write(by []byte) { + remaining := by + + for len(remaining) > 0 { + remaining_len := len(remaining) + availableSpace := len(buff.buffer) - buff.head + writeThisTime := remaining_len + + if writeThisTime > availableSpace { + writeThisTime = availableSpace + } + + copy( + buff.buffer[buff.head:buff.head+writeThisTime], // to + by, + ) + + buff.head += writeThisTime + + if buff.head == len(buff.buffer) { + buff.head = 0 + } + + remaining = remaining[writeThisTime:] + } +} diff --git a/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go b/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go new file mode 100644 index 0000000..6315989 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go @@ -0,0 +1,182 @@ +package circularbuffer + +import ( + "bytes" + "testing" +) + +const BLOCK_SIZE = 10 + +var incrementBlock = make([]byte, BLOCK_SIZE) +var incrementBlock2 = make([]byte, BLOCK_SIZE) + +func init() { + for i, _ := range incrementBlock { + incrementBlock[i] = byte(i) + incrementBlock2[i] = byte(i + BLOCK_SIZE) + } +} + +func TestCreateC2Buffer(t *testing.T) { + MakeC2Buffer(BLOCK_SIZE) +} + +func TestWriteBlock(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock) +} + +func TestWritingUnderCapacityGivesEmptyEvicted(t *testing.T) { + b := MakeC2Buffer(2) + b.Write([]byte{1, 2}) + + if len(b.Evicted()) != 0 { + t.Fatal("Evicted should have been empty:", b.Evicted()) + } +} + +func TestWritingMultipleBytesWhenBufferIsNotFull(t *testing.T) { + b := MakeC2Buffer(3) + b.Write([]byte{1, 2}) + b.Write([]byte{3, 4}) + + ev := b.Evicted() + + if len(ev) != 1 || ev[0] != 1 { + t.Fatal("Evicted should have been [1,]:", ev) + } +} + +func TestEvictedRegession1(t *testing.T) { + b := MakeC2Buffer(4) + + b.Write([]byte{7, 6}) + b.Write([]byte{5, 1, 2}) + b.Write([]byte{3, 4}) + + ev := b.Evicted() + if len(ev) != 2 || ev[0] != 6 || ev[1] != 5 { + t.Fatalf("Unexpected evicted [6,5]: %v", ev) + } +} + +func TestGetBlock(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock) + + block := b.GetBlock() + + if len(block) != BLOCK_SIZE { + t.Fatal("Wrong block size returned") + } + + for i, by := range block { + if byte(i) != by { + t.Errorf("byte %v does not match", i) + } + } +} + +func TestWriteTwoBlocksGet(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock) + b.Write(incrementBlock2) + + if bytes.Compare(b.GetBlock(), incrementBlock2) != 0 { + t.Errorf("Get block did not return the right value: %s", b.GetBlock()) + } +} + +func TestWriteSingleByteGetSingleByte(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + singleByte := []byte{0} + b.Write(singleByte) + + if bytes.Compare(b.GetBlock(), singleByte) != 0 { + t.Errorf("Get block did not return the right value: %s", b.GetBlock()) + } +} + +func TestWriteTwoBlocksGetEvicted(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock) + b.Write(incrementBlock2) + + if bytes.Compare(b.Evicted(), incrementBlock) != 0 { + t.Errorf("Evicted did not return the right value: %s", b.Evicted()) + } +} + +func TestWriteSingleByteReturnsSingleEvictedByte(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock2) + singleByte := []byte{0} + + b.Write(singleByte) + e := b.Evicted() + + if len(e) != 1 { + t.Fatalf("Evicted length is not correct: %s", e) + } + + if e[0] != byte(10) { + t.Errorf("Evicted content is not correct: %s", e) + } +} + +func TestTruncatingAfterWriting(t *testing.T) { + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock) + + evicted := b.Truncate(2) + + if len(evicted) != 2 { + t.Fatalf("Truncate did not return expected evicted length: %v", evicted) + } + + if evicted[0] != 0 || evicted[1] != 1 { + t.Errorf("Unexpected content in evicted: %v", evicted) + } +} + +func TestWritingAfterTruncating(t *testing.T) { + // test that after we truncate some content, the next operations + // on the buffer give us the expected results + b := MakeC2Buffer(BLOCK_SIZE) + b.Write(incrementBlock) + b.Truncate(4) + + b.Write([]byte{34, 46}) + + block := b.GetBlock() + + if len(block) != BLOCK_SIZE-2 { + t.Fatalf( + "Unexpected block length after truncation: %v (%v)", + block, + len(block), + ) + } + + if bytes.Compare(block, []byte{4, 5, 6, 7, 8, 9, 34, 46}) != 0 { + t.Errorf( + "Unexpected block content after truncation: %v (%v)", + block, + len(block)) + } +} + +// This should have no allocations! +func BenchmarkSingleWrites(b *testing.B) { + buffer := MakeC2Buffer(BLOCK_SIZE) + buffer.Write(incrementBlock) + b.ReportAllocs() + + singleByte := []byte{0} + b.StartTimer() + for i := 0; i < b.N; i++ { + buffer.Write(singleByte) + buffer.Evicted() + } + b.StopTimer() +} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/comparer.go b/vendor/github.com/Redundancy/go-sync/comparer/comparer.go new file mode 100644 index 0000000..b793818 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/comparer/comparer.go @@ -0,0 +1,219 @@ +/* +package comparer is responsible for using a FileChecksumGenerator (filechecksum) and an index +to move through a file and compare it to the index, producing a FileDiffSummary +*/ +package comparer + +import ( + "fmt" + "io" + "sync/atomic" + + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/circularbuffer" + "github.com/Redundancy/go-sync/filechecksum" +) + +const ( + READ_NEXT_BYTE = iota + READ_NEXT_BLOCK + READ_NONE +) + +// If the weak Hash object satisfies this interface, then +// StartFindMatchingBlocks will not allocate a circular buffer +type BlockBuffer interface { + Write([]byte) (int, error) + // the last set of bytes of the size of the circular buffer + // oldest to newest + GetLastBlock() []byte +} + +type BlockMatchResult struct { + // In case of error + Err error + + // The offset the comparison + baseOffset + ComparisonOffset int64 + + // The block from the index that it matched + BlockIdx uint +} + +type Index interface { + FindWeakChecksum2(chk []byte) interface{} + FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum +} + +/* +Iterates though comparison looking for blocks that match ones from the index +it emits each block to be read from the returned channel. Callers should check for +.Err != nil on the results, in which case reading will end immediately. + +StartFindMatchingBlocks is capable of running asyncronously +on sub-sections of a larger file. When doing this, you must overlap +by the block size, and use seperate checksum generators. +*/ + +type Comparer struct { + Comparisons int64 + WeakHashHits int64 + StrongHashHits int64 +} + +func (c *Comparer) StartFindMatchingBlocks( + comparison io.Reader, + baseOffset int64, + generator *filechecksum.FileChecksumGenerator, + referenceIndex Index, +) <-chan BlockMatchResult { + + resultStream := make(chan BlockMatchResult) + + go c.startFindMatchingBlocks_int( + resultStream, + comparison, + baseOffset, + generator, + referenceIndex, + ) + + return resultStream +} + +/* +TODO: When matching duplicated blocks, a channel of BlockMatchResult slices would be more efficient +*/ +func (c *Comparer) startFindMatchingBlocks_int( + results chan<- BlockMatchResult, + comparison io.Reader, + baseOffset int64, + generator *filechecksum.FileChecksumGenerator, + reference Index, +) { + defer close(results) + + block := make([]byte, generator.BlockSize) + var err error + + ReportErr := func(err error) { + results <- BlockMatchResult{ + Err: err, + } + } + + _, err = io.ReadFull(comparison, block) + + if err != nil { + ReportErr( + fmt.Errorf("Error reading first block in comparison: %v", err), + ) + return + } + + generator.WeakRollingHash.SetBlock(block) + singleByte := make([]byte, 1) + weaksum := make([]byte, generator.WeakRollingHash.Size()) + strongSum := make([]byte, 0, generator.GetStrongHash().Size()) + + blockMemory := circularbuffer.MakeC2Buffer(int(generator.BlockSize)) + blockMemory.Write(block) + + strong := generator.GetStrongHash() + // All the bytes + i := int64(0) + next := READ_NEXT_BYTE + + //ReadLoop: + for { + + atomic.AddInt64(&c.Comparisons, 1) + + // look for a weak match + generator.WeakRollingHash.GetSum(weaksum) + if weakMatchList := reference.FindWeakChecksum2(weaksum); weakMatchList != nil { + atomic.AddInt64(&c.WeakHashHits, 1) + + block = blockMemory.GetBlock() + + strong.Reset() + strong.Write(block) + strongSum = strong.Sum(strongSum) + strongList := reference.FindStrongChecksum2(strongSum, weakMatchList) + + // clear the slice + strongSum = strongSum[:0] + + // If there are many matches, it means that this block is + // duplicated in the reference. + // since we care about finding all the blocks in the reference, + // we must report all of them + off := i + baseOffset + for _, strongMatch := range strongList { + results <- BlockMatchResult{ + ComparisonOffset: off, + BlockIdx: strongMatch.ChunkOffset, + } + } + + if len(strongList) > 0 { + atomic.AddInt64(&c.StrongHashHits, 1) + if next == READ_NONE { + // found the match at the end, so exit + break + } + // No point looking for a match that overlaps this block + next = READ_NEXT_BLOCK + } + } + + var n int + var readBytes []byte + + switch next { + case READ_NEXT_BYTE: + n, err = comparison.Read(singleByte) + readBytes = singleByte + case READ_NEXT_BLOCK: + n, err = io.ReadFull(comparison, block) + readBytes = block[:n] + next = READ_NEXT_BYTE + } + + if uint(n) == generator.BlockSize { + generator.WeakRollingHash.SetBlock(block) + blockMemory.Write(block) + i += int64(n) + } else if n > 0 { + b_len := blockMemory.Len() + blockMemory.Write(readBytes) + generator.WeakRollingHash.AddAndRemoveBytes( + readBytes, + blockMemory.Evicted(), + b_len, + ) + i += int64(n) + } + + if next != READ_NONE && (err == io.EOF || err == io.ErrUnexpectedEOF) { + err = io.EOF + next = READ_NONE + } + + if next == READ_NONE { + if blockMemory.Empty() { + break + } + + b_len := blockMemory.Len() + removedByte := blockMemory.Truncate(1) + generator.WeakRollingHash.RemoveBytes(removedByte, b_len) + i += 1 + } + } + + if err != io.EOF { + ReportErr(err) + return + } +} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go b/vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go new file mode 100644 index 0000000..cd3b7b5 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go @@ -0,0 +1,76 @@ +package comparer + +import ( + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/util/readers" + "testing" +) + +var test = []byte{0, 1, 2, 3} + +type NegativeWeakIndex struct { +} + +func (i *NegativeWeakIndex) FindWeakChecksum2(chk []byte) interface{} { + return nil +} + +func (i *NegativeWeakIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { + return nil +} + +type NegativeStrongIndex struct { +} + +func (i *NegativeStrongIndex) FindWeakChecksum2(chk []byte) interface{} { + return i +} + +func (i *NegativeStrongIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { + return nil +} + +func BenchmarkWeakComparison(b *testing.B) { + b.ReportAllocs() + b.SetBytes(1) + + const BLOCK_SIZE = 8 + generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) + + b.StartTimer() + + results := (&Comparer{}).StartFindMatchingBlocks( + readers.OneReader(b.N+BLOCK_SIZE), + 0, + generator, + &NegativeWeakIndex{}, + ) + + for _, ok := <-results; ok; { + } + + b.StopTimer() +} + +func BenchmarkStrongComparison(b *testing.B) { + b.ReportAllocs() + b.SetBytes(1) + + const BLOCK_SIZE = 8 + generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) + + b.StartTimer() + + results := (&Comparer{}).StartFindMatchingBlocks( + readers.OneReader(b.N+BLOCK_SIZE), + 0, + generator, + &NegativeStrongIndex{}, + ) + + for _, ok := <-results; ok; { + } + + b.StopTimer() +} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go b/vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go new file mode 100644 index 0000000..9337a5e --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go @@ -0,0 +1,445 @@ +package comparer + +import ( + "bufio" + "bytes" + "io" + "reflect" + "testing" + + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/indexbuilder" +) + +func CheckResults( + t *testing.T, + original, modified string, + results <-chan BlockMatchResult, + block_size uint, + expectedResults []string, +) { + var resultStrings []string + + for i := range results { + + if i.Err != nil { + t.Fatal(i.Err) + } + + os := i.BlockIdx * block_size + original_max := os + block_size + if original_max > uint(len(original)) { + original_max = uint(len(original)) + } + + orignal := original[os:original_max] + + compare_max := i.ComparisonOffset + int64(block_size) + if compare_max > int64(len(modified)) { + compare_max = int64(len(modified)) + } + + compare := modified[i.ComparisonOffset:compare_max] + + if orignal != compare { + t.Errorf( + "Bad match: \"%v\" to \"%v\" : %#v", + orignal, compare, + i, + ) + } + + resultStrings = append(resultStrings, orignal) + } + + if len(resultStrings) != len(expectedResults) { + t.Fatalf( + "%#v blocks should have matched, got: %#v", + len(expectedResults), + resultStrings, + ) + } + + for i, v := range expectedResults { + if resultStrings[i] != v { + t.Errorf("%#v != %#v", resultStrings[i], v) + } + } +} + +func compare( + original string, + modified string, + block_size uint, +) (results <-chan BlockMatchResult, err error) { + + originalFileContent := bytes.NewBufferString(original) + generator := filechecksum.NewFileChecksumGenerator(block_size) + + _, reference, _, err := indexbuilder.BuildChecksumIndex( + generator, + originalFileContent, + ) + + if err != nil { + return + } + + modifiedContent := bytes.NewBufferString(modified) + + results = (&Comparer{}).StartFindMatchingBlocks( + modifiedContent, + 0, + generator, + reference, + ) + + return +} + +// Splits successive strings into blocks of size n +// 2, ABCD -> AB, CD +// 2, ABCD, E, FG -> AB, CD, E, FG +func split(n int, ss ...string) (result []string) { + totalLength := 0 + for _, s := range ss { + totalLength += len(s)/n + 1 + } + result = make([]string, 0, totalLength) + + for _, x := range ss { + i := int(0) + for i+n < len(x) { + result = append( + result, + x[i:i+n], + ) + + i += n + } + + if i < len(x)-1 { + result = append( + result, + x[i:], + ) + } + } + + return +} + +func TestSplit(t *testing.T) { + INPUT := "abcdef" + EXPECTED := []string{"ab", "cd", "ef"} + result := split(2, INPUT) + + if !reflect.DeepEqual(result, EXPECTED) { + t.Errorf( + "Lists differ: %v vs %v", + result, + EXPECTED, + ) + } +} + +func TestSplitWithPartial(t *testing.T) { + INPUT := "abcdef" + EXPECTED := []string{"abcd", "ef"} + result := split(4, INPUT) + + if !reflect.DeepEqual(result, EXPECTED) { + t.Errorf( + "Lists differ: %v vs %v", + result, + EXPECTED, + ) + } +} + +func TestMultiSplit(t *testing.T) { + INPUT := []string{"abcdef", "ghij"} + EXPECTED := []string{"abcd", "ef", "ghij"} + result := split(4, INPUT...) + + if !reflect.DeepEqual(result, EXPECTED) { + t.Errorf( + "Lists differ: %v vs %v", + result, + EXPECTED, + ) + } +} + +func TestDetectsPrependedContent(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + + const ORIGINAL_STRING = "abcdefghijklmnop" + const PREPENDED_STRING = "12" + ORIGINAL_STRING + + results, err := compare(ORIGINAL_STRING, PREPENDED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + PREPENDED_STRING, + results, + BLOCK_SIZE, + split(4, ORIGINAL_STRING), + ) +} + +func TestDetectsInjectedContent(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const A = "abcdefgh" + const B = "ijklmnop" + const ORIGINAL_STRING = A + B + const MODIFIED_STRING = A + "23" + B + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + split(4, A, B), + ) +} + +func TestDetectsAppendedContent(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + + const ORIGINAL_STRING = "abcdefghijklmnop" + const MODIFIED_STRING = ORIGINAL_STRING + "23" + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + split(4, ORIGINAL_STRING), + ) +} + +func TestDetectsModifiedContent(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const A = "abcdefgh" + const B = "ijkl" + const C = "mnop" + const ORIGINAL_STRING = A + B + C + const MODIFIED_STRING = A + "i2kl" + C + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + split(4, A, C), + ) +} + +func TestDetectsPartialBlockAtEnd(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const A = "abcdefghijklmnopqrstuvwxyz" + const ORIGINAL_STRING = A + const MODIFIED_STRING = A + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + split(4, A), + ) +} + +func TestDetectsModifiedPartialBlockAtEnd(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const A = "abcdefghijklmnopqrstuvwx" + const ORIGINAL_STRING = A + "yz" + const MODIFIED_STRING = A + "23" + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + split(4, A), + ) +} + +func TestDetectsUnmodifiedPartialBlockAtEnd(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const A = "abcdefghijklmnopqrst" + const ORIGINAL_STRING = A + "uvwx" + "yz" + const MODIFIED_STRING = A + "us6x" + "yz" + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + split(4, A, "yz"), + ) +} + +func TestMultipleResultsForDuplicatedBlocks(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const A = "abcd" + const ORIGINAL_STRING = A + A + const MODIFIED_STRING = A + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + []string{A, A}, + ) +} + +func TestRegression1(t *testing.T) { + const BLOCK_SIZE = 4 + var err error + const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" + const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" + + results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + if err != nil { + t.Fatal(err) + } + + CheckResults( + t, + ORIGINAL_STRING, + MODIFIED_STRING, + results, + BLOCK_SIZE, + []string{"The ", "k br", "own ", "fox ", "jump", "the ", "lazy"}, + ) +} + +func TestTwoComparisons(t *testing.T) { + const BLOCK_SIZE = 4 + const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" + const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" + + numMatchers := int64(4) + sectionSize := int64(len(ORIGINAL_STRING)) / numMatchers + sectionSize += int64(BLOCK_SIZE) - (sectionSize % int64(BLOCK_SIZE)) + + merger := &MatchMerger{} + + originalFile := bytes.NewReader([]byte(ORIGINAL_STRING)) + modifiedFile := bytes.NewReader([]byte(MODIFIED_STRING)) + generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) + + _, reference, _, _ := indexbuilder.BuildChecksumIndex( + generator, + originalFile, + ) + + for i := int64(0); i < numMatchers; i++ { + compare := &Comparer{} + offset := sectionSize * i + + t.Logf("Section %v: %v-%v", i, offset, offset+sectionSize) + + sectionReader := bufio.NewReaderSize( + io.NewSectionReader(modifiedFile, offset, sectionSize+BLOCK_SIZE), + 100000, // 1 MB buffer + ) + + // Bakes in the assumption about how to generate checksums (extract) + sectionGenerator := filechecksum.NewFileChecksumGenerator( + uint(BLOCK_SIZE), + ) + + matchStream := compare.StartFindMatchingBlocks( + sectionReader, offset, sectionGenerator, reference, + ) + + merger.StartMergeResultStream(matchStream, int64(BLOCK_SIZE)) + } + + merged := merger.GetMergedBlocks() + missing := merged.GetMissingBlocks(uint(len(ORIGINAL_STRING) / BLOCK_SIZE)) + + expected := []string{ + "quic", "ed over ", " dog", + } + + t.Logf("Missing blocks: %v", len(missing)) + + for x, v := range missing { + start := v.StartBlock * BLOCK_SIZE + end := (v.EndBlock + 1) * BLOCK_SIZE + if end > uint(len(ORIGINAL_STRING)) { + end = uint(len(ORIGINAL_STRING)) + } + s := ORIGINAL_STRING[start:end] + + if s != expected[x] { + t.Errorf( + "Wrong block %v (%v-%v): %#v (expected %#v)", + x, v.StartBlock, v.EndBlock, s, expected[x], + ) + } else { + t.Logf( + "Correct block %v (%v-%v): %#v (expected %#v)", + x, v.StartBlock, v.EndBlock, s, expected[x], + ) + } + } +} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/merger.go b/vendor/github.com/Redundancy/go-sync/comparer/merger.go new file mode 100644 index 0000000..59f108b --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/comparer/merger.go @@ -0,0 +1,309 @@ +package comparer + +import ( + "sort" + "sync" + + "github.com/petar/GoLLRB/llrb" +) + +// The result merger takes many BlockMatchResult +// and combines adjoining results into spans of blocks +type MatchMerger struct { + sync.Mutex + + wait sync.WaitGroup + + // Porting from map to llrb, to enable finding of the next + // largest blockID (What's the first blockID after 5?) + startEndBlockMap *llrb.LLRB + + blockCount uint +} + +// a span of multiple blocks, from start to end, which match the blocks +// starting at an offset of ComparisonStartOffset +type BlockSpan struct { + StartBlock uint + EndBlock uint + + // byte offset in the comparison for the match + ComparisonStartOffset int64 +} + +type BlockSpanIndex interface { + Position() uint +} + +// Wraps a blockspan so that it may easily be used +// in llrb. Corresponds to the start block of the blockspan. +type BlockSpanStart BlockSpan + +func (s BlockSpanStart) Position() uint { + return s.StartBlock +} + +func (s BlockSpanStart) Less(than llrb.Item) bool { + return s.StartBlock < than.(BlockSpanIndex).Position() +} + +// Wraps a blockspan so that it may easily be used +// in llrb. Corresponds to the end of the blockspan. +type BlockSpanEnd BlockSpan + +func (s BlockSpanEnd) Position() uint { + return s.EndBlock +} + +func (s BlockSpanEnd) Less(than llrb.Item) bool { + return s.EndBlock < than.(BlockSpanIndex).Position() +} + +// Wraps a block index, allowing easy use of llrb.Get() +type BlockSpanKey uint + +func (s BlockSpanKey) Position() uint { + return uint(s) +} + +func (k BlockSpanKey) Less(than llrb.Item) bool { + return uint(k) < than.(BlockSpanIndex).Position() +} + +func (b BlockSpan) EndOffset(blockSize int64) int64 { + return b.ComparisonStartOffset + blockSize*int64(b.EndBlock-b.StartBlock+1) +} + +func toBlockSpan(b BlockMatchResult) *BlockSpan { + return &BlockSpan{ + StartBlock: b.BlockIdx, + EndBlock: b.BlockIdx, + ComparisonStartOffset: b.ComparisonOffset, + } +} + +func isBordering(a, b *BlockSpan, blockSize int64) bool { + if a.EndBlock == b.StartBlock-1 && a.EndOffset(blockSize) == b.ComparisonStartOffset { + return true + } else if b.EndBlock == a.StartBlock-1 && b.EndOffset(blockSize) == a.ComparisonStartOffset { + return true + } + + return false +} + +func itemToBlockSpan(in llrb.Item) BlockSpan { + switch i := in.(type) { + case BlockSpanStart: + return BlockSpan(i) + case BlockSpanEnd: + return BlockSpan(i) + } + return BlockSpan{} +} + +// if merged, the block span remaining is the one with the lower start block +func (merger *MatchMerger) merge(block1, block2 *BlockSpan, blockSize int64) { + var a, b *BlockSpan = block1, block2 + + if block1.StartBlock > block2.StartBlock { + a, b = block2, block1 + } + + if isBordering(a, b, blockSize) { + // bordering, merge + // A ------ A B ------ B > A ---------------- A + merger.startEndBlockMap.Delete(BlockSpanKey(a.EndBlock)) + merger.startEndBlockMap.Delete(BlockSpanKey(b.StartBlock)) + a.EndBlock = b.EndBlock + + merger.startEndBlockMap.ReplaceOrInsert(BlockSpanStart(*a)) + merger.startEndBlockMap.ReplaceOrInsert(BlockSpanEnd(*a)) + } +} + +// Can be used on multiple streams of results simultaneously +// starts working asyncronously, call from the initiating goroutine +func (merger *MatchMerger) StartMergeResultStream( + resultStream <-chan BlockMatchResult, + blockSize int64, +) { + // Add should be called on the main goroutine + // to ensure that it has happened before wait is called + merger.wait.Add(1) + + if merger.startEndBlockMap == nil { + merger.startEndBlockMap = llrb.New() + } + + // used by the llrb iterator to signal that it found/didn't find + // an existing key on or spanning the given block + //foundExisting := make(chan bool) + + go func() { + defer merger.wait.Done() + + for result := range resultStream { + if result.Err != nil { + return + } + + merger.Lock() + merger.blockCount += 1 + + blockID := result.BlockIdx + preceeding := merger.startEndBlockMap.Get(BlockSpanKey(blockID - 1)) + following := merger.startEndBlockMap.Get(BlockSpanKey(blockID + 1)) + + asBlockSpan := toBlockSpan(result) + + var foundExisting bool + // Exists, or within an existing span + merger.startEndBlockMap.AscendGreaterOrEqual( + BlockSpanKey(blockID), + // iterator + func(i llrb.Item) bool { + j, ok := i.(BlockSpanIndex) + + if !ok { + foundExisting = true + return false + } + + switch k := j.(type) { + case BlockSpanStart: + // it's only overlapping if its the same blockID + foundExisting = k.StartBlock == blockID + return false + + case BlockSpanEnd: + // we didn't find a start, so there's an end that overlaps + foundExisting = true + return false + default: + foundExisting = true + return false + } + + }, + ) + + if foundExisting { + merger.Unlock() + continue + } + + merger.startEndBlockMap.ReplaceOrInsert( + BlockSpanStart(*toBlockSpan(result)), + ) + + if preceeding != nil && following != nil { + a := itemToBlockSpan(preceeding) + merger.merge( + asBlockSpan, + &a, + blockSize, + ) + + b := itemToBlockSpan(following) + merger.merge( + &a, + &b, + blockSize, + ) + + } else if preceeding != nil { + a := itemToBlockSpan(preceeding) + merger.merge( + asBlockSpan, + &a, + blockSize, + ) + } else if following != nil { + b := itemToBlockSpan(following) + merger.merge( + asBlockSpan, + &b, + blockSize, + ) + } + + merger.Unlock() + } + }() +} + +type BlockSpanList []BlockSpan + +func (l BlockSpanList) Len() int { + return len(l) +} + +func (l BlockSpanList) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +func (l BlockSpanList) Less(i, j int) bool { + return l[i].StartBlock < l[j].StartBlock +} + +// Sorted list of blocks, based on StartBlock +func (merger *MatchMerger) GetMergedBlocks() (sorted BlockSpanList) { + merger.wait.Wait() + var smallestKey uint = 0 + m := merger.startEndBlockMap + + m.AscendGreaterOrEqual(m.Min(), func(item llrb.Item) bool { + switch block := item.(type) { + case BlockSpanStart: + sorted = append(sorted, BlockSpan(block)) + smallestKey = block.StartBlock + 1 + } + return true + }) + + sort.Sort(sorted) + return +} + +// Creates a list of spans that are missing. +// note that maxBlock is blockCount-1 +func (l BlockSpanList) GetMissingBlocks(maxBlock uint) (sorted BlockSpanList) { + // it's difficult to know how many spans we will need + sorted = make(BlockSpanList, 0) + + lastBlockSpanIndex := -1 + for _, blockSpan := range l { + if int(blockSpan.StartBlock) > lastBlockSpanIndex+1 { + sorted = append( + sorted, + BlockSpan{ + StartBlock: uint(lastBlockSpanIndex + 1), + EndBlock: blockSpan.StartBlock - 1, + }, + ) + } + + lastBlockSpanIndex = int(blockSpan.EndBlock) + } + + if lastBlockSpanIndex == -1 { + sorted = append( + sorted, + BlockSpan{ + StartBlock: 0, + EndBlock: maxBlock, + }, + ) + } else if uint(lastBlockSpanIndex) < maxBlock { + sorted = append( + sorted, + BlockSpan{ + StartBlock: uint(lastBlockSpanIndex + 1), + EndBlock: maxBlock, + }, + ) + } + + return sorted +} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/merger_test.go b/vendor/github.com/Redundancy/go-sync/comparer/merger_test.go new file mode 100644 index 0000000..01ca053 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/comparer/merger_test.go @@ -0,0 +1,401 @@ +package comparer + +import ( + "testing" +) + +func TestMergeAdjacentBlocksAfter(t *testing.T) { + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 1, + } + + close(mergeChan) + + merged := merger.GetMergedBlocks() + + if len(merged) != 1 { + t.Fatalf("Wrong number of blocks returned: %#v", merged) + } + + if merged[0].EndBlock != 1 { + t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0]) + } +} + +func TestMergeAdjacentBlocksBefore(t *testing.T) { + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 1, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + close(mergeChan) + + merged := merger.GetMergedBlocks() + + if len(merged) != 1 { + t.Fatalf("Wrong number of blocks returned: %#v", merged) + } + + if merged[0].EndBlock != 1 { + t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0]) + } + + // start and end + if merger.startEndBlockMap.Len() != 2 { + t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) + } +} + +func TestMergeAdjacentBlocksBetween(t *testing.T) { + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 2 * BLOCK_SIZE, + BlockIdx: 2, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + // match in the center + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 1, + } + + close(mergeChan) + + merged := merger.GetMergedBlocks() + + if len(merged) != 1 { + t.Fatalf("Wrong number of blocks returned: %#v", merged) + } + + if merged[0].EndBlock != 2 { + t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0]) + } + if merged[0].StartBlock != 0 { + t.Errorf("Wrong StartBlock, expected 0, got %#v", merged[0]) + } + if merger.startEndBlockMap.Len() != 2 { + t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) + } +} + +func TestMissingBlocksOffsetStart(t *testing.T) { + b := BlockSpanList{ + { + StartBlock: 2, + EndBlock: 3, + }, + } + + m := b.GetMissingBlocks(3) + + if len(m) != 1 { + t.Fatalf("Wrong number of missing blocks: %v", len(m)) + } + + if m[0].StartBlock != 0 { + t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) + } + if m[0].EndBlock != 1 { + t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) + } +} + +func TestMissingCenterBlock(t *testing.T) { + b := BlockSpanList{ + { + StartBlock: 0, + EndBlock: 0, + }, + { + StartBlock: 2, + EndBlock: 3, + }, + } + + m := b.GetMissingBlocks(3) + + if len(m) != 1 { + t.Fatalf("Wrong number of missing blocks: %v", len(m)) + } + + if m[0].StartBlock != 1 { + t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) + } + if m[0].EndBlock != 1 { + t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) + } +} + +func TestMissingEndBlock(t *testing.T) { + b := BlockSpanList{ + { + StartBlock: 0, + EndBlock: 1, + }, + } + + m := b.GetMissingBlocks(3) + + if len(m) != 1 { + t.Fatalf("Wrong number of missing blocks: %v", len(m)) + } + + if m[0].StartBlock != 2 { + t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) + } + if m[0].EndBlock != 3 { + t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) + } +} + +func TestDuplicatedReferenceBlocks(t *testing.T) { + // Reference = AA + // Local = A + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + // When we find multiple strong matches, we send each of them + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 1, + } + + close(mergeChan) + + merged := merger.GetMergedBlocks() + + if len(merged) != 2 { + t.Errorf("Duplicated blocks cannot be merged: %#v", merged) + } + + missing := merged.GetMissingBlocks(1) + + if len(missing) > 0 { + t.Errorf("There were no missing blocks: %#v", missing) + } +} + +func TestDuplicatedLocalBlocks(t *testing.T) { + // Reference = A + // Local = AA + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + // When we find multiple strong matches, we send each of them + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 0, + } + + close(mergeChan) + + // We only need one of the matches in the resulting file + merged := merger.GetMergedBlocks() + + if len(merged) != 1 { + t.Errorf("Duplicated blocks cannot be merged: %#v", merged) + } + + missing := merged.GetMissingBlocks(0) + + if len(missing) > 0 { + t.Errorf("There were no missing blocks: %#v", missing) + } +} + +func TestDoublyDuplicatedBlocks(t *testing.T) { + // Reference = AA + // Local = AA + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + // When we find multiple strong matches, we send each of them + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 1, + } + + // Second local match + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 0, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 1, + } + + close(mergeChan) + + merged := merger.GetMergedBlocks() + + if len(merged) != 2 { + t.Errorf("Duplicated blocks cannot be merged: %#v", merged) + } + + missing := merged.GetMissingBlocks(1) + + if len(missing) > 0 { + t.Errorf("There were no missing blocks: %#v", missing) + } +} + +func TestBlockWithinSpan(t *testing.T) { + // catch the case where we're informed about a block, + // after we've merged blocks around it, so that the start and end + // are within a span, not bordering one + const BLOCK_SIZE = 4 + + mergeChan := make(chan BlockMatchResult) + merger := &MatchMerger{} + merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 0, + BlockIdx: 0, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 1, + } + + mergeChan <- BlockMatchResult{ + ComparisonOffset: 2 * BLOCK_SIZE, + BlockIdx: 2, + } + + // This one is a duplicate of an earlier one + mergeChan <- BlockMatchResult{ + ComparisonOffset: BLOCK_SIZE, + BlockIdx: 1, + } + + close(mergeChan) + + merged := merger.GetMergedBlocks() + + if len(merged) != 1 { + t.Fatalf("Wrong number of blocks returned: %#v", merged) + } + + if merged[0].EndBlock != 2 { + t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0]) + } + + // start and end + if merger.startEndBlockMap.Len() != 2 { + t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) + } +} + +func TestNilBlockSpanList(t *testing.T) { + s := BlockSpanList(nil) + + missing := s.GetMissingBlocks(1) + + if missing == nil { + t.Fail() + } + + if len(missing) == 0 { + t.Fatal("missing should not be empty") + } + + missingItem := missing[0] + + if missingItem.StartBlock != 0 { + t.Errorf("Wrong startblock: %v", missingItem.StartBlock) + } + if missingItem.EndBlock != 1 { + t.Errorf("Wrong endblock: %v", missingItem.EndBlock) + } +} + +func TestRegression1Merger(t *testing.T) { + const BLOCK_SIZE = 4 + const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" + const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" + + results, _ := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) + merger := &MatchMerger{} + merger.StartMergeResultStream(results, BLOCK_SIZE) + + merged := merger.GetMergedBlocks() + missing := merged.GetMissingBlocks(uint(len(ORIGINAL_STRING) / BLOCK_SIZE)) + + expected := []string{ + "quic", "ed over ", " dog", + } + + for i, v := range missing { + start := v.StartBlock * BLOCK_SIZE + end := (v.EndBlock + 1) * BLOCK_SIZE + if end > uint(len(ORIGINAL_STRING)) { + end = uint(len(ORIGINAL_STRING)) + } + s := ORIGINAL_STRING[start:end] + + if s != expected[i] { + t.Errorf("Wrong block %v: %v (expected %v)", i, expected[i]) + } + } +} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go b/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go new file mode 100644 index 0000000..4f69701 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go @@ -0,0 +1,237 @@ +/* +package filechecksum provides the FileChecksumGenerator, whose main responsibility is to read a file, +and generate both weak and strong checksums for every block. It is also used by the comparer, which +will generate weak checksums for potential byte ranges that could match the index, and strong checksums +if needed. +*/ +package filechecksum + +import ( + "crypto/md5" + "hash" + "io" + + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/rollsum" +) + +// Rsync swapped to this after version 30 +// this is a factory function, because we don't actually want to share hash state +var DefaultStrongHashGenerator = func() hash.Hash { + return md5.New() +} + +// We provide an overall hash of individual files +var DefaultFileHashGenerator = func() hash.Hash { + return md5.New() +} + +// Uses all default hashes (MD5 & rollsum16) +func NewFileChecksumGenerator(blocksize uint) *FileChecksumGenerator { + return &FileChecksumGenerator{ + BlockSize: blocksize, + WeakRollingHash: rollsum.NewRollsum32Base(blocksize), + //WeakRollingHash: rollsum.NewRollsum16Base(blocksize), + StrongHash: DefaultStrongHashGenerator(), + FileChecksumHash: DefaultFileHashGenerator(), + } +} + +type RollingHash interface { + // the size of the hash output + Size() int + + AddByte(b byte) + RemoveByte(b byte, length int) + + AddBytes(bs []byte) + RemoveBytes(bs []byte, length int) + + // pairs up bytes to do remove/add in the right order + AddAndRemoveBytes(add []byte, remove []byte, length int) + + SetBlock(block []byte) + + GetSum(b []byte) + Reset() +} + +/* +FileChecksumGenerator provides a description of what hashing functions to use to +evaluate a file. Since the hashes store state, it is NOT safe to use a generator concurrently +for different things. +*/ +type FileChecksumGenerator struct { + // See BlockBuffer + WeakRollingHash RollingHash + StrongHash hash.Hash + FileChecksumHash hash.Hash + BlockSize uint +} + +// Reset all hashes to initial state +func (check *FileChecksumGenerator) Reset() { + check.WeakRollingHash.Reset() + check.StrongHash.Reset() + check.FileChecksumHash.Reset() +} + +func (check *FileChecksumGenerator) ChecksumSize() int { + return check.WeakRollingHash.Size() + check.GetStrongHash().Size() +} + +func (check *FileChecksumGenerator) GetChecksumSizes() (int, int) { + return check.WeakRollingHash.Size(), check.GetStrongHash().Size() +} + +// Gets the Hash function for the overall file used on each block +// defaults to md5 +func (check *FileChecksumGenerator) GetFileHash() hash.Hash { + return check.FileChecksumHash +} + +// Gets the Hash function for the strong hash used on each block +// defaults to md5, but can be overriden by the generator +func (check *FileChecksumGenerator) GetStrongHash() hash.Hash { + return check.StrongHash +} + +// GenerateChecksums reads each block of the input file, and outputs first the weak, then the strong checksum +// to the output writer. It will return a checksum for the whole file. +// Potentially speaking, this might be better producing a channel of blocks, which would remove the need for io from +// a number of other places. +func (check *FileChecksumGenerator) GenerateChecksums(inputFile io.Reader, output io.Writer) (fileChecksum []byte, err error) { + for chunkResult := range check.StartChecksumGeneration(inputFile, 64, nil) { + if chunkResult.Err != nil { + return nil, chunkResult.Err + } else if chunkResult.Filechecksum != nil { + return chunkResult.Filechecksum, nil + } + + for _, chunk := range chunkResult.Checksums { + output.Write(chunk.WeakChecksum) + output.Write(chunk.StrongChecksum) + } + } + + return nil, nil +} + +type ChecksumResults struct { + // Return multiple chunks at once for performance + Checksums []chunks.ChunkChecksum + // only used for the last item + Filechecksum []byte + // signals that this is the last item + Err error +} + +// A function or object that can compress blocks +// the compression function must also write out the compressed blocks somewhere! +// Compressed blocks should be independently inflatable +type CompressionFunction func([]byte) (compressedSize int64, err error) + +func (check *FileChecksumGenerator) StartChecksumGeneration( + inputFile io.Reader, + blocksPerResult uint, + compressionFunction CompressionFunction, +) <-chan ChecksumResults { + resultChan := make(chan ChecksumResults) + go check.generate(resultChan, blocksPerResult, compressionFunction, inputFile) + return resultChan +} + +func (check *FileChecksumGenerator) generate( + resultChan chan ChecksumResults, + blocksPerResult uint, + compressionFunction CompressionFunction, + inputFile io.Reader, +) { + defer close(resultChan) + + fullChecksum := check.GetFileHash() + strongHash := check.GetStrongHash() + + buffer := make([]byte, check.BlockSize) + + // ensure that the hashes are clean + strongHash.Reset() + fullChecksum.Reset() + + // We reset the hashes when done do we can reuse the generator + defer check.WeakRollingHash.Reset() + defer strongHash.Reset() + defer fullChecksum.Reset() + + results := make([]chunks.ChunkChecksum, 0, blocksPerResult) + + i := uint(0) + for { + n, err := io.ReadFull(inputFile, buffer) + section := buffer[:n] + + if n == 0 { + break + } + + // As hashes, the assumption is that they never error + // additionally, we assume that the only reason not + // to write a full block would be reaching the end of the file + fullChecksum.Write(section) + check.WeakRollingHash.SetBlock(section) + strongHash.Write(section) + + strongChecksumValue := make([]byte, 0, strongHash.Size()) + weakChecksumValue := make([]byte, check.WeakRollingHash.Size()) + + check.WeakRollingHash.GetSum(weakChecksumValue) + strongChecksumValue = strongHash.Sum(strongChecksumValue) + + blockSize := int64(check.BlockSize) + + if compressionFunction != nil { + blockSize, err = compressionFunction(section) + } + + results = append( + results, + chunks.ChunkChecksum{ + ChunkOffset: i, + Size: blockSize, + WeakChecksum: weakChecksumValue, + StrongChecksum: strongChecksumValue, + }, + ) + + i++ + + if len(results) == cap(results) { + resultChan <- ChecksumResults{ + Checksums: results, + } + results = make([]chunks.ChunkChecksum, 0, blocksPerResult) + } + + // clear it again + strongChecksumValue = strongChecksumValue[:0] + + // Reset the strong + strongHash.Reset() + + if n != len(buffer) || err == io.EOF { + break + } + } + + if len(results) > 0 { + resultChan <- ChecksumResults{ + Checksums: results, + } + } + + resultChan <- ChecksumResults{ + Filechecksum: fullChecksum.Sum(nil), + } + + return +} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go b/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go new file mode 100644 index 0000000..76d1e74 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go @@ -0,0 +1,309 @@ +package filechecksum + +import ( + "bytes" + "io" + "os" + "testing" + + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/index" + "github.com/Redundancy/go-sync/util/readers" +) + +func TestChecksumGenerationEndsWithFilechecksum(t *testing.T) { + const BLOCKSIZE = 100 + const BLOCK_COUNT = 20 + emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + lastResult := ChecksumResults{} + + for lastResult = range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { + } + + if lastResult.Checksums != nil { + t.Errorf("Last result had checksums: %#v", lastResult) + } + + if lastResult.Filechecksum == nil { + t.Errorf("Last result did not contain the filechecksum: %#v", lastResult) + } +} + +func TestChecksumGenerationReturnsCorrectChecksumCount(t *testing.T) { + const BLOCKSIZE = 100 + const BLOCK_COUNT = 20 + + emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + resultCount := 0 + + for r := range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { + resultCount += len(r.Checksums) + } + + if resultCount != BLOCK_COUNT { + t.Errorf("Unexpected block count returned: %v", resultCount) + } +} + +func TestChecksumGenerationContainsHashes(t *testing.T) { + const BLOCKSIZE = 100 + const BLOCK_COUNT = 20 + + emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + for r := range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { + for _, r2 := range r.Checksums { + if len(r2.WeakChecksum) != checksum.WeakRollingHash.Size() { + t.Fatalf( + "Wrong length weak checksum: %v vs %v", + len(r2.WeakChecksum), + checksum.WeakRollingHash.Size(), + ) + } + + if len(r2.StrongChecksum) != checksum.StrongHash.Size() { + t.Fatalf( + "Wrong length strong checksum: %v vs %v", + len(r2.StrongChecksum), + checksum.StrongHash.Size(), + ) + } + } + } +} + +func TestRollsumLength(t *testing.T) { + const BLOCKSIZE = 100 + const BLOCK_COUNT = 20 + + emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) + output := bytes.NewBuffer(nil) + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + // output length is expected to be 20 blocks + expectedLength := (BLOCK_COUNT * checksum.GetStrongHash().Size()) + + (BLOCK_COUNT * checksum.WeakRollingHash.Size()) + + _, err := checksum.GenerateChecksums(emptybuffer, output) + + if err != nil { + t.Fatal(err) + } + + if output.Len() != expectedLength { + t.Errorf( + "output length (%v) did not match expected length (%v)", + output.Len(), + expectedLength, + ) + } +} + +func TestRollsumLengthWithPartialBlockAtEnd(t *testing.T) { + const BLOCKSIZE = 100 + const FULL_BLOCK_COUNT = 20 + const BLOCK_COUNT = FULL_BLOCK_COUNT + 1 + + emptybuffer := bytes.NewBuffer(make([]byte, FULL_BLOCK_COUNT*BLOCKSIZE+50)) + output := bytes.NewBuffer(nil) + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + // output length is expected to be 20 blocks + expectedLength := (BLOCK_COUNT * checksum.GetStrongHash().Size()) + + (BLOCK_COUNT * checksum.WeakRollingHash.Size()) + + _, err := checksum.GenerateChecksums(emptybuffer, output) + + if err != nil { + t.Fatal(err) + } + + if output.Len() != expectedLength { + t.Errorf( + "output length (%v) did not match expected length (%v)", + output.Len(), + expectedLength, + ) + } +} + +// Each of the data blocks is the same, so the checksums for the blocks should be the same +func TestChecksumBlocksTheSame(t *testing.T) { + const BLOCKSIZE = 100 + const BLOCK_COUNT = 20 + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + output := bytes.NewBuffer(nil) + + _, err := checksum.GenerateChecksums( + readers.OneReader(BLOCKSIZE*BLOCK_COUNT), + output, + ) + + if err != nil { + t.Fatal(err) + } + + weakSize, strongSize := checksum.GetChecksumSizes() + + if output.Len() != BLOCK_COUNT*(strongSize+weakSize) { + t.Errorf( + "Unexpected output length: %v, expected %v", + output.Len(), + BLOCK_COUNT*(strongSize+weakSize), + ) + } + + results, err := chunks.LoadChecksumsFromReader(output, weakSize, strongSize) + + if err != nil { + t.Fatal(err) + } + + if len(results) != BLOCK_COUNT { + t.Fatalf("Results too short! %v", len(results)) + } + + first := results[0] + + for i, chk := range results { + if chk.ChunkOffset != uint(i) { + t.Errorf("Unexpected offset %v on chunk %v", chk.ChunkOffset, i) + } + if !first.Match(chk) { + t.Fatalf("Chunks have different checksums on %v", i) + } + } +} + +func TestPrependedBlocks(t *testing.T) { + const BLOCKSIZE = 100 + const BLOCK_COUNT = 20 + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + file1 := io.LimitReader( + readers.NewNonRepeatingSequence(0), + BLOCKSIZE*BLOCK_COUNT, + ) + + file2 := io.LimitReader( + io.MultiReader( + readers.OneReader(BLOCKSIZE), // Off by one block + readers.NewNonRepeatingSequence(0), + ), + BLOCKSIZE*BLOCK_COUNT, + ) + + output1 := bytes.NewBuffer(nil) + chksum1, _ := checksum.GenerateChecksums(file1, output1) + + output2 := bytes.NewBuffer(nil) + chksum2, _ := checksum.GenerateChecksums(file2, output2) + + if bytes.Compare(chksum1, chksum2) == 0 { + t.Fatal("Checksums should be different") + } + + weaksize, strongSize := checksum.GetChecksumSizes() + sums1, _ := chunks.LoadChecksumsFromReader(output1, weaksize, strongSize) + sums2, _ := chunks.LoadChecksumsFromReader(output2, weaksize, strongSize) + + if len(sums1) != len(sums2) { + t.Fatalf("Checksum lengths differ %v vs %v", len(sums1), len(sums2)) + } + + if sums1[0].Match(sums2[0]) { + t.Error("Chunk sums1[0] should differ from sums2[0]") + } + + for i, _ := range sums2 { + if i == 0 { + continue + } + + if !sums1[i-1].Match(sums2[i]) { + t.Errorf("Chunk sums1[%v] equal sums2[%v]", i-1, i) + } + + } +} + +// Test that partial content that ends in the middle of a weak +// hash is caught correctly +func TestInvalidReaderLength_Weak(t *testing.T) { + const BLOCKSIZE = 100 + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + file1 := io.LimitReader( + readers.NewNonRepeatingSequence(0), + int64(checksum.ChecksumSize())+2, + ) + + ws, ss := checksum.GetChecksumSizes() + r, err := chunks.LoadChecksumsFromReader(file1, ws, ss) + + if r != nil || err != chunks.ErrPartialChecksum { + t.Error("Expected partial checksum error") + } +} + +// Test that partial content that ends in the middle of a strong +// hash is caught correctly +func TestInvalidReaderLength_Strong(t *testing.T) { + const BLOCKSIZE = 100 + + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + file1 := io.LimitReader( + readers.NewNonRepeatingSequence(0), + int64(checksum.ChecksumSize())+int64(checksum.WeakRollingHash.Size())+2, + ) + + ws, ss := checksum.GetChecksumSizes() + r, err := chunks.LoadChecksumsFromReader(file1, ws, ss) + + if r != nil || err != chunks.ErrPartialChecksum { + t.Error("Expected partial checksum error") + } +} + +func ExampleFileChecksumGenerator_LoadChecksumsFromReader() { + const BLOCKSIZE = 8096 + checksum := NewFileChecksumGenerator(BLOCKSIZE) + + // This could be any source that conforms to io.Reader + // sections of a file, or the body of an http response + file1, err := os.Open("fileChecksums.chk") + + if err != nil { + return + } + + defer file1.Close() + + ws, ss := checksum.GetChecksumSizes() + checksums, err := chunks.LoadChecksumsFromReader(file1, ws, ss) + + if err != nil { + return + } + + // Make an index that we can use against our local + // checksums + i := index.MakeChecksumIndex(checksums) + + // example checksum from a local file + // look for the chunk in the index + i.FindWeakChecksumInIndex([]byte("a")) + +} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go b/vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go new file mode 100644 index 0000000..9d604a5 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go @@ -0,0 +1,48 @@ +package filechecksum + +import ( + "bytes" + "hash" +) + +type ChecksumLookup interface { + GetStrongChecksumForBlock(blockID int) []byte +} + +type HashVerifier struct { + BlockSize uint + Hash hash.Hash + BlockChecksumGetter ChecksumLookup +} + +func (v *HashVerifier) VerifyBlockRange(startBlockID uint, data []byte) bool { + for i := 0; i*int(v.BlockSize) < len(data); i++ { + start := i * int(v.BlockSize) + end := start + int(v.BlockSize) + + if end > len(data) { + end = len(data) + } + + blockData := data[start:end] + + expectedChecksum := v.BlockChecksumGetter.GetStrongChecksumForBlock( + int(startBlockID) + i, + ) + + if expectedChecksum == nil { + return true + } + + v.Hash.Write(blockData) + hashedData := v.Hash.Sum(nil) + + if bytes.Compare(expectedChecksum, hashedData) != 0 { + return false + } + + v.Hash.Reset() + } + + return true +} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go b/vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go new file mode 100644 index 0000000..ce3938f --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go @@ -0,0 +1,72 @@ +package filechecksum + +import ( + "crypto/md5" + "testing" +) + +type SingleBlockSource []byte + +func (d SingleBlockSource) GetStrongChecksumForBlock(blockID int) []byte { + m := md5.New() + m.Write(d) + return m.Sum(nil) +} + +func TestBlockEqualsItself(t *testing.T) { + data := []byte("fooooo") + + h := HashVerifier{ + Hash: md5.New(), + BlockSize: uint(len(data)), + BlockChecksumGetter: SingleBlockSource(data), + } + + if !h.VerifyBlockRange(0, data) { + t.Error("data did not verify") + } +} + +type FourByteBlockSource []byte + +func (d FourByteBlockSource) GetStrongChecksumForBlock(blockID int) []byte { + m := md5.New() + + start := blockID * 4 + end := start + 4 + + if end >= len(d) { + end = len(d) + } + + m.Write(d[start:end]) + return m.Sum(nil) +} + +func TestSplitBlocksEqualThemselves(t *testing.T) { + data := []byte("foooBaar") + + h := HashVerifier{ + Hash: md5.New(), + BlockSize: uint(4), + BlockChecksumGetter: FourByteBlockSource(data), + } + + if !h.VerifyBlockRange(0, data) { + t.Error("data did not verify") + } +} + +func TestPartialBlock(t *testing.T) { + data := []byte("fo") + + h := HashVerifier{ + Hash: md5.New(), + BlockSize: uint(4), + BlockChecksumGetter: SingleBlockSource(data), + } + + if !h.VerifyBlockRange(0, data) { + t.Error("data did not verify") + } +} diff --git a/vendor/github.com/Redundancy/go-sync/gosync.go b/vendor/github.com/Redundancy/go-sync/gosync.go new file mode 100644 index 0000000..ea07157 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync.go @@ -0,0 +1,16 @@ +/* +Package gosync is inspired by zsync, and rsync. It aims to take the fundamentals and create a very flexible library that can be adapted +to work in many ways. + +We rely heavily on built in Go abstractions like io.Reader, hash.Hash and our own interfaces - this makes the code easier to change, and to test. +In particular, no part of the core library should know anything about the transport or layout of the reference data. If you want +to do rsync and do http/https range requests, that's just as good as zsync client-server over an SSH tunnel. The goal is also to allow +support for multiple concurrent connections, so that you can make the best use of your line in the face of the bandwidth latency product +(or other concerns that require concurrency to solve). + +The following optimizations are possible: +* Generate hashes with multiple threads (both during reference generation and local file interrogation) +* Multiple ranged requests (can even be used to get the hashes) + +*/ +package gosync diff --git a/vendor/github.com/Redundancy/go-sync/gosync/build.go b/vendor/github.com/Redundancy/go-sync/gosync/build.go new file mode 100644 index 0000000..7dfb000 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync/build.go @@ -0,0 +1,114 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/Redundancy/go-sync/filechecksum" + "github.com/codegangsta/cli" +) + +func init() { + app.Commands = append( + app.Commands, + cli.Command{ + Name: "build", + ShortName: "b", + Usage: "build a .gosync file for a file", + Action: Build, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "blocksize", + Value: DefaultBlockSize, + Usage: "The block size to use for the gosync file", + }, + }, + }, + ) +} + +func Build(c *cli.Context) { + filename := c.Args()[0] + blocksize := uint32(c.Int("blocksize")) + generator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) + inputFile, err := os.Open(filename) + + if err != nil { + absInputPath, err2 := filepath.Abs(filename) + if err2 == nil { + handleFileError(absInputPath, err) + } else { + handleFileError(filename, err) + } + + os.Exit(1) + } + + s, _ := inputFile.Stat() + // TODO: Error? + file_size := s.Size() + + defer inputFile.Close() + + ext := filepath.Ext(filename) + outfilePath := filename[:len(filename)-len(ext)] + ".gosync" + outputFile, err := os.Create(outfilePath) + + if err != nil { + handleFileError(outfilePath, err) + os.Exit(1) + } + + defer outputFile.Close() + + if err = writeHeaders( + outputFile, + magicString, + blocksize, + file_size, + []uint16{majorVersion, minorVersion, patchVersion}, + ); err != nil { + fmt.Fprintf( + os.Stderr, + "Error getting file info: %v\n", + filename, + err, + ) + os.Exit(2) + } + + start := time.Now() + _, err = generator.GenerateChecksums(inputFile, outputFile) + end := time.Now() + + if err != nil { + fmt.Fprintf( + os.Stderr, + "Error generating checksum: %v\n", + filename, + err, + ) + os.Exit(2) + } + + inputFileInfo, err := os.Stat(filename) + if err != nil { + fmt.Fprintf( + os.Stderr, + "Error getting file info: %v\n", + filename, + err, + ) + os.Exit(2) + } + + fmt.Fprintf( + os.Stderr, + "Index for %v file generated in %v (%v bytes/S)\n", + inputFileInfo.Size(), + end.Sub(start), + float64(inputFileInfo.Size())/end.Sub(start).Seconds(), + ) +} diff --git a/vendor/github.com/Redundancy/go-sync/gosync/common.go b/vendor/github.com/Redundancy/go-sync/gosync/common.go new file mode 100644 index 0000000..f8091a9 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync/common.go @@ -0,0 +1,277 @@ +package main + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/comparer" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/index" + "github.com/Redundancy/go-sync/patcher" + "github.com/codegangsta/cli" +) + +const ( + // KB - One Kilobyte + KB = 1024 + // MB - One Megabyte + MB = 1000000 +) + +func errorWrapper(c *cli.Context, f func(*cli.Context) error) { + defer func() { + if p := recover(); p != nil { + fmt.Fprintln(os.Stderr, p) + os.Exit(1) + } + }() + + if err := f(c); err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + os.Exit(1) + } + + return +} + +func openFileAndHandleError(filename string) (f *os.File) { + var err error + f, err = os.Open(filename) + + if err != nil { + f = nil + handleFileError(filename, err) + } + + return +} + +func formatFileError(filename string, err error) error { + switch { + case os.IsExist(err): + return fmt.Errorf( + "Could not open %v (already exists): %v", + filename, + err, + ) + case os.IsNotExist(err): + return fmt.Errorf( + "Could not find %v: %v\n", + filename, + err, + ) + case os.IsPermission(err): + return fmt.Errorf( + "Could not open %v (permission denied): %v\n", + filename, + err, + ) + default: + return fmt.Errorf( + "Unknown error opening %v: %v\n", + filename, + err, + ) + } +} + +func handleFileError(filename string, err error) { + e := formatFileError(filename, err) + fmt.Fprintln(os.Stderr, e) +} + +func getLocalOrRemoteFile(path string) (io.ReadCloser, error) { + url, err := url.Parse(path) + + switch { + case err != nil: + return os.Open(path) + case url.Scheme == "": + return os.Open(path) + default: + response, err := http.Get(path) + + if err != nil { + return nil, err + } + + if response.StatusCode < 200 || response.StatusCode > 299 { + return nil, fmt.Errorf("Request to %v returned status: %v", path, response.Status) + } + + return response.Body, nil + } +} + +func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { + result := make([]patcher.FoundBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].MatchOffset = v.ComparisonStartOffset + result[i].BlockSize = blockSize + } + + return result +} + +func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { + result := make([]patcher.MissingBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].BlockSize = blockSize + } + + return result +} + +func writeHeaders( + f *os.File, + magic string, + blocksize uint32, + filesize int64, + versions []uint16, +) (err error) { + if _, err = f.WriteString(magicString); err != nil { + return + } + + for _, v := range versions { + if err = binary.Write(f, binary.LittleEndian, v); err != nil { + return + } + } + + if err = binary.Write(f, binary.LittleEndian, filesize); err != nil { + return + } + + err = binary.Write(f, binary.LittleEndian, blocksize) + return +} + +// reads the file headers and checks the magic string, then the semantic versioning +func readHeadersAndCheck( + r io.Reader, + magic string, + requiredMajorVersion uint16, +) ( + major, minor, patch uint16, + filesize int64, + blocksize uint32, + err error, +) { + b := make([]byte, len(magicString)) + + if _, err = r.Read(b); err != nil { + return + } else if string(b) != magicString { + err = errors.New( + "file header does not match magic string. Not a valid gosync file", + ) + return + } + + for _, v := range []*uint16{&major, &minor, &patch} { + err = binary.Read(r, binary.LittleEndian, v) + if err != nil { + return + } + } + + if requiredMajorVersion != major { + err = fmt.Errorf( + "The major version of the gosync file (%v.%v.%v) does not match the tool (%v.%v.%v).", + major, minor, patch, + majorVersion, minorVersion, patchVersion, + ) + + return + } + + err = binary.Read(r, binary.LittleEndian, &filesize) + if err != nil { + return + } + + err = binary.Read(r, binary.LittleEndian, &blocksize) + return +} + +func readIndex(r io.Reader, blocksize uint) ( + i *index.ChecksumIndex, + checksumLookup filechecksum.ChecksumLookup, + blockCount uint, + err error, +) { + generator := filechecksum.NewFileChecksumGenerator(blocksize) + + readChunks, e := chunks.LoadChecksumsFromReader( + r, + generator.WeakRollingHash.Size(), + generator.StrongHash.Size(), + ) + + err = e + + if err != nil { + return + } + + checksumLookup = chunks.StrongChecksumGetter(readChunks) + i = index.MakeChecksumIndex(readChunks) + blockCount = uint(len(readChunks)) + + return +} + +func multithreadedMatching( + localFile *os.File, + idx *index.ChecksumIndex, + localFileSize, + matcherCount int64, + blocksize uint, +) (*comparer.MatchMerger, *comparer.Comparer) { + // Note: Since not all sections of the file are equal in work + // it would be better to divide things up into more sections and + // pull work from a queue channel as each finish + sectionSize := localFileSize / matcherCount + sectionSize += int64(blocksize) - (sectionSize % int64(blocksize)) + merger := &comparer.MatchMerger{} + compare := &comparer.Comparer{} + + for i := int64(0); i < matcherCount; i++ { + offset := sectionSize * i + + // Sections must overlap by blocksize (strictly blocksize - 1?) + if i > 0 { + offset -= int64(blocksize) + } + + sectionReader := bufio.NewReaderSize( + io.NewSectionReader(localFile, offset, sectionSize), + MB, + ) + + sectionGenerator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) + + matchStream := compare.StartFindMatchingBlocks( + sectionReader, offset, sectionGenerator, idx) + + merger.StartMergeResultStream(matchStream, int64(blocksize)) + } + + return merger, compare +} + +// better way to do this? diff --git a/vendor/github.com/Redundancy/go-sync/gosync/diff.go b/vendor/github.com/Redundancy/go-sync/gosync/diff.go new file mode 100644 index 0000000..14a4acf --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync/diff.go @@ -0,0 +1,145 @@ +package main + +import ( + "fmt" + "os" + "runtime" + "time" + + "github.com/codegangsta/cli" +) + +func init() { + app.Commands = append( + app.Commands, + cli.Command{ + Name: "diff", + ShortName: "d", + Usage: "gosync diff ", + Description: `Compare a file with a reference index, and print statistics on the comparison and performance.`, + Action: Diff, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "p", + Value: runtime.NumCPU(), + Usage: "The number of streams to use concurrently", + }, + }, + }, + ) +} + +func Diff(c *cli.Context) { + localFilename := c.Args()[0] + referenceFilename := c.Args()[1] + startTime := time.Now() + + localFile := openFileAndHandleError(localFilename) + + if localFile == nil { + os.Exit(1) + } + + defer localFile.Close() + + var blocksize uint32 + referenceFile := openFileAndHandleError(referenceFilename) + + if referenceFile == nil { + os.Exit(1) + } + + defer referenceFile.Close() + + _, _, _, _, blocksize, e := readHeadersAndCheck( + referenceFile, + magicString, + majorVersion, + ) + + if e != nil { + fmt.Printf("Error loading index: %v", e) + os.Exit(1) + } + + fmt.Println("Blocksize: ", blocksize) + + index, _, _, err := readIndex(referenceFile, uint(blocksize)) + referenceFile.Close() + + if err != nil { + return + } + + fmt.Println("Weak hash count:", index.WeakCount()) + + fi, err := localFile.Stat() + + if err != nil { + fmt.Println("Could not get info on file:", err) + os.Exit(1) + } + + num_matchers := int64(c.Int("p")) + + localFile_size := fi.Size() + + // Don't split up small files + if localFile_size < 1024*1024 { + num_matchers = 1 + } + + merger, compare := multithreadedMatching( + localFile, + index, + localFile_size, + num_matchers, + uint(blocksize), + ) + + mergedBlocks := merger.GetMergedBlocks() + + fmt.Println("\nMatched:") + totalMatchingSize := uint64(0) + matchedBlockCountAfterMerging := uint(0) + + for _, b := range mergedBlocks { + totalMatchingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) + matchedBlockCountAfterMerging += b.EndBlock - b.StartBlock + 1 + } + + fmt.Println("Comparisons:", compare.Comparisons) + fmt.Println("Weak hash hits:", compare.WeakHashHits) + + if compare.Comparisons > 0 { + fmt.Printf( + "Weak hit rate: %.2f%%\n", + 100.0*float64(compare.WeakHashHits)/float64(compare.Comparisons), + ) + } + + fmt.Println("Strong hash hits:", compare.StrongHashHits) + if compare.WeakHashHits > 0 { + fmt.Printf( + "Weak hash error rate: %.2f%%\n", + 100.0*float64(compare.WeakHashHits-compare.StrongHashHits)/float64(compare.WeakHashHits), + ) + } + + fmt.Println("Total matched bytes:", totalMatchingSize) + fmt.Println("Total matched blocks:", matchedBlockCountAfterMerging) + + // TODO: GetMissingBlocks uses the highest index, not the count, this can be pretty confusing + // Should clean up this interface to avoid that + missing := mergedBlocks.GetMissingBlocks(uint(index.BlockCount) - 1) + fmt.Println("Index blocks:", index.BlockCount) + + totalMissingSize := uint64(0) + for _, b := range missing { + //fmt.Printf("%#v\n", b) + totalMissingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) + } + + fmt.Println("Approximate missing bytes:", totalMissingSize) + fmt.Println("Time taken:", time.Now().Sub(startTime)) +} diff --git a/vendor/github.com/Redundancy/go-sync/gosync/fileformat.md b/vendor/github.com/Redundancy/go-sync/gosync/fileformat.md new file mode 100644 index 0000000..0fd5522 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync/fileformat.md @@ -0,0 +1,21 @@ +*NB: I'm documenting the current format of the gosync files merely as a point in time +reference of format that is in use in the tool that is meant to serve as a practical reference and acceptance testing tool. The gosync tool is not intended as a production-worthy, well supported, tested tool.* + +*The format used exists entirely in service of being able to test the implementation of the gosync library as a cohesive whole in the real world, and therefore backwards and forwards compatibility (or even efficiency) are not primary concerns.* + +# Version 0.2.0 +### The header +(LE = little endian) +* The string "G0S9NC" in UTF-8 +* versions*3 (eg. 0.1.2), uint16 LE +* filesize, int64 LE +* blocksize uint32 LE + +### The body +Repeating: +* WeakChecksum +* StrongChecksum + +each referring to blocks, starting at 0 (file start) and going upwards. + +In the current implementation of the FileChecksumGenerator used the WeakChecksum is the rolling checksum (4 bytes), and StrongChecksum is MD5 (16 bytes). diff --git a/vendor/github.com/Redundancy/go-sync/gosync/main.go b/vendor/github.com/Redundancy/go-sync/gosync/main.go new file mode 100644 index 0000000..3164a8d --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync/main.go @@ -0,0 +1,65 @@ +/* +gosync is a command-line implementation of the gosync package functionality, primarily as a demonstration of usage +but supposed to be functional in itself. +*/ +package main + +import ( + "fmt" + "log" + "net/http" + _ "net/http/pprof" + "os" + "runtime" + + "github.com/codegangsta/cli" +) + +const ( + DefaultBlockSize = 8192 + magicString = "G0S9NC" // just to confirm the file type is used correctly + majorVersion = uint16(0) + minorVersion = uint16(2) + patchVersion = uint16(1) +) + +var app = cli.NewApp() + +func main() { + app.Name = "gosync" + app.Usage = "Build indexes, patches, patch files" + app.Flags = []cli.Flag{ + cli.BoolFlag{ + Name: "profile", + Usage: "enable HTTP profiling", + }, + cli.IntFlag{ + Name: "profilePort", + Value: 6060, + Usage: "The number of streams to use concurrently", + }, + } + + app.Version = fmt.Sprintf( + "%v.%v.%v", + majorVersion, + minorVersion, + patchVersion, + ) + + runtime.GOMAXPROCS(runtime.NumCPU()) + + app.Before = func(c *cli.Context) error { + if c.Bool("profile") { + port := fmt.Sprint(c.Int("profilePort")) + + go func() { + log.Println(http.ListenAndServe("localhost:"+port, nil)) + }() + } + + return nil + } + + app.Run(os.Args) +} diff --git a/vendor/github.com/Redundancy/go-sync/gosync/patch.go b/vendor/github.com/Redundancy/go-sync/gosync/patch.go new file mode 100644 index 0000000..16730d3 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync/patch.go @@ -0,0 +1,105 @@ +package main + +import ( + "fmt" + "os" + "runtime" + + gosync_main "github.com/Redundancy/go-sync" + "github.com/codegangsta/cli" +) + +const usage = "gosync patch []" + +func init() { + app.Commands = append( + app.Commands, + cli.Command{ + Name: "patch", + ShortName: "p", + Usage: usage, + Description: `Recreate the reference source file, using an index and a local file that is believed to be similar. +The index should be produced by "gosync build". + + is a .gosync file and may be a local, unc network path or http/https url + is corresponding target and may be a local, unc network path or http/https url + is optional. If not specified, the local file will be overwritten when done.`, + Action: Patch, + Flags: []cli.Flag{ + cli.IntFlag{ + Name: "p", + Value: runtime.NumCPU(), + Usage: "The number of streams to use concurrently", + }, + }, + }, + ) +} + +// Patch a file +func Patch(c *cli.Context) { + errorWrapper(c, func(c *cli.Context) error { + + fmt.Fprintln(os.Stderr, "Starting patching process") + + if l := len(c.Args()); l < 3 || l > 4 { + return fmt.Errorf( + "Usage is \"%v\" (invalid number of arguments)", + usage, + ) + } + + localFilename := c.Args()[0] + summaryFile := c.Args()[1] + referencePath := c.Args()[2] + + outFilename := localFilename + if len(c.Args()) == 4 { + outFilename = c.Args()[3] + } + + indexReader, e := os.Open(summaryFile) + if e != nil { + return e + } + defer indexReader.Close() + + _, _, _, filesize, blocksize, e := readHeadersAndCheck( + indexReader, + magicString, + majorVersion, + ) + + index, checksumLookup, blockCount, err := readIndex( + indexReader, + uint(blocksize), + ) + + fs := &gosync_main.BasicSummary{ + ChecksumIndex: index, + ChecksumLookup: checksumLookup, + BlockCount: blockCount, + BlockSize: uint(blocksize), + FileSize: filesize, + } + + rsync, err := gosync_main.MakeRSync( + localFilename, + referencePath, + outFilename, + fs, + ) + + if err != nil { + return err + } + + err = rsync.Patch() + + if err != nil { + return err + } + + return rsync.Close() + }) +} diff --git a/vendor/github.com/Redundancy/go-sync/gosync_test.go b/vendor/github.com/Redundancy/go-sync/gosync_test.go new file mode 100644 index 0000000..72195db --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/gosync_test.go @@ -0,0 +1,111 @@ +package gosync + +import ( + "fmt" + "testing" + + "bytes" + + "github.com/Redundancy/go-sync/blocksources" + "github.com/Redundancy/go-sync/comparer" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/indexbuilder" + "github.com/Redundancy/go-sync/util/readers" +) + +func Example() { + // due to short example strings, use a very small block size + // using one this small in practice would increase your file transfer! + const blockSize = 4 + + // This is the "file" as described by the authoritive version + const reference = "The quick brown fox jumped over the lazy dog" + + // This is what we have locally. Not too far off, but not correct. + const localVersion = "The qwik brown fox jumped 0v3r the lazy" + + generator := filechecksum.NewFileChecksumGenerator(blockSize) + _, referenceFileIndex, _, err := indexbuilder.BuildIndexFromString( + generator, + reference, + ) + + if err != nil { + return + } + + referenceAsBytes := []byte(reference) + localVersionAsBytes := []byte(localVersion) + + blockCount := len(referenceAsBytes) / blockSize + if len(referenceAsBytes)%blockSize != 0 { + blockCount++ + } + + inputFile := bytes.NewReader(localVersionAsBytes) + patchedFile := bytes.NewBuffer(nil) + + // This is more complicated than usual, because we're using in-memory + // "files" and sources. Normally you would use MakeRSync + summary := &BasicSummary{ + ChecksumIndex: referenceFileIndex, + ChecksumLookup: nil, + BlockCount: uint(blockCount), + BlockSize: blockSize, + FileSize: int64(len(referenceAsBytes)), + } + + rsync := &RSync{ + Input: inputFile, + Output: patchedFile, + Source: blocksources.NewReadSeekerBlockSource( + bytes.NewReader(referenceAsBytes), + blocksources.MakeNullFixedSizeResolver(uint64(blockSize)), + ), + Summary: summary, + OnClose: nil, + } + + if err := rsync.Patch(); err != nil { + fmt.Printf("Error: %v", err) + return + } + + fmt.Printf("Patched result: \"%s\"\n", patchedFile.Bytes()) + // Output: + // Patched result: "The quick brown fox jumped over the lazy dog" +} + +const ( + BYTE = 1 + KB = 1024 * BYTE + MB = 1024 * KB +) + +func BenchmarkIndexComparisons(b *testing.B) { + b.ReportAllocs() + + const SIZE = 200 * KB + b.SetBytes(SIZE) + + file := readers.NewSizedNonRepeatingSequence(6, SIZE) + generator := filechecksum.NewFileChecksumGenerator(8 * KB) + _, index, _, err := indexbuilder.BuildChecksumIndex(generator, file) + + if err != nil { + b.Fatal(err) + } + + b.StartTimer() + for i := 0; i < b.N; i++ { + // must reinitialize the file for each comparison + otherFile := readers.NewSizedNonRepeatingSequence(745656, SIZE) + compare := &comparer.Comparer{} + m := compare.StartFindMatchingBlocks(otherFile, 0, generator, index) + + for _, ok := <-m; ok; { + } + } + + b.StopTimer() +} diff --git a/vendor/github.com/Redundancy/go-sync/http_test.go b/vendor/github.com/Redundancy/go-sync/http_test.go new file mode 100644 index 0000000..6d7c047 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/http_test.go @@ -0,0 +1,174 @@ +package gosync + +import ( + "bytes" + "crypto/md5" + "fmt" + "net" + "net/http" + "time" + + "github.com/Redundancy/go-sync/blocksources" + "github.com/Redundancy/go-sync/comparer" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/indexbuilder" + "github.com/Redundancy/go-sync/patcher" +) + +// due to short example strings, use a very small block size +// using one this small in practice would increase your file transfer! +const BLOCK_SIZE = 4 + +// This is the "file" as described by the authoritive version +const REFERENCE = "The quick brown fox jumped over the lazy dog" + +// This is what we have locally. Not too far off, but not correct. +const LOCAL_VERSION = "The qwik brown fox jumped 0v3r the lazy" + +var content = bytes.NewReader([]byte(REFERENCE)) + +func handler(w http.ResponseWriter, req *http.Request) { + http.ServeContent(w, req, "", time.Now(), content) +} + +// set up a http server locally that will respond predictably to ranged requests +func setupServer() <-chan int { + var PORT = 8000 + s := http.NewServeMux() + s.HandleFunc("/content", handler) + + portChan := make(chan int) + + go func() { + var listener net.Listener + var err error + + for { + PORT++ + p := fmt.Sprintf(":%v", PORT) + listener, err = net.Listen("tcp", p) + + if err == nil { + break + } + } + portChan <- PORT + http.Serve(listener, s) + }() + + return portChan +} + +// This is exceedingly similar to the module Example, but uses the http blocksource and a local http server +func Example_httpBlockSource() { + PORT := <-setupServer() + LOCAL_URL := fmt.Sprintf("http://localhost:%v/content", PORT) + + generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) + _, referenceFileIndex, checksumLookup, err := indexbuilder.BuildIndexFromString(generator, REFERENCE) + + if err != nil { + return + } + + fileSize := int64(len([]byte(REFERENCE))) + + // This would normally be saved in a file + + blockCount := fileSize / BLOCK_SIZE + if fileSize%BLOCK_SIZE != 0 { + blockCount++ + } + + fs := &BasicSummary{ + ChecksumIndex: referenceFileIndex, + ChecksumLookup: checksumLookup, + BlockCount: uint(blockCount), + BlockSize: uint(BLOCK_SIZE), + FileSize: fileSize, + } + + /* + // Normally, this would be: + rsync, err := MakeRSync( + "toPatch.file", + "http://localhost/content", + "out.file", + fs, + ) + */ + // Need to replace the output and the input + inputFile := bytes.NewReader([]byte(LOCAL_VERSION)) + patchedFile := bytes.NewBuffer(nil) + + resolver := blocksources.MakeFileSizedBlockResolver( + uint64(fs.GetBlockSize()), + fs.GetFileSize(), + ) + + rsync := &RSync{ + Input: inputFile, + Output: patchedFile, + Source: blocksources.NewHttpBlockSource( + LOCAL_URL, + 1, + resolver, + &filechecksum.HashVerifier{ + Hash: md5.New(), + BlockSize: fs.GetBlockSize(), + BlockChecksumGetter: fs, + }, + ), + Summary: fs, + OnClose: nil, + } + + err = rsync.Patch() + + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + + err = rsync.Close() + + if err != nil { + fmt.Printf("Error: %v\n", err) + return + } + + fmt.Printf("Patched content: \"%v\"\n", patchedFile.String()) + + // Just for inspection + remoteReferenceSource := rsync.Source.(*blocksources.BlockSourceBase) + fmt.Printf("Downloaded Bytes: %v\n", remoteReferenceSource.ReadBytes()) + + // Output: + // Patched content: "The quick brown fox jumped over the lazy dog" + // Downloaded Bytes: 16 +} + +func ToPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { + result := make([]patcher.FoundBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].MatchOffset = v.ComparisonStartOffset + result[i].BlockSize = blockSize + } + + return result +} + +func ToPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { + result := make([]patcher.MissingBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].BlockSize = blockSize + } + + return result +} diff --git a/vendor/github.com/Redundancy/go-sync/index/index.go b/vendor/github.com/Redundancy/go-sync/index/index.go new file mode 100644 index 0000000..6711024 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/index/index.go @@ -0,0 +1,183 @@ +/* +Package index provides the functionality to describe a reference 'file' and its contents in terms of +the weak and strong checksums, in such a way that you can check if a weak checksum is present, +then check if there is a strong checksum that matches. + +It also allows lookups in terms of block offsets, so that upon finding a match, you can more efficiently +check if the next block follows it. + +The index structure does not lend itself to being an interface - the pattern of taking the result of looking for +the weak checksum and looking up the strong checksum in that requires us to return an object matching an interface which +both packages must know about. + +Here's the interface: + +type Index interface { + FindWeakChecksum(chk []byte) interface{} + FindStrongChecksum(chk []byte, weak interface{}) []chunks.ChunkChecksum +} + +This allows the implementation to rely on a previously generated value, without the users knowing what it is. +This breaks the dependency that requires so many packages to import index. +*/ +package index + +import ( + "bytes" + "encoding/binary" + "github.com/Redundancy/go-sync/chunks" + "sort" +) + +type ChecksumIndex struct { + BlockCount int + + /* + This datastructure is based on some benchmarking that indicates that it outperforms + a basic map 70ns vs 130ns for ~8192 checksums (which is reasonably large - + say 64 MB with no weak collisions @8192 bytes per block). + + We use a 256 element slice, and the value of the least significant byte to determine + which map to look up into. + */ + weakChecksumLookup []map[uint32]StrongChecksumList + + MaxStrongLength int + AverageStrongLength float32 + Count int +} + +// Builds an index in which chunks can be found, with their corresponding offsets +// We use this for the +func MakeChecksumIndex(checksums []chunks.ChunkChecksum) *ChecksumIndex { + n := &ChecksumIndex{ + BlockCount: len(checksums), + weakChecksumLookup: make([]map[uint32]StrongChecksumList, 256), + } + + for _, chunk := range checksums { + weakChecksumAsInt := binary.LittleEndian.Uint32(chunk.WeakChecksum) + arrayOffset := weakChecksumAsInt & 255 + + if n.weakChecksumLookup[arrayOffset] == nil { + n.weakChecksumLookup[arrayOffset] = make(map[uint32]StrongChecksumList) + } + + n.weakChecksumLookup[arrayOffset][weakChecksumAsInt] = append( + n.weakChecksumLookup[arrayOffset][weakChecksumAsInt], + chunk, + ) + + } + + sum := 0 + count := 0 + + for _, a := range n.weakChecksumLookup { + for _, c := range a { + sort.Sort(c) + if len(c) > n.MaxStrongLength { + n.MaxStrongLength = len(c) + } + sum += len(c) + count += 1 + n.Count += len(c) + } + } + + n.AverageStrongLength = float32(sum) / float32(count) + + return n +} + +func (index *ChecksumIndex) WeakCount() int { + return index.Count +} + +func (index *ChecksumIndex) FindWeakChecksumInIndex(weak []byte) StrongChecksumList { + x := binary.LittleEndian.Uint32(weak) + if index.weakChecksumLookup[x&255] != nil { + if v, ok := index.weakChecksumLookup[x&255][x]; ok { + return v + } + } + return nil +} + +func (index *ChecksumIndex) FindWeakChecksum2(chk []byte) interface{} { + w := index.FindWeakChecksumInIndex(chk) + + if len(w) == 0 { + return nil + } else { + return w + } +} + +func (index *ChecksumIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { + if strongList, ok := weak.(StrongChecksumList); ok { + return strongList.FindStrongChecksum(chk) + } else { + return nil + } +} + +type StrongChecksumList []chunks.ChunkChecksum + +// Sortable interface +func (s StrongChecksumList) Len() int { + return len(s) +} + +// Sortable interface +func (s StrongChecksumList) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +// Sortable interface +func (s StrongChecksumList) Less(i, j int) bool { + return bytes.Compare(s[i].StrongChecksum, s[j].StrongChecksum) == -1 +} + +func (s StrongChecksumList) FindStrongChecksum(strong []byte) (result []chunks.ChunkChecksum) { + n := len(s) + + // average length is 1, so fast path comparison + if n == 1 { + if bytes.Compare(s[0].StrongChecksum, strong) == 0 { + return s + } else { + return nil + } + } + + // find the first possible occurance + first_gte_checksum := sort.Search( + n, + func(i int) bool { + return bytes.Compare(s[i].StrongChecksum, strong) >= 0 + }, + ) + + // out of bounds + if first_gte_checksum == -1 || first_gte_checksum == n { + return nil + } + + // Somewhere in the middle, but the next one didn't match + if bytes.Compare(s[first_gte_checksum].StrongChecksum, strong) != 0 { + return nil + } + + end := first_gte_checksum + 1 + for end < n { + if bytes.Compare(s[end].StrongChecksum, strong) == 0 { + end += 1 + } else { + break + } + + } + + return s[first_gte_checksum:end] +} diff --git a/vendor/github.com/Redundancy/go-sync/index/index_bench_test.go b/vendor/github.com/Redundancy/go-sync/index/index_bench_test.go new file mode 100644 index 0000000..aace8a1 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/index/index_bench_test.go @@ -0,0 +1,148 @@ +package index + +import ( + "github.com/Redundancy/go-sync/chunks" + "math/rand" + "sort" + "testing" +) + +var T = []byte{1, 2, 3, 4} + +func BenchmarkIndex1024(b *testing.B) { + i := ChecksumIndex{} + i.weakChecksumLookup = make([]map[uint32]StrongChecksumList, 256) + + for x := 0; x < 1024; x++ { + w := rand.Uint32() + + if i.weakChecksumLookup[w&255] == nil { + i.weakChecksumLookup[w&255] = make(map[uint32]StrongChecksumList) + } + + i.weakChecksumLookup[w&255][w] = append( + i.weakChecksumLookup[w&255][w], + chunks.ChunkChecksum{}, + ) + } + + b.SetBytes(1) + b.StartTimer() + for x := 0; x < b.N; x++ { + i.FindWeakChecksum2(T) + } + b.StopTimer() + +} + +func BenchmarkIndex8192(b *testing.B) { + i := ChecksumIndex{} + i.weakChecksumLookup = make([]map[uint32]StrongChecksumList, 256) + + for x := 0; x < 8192; x++ { + w := rand.Uint32() + + if i.weakChecksumLookup[w&255] == nil { + i.weakChecksumLookup[w&255] = make(map[uint32]StrongChecksumList) + } + + i.weakChecksumLookup[w&255][w] = append( + i.weakChecksumLookup[w&255][w], + chunks.ChunkChecksum{}, + ) + } + + b.SetBytes(1) + b.StartTimer() + for x := 0; x < b.N; x++ { + i.FindWeakChecksum2(T) + } + b.StopTimer() +} + +// Check how fast a sorted list of 8192 items would be +func BenchmarkIndexAsListBinarySearch8192(b *testing.B) { + b.SkipNow() + + s := make([]int, 8192) + for x := 0; x < 8192; x++ { + s[x] = rand.Int() + } + + sort.Ints(s) + + b.StartTimer() + for x := 0; x < b.N; x++ { + sort.SearchInts(s, rand.Int()) + } + b.StopTimer() +} + +// Check how fast a sorted list of 8192 items would be +// Checking for cache coherency gains +func BenchmarkIndexAsListLinearSearch8192(b *testing.B) { + s := make([]int, 8192) + for x := 0; x < 8192; x++ { + s[x] = rand.Int() + } + + sort.Ints(s) + + l := len(s) + b.StartTimer() + for x := 0; x < b.N; x++ { + v := rand.Int() + for i := 0; i < l; i++ { + if v < s[i] { + break + } + } + } + b.StopTimer() +} + +func Benchmark_256SplitBinarySearch(b *testing.B) { + a := make([][]int, 256) + for x := 0; x < 8192; x++ { + i := rand.Int() + a[i&255] = append( + a[i&255], + i, + ) + } + + for x := 0; x < 256; x++ { + sort.Ints(a[x]) + } + + b.StartTimer() + for x := 0; x < b.N; x++ { + v := rand.Int() + sort.SearchInts(a[v&255], v) + } + b.StopTimer() +} + +/* +This is currently the best performing contender for the index data structure for +weak checksum lookups. +*/ +func Benchmark_256Split_Map(b *testing.B) { + a := make([]map[int]interface{}, 256) + for x := 0; x < 8192; x++ { + i := rand.Int() + if a[i&255] == nil { + a[i&255] = make(map[int]interface{}) + } + a[i&255][i] = nil + } + + b.StartTimer() + for x := 0; x < b.N; x++ { + v := rand.Int() + if _, ok := a[v&255][v]; ok { + + } + } + b.StopTimer() +} diff --git a/vendor/github.com/Redundancy/go-sync/index/index_test.go b/vendor/github.com/Redundancy/go-sync/index/index_test.go new file mode 100644 index 0000000..3336134 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/index/index_test.go @@ -0,0 +1,206 @@ +package index + +import ( + "testing" + + "github.com/Redundancy/go-sync/chunks" +) + +// Weak checksums must be 4 bytes +var WEAK_A = []byte("aaaa") +var WEAK_B = []byte("bbbb") + +/* +ChunkOffset uint +// the size of the block +Size int64 +WeakChecksum []byte +StrongChecksum []byte +*/ + +func TestMakeIndex(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + }, + ) + + if i.Count != 2 { + t.Fatalf("Wrong count on index %v", i.Count) + } +} + +func TestFindWeakInIndex(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + }, + ) + + result := i.FindWeakChecksumInIndex(WEAK_B) + + if result == nil { + t.Error("Did not find lookfor in the index") + } else if len(result) != 2 { + t.Errorf("Wrong number of possible matches found: %v", len(result)) + } else if result[0].ChunkOffset != 1 { + t.Errorf("Found chunk had offset %v expected 1", result[0].ChunkOffset) + } +} + +func TestWeakNotInIndex(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + }, + ) + + result := i.FindWeakChecksumInIndex([]byte("afgh")) + + if result != nil { + t.Error("Result from FindWeakChecksumInIndex should be nil") + } + + result2 := i.FindWeakChecksum2([]byte("afgh")) + + if result2 != nil { + t.Errorf("Result from FindWeakChecksum2 should be nil: %#v", result2) + } +} + +func TestWeakNotInIndex2(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + }, + ) + + result := i.FindWeakChecksumInIndex([]byte("llll")) + + if result != nil { + t.Error("Result should be nil") + } +} + +func TestFindStrongInIndex(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + }, + ) + + // builds upon TestFindWeakInIndex + result := i.FindWeakChecksumInIndex(WEAK_B) + strongs := result.FindStrongChecksum([]byte("c")) + + if len(strongs) != 1 { + t.Errorf("Incorrect number of strong checksums found: %v", len(strongs)) + } else if strongs[0].ChunkOffset != 1 { + t.Errorf("Wrong chunk found, had offset %v", strongs[0].ChunkOffset) + } +} + +func TestNotFoundStrongInIndexAtEnd(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + }, + ) + + // builds upon TestFindWeakInIndex + result := i.FindWeakChecksumInIndex(WEAK_B) + strongs := result.FindStrongChecksum([]byte("e")) + + if len(strongs) != 0 { + t.Errorf("Incorrect number of strong checksums found: %v", strongs) + } +} + +func TestNotFoundStrongInIndexInCenter(t *testing.T) { + // The strong checksum we're looking for is not found + // but is < another checksum in the strong list + + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + {ChunkOffset: 3, WeakChecksum: WEAK_B, StrongChecksum: []byte("f")}, + }, + ) + + // builds upon TestFindWeakInIndex + result := i.FindWeakChecksumInIndex(WEAK_B) + strongs := result.FindStrongChecksum([]byte("e")) + + if len(strongs) != 0 { + t.Errorf("Incorrect number of strong checksums found: %v", strongs) + } +} + +func TestFindDuplicatedBlocksInIndex(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 3, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, + }, + ) + + // builds upon TestFindWeakInIndex + result := i.FindWeakChecksumInIndex(WEAK_B) + strongs := result.FindStrongChecksum([]byte("c")) + + if len(strongs) != 2 { + t.Fatalf("Incorrect number of strong checksums found: %v", strongs) + } + + first := strongs[0] + if first.ChunkOffset != 1 { + t.Errorf("Wrong chunk found, had offset %v", first.ChunkOffset) + } + + second := strongs[1] + if second.ChunkOffset != 3 { + t.Errorf("Wrong chunk found, had offset %v", second.ChunkOffset) + } +} + +func TestFindTwoDuplicatedBlocksInIndex(t *testing.T) { + i := MakeChecksumIndex( + []chunks.ChunkChecksum{ + {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, + }, + ) + + // builds upon TestFindWeakInIndex + result := i.FindWeakChecksumInIndex(WEAK_B) + strongs := result.FindStrongChecksum([]byte("c")) + + if len(strongs) != 2 { + t.Fatalf("Incorrect number of strong checksums found: %v", strongs) + } + + first := strongs[0] + if first.ChunkOffset != 1 { + t.Errorf("Wrong chunk found, had offset %v", first.ChunkOffset) + } + + second := strongs[1] + if second.ChunkOffset != 2 { + t.Errorf("Wrong chunk found, had offset %v", second.ChunkOffset) + } +} diff --git a/vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go b/vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go new file mode 100644 index 0000000..8d2ebef --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go @@ -0,0 +1,55 @@ +/* +Package indexbuilder provides a few shortbuts to building a checksum index by generating and then loading +the checksums, and building an index from that. It's potentially a sign that the responsibilities here need refactoring. +*/ +package indexbuilder + +import ( + "bytes" + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/index" + "io" +) + +// Generates an index from a reader +// This is mostly a utility function to avoid being overly verbose in tests that need +// an index to work, but don't want to construct one by hand in order to avoid the dependencies +// obviously this means that those tests are likely to fail if there are issues with any of the other +// modules, which is not ideal. +// TODO: move to util? +func BuildChecksumIndex(check *filechecksum.FileChecksumGenerator, r io.Reader) ( + fcheck []byte, + i *index.ChecksumIndex, + lookup filechecksum.ChecksumLookup, + err error, +) { + b := bytes.NewBuffer(nil) + fcheck, err = check.GenerateChecksums(r, b) + + if err != nil { + return + } + + weakSize := check.WeakRollingHash.Size() + strongSize := check.GetStrongHash().Size() + readChunks, err := chunks.LoadChecksumsFromReader(b, weakSize, strongSize) + + if err != nil { + return + } + + i = index.MakeChecksumIndex(readChunks) + lookup = chunks.StrongChecksumGetter(readChunks) + + return +} + +func BuildIndexFromString(generator *filechecksum.FileChecksumGenerator, reference string) ( + fileCheckSum []byte, + referenceIndex *index.ChecksumIndex, + lookup filechecksum.ChecksumLookup, + err error, +) { + return BuildChecksumIndex(generator, bytes.NewBufferString(reference)) +} diff --git a/vendor/github.com/Redundancy/go-sync/patcher/blocksource.go b/vendor/github.com/Redundancy/go-sync/patcher/blocksource.go new file mode 100644 index 0000000..c50241d --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/patcher/blocksource.go @@ -0,0 +1,58 @@ +/* +Package patcher follows a pattern established by hash, which defines the interface in the top level package, and then provides implementations +below it. +*/ +package patcher + +import ( + "hash" +) + +/* +BlockSource is an interface used by the patchers to obtain blocks from the reference +It does not stipulate where the reference data might be (it could be local, in a pre-built patch file, on S3 or somewhere else) + +It is assumed that the BlockSource may be slow, and may benefit from request pipelining & concurrency. +Therefore patchers should feel free to request as many block spans as they can handle. + +A BlockSource may be a view onto a larger transport concept, so that multiple files can be handled with wider +knowledge of the number of simultaneous requests allowed, etc. The BlockSource may decide to split BlockSpans +into smaller sizes if it wants. + +It is up to the patcher to receive blocks in a timely manner, and decide what to do with them, rather than +bother the BlockSource with more memory management and buffering logic. + +Since these interfaces require specific structs to satisfy, it's expected that implementers will import this module. + +*/ +type BlockSource interface { + RequestBlocks(MissingBlockSpan) error + + GetResultChannel() <-chan BlockReponse + + // If the block source encounters an unsurmountable problem + EncounteredError() <-chan error +} + +type FoundBlockSpan struct { + StartBlock uint + EndBlock uint + BlockSize int64 + MatchOffset int64 +} + +type MissingBlockSpan struct { + StartBlock uint + EndBlock uint + + BlockSize int64 + // a hasher to use to ensure that the block response matches + Hasher hash.Hash + // the hash values that the blocks should have + ExpectedSums [][]byte +} + +type BlockReponse struct { + StartBlock uint + Data []byte +} diff --git a/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go b/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go new file mode 100644 index 0000000..b8a3018 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go @@ -0,0 +1,145 @@ +/* +Sequential Patcher will stream the patched version of the file to output, +since it works strictly in order, it cannot patch the local file directly +(since it might overwrite a block needed later), +so there would have to be a final copy once the patching was done. +*/ +package sequential + +import ( + "fmt" + "github.com/Redundancy/go-sync/patcher" + "io" +) + +const ( + ABSOLUTE_POSITION = 0 +) + +/* +This simple example currently doesn't do any pipelining of needed blocks, nor does it deal with +blocks being delivered out of order. +*/ +func SequentialPatcher( + localFile io.ReadSeeker, + reference patcher.BlockSource, + requiredRemoteBlocks []patcher.MissingBlockSpan, + locallyAvailableBlocks []patcher.FoundBlockSpan, + maxBlockStorage uint64, // the amount of memory we're allowed to use for temporary data storage + output io.Writer, +) error { + + maxBlockMissing := uint(0) + if len(requiredRemoteBlocks) > 0 { + maxBlockMissing = requiredRemoteBlocks[len(requiredRemoteBlocks)-1].EndBlock + } + + maxBlockFound := uint(0) + if len(locallyAvailableBlocks) > 0 { + maxBlockFound = locallyAvailableBlocks[len(locallyAvailableBlocks)-1].EndBlock + } + + if reference == nil { + return fmt.Errorf("No BlockSource set for obtaining reference blocks") + } + + maxBlock := maxBlockMissing + if maxBlockFound > maxBlock { + maxBlock = maxBlockFound + } + + currentBlock := uint(0) + + // TODO: find a way to test this, since it seemed to be the cause of an issue + for currentBlock <= maxBlock { + // where is the next block supposed to come from? + if withinFirstBlockOfLocalBlocks(currentBlock, locallyAvailableBlocks) { + firstMatched := locallyAvailableBlocks[0] + + // we have the current block range in the local file + localFile.Seek(firstMatched.MatchOffset, ABSOLUTE_POSITION) + blockSizeToRead := int64(firstMatched.EndBlock-firstMatched.StartBlock+1) * firstMatched.BlockSize + + if _, err := io.Copy(output, io.LimitReader(localFile, blockSizeToRead)); err != nil { + return fmt.Errorf("Could not copy %v bytes to output: %v", blockSizeToRead, err) + } + + currentBlock = firstMatched.EndBlock + 1 + locallyAvailableBlocks = locallyAvailableBlocks[1:] + + } else if withinFirstBlockOfRemoteBlocks(currentBlock, requiredRemoteBlocks) { + firstMissing := requiredRemoteBlocks[0] + reference.RequestBlocks(firstMissing) + + select { + case result := <-reference.GetResultChannel(): + if result.StartBlock == currentBlock { + if _, err := output.Write(result.Data); err != nil { + return fmt.Errorf( + "Could not write data to output: %v", + err, + ) + } else { + + completed := calculateNumberOfCompletedBlocks( + len(result.Data), + firstMissing.BlockSize, + ) + + if completed != (firstMissing.EndBlock-firstMissing.StartBlock) + 1 { + return fmt.Errorf( + "Unexpected reponse length from remote source: blocks %v-%v (got %v blocks)", + firstMissing.StartBlock, + firstMissing.EndBlock, + completed, + ) + } + + currentBlock += completed + requiredRemoteBlocks = requiredRemoteBlocks[1:] + } + } else { + return fmt.Errorf( + "Received unexpected block: %v", + result.StartBlock, + ) + } + case err := <-reference.EncounteredError(): + return fmt.Errorf( + "Failed to read from reference file: %v", + err, + ) + } + + } else { + return fmt.Errorf( + "Could not find block in missing or matched list: %v\nRemote: %v\nLocal: %v\n", + currentBlock, + requiredRemoteBlocks, + locallyAvailableBlocks, + ) + } + } + + return nil +} + +func withinFirstBlockOfRemoteBlocks(currentBlock uint, remoteBlocks []patcher.MissingBlockSpan) bool { + return len(remoteBlocks) > 0 && remoteBlocks[0].StartBlock <= currentBlock && remoteBlocks[0].EndBlock >= currentBlock +} + +func withinFirstBlockOfLocalBlocks(currentBlock uint, localBlocks []patcher.FoundBlockSpan) bool { + return len(localBlocks) > 0 && localBlocks[0].StartBlock <= currentBlock && localBlocks[0].EndBlock >= currentBlock +} + +func calculateNumberOfCompletedBlocks(resultLength int, blockSize int64) (completedBlockCount uint) { + // TODO: lots of casting to uint here, is it safe? + completedBlockCount = uint(resultLength) / uint(blockSize) + + // round up in the case of a partial block (last block may not be full sized) + if uint(resultLength)%uint(blockSize) != 0 { + completedBlockCount += 1 + } + + return +} diff --git a/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go b/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go new file mode 100644 index 0000000..7608f8b --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go @@ -0,0 +1,188 @@ +package sequential + +import ( + "bytes" + "crypto/md5" + "github.com/Redundancy/go-sync/blocksources" + "github.com/Redundancy/go-sync/patcher" + "io" + "io/ioutil" + "strings" + "testing" +) + +const ( + BLOCKSIZE = 4 + REFERENCE_STRING = "The quick brown fox jumped over the lazy dog" +) + +var ( + REFERENCE_BUFFER = bytes.NewBufferString(REFERENCE_STRING) + REFERENCE_BLOCKS []string + BLOCK_COUNT int + REFERENCE_HASHES [][]byte +) + +func init() { + maxLen := len(REFERENCE_STRING) + m := md5.New() + for i := 0; i < maxLen; i += BLOCKSIZE { + last := i + 4 + + if last >= maxLen { + last = maxLen - 1 + } + + block := REFERENCE_STRING[i:last] + + REFERENCE_BLOCKS = append(REFERENCE_BLOCKS, block) + m.Write([]byte(block)) + REFERENCE_HASHES = append(REFERENCE_HASHES, m.Sum(nil)) + m.Reset() + } + + BLOCK_COUNT = len(REFERENCE_BLOCKS) +} + +func stringToReadSeeker(input string) io.ReadSeeker { + return bytes.NewReader([]byte(input)) +} + +func TestPatchingStart(t *testing.T) { + LOCAL := bytes.NewReader([]byte("48 brown fox jumped over the lazy dog")) + out := bytes.NewBuffer(nil) + + missing := []patcher.MissingBlockSpan{ + { + BlockSize: BLOCKSIZE, + StartBlock: 0, + EndBlock: 2, + Hasher: md5.New(), + ExpectedSums: REFERENCE_HASHES[0:3], + }, + } + + matched := []patcher.FoundBlockSpan{ + { + BlockSize: BLOCKSIZE, + StartBlock: 3, + EndBlock: 11, + MatchOffset: 5, + }, + } + + err := SequentialPatcher( + LOCAL, + blocksources.NewReadSeekerBlockSource( + stringToReadSeeker(REFERENCE_STRING), + blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), + ), + missing, + matched, + 1024, + out, + ) + + if err != nil { + t.Fatal(err) + } + + if result, err := ioutil.ReadAll(out); err == nil { + t.Logf("String split is: \"%v\"", strings.Join(REFERENCE_BLOCKS, "\", \"")) + if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { + t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) + } + } else { + t.Fatal(err) + } + // +} + +func TestPatchingEnd(t *testing.T) { + LOCAL := bytes.NewReader([]byte("The quick brown fox jumped over the l4zy d0g")) + out := bytes.NewBuffer(nil) + + missing := []patcher.MissingBlockSpan{ + { + BlockSize: BLOCKSIZE, + StartBlock: 9, + EndBlock: 10, + Hasher: md5.New(), + ExpectedSums: REFERENCE_HASHES[0:3], + }, + } + + matched := []patcher.FoundBlockSpan{ + { + BlockSize: BLOCKSIZE, + StartBlock: 0, + EndBlock: 8, + MatchOffset: 0, + }, + } + + err := SequentialPatcher( + LOCAL, + blocksources.NewReadSeekerBlockSource( + stringToReadSeeker(REFERENCE_STRING), + blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), + ), + missing, + matched, + 1024, + out, + ) + + if err != nil { + t.Fatal(err) + } + + if result, err := ioutil.ReadAll(out); err == nil { + if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { + t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) + } + } else { + t.Fatal(err) + } +} + +func TestPatchingEntirelyMissing(t *testing.T) { + LOCAL := bytes.NewReader([]byte("")) + out := bytes.NewBuffer(nil) + + missing := []patcher.MissingBlockSpan{ + { + BlockSize: BLOCKSIZE, + StartBlock: 0, + EndBlock: 10, + Hasher: md5.New(), + ExpectedSums: REFERENCE_HASHES[0:10], + }, + } + + matched := []patcher.FoundBlockSpan{} + + err := SequentialPatcher( + LOCAL, + blocksources.NewReadSeekerBlockSource( + stringToReadSeeker(REFERENCE_STRING), + blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), + ), + missing, + matched, + 1024, + out, + ) + + if err != nil { + t.Fatal(err) + } + + if result, err := ioutil.ReadAll(out); err == nil { + if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { + t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) + } + } else { + t.Fatal(err) + } +} diff --git a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go new file mode 100644 index 0000000..1fbe7b8 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go @@ -0,0 +1,85 @@ +/* +rollsum provides an implementation of a rolling checksum - a checksum that's efficient to advance a byte +or more at a time. It is inspired by the rollsum in rsync, but differs in that the internal values used +are 32bit integers - to make a conformant implementation, a find a replace on "32" should be almost sufficient +(although it would be highly recommended to test against known values from the original implementation). + +Rollsum32 supports the hash.Hash implementation, but is not used much in go-sync, mostly in order to +share and access the underlying circular buffer storage, and use the implementation as efficiently as possible. +*/ +package rollsum + +import ( + "github.com/Redundancy/go-sync/circularbuffer" +) + +func NewRollsum32(blocksize uint) *Rollsum32 { + return &Rollsum32{ + Rollsum32Base: Rollsum32Base{ + blockSize: blocksize, + }, + buffer: circularbuffer.MakeC2Buffer(int(blocksize)), + } +} + +// Rollsum32 is a rolling checksum implemenation +// inspired by rsync, but with 32bit internal values +// Create one using NewRollsum32 +type Rollsum32 struct { + Rollsum32Base + buffer *circularbuffer.C2 +} + +// cannot be called concurrently +func (r *Rollsum32) Write(p []byte) (n int, err error) { + ulen_p := uint(len(p)) + + if ulen_p >= r.blockSize { + // if it's really long, we can just ignore a load of it + remaining := p[ulen_p-r.blockSize:] + r.buffer.Write(remaining) + r.Rollsum32Base.SetBlock(remaining) + } else { + b_len := r.buffer.Len() + r.buffer.Write(p) + evicted := r.buffer.Evicted() + r.Rollsum32Base.AddAndRemoveBytes(p, evicted, b_len) + } + + return len(p), nil +} + +// The most efficient byte length to call Write with +func (r *Rollsum32) BlockSize() int { + return int(r.blockSize) +} + +// the number of bytes +func (r *Rollsum32) Size() int { + return 4 +} + +func (r *Rollsum32) Reset() { + r.Rollsum32Base.Reset() + r.buffer.Reset() +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +// Note that this is to allow Sum() to reuse a preallocated buffer +func (r *Rollsum32) Sum(b []byte) []byte { + if b != nil && cap(b)-len(b) >= 4 { + p := len(b) + b = b[:len(b)+4] + r.Rollsum32Base.GetSum(b[p:]) + return b + } else { + result := []byte{0, 0, 0, 0} + r.Rollsum32Base.GetSum(result) + return append(b, result...) + } +} + +func (r *Rollsum32) GetLastBlock() []byte { + return r.buffer.GetBlock() +} diff --git a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go new file mode 100644 index 0000000..8153e72 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go @@ -0,0 +1,86 @@ +package rollsum + +import ( + "encoding/binary" +) + +const FULL_BYTES_16 = (1 << 16) - 1 + +// Rollsum32Base decouples the rollsum algorithm from the implementation of +// hash.Hash and the storage the rolling checksum window +// this allows us to write different versions of the storage for the distinctly different +// use-cases and optimize the storage with the usage pattern. +func NewRollsum32Base(blockSize uint) *Rollsum32Base { + return &Rollsum32Base{blockSize: blockSize} +} + +// The specification of hash.Hash is such that it cannot be implemented without implementing storage +// but the most optimal storage scheme depends on usage of the circular buffer & hash +type Rollsum32Base struct { + blockSize uint + a, b uint32 +} + +// Add a single byte into the rollsum +func (r *Rollsum32Base) AddByte(b byte) { + r.a += uint32(b) + r.b += r.a +} + +func (r *Rollsum32Base) AddBytes(bs []byte) { + for _, b := range bs { + r.a += uint32(b) + r.b += r.a + } +} + +// Remove a byte from the end of the rollsum +// Use the previous length (before removal) +func (r *Rollsum32Base) RemoveByte(b byte, length int) { + r.a -= uint32(b) + r.b -= uint32(uint(length) * uint(b)) +} + +func (r *Rollsum32Base) RemoveBytes(bs []byte, length int) { + for _, b := range bs { + r.a -= uint32(b) + r.b -= uint32(uint(length) * uint(b)) + length -= 1 + } +} + +func (r *Rollsum32Base) AddAndRemoveBytes(add []byte, remove []byte, length int) { + len_added := len(add) + len_removed := len(remove) + + startEvicted := len_added - len_removed + r.AddBytes(add[:startEvicted]) + length += startEvicted + + for i := startEvicted; i < len_added; i++ { + r.RemoveByte(remove[i-startEvicted], length) + r.AddByte(add[i]) + } +} + +// Set a whole block of blockSize +func (r *Rollsum32Base) SetBlock(block []byte) { + r.Reset() + r.AddBytes(block) +} + +// Reset the hash to the initial state +func (r *Rollsum32Base) Reset() { + r.a, r.b = 0, 0 +} + +// size of the hash in bytes +func (r *Rollsum32Base) Size() int { + return 4 +} + +// Puts the sum into b. Avoids allocation. b must have length >= 4 +func (r *Rollsum32Base) GetSum(b []byte) { + value := uint32((r.a & FULL_BYTES_16) + ((r.b & FULL_BYTES_16) << 16)) + binary.LittleEndian.PutUint32(b, value) +} diff --git a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go new file mode 100644 index 0000000..cbf4c93 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go @@ -0,0 +1,298 @@ +package rollsum + +import ( + "bytes" + "github.com/Redundancy/go-sync/circularbuffer" + "hash" + "io" + "testing" +) + +func TestThatRollsum32SatisfiesHashInterface(t *testing.T) { + var i hash.Hash = NewRollsum32(10) + i.Reset() +} + +func TestThatRollsum32SatisfiedWriterInterface(t *testing.T) { + var i io.Writer = NewRollsum32(10) + n, err := i.Write([]byte{1, 2, 3, 4}) + + if n != 4 { + t.Error("Did not report writing 4 bytes") + } + + if err != nil { + t.Error(err) + } +} + +func TestThatRollsum32IsTheSameAfterBlockSizeBytes(t *testing.T) { + r1 := NewRollsum32(4) + r2 := NewRollsum32(4) + + r1.Write([]byte{1, 2, 3, 4}) + + r2.Write([]byte{7, 6}) + r2.Write([]byte{5, 1, 2}) + r2.Write([]byte{3, 4}) + + sum1 := r1.Sum(nil) + sum2 := r2.Sum(nil) + + if bytes.Compare(sum1, sum2) != 0 { + t.Errorf( + "Rollsums are different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestThatRollsum32IsTheSameAfterBlockSizeBytesWithPartialEviction(t *testing.T) { + r1 := NewRollsum32(4) + r2 := NewRollsum32(4) + + r1.Write([]byte{1, 2, 3, 4}) + + r2.Write([]byte{7, 5}) + r2.Write([]byte{1, 2, 3, 4}) + + sum1 := r1.Sum(nil) + sum2 := r2.Sum(nil) + + if bytes.Compare(sum1, sum2) != 0 { + t.Errorf( + "Rollsums are different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestRegression2(t *testing.T) { + const A = "The quick br" + const B = "The qwik br" + + r1 := NewRollsum32(4) + r2 := NewRollsum32(4) + + r1.Write([]byte(A[:4])) + r1.Reset() + r1.Write([]byte(A[4:8])) + r1.Reset() + r1.Write([]byte(A[8:12])) + + r2.Write([]byte(B[:4])) + r2.Write([]byte(B[4:8])) + for _, c := range B[8:] { + r2.Write([]byte{byte(c)}) + } + + sum1 := r1.Sum(nil) + sum2 := r2.Sum(nil) + + if bytes.Compare(sum1, sum2) != 0 { + t.Errorf( + "Rollsums are different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestThatRollsum32RemovesBytesCorrectly(t *testing.T) { + r1 := NewRollsum32Base(2) + + r1.AddByte(255) + r1.AddByte(10) + r1.RemoveByte(255, 2) + r1.AddByte(0) + r1.RemoveByte(10, 2) + r1.AddByte(0) + + if r1.a != 0 || r1.b != 0 { + t.Errorf("Values are not reset: %v %v", r1.a, r1.b) + } +} + +func TestThatRollsum32IsDifferentForDifferentInput(t *testing.T) { + r1 := NewRollsum32(4) + r2 := NewRollsum32(4) + + r1.Write([]byte{1, 2, 3, 4}) + r2.Write([]byte{7, 6, 5, 1}) + + sum1 := r1.Sum(nil) + sum2 := r2.Sum(nil) + + if bytes.Compare(sum1, sum2) == 0 { + t.Errorf( + "Rollsums should be different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestResettingTheRollsum32(t *testing.T) { + r1 := NewRollsum32(4) + r2 := NewRollsum32(4) + + r1.Write([]byte{1, 2, 3}) + + r2.Write([]byte{7, 6}) + r2.Reset() + r2.Write([]byte{1, 2, 3}) + + sum1 := r1.Sum(nil) + sum2 := r2.Sum(nil) + + if bytes.Compare(sum1, sum2) != 0 { + t.Errorf( + "Rollsums should not be different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestTruncatingPartiallyFilledBufferResultsInSameState(t *testing.T) { + r1 := NewRollsum32Base(4) + r2 := NewRollsum32Base(4) + + r1.AddByte(2) + sum1 := make([]byte, 4) + r1.GetSum(sum1) + + r2.AddByte(1) + r2.AddByte(2) + // Removal works from the left + r2.RemoveByte(1, 2) + sum2 := make([]byte, 4) + r2.GetSum(sum2) + + if bytes.Compare(sum1, sum2) != 0 { + t.Errorf( + "Rollsums should not be different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestThat32SumDoesNotChangeTheHashState(t *testing.T) { + r1 := NewRollsum32(4) + + sum1 := r1.Sum([]byte{1, 2, 3}) + sum2 := r1.Sum([]byte{3, 4, 5}) + + if bytes.Compare(sum1[3:], sum2[3:]) != 0 { + t.Errorf( + "Rollsums should not be different \"%v\" vs \"%v\"", + sum1, + sum2, + ) + } +} + +func TestThat32OutputLengthMatchesSize(t *testing.T) { + r1 := NewRollsum32(4) + sumLength := len(r1.Sum(nil)) + + if sumLength != r1.Size() { + t.Errorf("Unexpected length: %v vs expected %v", sumLength, r1.Size()) + } +} + +func BenchmarkRollsum32(b *testing.B) { + r := NewRollsum32(100) + buffer := make([]byte, 100) + b.ReportAllocs() + b.SetBytes(int64(len(buffer))) + checksum := make([]byte, 16) + + b.StartTimer() + for i := 0; i < b.N; i++ { + r.Write(buffer) + r.Sum(checksum) + checksum = checksum[:0] + } + b.StopTimer() +} + +func BenchmarkRollsum32_8096(b *testing.B) { + r := NewRollsum32(8096) + buffer := make([]byte, 8096) + b.ReportAllocs() + b.SetBytes(int64(len(buffer))) + checksum := make([]byte, 16) + + b.StartTimer() + for i := 0; i < b.N; i++ { + r.Write(buffer) + r.Sum(checksum) + checksum = checksum[:0] + } + b.StopTimer() +} + +func BenchmarkRollsum32Base(b *testing.B) { + r := Rollsum32Base{blockSize: 100} + buffer := make([]byte, 100) + checksum := make([]byte, 16) + b.ReportAllocs() + b.SetBytes(int64(len(buffer))) + + b.StartTimer() + for i := 0; i < b.N; i++ { + r.SetBlock(buffer) + r.GetSum(checksum) + } + b.StopTimer() + +} + +// This is the benchmark where Rollsum should beat a full MD5 for each blocksize +func BenchmarkIncrementalRollsum32(b *testing.B) { + r := NewRollsum32(100) + buffer := make([]byte, 100) + r.Write(buffer) + b.SetBytes(1) + + b.ReportAllocs() + checksum := make([]byte, 16) + increment := make([]byte, 1) + + b.StartTimer() + for i := 0; i < b.N; i++ { + r.Write(increment) + r.Sum(checksum) + checksum = checksum[:0] + } + b.StopTimer() +} + +// The C2 veersion should avoid all allocations in the main loop, and beat the pants off the +// other versions +func BenchmarkIncrementalRollsum32WithC2(b *testing.B) { + const BLOCK_SIZE = 100 + r := NewRollsum32Base(BLOCK_SIZE) + buffer := make([]byte, BLOCK_SIZE) + b.SetBytes(1) + cbuffer := circularbuffer.MakeC2Buffer(BLOCK_SIZE) + + r.AddBytes(buffer) + cbuffer.Write(buffer) + + b.ReportAllocs() + checksum := make([]byte, 16) + increment := make([]byte, 1) + + b.StartTimer() + for i := 0; i < b.N; i++ { + cbuffer.Write(increment) + r.AddAndRemoveBytes(increment, cbuffer.Evicted(), BLOCK_SIZE) + r.GetSum(checksum) + } + b.StopTimer() +} diff --git a/vendor/github.com/Redundancy/go-sync/rsync.go b/vendor/github.com/Redundancy/go-sync/rsync.go new file mode 100644 index 0000000..59db7c1 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/rsync.go @@ -0,0 +1,347 @@ +package gosync + +import ( + "bufio" + "crypto/md5" + "fmt" + "io" + "io/ioutil" + "os" + "runtime" + + "github.com/Redundancy/go-sync/blocksources" + "github.com/Redundancy/go-sync/chunks" + "github.com/Redundancy/go-sync/comparer" + "github.com/Redundancy/go-sync/filechecksum" + "github.com/Redundancy/go-sync/index" + "github.com/Redundancy/go-sync/patcher" + "github.com/Redundancy/go-sync/patcher/sequential" +) + +const ( + megabyte = 1000000 +) + +var ( + // DefaultConcurrency is the default concurrency level used by patching and downloading + DefaultConcurrency = runtime.NumCPU() +) + +// ReadSeekerAt is the combinaton of ReadSeeker and ReaderAt interfaces +type ReadSeekerAt interface { + io.ReadSeeker + io.ReaderAt +} + +/* +RSync is an object designed to make the standard use-case for gosync as +easy as possible. + +To this end, it hides away many low level choices by default, and makes some +assumptions. +*/ +type RSync struct { + Input ReadSeekerAt + Source patcher.BlockSource + Output io.Writer + + Summary FileSummary + + OnClose []closer +} + +type closer interface { + Close() error +} + +// FileSummary combines many of the interfaces that are needed +// It is expected that you might implement it by embedding existing structs +type FileSummary interface { + GetBlockSize() uint + GetBlockCount() uint + GetFileSize() int64 + FindWeakChecksum2(bytes []byte) interface{} + FindStrongChecksum2(bytes []byte, weak interface{}) []chunks.ChunkChecksum + GetStrongChecksumForBlock(blockID int) []byte +} + +// BasicSummary implements a version of the FileSummary interface +type BasicSummary struct { + BlockSize uint + BlockCount uint + FileSize int64 + *index.ChecksumIndex + filechecksum.ChecksumLookup +} + +// GetBlockSize gets the size of each block +func (fs *BasicSummary) GetBlockSize() uint { + return fs.BlockSize +} + +// GetBlockCount gets the number of blocks +func (fs *BasicSummary) GetBlockCount() uint { + return fs.BlockCount +} + +// GetFileSize gets the file size of the file +func (fs *BasicSummary) GetFileSize() int64 { + return fs.FileSize +} + +// MakeRSync creates an RSync object using string paths, +// inferring most of the configuration +func MakeRSync( + InputFile, + Source, + OutFile string, + Summary FileSummary, +) (r *RSync, err error) { + useTempFile := false + if useTempFile, err = IsSameFile(InputFile, OutFile); err != nil { + return nil, err + } + + inputFile, err := os.Open(InputFile) + + if err != nil { + return + } + + var out io.WriteCloser + var outFilename = OutFile + var copier closer + + if useTempFile { + out, outFilename, err = getTempFile() + + if err != nil { + return + } + + copier = &fileCopyCloser{ + from: outFilename, + to: OutFile, + } + } else { + out, err = getOutFile(OutFile) + + if err != nil { + return + } + + copier = nullCloser{} + } + + // blocksource + var source *blocksources.BlockSourceBase + + resolver := blocksources.MakeFileSizedBlockResolver( + uint64(Summary.GetBlockSize()), + Summary.GetFileSize(), + ) + + source = blocksources.NewHttpBlockSource( + Source, + DefaultConcurrency, + resolver, + &filechecksum.HashVerifier{ + Hash: md5.New(), + BlockSize: Summary.GetBlockSize(), + BlockChecksumGetter: Summary, + }, + ) + + r = &RSync{ + Input: inputFile, + Output: out, + Source: source, + Summary: Summary, + OnClose: []closer{ + &fileCloser{inputFile, InputFile}, + &fileCloser{out, outFilename}, + copier, + }, + } + + return +} + +// Patch the files +func (rsync *RSync) Patch() (err error) { + numMatchers := int64(DefaultConcurrency) + blockSize := rsync.Summary.GetBlockSize() + sectionSize := rsync.Summary.GetFileSize() / numMatchers + sectionSize += int64(blockSize) - (sectionSize % int64(blockSize)) + + merger := &comparer.MatchMerger{} + + for i := int64(0); i < numMatchers; i++ { + compare := &comparer.Comparer{} + offset := sectionSize * i + + sectionReader := bufio.NewReaderSize( + io.NewSectionReader(rsync.Input, offset, sectionSize+int64(blockSize)), + megabyte, // 1 MB buffer + ) + + // Bakes in the assumption about how to generate checksums (extract) + sectionGenerator := filechecksum.NewFileChecksumGenerator( + uint(blockSize), + ) + + matchStream := compare.StartFindMatchingBlocks( + sectionReader, offset, sectionGenerator, rsync.Summary, + ) + + merger.StartMergeResultStream(matchStream, int64(blockSize)) + } + + mergedBlocks := merger.GetMergedBlocks() + missing := mergedBlocks.GetMissingBlocks(rsync.Summary.GetBlockCount() - 1) + + return sequential.SequentialPatcher( + rsync.Input, + rsync.Source, + toPatcherMissingSpan(missing, int64(blockSize)), + toPatcherFoundSpan(mergedBlocks, int64(blockSize)), + 20*megabyte, + rsync.Output, + ) +} + +func getOutFile(filename string) (f io.WriteCloser, err error) { + if _, err = os.Stat(filename); os.IsNotExist(err) { + return os.Create(filename) + } + + return os.OpenFile(filename, os.O_WRONLY, 0) +} + +func getTempFile() (f io.WriteCloser, filename string, err error) { + ft, err := ioutil.TempFile(".", "tmp_") + filename = ft.Name() + f = ft + return +} + +// IsSameFile checks if two file paths are the same file +func IsSameFile(path1, path2 string) (same bool, err error) { + + fi1, err := os.Stat(path1) + + switch { + case os.IsNotExist(err): + return false, nil + case err != nil: + return + } + + fi2, err := os.Stat(path2) + + switch { + case os.IsNotExist(err): + return false, nil + case err != nil: + return + } + + return os.SameFile(fi1, fi2), nil +} + +// Close - close open files, copy to the final location from +// a temporary one if neede +func (rsync *RSync) Close() error { + for _, f := range rsync.OnClose { + if err := f.Close(); err != nil { + return err + } + } + return nil +} + +type fileCloser struct { + f io.Closer + path string +} + +// Close - add file path information to closing a file +func (f *fileCloser) Close() error { + err := f.f.Close() + if err != nil { + return fmt.Errorf( + "Could not close file %v: %v", + f.path, + err, + ) + } + return nil +} + +type nullCloser struct{} + +func (n nullCloser) Close() error { + return nil +} + +type fileCopyCloser struct { + from string + to string +} + +func (f *fileCopyCloser) Close() (err error) { + from, err := os.OpenFile(f.from, os.O_RDONLY, 0) + + if err != nil { + return err + } + + defer func() { + e := from.Close() + if err != nil { + err = e + } + }() + + to, err := os.OpenFile(f.to, os.O_TRUNC|os.O_WRONLY, 0) + + if err != nil { + return err + } + + defer func() { + e := to.Close() + if err != nil { + err = e + } + }() + + bufferedReader := bufio.NewReaderSize(from, megabyte) + _, err = io.Copy(to, bufferedReader) + return +} + +func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { + result := make([]patcher.FoundBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].MatchOffset = v.ComparisonStartOffset + result[i].BlockSize = blockSize + } + + return result +} + +func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { + result := make([]patcher.MissingBlockSpan, len(sl)) + + for i, v := range sl { + result[i].StartBlock = v.StartBlock + result[i].EndBlock = v.EndBlock + result[i].BlockSize = blockSize + } + + return result +} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go b/vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go new file mode 100644 index 0000000..5da00af --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go @@ -0,0 +1,18 @@ +package readers + +import ( + "io" +) + +// Injects the second reader into the first at an offset +func InjectedReader( + offsetFromStart int64, + base io.Reader, + inject io.Reader, +) io.Reader { + return io.MultiReader( + io.LimitReader(base, offsetFromStart), + inject, + base, + ) +} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go b/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go new file mode 100644 index 0000000..f1edc0d --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go @@ -0,0 +1,35 @@ +package readers + +import ( + "encoding/binary" + "io" +) + +const nonRepeatingModulo = 87178291199 +const nonRepeatingIncrement = 17180131327 + +// *should* produce a non-repeating sequence of bytes in a deterministic fashion +// use io.LimitReader to limit it to a specific length +type nonRepeatingSequenceReader struct { + value int +} + +func NewNonRepeatingSequence(i int) io.Reader { + return &nonRepeatingSequenceReader{i} +} + +func NewSizedNonRepeatingSequence(i int, s int64) io.Reader { + return io.LimitReader(NewNonRepeatingSequence(i), s) +} + +func (r *nonRepeatingSequenceReader) Read(p []byte) (n int, err error) { + lenp := len(p) + b := []byte{1, 2, 3, 4} + + for i := 0; i < lenp; i++ { + binary.LittleEndian.PutUint32(b, uint32(r.value)) + p[i] = b[0] + r.value = (r.value + nonRepeatingIncrement) % nonRepeatingModulo + } + return lenp, nil +} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go b/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go new file mode 100644 index 0000000..3c65b1c --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go @@ -0,0 +1,58 @@ +package readers + +import ( + "io" + "io/ioutil" + "testing" +) + +// This is only a very basic test +func TestNonRepeatingSequenceReader(t *testing.T) { + i := NewNonRepeatingSequence(0) + a := []byte{0} + b := []byte{0} + + i.Read(a) + i.Read(b) + + if a[0] == b[0] { + t.Fatalf("Bytes should not be the same! %s vs %s", a, b) + } +} + +func TestNonRepeatingSequenceIsDifferent(t *testing.T) { + i := NewNonRepeatingSequence(0) + i2 := NewNonRepeatingSequence(5) + + a := []byte{0} + b := []byte{0} + + commonalities := 0 + + for x := 0; x < 100; x++ { + i.Read(a) + i2.Read(b) + + if a[0] == b[0] { + commonalities += 1 + } + } + + if commonalities > 5 { + t.Fatal("Sequences are too similar") + } +} + +func BenchmarkNonRepeatingSequence(b *testing.B) { + b.SetBytes(1) + + s := NewSizedNonRepeatingSequence(0, int64(b.N)) + + b.StartTimer() + _, err := io.Copy(ioutil.Discard, s) + b.StopTimer() + + if err != nil { + b.Fatal(err) + } +} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/readers.go b/vendor/github.com/Redundancy/go-sync/util/readers/readers.go new file mode 100644 index 0000000..c9db457 --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/readers.go @@ -0,0 +1,6 @@ +/* +util/readers exists to provide convenient and composable io.Reader compatible streams to allow testing +without having to check in large binary files. + +*/ +package readers diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go b/vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go new file mode 100644 index 0000000..523084e --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go @@ -0,0 +1 @@ +package readers diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go b/vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go new file mode 100644 index 0000000..96ba1bd --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go @@ -0,0 +1,12 @@ +package readers + +import ( + "io" +) + +// read from 'readers' in sequence up to a limit of 'size' +func SequenceLimit(size int64, readers ...io.Reader) io.Reader { + return io.LimitReader( + io.MultiReader(readers...), + size) +} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go b/vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go new file mode 100644 index 0000000..e24e69b --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go @@ -0,0 +1,90 @@ +package readers + +import ( + "io" + "io/ioutil" + "testing" +) + +func TestUniformReaderLength(t *testing.T) { + r, err := ioutil.ReadAll(OneReader(100)) + + if err != nil { + t.Fatal(err) + } + + if len(r) != 100 { + t.Errorf("Unexpected length: %v", len(r)) + } + + for i, b := range r { + if b != 1 { + t.Errorf("Byte at position %v is not 1: %v", i, b) + } + } +} + +func TestReadIntoLargerBuffer(t *testing.T) { + b := make([]byte, 100) + r := OneReader(10) + + n, err := r.Read(b) + + if n != 10 { + t.Errorf("Wrong read length: %v", n) + } + + if err != io.EOF { + t.Errorf("Did not raise EOF after reading: %v", err) + } +} + +func TestMultiUniformReader(t *testing.T) { + r := io.MultiReader( + OneReader(12), + NewSizedNonRepeatingSequence(0, 88), + ) + + b := make([]byte, 100) + + n, err := r.Read(b) + + if n != 12 { + t.Errorf("Wrong read length: %v", n) + } + + if err == io.EOF { + t.Errorf("Raised EOF after reading! %v", err) + } + + n, err = r.Read(b) + + if n != 88 { + t.Errorf("Wrong read length: %v", n) + } + + n, err = r.Read(b) + + if err != io.EOF { + t.Errorf("Really expected EOF by now: %v %v", err, n) + } +} + +func TestFillBuffer(t *testing.T) { + r := io.MultiReader( + OneReader(12), + NewSizedNonRepeatingSequence(0, 88), + ) + + b := make([]byte, 100) + _, err := io.ReadFull(r, b) + + if err != nil && err != io.EOF { + t.Error(err) + } + + if len(b) != cap(b) { + t.Errorf("Expected to fill b: %v", len(b)) + } + +} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go b/vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go new file mode 100644 index 0000000..e6045da --- /dev/null +++ b/vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go @@ -0,0 +1,55 @@ +package readers + +import ( + "io" +) + +// Reads a continuous stream of bytes with the same value, up to length +type uniformReader struct { + value byte + length int + read int +} + +func (r *uniformReader) Read(p []byte) (n int, err error) { + destinationLength := len(p) + readable := r.length - r.read + read := destinationLength + + if readable < destinationLength { + read = readable + } + + if read == 0 { + return 0, io.EOF + } + + for i := 0; i < read; i++ { + p[i] = r.value + } + + var result error = nil + if read == readable { + result = io.EOF + } + + r.read += read + + return read, result +} + +func ZeroReader(length int) io.Reader { + return &uniformReader{ + value: 0, + length: length, + read: 0, + } +} + +func OneReader(length int) io.Reader { + return &uniformReader{ + value: 1, + length: length, + read: 0, + } +} diff --git a/vendor/github.com/codegangsta/cli/LICENSE b/vendor/github.com/codegangsta/cli/LICENSE new file mode 100644 index 0000000..5515ccf --- /dev/null +++ b/vendor/github.com/codegangsta/cli/LICENSE @@ -0,0 +1,21 @@ +Copyright (C) 2013 Jeremy Saenz +All Rights Reserved. + +MIT LICENSE + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/codegangsta/cli/README.md b/vendor/github.com/codegangsta/cli/README.md new file mode 100644 index 0000000..364c964 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/README.md @@ -0,0 +1,394 @@ +[![Coverage](http://gocover.io/_badge/github.com/codegangsta/cli?0)](http://gocover.io/github.com/codegangsta/cli) +[![Build Status](https://travis-ci.org/codegangsta/cli.svg?branch=master)](https://travis-ci.org/codegangsta/cli) +[![GoDoc](https://godoc.org/github.com/codegangsta/cli?status.svg)](https://godoc.org/github.com/codegangsta/cli) + +# cli.go + +`cli.go` is simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way. + +## Overview + +Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app. + +**This is where `cli.go` comes into play.** `cli.go` makes command line programming fun, organized, and expressive! + +## Installation + +Make sure you have a working Go environment (go 1.1+ is *required*). [See the install instructions](http://golang.org/doc/install.html). + +To install `cli.go`, simply run: +``` +$ go get github.com/codegangsta/cli +``` + +Make sure your `PATH` includes to the `$GOPATH/bin` directory so your commands can be easily used: +``` +export PATH=$PATH:$GOPATH/bin +``` + +## Getting Started + +One of the philosophies behind `cli.go` is that an API should be playful and full of discovery. So a `cli.go` app can be as little as one line of code in `main()`. + +``` go +package main + +import ( + "os" + "github.com/codegangsta/cli" +) + +func main() { + cli.NewApp().Run(os.Args) +} +``` + +This app will run and show help text, but is not very useful. Let's give an action to execute and some help documentation: + +``` go +package main + +import ( + "os" + "github.com/codegangsta/cli" +) + +func main() { + app := cli.NewApp() + app.Name = "boom" + app.Usage = "make an explosive entrance" + app.Action = func(c *cli.Context) { + println("boom! I say!") + } + + app.Run(os.Args) +} +``` + +Running this already gives you a ton of functionality, plus support for things like subcommands and flags, which are covered below. + +## Example + +Being a programmer can be a lonely job. Thankfully by the power of automation that is not the case! Let's create a greeter app to fend off our demons of loneliness! + +Start by creating a directory named `greet`, and within it, add a file, `greet.go` with the following code in it: + +``` go +package main + +import ( + "os" + "github.com/codegangsta/cli" +) + +func main() { + app := cli.NewApp() + app.Name = "greet" + app.Usage = "fight the loneliness!" + app.Action = func(c *cli.Context) { + println("Hello friend!") + } + + app.Run(os.Args) +} +``` + +Install our command to the `$GOPATH/bin` directory: + +``` +$ go install +``` + +Finally run our new command: + +``` +$ greet +Hello friend! +``` + +`cli.go` also generates neat help text: + +``` +$ greet help +NAME: + greet - fight the loneliness! + +USAGE: + greet [global options] command [command options] [arguments...] + +VERSION: + 0.0.0 + +COMMANDS: + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS + --version Shows version information +``` + +### Arguments + +You can lookup arguments by calling the `Args` function on `cli.Context`. + +``` go +... +app.Action = func(c *cli.Context) { + println("Hello", c.Args()[0]) +} +... +``` + +### Flags + +Setting and querying flags is simple. + +``` go +... +app.Flags = []cli.Flag { + cli.StringFlag{ + Name: "lang", + Value: "english", + Usage: "language for the greeting", + }, +} +app.Action = func(c *cli.Context) { + name := "someone" + if c.NArg() > 0 { + name = c.Args()[0] + } + if c.String("lang") == "spanish" { + println("Hola", name) + } else { + println("Hello", name) + } +} +... +``` + +You can also set a destination variable for a flag, to which the content will be scanned. + +``` go +... +var language string +app.Flags = []cli.Flag { + cli.StringFlag{ + Name: "lang", + Value: "english", + Usage: "language for the greeting", + Destination: &language, + }, +} +app.Action = func(c *cli.Context) { + name := "someone" + if c.NArg() > 0 { + name = c.Args()[0] + } + if language == "spanish" { + println("Hola", name) + } else { + println("Hello", name) + } +} +... +``` + +See full list of flags at http://godoc.org/github.com/codegangsta/cli + +#### Alternate Names + +You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g. + +``` go +app.Flags = []cli.Flag { + cli.StringFlag{ + Name: "lang, l", + Value: "english", + Usage: "language for the greeting", + }, +} +``` + +That flag can then be set with `--lang spanish` or `-l spanish`. Note that giving two different forms of the same flag in the same command invocation is an error. + +#### Values from the Environment + +You can also have the default value set from the environment via `EnvVar`. e.g. + +``` go +app.Flags = []cli.Flag { + cli.StringFlag{ + Name: "lang, l", + Value: "english", + Usage: "language for the greeting", + EnvVar: "APP_LANG", + }, +} +``` + +The `EnvVar` may also be given as a comma-delimited "cascade", where the first environment variable that resolves is used as the default. + +``` go +app.Flags = []cli.Flag { + cli.StringFlag{ + Name: "lang, l", + Value: "english", + Usage: "language for the greeting", + EnvVar: "LEGACY_COMPAT_LANG,APP_LANG,LANG", + }, +} +``` + +#### Values from alternate input sources (YAML and others) + +There is a separate package altsrc that adds support for getting flag values from other input sources like YAML. + +In order to get values for a flag from an alternate input source the following code would be added to wrap an existing cli.Flag like below: + +``` go + altsrc.NewIntFlag(cli.IntFlag{Name: "test"}) +``` + +Initialization must also occur for these flags. Below is an example initializing getting data from a yaml file below. + +``` go + command.Before = altsrc.InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) +``` + +The code above will use the "load" string as a flag name to get the file name of a yaml file from the cli.Context. +It will then use that file name to initialize the yaml input source for any flags that are defined on that command. +As a note the "load" flag used would also have to be defined on the command flags in order for this code snipped to work. + +Currently only YAML files are supported but developers can add support for other input sources by implementing the +altsrc.InputSourceContext for their given sources. + +Here is a more complete sample of a command using YAML support: + +``` go + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) { + // Action to run + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test"}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + err := command.Run(c) +``` + +### Subcommands + +Subcommands can be defined for a more git-like command line app. + +```go +... +app.Commands = []cli.Command{ + { + Name: "add", + Aliases: []string{"a"}, + Usage: "add a task to the list", + Action: func(c *cli.Context) { + println("added task: ", c.Args().First()) + }, + }, + { + Name: "complete", + Aliases: []string{"c"}, + Usage: "complete a task on the list", + Action: func(c *cli.Context) { + println("completed task: ", c.Args().First()) + }, + }, + { + Name: "template", + Aliases: []string{"r"}, + Usage: "options for task templates", + Subcommands: []cli.Command{ + { + Name: "add", + Usage: "add a new template", + Action: func(c *cli.Context) { + println("new task template: ", c.Args().First()) + }, + }, + { + Name: "remove", + Usage: "remove an existing template", + Action: func(c *cli.Context) { + println("removed task template: ", c.Args().First()) + }, + }, + }, + }, +} +... +``` + +### Bash Completion + +You can enable completion commands by setting the `EnableBashCompletion` +flag on the `App` object. By default, this setting will only auto-complete to +show an app's subcommands, but you can write your own completion methods for +the App or its subcommands. + +```go +... +var tasks = []string{"cook", "clean", "laundry", "eat", "sleep", "code"} +app := cli.NewApp() +app.EnableBashCompletion = true +app.Commands = []cli.Command{ + { + Name: "complete", + Aliases: []string{"c"}, + Usage: "complete a task on the list", + Action: func(c *cli.Context) { + println("completed task: ", c.Args().First()) + }, + BashComplete: func(c *cli.Context) { + // This will complete if no args are passed + if c.NArg() > 0 { + return + } + for _, t := range tasks { + fmt.Println(t) + } + }, + } +} +... +``` + +#### To Enable + +Source the `autocomplete/bash_autocomplete` file in your `.bashrc` file while +setting the `PROG` variable to the name of your program: + +`PROG=myprogram source /.../cli/autocomplete/bash_autocomplete` + +#### To Distribute + +Copy `autocomplete/bash_autocomplete` into `/etc/bash_completion.d/` and rename +it to the name of the program you wish to add autocomplete support for (or +automatically install it there if you are distributing a package). Don't forget +to source the file to make it active in the current shell. + +``` +sudo cp src/bash_autocomplete /etc/bash_completion.d/ +source /etc/bash_completion.d/ +``` + +Alternatively, you can just document that users should source the generic +`autocomplete/bash_autocomplete` in their bash configuration with `$PROG` set +to the name of their program (as above). + +## Contribution Guidelines + +Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch. + +If you have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together. + +If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out. diff --git a/vendor/github.com/codegangsta/cli/altsrc/flag.go b/vendor/github.com/codegangsta/cli/altsrc/flag.go new file mode 100644 index 0000000..f13ffb4 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/flag.go @@ -0,0 +1,439 @@ +package altsrc + +import ( + "flag" + "fmt" + "os" + "strconv" + "strings" + + "github.com/codegangsta/cli" +) + +// FlagInputSourceExtension is an extension interface of cli.Flag that +// allows a value to be set on the existing parsed flags. +type FlagInputSourceExtension interface { + cli.Flag + ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error +} + +// ApplyInputSourceValues iterates over all provided flags and +// executes ApplyInputSourceValue on flags implementing the +// FlagInputSourceExtension interface to initialize these flags +// to an alternate input source. +func ApplyInputSourceValues(context *cli.Context, inputSourceContext InputSourceContext, flags []cli.Flag) error { + for _, f := range flags { + inputSourceExtendedFlag, isType := f.(FlagInputSourceExtension) + if isType { + err := inputSourceExtendedFlag.ApplyInputSourceValue(context, inputSourceContext) + if err != nil { + return err + } + } + } + + return nil +} + +// InitInputSource is used to to setup an InputSourceContext on a cli.Command Before method. It will create a new +// input source based on the func provided. If there is no error it will then apply the new input source to any flags +// that are supported by the input source +func InitInputSource(flags []cli.Flag, createInputSource func() (InputSourceContext, error)) func(context *cli.Context) error { + return func(context *cli.Context) error { + inputSource, err := createInputSource() + if err != nil { + return fmt.Errorf("Unable to create input source: inner error: \n'%v'", err.Error()) + } + + return ApplyInputSourceValues(context, inputSource, flags) + } +} + +// InitInputSourceWithContext is used to to setup an InputSourceContext on a cli.Command Before method. It will create a new +// input source based on the func provided with potentially using existing cli.Context values to initialize itself. If there is +// no error it will then apply the new input source to any flags that are supported by the input source +func InitInputSourceWithContext(flags []cli.Flag, createInputSource func(context *cli.Context) (InputSourceContext, error)) func(context *cli.Context) error { + return func(context *cli.Context) error { + inputSource, err := createInputSource(context) + if err != nil { + return fmt.Errorf("Unable to create input source with context: inner error: \n'%v'", err.Error()) + } + + return ApplyInputSourceValues(context, inputSource, flags) + } +} + +// GenericFlag is the flag type that wraps cli.GenericFlag to allow +// for other values to be specified +type GenericFlag struct { + cli.GenericFlag + set *flag.FlagSet +} + +// NewGenericFlag creates a new GenericFlag +func NewGenericFlag(flag cli.GenericFlag) *GenericFlag { + return &GenericFlag{GenericFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a generic value to the flagSet if required +func (f *GenericFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { + value, err := isc.Generic(f.GenericFlag.Name) + if err != nil { + return err + } + if value != nil { + eachName(f.Name, func(name string) { + f.set.Set(f.Name, value.String()) + }) + } + } + } + + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped GenericFlag.Apply +func (f *GenericFlag) Apply(set *flag.FlagSet) { + f.set = set + f.GenericFlag.Apply(set) +} + +// StringSliceFlag is the flag type that wraps cli.StringSliceFlag to allow +// for other values to be specified +type StringSliceFlag struct { + cli.StringSliceFlag + set *flag.FlagSet +} + +// NewStringSliceFlag creates a new StringSliceFlag +func NewStringSliceFlag(flag cli.StringSliceFlag) *StringSliceFlag { + return &StringSliceFlag{StringSliceFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a StringSlice value to the flagSet if required +func (f *StringSliceFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { + value, err := isc.StringSlice(f.StringSliceFlag.Name) + if err != nil { + return err + } + if value != nil { + var sliceValue cli.StringSlice = value + eachName(f.Name, func(name string) { + underlyingFlag := f.set.Lookup(f.Name) + if underlyingFlag != nil { + underlyingFlag.Value = &sliceValue + } + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped StringSliceFlag.Apply +func (f *StringSliceFlag) Apply(set *flag.FlagSet) { + f.set = set + f.StringSliceFlag.Apply(set) +} + +// IntSliceFlag is the flag type that wraps cli.IntSliceFlag to allow +// for other values to be specified +type IntSliceFlag struct { + cli.IntSliceFlag + set *flag.FlagSet +} + +// NewIntSliceFlag creates a new IntSliceFlag +func NewIntSliceFlag(flag cli.IntSliceFlag) *IntSliceFlag { + return &IntSliceFlag{IntSliceFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a IntSlice value if required +func (f *IntSliceFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { + value, err := isc.IntSlice(f.IntSliceFlag.Name) + if err != nil { + return err + } + if value != nil { + var sliceValue cli.IntSlice = value + eachName(f.Name, func(name string) { + underlyingFlag := f.set.Lookup(f.Name) + if underlyingFlag != nil { + underlyingFlag.Value = &sliceValue + } + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped IntSliceFlag.Apply +func (f *IntSliceFlag) Apply(set *flag.FlagSet) { + f.set = set + f.IntSliceFlag.Apply(set) +} + +// BoolFlag is the flag type that wraps cli.BoolFlag to allow +// for other values to be specified +type BoolFlag struct { + cli.BoolFlag + set *flag.FlagSet +} + +// NewBoolFlag creates a new BoolFlag +func NewBoolFlag(flag cli.BoolFlag) *BoolFlag { + return &BoolFlag{BoolFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a Bool value to the flagSet if required +func (f *BoolFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { + value, err := isc.Bool(f.BoolFlag.Name) + if err != nil { + return err + } + if value { + eachName(f.Name, func(name string) { + f.set.Set(f.Name, strconv.FormatBool(value)) + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped BoolFlag.Apply +func (f *BoolFlag) Apply(set *flag.FlagSet) { + f.set = set + f.BoolFlag.Apply(set) +} + +// BoolTFlag is the flag type that wraps cli.BoolTFlag to allow +// for other values to be specified +type BoolTFlag struct { + cli.BoolTFlag + set *flag.FlagSet +} + +// NewBoolTFlag creates a new BoolTFlag +func NewBoolTFlag(flag cli.BoolTFlag) *BoolTFlag { + return &BoolTFlag{BoolTFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a BoolT value to the flagSet if required +func (f *BoolTFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { + value, err := isc.BoolT(f.BoolTFlag.Name) + if err != nil { + return err + } + if !value { + eachName(f.Name, func(name string) { + f.set.Set(f.Name, strconv.FormatBool(value)) + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped BoolTFlag.Apply +func (f *BoolTFlag) Apply(set *flag.FlagSet) { + f.set = set + + f.BoolTFlag.Apply(set) +} + +// StringFlag is the flag type that wraps cli.StringFlag to allow +// for other values to be specified +type StringFlag struct { + cli.StringFlag + set *flag.FlagSet +} + +// NewStringFlag creates a new StringFlag +func NewStringFlag(flag cli.StringFlag) *StringFlag { + return &StringFlag{StringFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a String value to the flagSet if required +func (f *StringFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { + value, err := isc.String(f.StringFlag.Name) + if err != nil { + return err + } + if value != "" { + eachName(f.Name, func(name string) { + f.set.Set(f.Name, value) + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped StringFlag.Apply +func (f *StringFlag) Apply(set *flag.FlagSet) { + f.set = set + + f.StringFlag.Apply(set) +} + +// IntFlag is the flag type that wraps cli.IntFlag to allow +// for other values to be specified +type IntFlag struct { + cli.IntFlag + set *flag.FlagSet +} + +// NewIntFlag creates a new IntFlag +func NewIntFlag(flag cli.IntFlag) *IntFlag { + return &IntFlag{IntFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a int value to the flagSet if required +func (f *IntFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { + value, err := isc.Int(f.IntFlag.Name) + if err != nil { + return err + } + if value > 0 { + eachName(f.Name, func(name string) { + f.set.Set(f.Name, strconv.FormatInt(int64(value), 10)) + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped IntFlag.Apply +func (f *IntFlag) Apply(set *flag.FlagSet) { + f.set = set + f.IntFlag.Apply(set) +} + +// DurationFlag is the flag type that wraps cli.DurationFlag to allow +// for other values to be specified +type DurationFlag struct { + cli.DurationFlag + set *flag.FlagSet +} + +// NewDurationFlag creates a new DurationFlag +func NewDurationFlag(flag cli.DurationFlag) *DurationFlag { + return &DurationFlag{DurationFlag: flag, set: nil} +} + +// ApplyInputSourceValue applies a Duration value to the flagSet if required +func (f *DurationFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { + value, err := isc.Duration(f.DurationFlag.Name) + if err != nil { + return err + } + if value > 0 { + eachName(f.Name, func(name string) { + f.set.Set(f.Name, value.String()) + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped DurationFlag.Apply +func (f *DurationFlag) Apply(set *flag.FlagSet) { + f.set = set + + f.DurationFlag.Apply(set) +} + +// Float64Flag is the flag type that wraps cli.Float64Flag to allow +// for other values to be specified +type Float64Flag struct { + cli.Float64Flag + set *flag.FlagSet +} + +// NewFloat64Flag creates a new Float64Flag +func NewFloat64Flag(flag cli.Float64Flag) *Float64Flag { + return &Float64Flag{Float64Flag: flag, set: nil} +} + +// ApplyInputSourceValue applies a Float64 value to the flagSet if required +func (f *Float64Flag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { + if f.set != nil { + if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { + value, err := isc.Float64(f.Float64Flag.Name) + if err != nil { + return err + } + if value > 0 { + floatStr := float64ToString(value) + eachName(f.Name, func(name string) { + f.set.Set(f.Name, floatStr) + }) + } + } + } + return nil +} + +// Apply saves the flagSet for later usage then calls +// the wrapped Float64Flag.Apply +func (f *Float64Flag) Apply(set *flag.FlagSet) { + f.set = set + + f.Float64Flag.Apply(set) +} + +func isEnvVarSet(envVars string) bool { + for _, envVar := range strings.Split(envVars, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + // TODO: Can't use this for bools as + // set means that it was true or false based on + // Bool flag type, should work for other types + if len(envVal) > 0 { + return true + } + } + } + + return false +} + +func float64ToString(f float64) string { + return fmt.Sprintf("%v", f) +} + +func eachName(longName string, fn func(string)) { + parts := strings.Split(longName, ",") + for _, name := range parts { + name = strings.Trim(name, " ") + fn(name) + } +} diff --git a/vendor/github.com/codegangsta/cli/altsrc/flag_test.go b/vendor/github.com/codegangsta/cli/altsrc/flag_test.go new file mode 100644 index 0000000..ac4d1f5 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/flag_test.go @@ -0,0 +1,336 @@ +package altsrc + +import ( + "flag" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/codegangsta/cli" +) + +type testApplyInputSource struct { + Flag FlagInputSourceExtension + FlagName string + FlagSetName string + Expected string + ContextValueString string + ContextValue flag.Value + EnvVarValue string + EnvVarName string + MapValue interface{} +} + +func TestGenericApplyInputSourceValue(t *testing.T) { + v := &Parser{"abc", "def"} + c := runTest(t, testApplyInputSource{ + Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}}), + FlagName: "test", + MapValue: v, + }) + expect(t, v, c.Generic("test")) +} + +func TestGenericApplyInputSourceMethodContextSet(t *testing.T) { + p := &Parser{"abc", "def"} + c := runTest(t, testApplyInputSource{ + Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}}), + FlagName: "test", + MapValue: &Parser{"efg", "hig"}, + ContextValueString: p.String(), + }) + expect(t, p, c.Generic("test")) +} + +func TestGenericApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}, EnvVar: "TEST"}), + FlagName: "test", + MapValue: &Parser{"efg", "hij"}, + EnvVarName: "TEST", + EnvVarValue: "abc,def", + }) + expect(t, &Parser{"abc", "def"}, c.Generic("test")) +} + +func TestStringSliceApplyInputSourceValue(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test"}), + FlagName: "test", + MapValue: []string{"hello", "world"}, + }) + expect(t, c.StringSlice("test"), []string{"hello", "world"}) +} + +func TestStringSliceApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test"}), + FlagName: "test", + MapValue: []string{"hello", "world"}, + ContextValueString: "ohno", + }) + expect(t, c.StringSlice("test"), []string{"ohno"}) +} + +func TestStringSliceApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: []string{"hello", "world"}, + EnvVarName: "TEST", + EnvVarValue: "oh,no", + }) + expect(t, c.StringSlice("test"), []string{"oh", "no"}) +} + +func TestIntSliceApplyInputSourceValue(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test"}), + FlagName: "test", + MapValue: []int{1, 2}, + }) + expect(t, c.IntSlice("test"), []int{1, 2}) +} + +func TestIntSliceApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test"}), + FlagName: "test", + MapValue: []int{1, 2}, + ContextValueString: "3", + }) + expect(t, c.IntSlice("test"), []int{3}) +} + +func TestIntSliceApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: []int{1, 2}, + EnvVarName: "TEST", + EnvVarValue: "3,4", + }) + expect(t, c.IntSlice("test"), []int{3, 4}) +} + +func TestBoolApplyInputSourceMethodSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewBoolFlag(cli.BoolFlag{Name: "test"}), + FlagName: "test", + MapValue: true, + }) + expect(t, true, c.Bool("test")) +} + +func TestBoolApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewBoolFlag(cli.BoolFlag{Name: "test"}), + FlagName: "test", + MapValue: false, + ContextValueString: "true", + }) + expect(t, true, c.Bool("test")) +} + +func TestBoolApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewBoolFlag(cli.BoolFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: false, + EnvVarName: "TEST", + EnvVarValue: "true", + }) + expect(t, true, c.Bool("test")) +} + +func TestBoolTApplyInputSourceMethodSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test"}), + FlagName: "test", + MapValue: false, + }) + expect(t, false, c.BoolT("test")) +} + +func TestBoolTApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test"}), + FlagName: "test", + MapValue: true, + ContextValueString: "false", + }) + expect(t, false, c.BoolT("test")) +} + +func TestBoolTApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: true, + EnvVarName: "TEST", + EnvVarValue: "false", + }) + expect(t, false, c.BoolT("test")) +} + +func TestStringApplyInputSourceMethodSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewStringFlag(cli.StringFlag{Name: "test"}), + FlagName: "test", + MapValue: "hello", + }) + expect(t, "hello", c.String("test")) +} + +func TestStringApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewStringFlag(cli.StringFlag{Name: "test"}), + FlagName: "test", + MapValue: "hello", + ContextValueString: "goodbye", + }) + expect(t, "goodbye", c.String("test")) +} + +func TestStringApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewStringFlag(cli.StringFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: "hello", + EnvVarName: "TEST", + EnvVarValue: "goodbye", + }) + expect(t, "goodbye", c.String("test")) +} + +func TestIntApplyInputSourceMethodSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewIntFlag(cli.IntFlag{Name: "test"}), + FlagName: "test", + MapValue: 15, + }) + expect(t, 15, c.Int("test")) +} + +func TestIntApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewIntFlag(cli.IntFlag{Name: "test"}), + FlagName: "test", + MapValue: 15, + ContextValueString: "7", + }) + expect(t, 7, c.Int("test")) +} + +func TestIntApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewIntFlag(cli.IntFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: 15, + EnvVarName: "TEST", + EnvVarValue: "12", + }) + expect(t, 12, c.Int("test")) +} + +func TestDurationApplyInputSourceMethodSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewDurationFlag(cli.DurationFlag{Name: "test"}), + FlagName: "test", + MapValue: time.Duration(30 * time.Second), + }) + expect(t, time.Duration(30*time.Second), c.Duration("test")) +} + +func TestDurationApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewDurationFlag(cli.DurationFlag{Name: "test"}), + FlagName: "test", + MapValue: time.Duration(30 * time.Second), + ContextValueString: time.Duration(15 * time.Second).String(), + }) + expect(t, time.Duration(15*time.Second), c.Duration("test")) +} + +func TestDurationApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewDurationFlag(cli.DurationFlag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: time.Duration(30 * time.Second), + EnvVarName: "TEST", + EnvVarValue: time.Duration(15 * time.Second).String(), + }) + expect(t, time.Duration(15*time.Second), c.Duration("test")) +} + +func TestFloat64ApplyInputSourceMethodSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewFloat64Flag(cli.Float64Flag{Name: "test"}), + FlagName: "test", + MapValue: 1.3, + }) + expect(t, 1.3, c.Float64("test")) +} + +func TestFloat64ApplyInputSourceMethodContextSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewFloat64Flag(cli.Float64Flag{Name: "test"}), + FlagName: "test", + MapValue: 1.3, + ContextValueString: fmt.Sprintf("%v", 1.4), + }) + expect(t, 1.4, c.Float64("test")) +} + +func TestFloat64ApplyInputSourceMethodEnvVarSet(t *testing.T) { + c := runTest(t, testApplyInputSource{ + Flag: NewFloat64Flag(cli.Float64Flag{Name: "test", EnvVar: "TEST"}), + FlagName: "test", + MapValue: 1.3, + EnvVarName: "TEST", + EnvVarValue: fmt.Sprintf("%v", 1.4), + }) + expect(t, 1.4, c.Float64("test")) +} + +func runTest(t *testing.T, test testApplyInputSource) *cli.Context { + inputSource := &MapInputSource{valueMap: map[string]interface{}{test.FlagName: test.MapValue}} + set := flag.NewFlagSet(test.FlagSetName, flag.ContinueOnError) + c := cli.NewContext(nil, set, nil) + if test.EnvVarName != "" && test.EnvVarValue != "" { + os.Setenv(test.EnvVarName, test.EnvVarValue) + defer os.Setenv(test.EnvVarName, "") + } + + test.Flag.Apply(set) + if test.ContextValue != nil { + flag := set.Lookup(test.FlagName) + flag.Value = test.ContextValue + } + if test.ContextValueString != "" { + set.Set(test.FlagName, test.ContextValueString) + } + test.Flag.ApplyInputSourceValue(c, inputSource) + + return c +} + +type Parser [2]string + +func (p *Parser) Set(value string) error { + parts := strings.Split(value, ",") + if len(parts) != 2 { + return fmt.Errorf("invalid format") + } + + (*p)[0] = parts[0] + (*p)[1] = parts[1] + + return nil +} + +func (p *Parser) String() string { + return fmt.Sprintf("%s,%s", p[0], p[1]) +} diff --git a/vendor/github.com/codegangsta/cli/altsrc/helpers_test.go b/vendor/github.com/codegangsta/cli/altsrc/helpers_test.go new file mode 100644 index 0000000..3b7f7e9 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/helpers_test.go @@ -0,0 +1,18 @@ +package altsrc + +import ( + "reflect" + "testing" +) + +func expect(t *testing.T, a interface{}, b interface{}) { + if !reflect.DeepEqual(b, a) { + t.Errorf("Expected %#v (type %v) - Got %#v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) + } +} + +func refute(t *testing.T, a interface{}, b interface{}) { + if a == b { + t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) + } +} diff --git a/vendor/github.com/codegangsta/cli/altsrc/input_source_context.go b/vendor/github.com/codegangsta/cli/altsrc/input_source_context.go new file mode 100644 index 0000000..6d695ff --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/input_source_context.go @@ -0,0 +1,21 @@ +package altsrc + +import ( + "time" + + "github.com/codegangsta/cli" +) + +// InputSourceContext is an interface used to allow +// other input sources to be implemented as needed. +type InputSourceContext interface { + Int(name string) (int, error) + Duration(name string) (time.Duration, error) + Float64(name string) (float64, error) + String(name string) (string, error) + StringSlice(name string) ([]string, error) + IntSlice(name string) ([]int, error) + Generic(name string) (cli.Generic, error) + Bool(name string) (bool, error) + BoolT(name string) (bool, error) +} diff --git a/vendor/github.com/codegangsta/cli/altsrc/map_input_source.go b/vendor/github.com/codegangsta/cli/altsrc/map_input_source.go new file mode 100644 index 0000000..f1670fb --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/map_input_source.go @@ -0,0 +1,152 @@ +package altsrc + +import ( + "fmt" + "reflect" + "time" + + "github.com/codegangsta/cli" +) + +// MapInputSource implements InputSourceContext to return +// data from the map that is loaded. +type MapInputSource struct { + valueMap map[string]interface{} +} + +// Int returns an int from the map if it exists otherwise returns 0 +func (fsm *MapInputSource) Int(name string) (int, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(int) + if !isType { + return 0, incorrectTypeForFlagError(name, "int", otherGenericValue) + } + + return otherValue, nil + } + + return 0, nil +} + +// Duration returns a duration from the map if it exists otherwise returns 0 +func (fsm *MapInputSource) Duration(name string) (time.Duration, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(time.Duration) + if !isType { + return 0, incorrectTypeForFlagError(name, "duration", otherGenericValue) + } + return otherValue, nil + } + + return 0, nil +} + +// Float64 returns an float64 from the map if it exists otherwise returns 0 +func (fsm *MapInputSource) Float64(name string) (float64, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(float64) + if !isType { + return 0, incorrectTypeForFlagError(name, "float64", otherGenericValue) + } + return otherValue, nil + } + + return 0, nil +} + +// String returns a string from the map if it exists otherwise returns an empty string +func (fsm *MapInputSource) String(name string) (string, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(string) + if !isType { + return "", incorrectTypeForFlagError(name, "string", otherGenericValue) + } + return otherValue, nil + } + + return "", nil +} + +// StringSlice returns an []string from the map if it exists otherwise returns nil +func (fsm *MapInputSource) StringSlice(name string) ([]string, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.([]string) + if !isType { + return nil, incorrectTypeForFlagError(name, "[]string", otherGenericValue) + } + return otherValue, nil + } + + return nil, nil +} + +// IntSlice returns an []int from the map if it exists otherwise returns nil +func (fsm *MapInputSource) IntSlice(name string) ([]int, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.([]int) + if !isType { + return nil, incorrectTypeForFlagError(name, "[]int", otherGenericValue) + } + return otherValue, nil + } + + return nil, nil +} + +// Generic returns an cli.Generic from the map if it exists otherwise returns nil +func (fsm *MapInputSource) Generic(name string) (cli.Generic, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(cli.Generic) + if !isType { + return nil, incorrectTypeForFlagError(name, "cli.Generic", otherGenericValue) + } + return otherValue, nil + } + + return nil, nil +} + +// Bool returns an bool from the map otherwise returns false +func (fsm *MapInputSource) Bool(name string) (bool, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(bool) + if !isType { + return false, incorrectTypeForFlagError(name, "bool", otherGenericValue) + } + return otherValue, nil + } + + return false, nil +} + +// BoolT returns an bool from the map otherwise returns true +func (fsm *MapInputSource) BoolT(name string) (bool, error) { + otherGenericValue, exists := fsm.valueMap[name] + if exists { + otherValue, isType := otherGenericValue.(bool) + if !isType { + return true, incorrectTypeForFlagError(name, "bool", otherGenericValue) + } + return otherValue, nil + } + + return true, nil +} + +func incorrectTypeForFlagError(name, expectedTypeName string, value interface{}) error { + valueType := reflect.TypeOf(value) + valueTypeName := "" + if valueType != nil { + valueTypeName = valueType.Name() + } + + return fmt.Errorf("Mismatched type for flag '%s'. Expected '%s' but actual is '%s'", name, expectedTypeName, valueTypeName) +} diff --git a/vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go b/vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go new file mode 100644 index 0000000..c7ccbf7 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go @@ -0,0 +1,172 @@ +// Disabling building of yaml support in cases where golang is 1.0 or 1.1 +// as the encoding library is not implemented or supported. + +// +build !go1,!go1.1 + +package altsrc + +import ( + "flag" + "io/ioutil" + "os" + "testing" + + "github.com/codegangsta/cli" +) + +func TestCommandYamlFileTest(t *testing.T) { + app := cli.NewApp() + set := flag.NewFlagSet("test", 0) + ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) + defer os.Remove("current.yaml") + test := []string{"test-cmd", "--load", "current.yaml"} + set.Parse(test) + + c := cli.NewContext(app, set, nil) + + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) { + val := c.Int("test") + expect(t, val, 15) + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test"}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + err := command.Run(c) + + expect(t, err, nil) +} + +func TestCommandYamlFileTestGlobalEnvVarWins(t *testing.T) { + app := cli.NewApp() + set := flag.NewFlagSet("test", 0) + ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) + defer os.Remove("current.yaml") + + os.Setenv("THE_TEST", "10") + defer os.Setenv("THE_TEST", "") + test := []string{"test-cmd", "--load", "current.yaml"} + set.Parse(test) + + c := cli.NewContext(app, set, nil) + + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) { + val := c.Int("test") + expect(t, val, 10) + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test", EnvVar: "THE_TEST"}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + + err := command.Run(c) + + expect(t, err, nil) +} + +func TestCommandYamlFileTestSpecifiedFlagWins(t *testing.T) { + app := cli.NewApp() + set := flag.NewFlagSet("test", 0) + ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) + defer os.Remove("current.yaml") + + test := []string{"test-cmd", "--load", "current.yaml", "--test", "7"} + set.Parse(test) + + c := cli.NewContext(app, set, nil) + + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) { + val := c.Int("test") + expect(t, val, 7) + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test"}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + + err := command.Run(c) + + expect(t, err, nil) +} + +func TestCommandYamlFileTestDefaultValueFileWins(t *testing.T) { + app := cli.NewApp() + set := flag.NewFlagSet("test", 0) + ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) + defer os.Remove("current.yaml") + + test := []string{"test-cmd", "--load", "current.yaml"} + set.Parse(test) + + c := cli.NewContext(app, set, nil) + + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) { + val := c.Int("test") + expect(t, val, 15) + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test", Value: 7}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + + err := command.Run(c) + + expect(t, err, nil) +} + +func TestCommandYamlFileFlagHasDefaultGlobalEnvYamlSetGlobalEnvWins(t *testing.T) { + app := cli.NewApp() + set := flag.NewFlagSet("test", 0) + ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) + defer os.Remove("current.yaml") + + os.Setenv("THE_TEST", "11") + defer os.Setenv("THE_TEST", "") + + test := []string{"test-cmd", "--load", "current.yaml"} + set.Parse(test) + + c := cli.NewContext(app, set, nil) + + command := &cli.Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(c *cli.Context) { + val := c.Int("test") + expect(t, val, 11) + }, + Flags: []cli.Flag{ + NewIntFlag(cli.IntFlag{Name: "test", Value: 7, EnvVar: "THE_TEST"}), + cli.StringFlag{Name: "load"}}, + } + command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) + err := command.Run(c) + + expect(t, err, nil) +} diff --git a/vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go b/vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go new file mode 100644 index 0000000..1251aeb --- /dev/null +++ b/vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go @@ -0,0 +1,84 @@ +// Disabling building of yaml support in cases where golang is 1.0 or 1.1 +// as the encoding library is not implemented or supported. + +// +build !go1,!go1.1 + +package altsrc + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + + "github.com/codegangsta/cli" + + "gopkg.in/yaml.v2" +) + +type yamlSourceContext struct { + FilePath string +} + +// NewYamlSourceFromFile creates a new Yaml InputSourceContext from a filepath. +func NewYamlSourceFromFile(file string) (InputSourceContext, error) { + ymlLoader := &yamlSourceLoader{FilePath: file} + var results map[string]interface{} + err := readCommandYaml(ysl.FilePath, &results) + if err != nil { + return fmt.Errorf("Unable to load Yaml file '%s': inner error: \n'%v'", filePath, err.Error()) + } + + return &MapInputSource{valueMap: results}, nil +} + +// NewYamlSourceFromFlagFunc creates a new Yaml InputSourceContext from a provided flag name and source context. +func NewYamlSourceFromFlagFunc(flagFileName string) func(InputSourceContext, error) { + return func(context cli.Context) { + filePath := context.String(flagFileName) + return NewYamlSourceFromFile(filePath) + } +} + +func readCommandYaml(filePath string, container interface{}) (err error) { + b, err := loadDataFrom(filePath) + if err != nil { + return err + } + + err = yaml.Unmarshal(b, container) + if err != nil { + return err + } + + err = nil + return +} + +func loadDataFrom(filePath string) ([]byte, error) { + u, err := url.Parse(filePath) + if err != nil { + return nil, err + } + + if u.Host != "" { // i have a host, now do i support the scheme? + switch u.Scheme { + case "http", "https": + res, err := http.Get(filePath) + if err != nil { + return nil, err + } + return ioutil.ReadAll(res.Body) + default: + return nil, fmt.Errorf("scheme of %s is unsupported", filePath) + } + } else if u.Path != "" { // i dont have a host, but I have a path. I am a local file. + if _, notFoundFileErr := os.Stat(filePath); notFoundFileErr != nil { + return nil, fmt.Errorf("Cannot read from file: '%s' because it does not exist.", filePath) + } + return ioutil.ReadFile(filePath) + } else { + return nil, fmt.Errorf("unable to determine how to load from path %s", filePath) + } +} diff --git a/vendor/github.com/codegangsta/cli/app.go b/vendor/github.com/codegangsta/cli/app.go new file mode 100644 index 0000000..1ea3fd0 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/app.go @@ -0,0 +1,349 @@ +package cli + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path" + "time" +) + +// App is the main structure of a cli application. It is recommended that +// an app be created with the cli.NewApp() function +type App struct { + // The name of the program. Defaults to path.Base(os.Args[0]) + Name string + // Full name of command for help, defaults to Name + HelpName string + // Description of the program. + Usage string + // Text to override the USAGE section of help + UsageText string + // Description of the program argument format. + ArgsUsage string + // Version of the program + Version string + // List of commands to execute + Commands []Command + // List of flags to parse + Flags []Flag + // Boolean to enable bash completion commands + EnableBashCompletion bool + // Boolean to hide built-in help command + HideHelp bool + // Boolean to hide built-in version flag + HideVersion bool + // An action to execute when the bash-completion flag is set + BashComplete func(context *Context) + // An action to execute before any subcommands are run, but after the context is ready + // If a non-nil error is returned, no subcommands are run + Before func(context *Context) error + // An action to execute after any subcommands are run, but after the subcommand has finished + // It is run even if Action() panics + After func(context *Context) error + // The action to execute when no subcommands are specified + Action func(context *Context) + // Execute this function if the proper command cannot be found + CommandNotFound func(context *Context, command string) + // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. + // This function is able to replace the original error messages. + // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. + OnUsageError func(context *Context, err error, isSubcommand bool) error + // Compilation date + Compiled time.Time + // List of all authors who contributed + Authors []Author + // Copyright of the binary if any + Copyright string + // Name of Author (Note: Use App.Authors, this is deprecated) + Author string + // Email of Author (Note: Use App.Authors, this is deprecated) + Email string + // Writer writer to write output to + Writer io.Writer +} + +// Tries to find out when this binary was compiled. +// Returns the current time if it fails to find it. +func compileTime() time.Time { + info, err := os.Stat(os.Args[0]) + if err != nil { + return time.Now() + } + return info.ModTime() +} + +// Creates a new cli Application with some reasonable defaults for Name, Usage, Version and Action. +func NewApp() *App { + return &App{ + Name: path.Base(os.Args[0]), + HelpName: path.Base(os.Args[0]), + Usage: "A new cli application", + UsageText: "", + Version: "0.0.0", + BashComplete: DefaultAppComplete, + Action: helpCommand.Action, + Compiled: compileTime(), + Writer: os.Stdout, + } +} + +// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination +func (a *App) Run(arguments []string) (err error) { + if a.Author != "" || a.Email != "" { + a.Authors = append(a.Authors, Author{Name: a.Author, Email: a.Email}) + } + + newCmds := []Command{} + for _, c := range a.Commands { + if c.HelpName == "" { + c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name) + } + newCmds = append(newCmds, c) + } + a.Commands = newCmds + + // append help to commands + if a.Command(helpCommand.Name) == nil && !a.HideHelp { + a.Commands = append(a.Commands, helpCommand) + if (HelpFlag != BoolFlag{}) { + a.appendFlag(HelpFlag) + } + } + + //append version/help flags + if a.EnableBashCompletion { + a.appendFlag(BashCompletionFlag) + } + + if !a.HideVersion { + a.appendFlag(VersionFlag) + } + + // parse flags + set := flagSet(a.Name, a.Flags) + set.SetOutput(ioutil.Discard) + err = set.Parse(arguments[1:]) + nerr := normalizeFlags(a.Flags, set) + context := NewContext(a, set, nil) + if nerr != nil { + fmt.Fprintln(a.Writer, nerr) + ShowAppHelp(context) + return nerr + } + + if checkCompletions(context) { + return nil + } + + if err != nil { + if a.OnUsageError != nil { + err := a.OnUsageError(context, err, false) + return err + } else { + fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") + ShowAppHelp(context) + return err + } + } + + if !a.HideHelp && checkHelp(context) { + ShowAppHelp(context) + return nil + } + + if !a.HideVersion && checkVersion(context) { + ShowVersion(context) + return nil + } + + if a.After != nil { + defer func() { + if afterErr := a.After(context); afterErr != nil { + if err != nil { + err = NewMultiError(err, afterErr) + } else { + err = afterErr + } + } + }() + } + + if a.Before != nil { + err = a.Before(context) + if err != nil { + fmt.Fprintf(a.Writer, "%v\n\n", err) + ShowAppHelp(context) + return err + } + } + + args := context.Args() + if args.Present() { + name := args.First() + c := a.Command(name) + if c != nil { + return c.Run(context) + } + } + + // Run default Action + a.Action(context) + return nil +} + +// Another entry point to the cli app, takes care of passing arguments and error handling +func (a *App) RunAndExitOnError() { + if err := a.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags +func (a *App) RunAsSubcommand(ctx *Context) (err error) { + // append help to commands + if len(a.Commands) > 0 { + if a.Command(helpCommand.Name) == nil && !a.HideHelp { + a.Commands = append(a.Commands, helpCommand) + if (HelpFlag != BoolFlag{}) { + a.appendFlag(HelpFlag) + } + } + } + + newCmds := []Command{} + for _, c := range a.Commands { + if c.HelpName == "" { + c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name) + } + newCmds = append(newCmds, c) + } + a.Commands = newCmds + + // append flags + if a.EnableBashCompletion { + a.appendFlag(BashCompletionFlag) + } + + // parse flags + set := flagSet(a.Name, a.Flags) + set.SetOutput(ioutil.Discard) + err = set.Parse(ctx.Args().Tail()) + nerr := normalizeFlags(a.Flags, set) + context := NewContext(a, set, ctx) + + if nerr != nil { + fmt.Fprintln(a.Writer, nerr) + fmt.Fprintln(a.Writer) + if len(a.Commands) > 0 { + ShowSubcommandHelp(context) + } else { + ShowCommandHelp(ctx, context.Args().First()) + } + return nerr + } + + if checkCompletions(context) { + return nil + } + + if err != nil { + if a.OnUsageError != nil { + err = a.OnUsageError(context, err, true) + return err + } else { + fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") + ShowSubcommandHelp(context) + return err + } + } + + if len(a.Commands) > 0 { + if checkSubcommandHelp(context) { + return nil + } + } else { + if checkCommandHelp(ctx, context.Args().First()) { + return nil + } + } + + if a.After != nil { + defer func() { + afterErr := a.After(context) + if afterErr != nil { + if err != nil { + err = NewMultiError(err, afterErr) + } else { + err = afterErr + } + } + }() + } + + if a.Before != nil { + err := a.Before(context) + if err != nil { + return err + } + } + + args := context.Args() + if args.Present() { + name := args.First() + c := a.Command(name) + if c != nil { + return c.Run(context) + } + } + + // Run default Action + a.Action(context) + + return nil +} + +// Returns the named command on App. Returns nil if the command does not exist +func (a *App) Command(name string) *Command { + for _, c := range a.Commands { + if c.HasName(name) { + return &c + } + } + + return nil +} + +func (a *App) hasFlag(flag Flag) bool { + for _, f := range a.Flags { + if flag == f { + return true + } + } + + return false +} + +func (a *App) appendFlag(flag Flag) { + if !a.hasFlag(flag) { + a.Flags = append(a.Flags, flag) + } +} + +// Author represents someone who has contributed to a cli project. +type Author struct { + Name string // The Authors name + Email string // The Authors email +} + +// String makes Author comply to the Stringer interface, to allow an easy print in the templating process +func (a Author) String() string { + e := "" + if a.Email != "" { + e = "<" + a.Email + "> " + } + + return fmt.Sprintf("%v %v", a.Name, e) +} diff --git a/vendor/github.com/codegangsta/cli/app_test.go b/vendor/github.com/codegangsta/cli/app_test.go new file mode 100644 index 0000000..7feaf1f --- /dev/null +++ b/vendor/github.com/codegangsta/cli/app_test.go @@ -0,0 +1,1047 @@ +package cli + +import ( + "bytes" + "errors" + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "testing" +) + +func ExampleApp_Run() { + // set args for examples sake + os.Args = []string{"greet", "--name", "Jeremy"} + + app := NewApp() + app.Name = "greet" + app.Flags = []Flag{ + StringFlag{Name: "name", Value: "bob", Usage: "a name to say"}, + } + app.Action = func(c *Context) { + fmt.Printf("Hello %v\n", c.String("name")) + } + app.UsageText = "app [first_arg] [second_arg]" + app.Author = "Harrison" + app.Email = "harrison@lolwut.com" + app.Authors = []Author{Author{Name: "Oliver Allen", Email: "oliver@toyshop.com"}} + app.Run(os.Args) + // Output: + // Hello Jeremy +} + +func ExampleApp_Run_subcommand() { + // set args for examples sake + os.Args = []string{"say", "hi", "english", "--name", "Jeremy"} + app := NewApp() + app.Name = "say" + app.Commands = []Command{ + { + Name: "hello", + Aliases: []string{"hi"}, + Usage: "use it to see a description", + Description: "This is how we describe hello the function", + Subcommands: []Command{ + { + Name: "english", + Aliases: []string{"en"}, + Usage: "sends a greeting in english", + Description: "greets someone in english", + Flags: []Flag{ + StringFlag{ + Name: "name", + Value: "Bob", + Usage: "Name of the person to greet", + }, + }, + Action: func(c *Context) { + fmt.Println("Hello,", c.String("name")) + }, + }, + }, + }, + } + + app.Run(os.Args) + // Output: + // Hello, Jeremy +} + +func ExampleApp_Run_help() { + // set args for examples sake + os.Args = []string{"greet", "h", "describeit"} + + app := NewApp() + app.Name = "greet" + app.Flags = []Flag{ + StringFlag{Name: "name", Value: "bob", Usage: "a name to say"}, + } + app.Commands = []Command{ + { + Name: "describeit", + Aliases: []string{"d"}, + Usage: "use it to see a description", + Description: "This is how we describe describeit the function", + Action: func(c *Context) { + fmt.Printf("i like to describe things") + }, + }, + } + app.Run(os.Args) + // Output: + // NAME: + // greet describeit - use it to see a description + // + // USAGE: + // greet describeit [arguments...] + // + // DESCRIPTION: + // This is how we describe describeit the function +} + +func ExampleApp_Run_bashComplete() { + // set args for examples sake + os.Args = []string{"greet", "--generate-bash-completion"} + + app := NewApp() + app.Name = "greet" + app.EnableBashCompletion = true + app.Commands = []Command{ + { + Name: "describeit", + Aliases: []string{"d"}, + Usage: "use it to see a description", + Description: "This is how we describe describeit the function", + Action: func(c *Context) { + fmt.Printf("i like to describe things") + }, + }, { + Name: "next", + Usage: "next example", + Description: "more stuff to see when generating bash completion", + Action: func(c *Context) { + fmt.Printf("the next example") + }, + }, + } + + app.Run(os.Args) + // Output: + // describeit + // d + // next + // help + // h +} + +func TestApp_Run(t *testing.T) { + s := "" + + app := NewApp() + app.Action = func(c *Context) { + s = s + c.Args().First() + } + + err := app.Run([]string{"command", "foo"}) + expect(t, err, nil) + err = app.Run([]string{"command", "bar"}) + expect(t, err, nil) + expect(t, s, "foobar") +} + +var commandAppTests = []struct { + name string + expected bool +}{ + {"foobar", true}, + {"batbaz", true}, + {"b", true}, + {"f", true}, + {"bat", false}, + {"nothing", false}, +} + +func TestApp_Command(t *testing.T) { + app := NewApp() + fooCommand := Command{Name: "foobar", Aliases: []string{"f"}} + batCommand := Command{Name: "batbaz", Aliases: []string{"b"}} + app.Commands = []Command{ + fooCommand, + batCommand, + } + + for _, test := range commandAppTests { + expect(t, app.Command(test.name) != nil, test.expected) + } +} + +func TestApp_CommandWithArgBeforeFlags(t *testing.T) { + var parsedOption, firstArg string + + app := NewApp() + command := Command{ + Name: "cmd", + Flags: []Flag{ + StringFlag{Name: "option", Value: "", Usage: "some option"}, + }, + Action: func(c *Context) { + parsedOption = c.String("option") + firstArg = c.Args().First() + }, + } + app.Commands = []Command{command} + + app.Run([]string{"", "cmd", "my-arg", "--option", "my-option"}) + + expect(t, parsedOption, "my-option") + expect(t, firstArg, "my-arg") +} + +func TestApp_RunAsSubcommandParseFlags(t *testing.T) { + var context *Context + + a := NewApp() + a.Commands = []Command{ + { + Name: "foo", + Action: func(c *Context) { + context = c + }, + Flags: []Flag{ + StringFlag{ + Name: "lang", + Value: "english", + Usage: "language for the greeting", + }, + }, + Before: func(_ *Context) error { return nil }, + }, + } + a.Run([]string{"", "foo", "--lang", "spanish", "abcd"}) + + expect(t, context.Args().Get(0), "abcd") + expect(t, context.String("lang"), "spanish") +} + +func TestApp_CommandWithFlagBeforeTerminator(t *testing.T) { + var parsedOption string + var args []string + + app := NewApp() + command := Command{ + Name: "cmd", + Flags: []Flag{ + StringFlag{Name: "option", Value: "", Usage: "some option"}, + }, + Action: func(c *Context) { + parsedOption = c.String("option") + args = c.Args() + }, + } + app.Commands = []Command{command} + + app.Run([]string{"", "cmd", "my-arg", "--option", "my-option", "--", "--notARealFlag"}) + + expect(t, parsedOption, "my-option") + expect(t, args[0], "my-arg") + expect(t, args[1], "--") + expect(t, args[2], "--notARealFlag") +} + +func TestApp_CommandWithDash(t *testing.T) { + var args []string + + app := NewApp() + command := Command{ + Name: "cmd", + Action: func(c *Context) { + args = c.Args() + }, + } + app.Commands = []Command{command} + + app.Run([]string{"", "cmd", "my-arg", "-"}) + + expect(t, args[0], "my-arg") + expect(t, args[1], "-") +} + +func TestApp_CommandWithNoFlagBeforeTerminator(t *testing.T) { + var args []string + + app := NewApp() + command := Command{ + Name: "cmd", + Action: func(c *Context) { + args = c.Args() + }, + } + app.Commands = []Command{command} + + app.Run([]string{"", "cmd", "my-arg", "--", "notAFlagAtAll"}) + + expect(t, args[0], "my-arg") + expect(t, args[1], "--") + expect(t, args[2], "notAFlagAtAll") +} + +func TestApp_Float64Flag(t *testing.T) { + var meters float64 + + app := NewApp() + app.Flags = []Flag{ + Float64Flag{Name: "height", Value: 1.5, Usage: "Set the height, in meters"}, + } + app.Action = func(c *Context) { + meters = c.Float64("height") + } + + app.Run([]string{"", "--height", "1.93"}) + expect(t, meters, 1.93) +} + +func TestApp_ParseSliceFlags(t *testing.T) { + var parsedOption, firstArg string + var parsedIntSlice []int + var parsedStringSlice []string + + app := NewApp() + command := Command{ + Name: "cmd", + Flags: []Flag{ + IntSliceFlag{Name: "p", Value: &IntSlice{}, Usage: "set one or more ip addr"}, + StringSliceFlag{Name: "ip", Value: &StringSlice{}, Usage: "set one or more ports to open"}, + }, + Action: func(c *Context) { + parsedIntSlice = c.IntSlice("p") + parsedStringSlice = c.StringSlice("ip") + parsedOption = c.String("option") + firstArg = c.Args().First() + }, + } + app.Commands = []Command{command} + + app.Run([]string{"", "cmd", "my-arg", "-p", "22", "-p", "80", "-ip", "8.8.8.8", "-ip", "8.8.4.4"}) + + IntsEquals := func(a, b []int) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true + } + + StrsEquals := func(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i, v := range a { + if v != b[i] { + return false + } + } + return true + } + var expectedIntSlice = []int{22, 80} + var expectedStringSlice = []string{"8.8.8.8", "8.8.4.4"} + + if !IntsEquals(parsedIntSlice, expectedIntSlice) { + t.Errorf("%v does not match %v", parsedIntSlice, expectedIntSlice) + } + + if !StrsEquals(parsedStringSlice, expectedStringSlice) { + t.Errorf("%v does not match %v", parsedStringSlice, expectedStringSlice) + } +} + +func TestApp_ParseSliceFlagsWithMissingValue(t *testing.T) { + var parsedIntSlice []int + var parsedStringSlice []string + + app := NewApp() + command := Command{ + Name: "cmd", + Flags: []Flag{ + IntSliceFlag{Name: "a", Usage: "set numbers"}, + StringSliceFlag{Name: "str", Usage: "set strings"}, + }, + Action: func(c *Context) { + parsedIntSlice = c.IntSlice("a") + parsedStringSlice = c.StringSlice("str") + }, + } + app.Commands = []Command{command} + + app.Run([]string{"", "cmd", "my-arg", "-a", "2", "-str", "A"}) + + var expectedIntSlice = []int{2} + var expectedStringSlice = []string{"A"} + + if parsedIntSlice[0] != expectedIntSlice[0] { + t.Errorf("%v does not match %v", parsedIntSlice[0], expectedIntSlice[0]) + } + + if parsedStringSlice[0] != expectedStringSlice[0] { + t.Errorf("%v does not match %v", parsedIntSlice[0], expectedIntSlice[0]) + } +} + +func TestApp_DefaultStdout(t *testing.T) { + app := NewApp() + + if app.Writer != os.Stdout { + t.Error("Default output writer not set.") + } +} + +type mockWriter struct { + written []byte +} + +func (fw *mockWriter) Write(p []byte) (n int, err error) { + if fw.written == nil { + fw.written = p + } else { + fw.written = append(fw.written, p...) + } + + return len(p), nil +} + +func (fw *mockWriter) GetWritten() (b []byte) { + return fw.written +} + +func TestApp_SetStdout(t *testing.T) { + w := &mockWriter{} + + app := NewApp() + app.Name = "test" + app.Writer = w + + err := app.Run([]string{"help"}) + + if err != nil { + t.Fatalf("Run error: %s", err) + } + + if len(w.written) == 0 { + t.Error("App did not write output to desired writer.") + } +} + +func TestApp_BeforeFunc(t *testing.T) { + beforeRun, subcommandRun := false, false + beforeError := fmt.Errorf("fail") + var err error + + app := NewApp() + + app.Before = func(c *Context) error { + beforeRun = true + s := c.String("opt") + if s == "fail" { + return beforeError + } + + return nil + } + + app.Commands = []Command{ + Command{ + Name: "sub", + Action: func(c *Context) { + subcommandRun = true + }, + }, + } + + app.Flags = []Flag{ + StringFlag{Name: "opt"}, + } + + // run with the Before() func succeeding + err = app.Run([]string{"command", "--opt", "succeed", "sub"}) + + if err != nil { + t.Fatalf("Run error: %s", err) + } + + if beforeRun == false { + t.Errorf("Before() not executed when expected") + } + + if subcommandRun == false { + t.Errorf("Subcommand not executed when expected") + } + + // reset + beforeRun, subcommandRun = false, false + + // run with the Before() func failing + err = app.Run([]string{"command", "--opt", "fail", "sub"}) + + // should be the same error produced by the Before func + if err != beforeError { + t.Errorf("Run error expected, but not received") + } + + if beforeRun == false { + t.Errorf("Before() not executed when expected") + } + + if subcommandRun == true { + t.Errorf("Subcommand executed when NOT expected") + } + +} + +func TestApp_AfterFunc(t *testing.T) { + afterRun, subcommandRun := false, false + afterError := fmt.Errorf("fail") + var err error + + app := NewApp() + + app.After = func(c *Context) error { + afterRun = true + s := c.String("opt") + if s == "fail" { + return afterError + } + + return nil + } + + app.Commands = []Command{ + Command{ + Name: "sub", + Action: func(c *Context) { + subcommandRun = true + }, + }, + } + + app.Flags = []Flag{ + StringFlag{Name: "opt"}, + } + + // run with the After() func succeeding + err = app.Run([]string{"command", "--opt", "succeed", "sub"}) + + if err != nil { + t.Fatalf("Run error: %s", err) + } + + if afterRun == false { + t.Errorf("After() not executed when expected") + } + + if subcommandRun == false { + t.Errorf("Subcommand not executed when expected") + } + + // reset + afterRun, subcommandRun = false, false + + // run with the Before() func failing + err = app.Run([]string{"command", "--opt", "fail", "sub"}) + + // should be the same error produced by the Before func + if err != afterError { + t.Errorf("Run error expected, but not received") + } + + if afterRun == false { + t.Errorf("After() not executed when expected") + } + + if subcommandRun == false { + t.Errorf("Subcommand not executed when expected") + } +} + +func TestAppNoHelpFlag(t *testing.T) { + oldFlag := HelpFlag + defer func() { + HelpFlag = oldFlag + }() + + HelpFlag = BoolFlag{} + + app := NewApp() + app.Writer = ioutil.Discard + err := app.Run([]string{"test", "-h"}) + + if err != flag.ErrHelp { + t.Errorf("expected error about missing help flag, but got: %s (%T)", err, err) + } +} + +func TestAppHelpPrinter(t *testing.T) { + oldPrinter := HelpPrinter + defer func() { + HelpPrinter = oldPrinter + }() + + var wasCalled = false + HelpPrinter = func(w io.Writer, template string, data interface{}) { + wasCalled = true + } + + app := NewApp() + app.Run([]string{"-h"}) + + if wasCalled == false { + t.Errorf("Help printer expected to be called, but was not") + } +} + +func TestAppVersionPrinter(t *testing.T) { + oldPrinter := VersionPrinter + defer func() { + VersionPrinter = oldPrinter + }() + + var wasCalled = false + VersionPrinter = func(c *Context) { + wasCalled = true + } + + app := NewApp() + ctx := NewContext(app, nil, nil) + ShowVersion(ctx) + + if wasCalled == false { + t.Errorf("Version printer expected to be called, but was not") + } +} + +func TestAppCommandNotFound(t *testing.T) { + beforeRun, subcommandRun := false, false + app := NewApp() + + app.CommandNotFound = func(c *Context, command string) { + beforeRun = true + } + + app.Commands = []Command{ + Command{ + Name: "bar", + Action: func(c *Context) { + subcommandRun = true + }, + }, + } + + app.Run([]string{"command", "foo"}) + + expect(t, beforeRun, true) + expect(t, subcommandRun, false) +} + +func TestGlobalFlag(t *testing.T) { + var globalFlag string + var globalFlagSet bool + app := NewApp() + app.Flags = []Flag{ + StringFlag{Name: "global, g", Usage: "global"}, + } + app.Action = func(c *Context) { + globalFlag = c.GlobalString("global") + globalFlagSet = c.GlobalIsSet("global") + } + app.Run([]string{"command", "-g", "foo"}) + expect(t, globalFlag, "foo") + expect(t, globalFlagSet, true) + +} + +func TestGlobalFlagsInSubcommands(t *testing.T) { + subcommandRun := false + parentFlag := false + app := NewApp() + + app.Flags = []Flag{ + BoolFlag{Name: "debug, d", Usage: "Enable debugging"}, + } + + app.Commands = []Command{ + Command{ + Name: "foo", + Flags: []Flag{ + BoolFlag{Name: "parent, p", Usage: "Parent flag"}, + }, + Subcommands: []Command{ + { + Name: "bar", + Action: func(c *Context) { + if c.GlobalBool("debug") { + subcommandRun = true + } + if c.GlobalBool("parent") { + parentFlag = true + } + }, + }, + }, + }, + } + + app.Run([]string{"command", "-d", "foo", "-p", "bar"}) + + expect(t, subcommandRun, true) + expect(t, parentFlag, true) +} + +func TestApp_Run_CommandWithSubcommandHasHelpTopic(t *testing.T) { + var subcommandHelpTopics = [][]string{ + {"command", "foo", "--help"}, + {"command", "foo", "-h"}, + {"command", "foo", "help"}, + } + + for _, flagSet := range subcommandHelpTopics { + t.Logf("==> checking with flags %v", flagSet) + + app := NewApp() + buf := new(bytes.Buffer) + app.Writer = buf + + subCmdBar := Command{ + Name: "bar", + Usage: "does bar things", + } + subCmdBaz := Command{ + Name: "baz", + Usage: "does baz things", + } + cmd := Command{ + Name: "foo", + Description: "descriptive wall of text about how it does foo things", + Subcommands: []Command{subCmdBar, subCmdBaz}, + } + + app.Commands = []Command{cmd} + err := app.Run(flagSet) + + if err != nil { + t.Error(err) + } + + output := buf.String() + t.Logf("output: %q\n", buf.Bytes()) + + if strings.Contains(output, "No help topic for") { + t.Errorf("expect a help topic, got none: \n%q", output) + } + + for _, shouldContain := range []string{ + cmd.Name, cmd.Description, + subCmdBar.Name, subCmdBar.Usage, + subCmdBaz.Name, subCmdBaz.Usage, + } { + if !strings.Contains(output, shouldContain) { + t.Errorf("want help to contain %q, did not: \n%q", shouldContain, output) + } + } + } +} + +func TestApp_Run_SubcommandFullPath(t *testing.T) { + app := NewApp() + buf := new(bytes.Buffer) + app.Writer = buf + app.Name = "command" + subCmd := Command{ + Name: "bar", + Usage: "does bar things", + } + cmd := Command{ + Name: "foo", + Description: "foo commands", + Subcommands: []Command{subCmd}, + } + app.Commands = []Command{cmd} + + err := app.Run([]string{"command", "foo", "bar", "--help"}) + if err != nil { + t.Error(err) + } + + output := buf.String() + if !strings.Contains(output, "command foo bar - does bar things") { + t.Errorf("expected full path to subcommand: %s", output) + } + if !strings.Contains(output, "command foo bar [arguments...]") { + t.Errorf("expected full path to subcommand: %s", output) + } +} + +func TestApp_Run_SubcommandHelpName(t *testing.T) { + app := NewApp() + buf := new(bytes.Buffer) + app.Writer = buf + app.Name = "command" + subCmd := Command{ + Name: "bar", + HelpName: "custom", + Usage: "does bar things", + } + cmd := Command{ + Name: "foo", + Description: "foo commands", + Subcommands: []Command{subCmd}, + } + app.Commands = []Command{cmd} + + err := app.Run([]string{"command", "foo", "bar", "--help"}) + if err != nil { + t.Error(err) + } + + output := buf.String() + if !strings.Contains(output, "custom - does bar things") { + t.Errorf("expected HelpName for subcommand: %s", output) + } + if !strings.Contains(output, "custom [arguments...]") { + t.Errorf("expected HelpName to subcommand: %s", output) + } +} + +func TestApp_Run_CommandHelpName(t *testing.T) { + app := NewApp() + buf := new(bytes.Buffer) + app.Writer = buf + app.Name = "command" + subCmd := Command{ + Name: "bar", + Usage: "does bar things", + } + cmd := Command{ + Name: "foo", + HelpName: "custom", + Description: "foo commands", + Subcommands: []Command{subCmd}, + } + app.Commands = []Command{cmd} + + err := app.Run([]string{"command", "foo", "bar", "--help"}) + if err != nil { + t.Error(err) + } + + output := buf.String() + if !strings.Contains(output, "command foo bar - does bar things") { + t.Errorf("expected full path to subcommand: %s", output) + } + if !strings.Contains(output, "command foo bar [arguments...]") { + t.Errorf("expected full path to subcommand: %s", output) + } +} + +func TestApp_Run_CommandSubcommandHelpName(t *testing.T) { + app := NewApp() + buf := new(bytes.Buffer) + app.Writer = buf + app.Name = "base" + subCmd := Command{ + Name: "bar", + HelpName: "custom", + Usage: "does bar things", + } + cmd := Command{ + Name: "foo", + Description: "foo commands", + Subcommands: []Command{subCmd}, + } + app.Commands = []Command{cmd} + + err := app.Run([]string{"command", "foo", "--help"}) + if err != nil { + t.Error(err) + } + + output := buf.String() + if !strings.Contains(output, "base foo - foo commands") { + t.Errorf("expected full path to subcommand: %s", output) + } + if !strings.Contains(output, "base foo command [command options] [arguments...]") { + t.Errorf("expected full path to subcommand: %s", output) + } +} + +func TestApp_Run_Help(t *testing.T) { + var helpArguments = [][]string{{"boom", "--help"}, {"boom", "-h"}, {"boom", "help"}} + + for _, args := range helpArguments { + buf := new(bytes.Buffer) + + t.Logf("==> checking with arguments %v", args) + + app := NewApp() + app.Name = "boom" + app.Usage = "make an explosive entrance" + app.Writer = buf + app.Action = func(c *Context) { + buf.WriteString("boom I say!") + } + + err := app.Run(args) + if err != nil { + t.Error(err) + } + + output := buf.String() + t.Logf("output: %q\n", buf.Bytes()) + + if !strings.Contains(output, "boom - make an explosive entrance") { + t.Errorf("want help to contain %q, did not: \n%q", "boom - make an explosive entrance", output) + } + } +} + +func TestApp_Run_Version(t *testing.T) { + var versionArguments = [][]string{{"boom", "--version"}, {"boom", "-v"}} + + for _, args := range versionArguments { + buf := new(bytes.Buffer) + + t.Logf("==> checking with arguments %v", args) + + app := NewApp() + app.Name = "boom" + app.Usage = "make an explosive entrance" + app.Version = "0.1.0" + app.Writer = buf + app.Action = func(c *Context) { + buf.WriteString("boom I say!") + } + + err := app.Run(args) + if err != nil { + t.Error(err) + } + + output := buf.String() + t.Logf("output: %q\n", buf.Bytes()) + + if !strings.Contains(output, "0.1.0") { + t.Errorf("want version to contain %q, did not: \n%q", "0.1.0", output) + } + } +} + +func TestApp_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) { + app := NewApp() + app.Action = func(c *Context) {} + app.Before = func(c *Context) error { return fmt.Errorf("before error") } + app.After = func(c *Context) error { return fmt.Errorf("after error") } + + err := app.Run([]string{"foo"}) + if err == nil { + t.Fatalf("expected to receive error from Run, got none") + } + + if !strings.Contains(err.Error(), "before error") { + t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) + } + if !strings.Contains(err.Error(), "after error") { + t.Errorf("expected text of error from After method, but got none in \"%v\"", err) + } +} + +func TestApp_Run_SubcommandDoesNotOverwriteErrorFromBefore(t *testing.T) { + app := NewApp() + app.Commands = []Command{ + Command{ + Subcommands: []Command{ + Command{ + Name: "sub", + }, + }, + Name: "bar", + Before: func(c *Context) error { return fmt.Errorf("before error") }, + After: func(c *Context) error { return fmt.Errorf("after error") }, + }, + } + + err := app.Run([]string{"foo", "bar"}) + if err == nil { + t.Fatalf("expected to receive error from Run, got none") + } + + if !strings.Contains(err.Error(), "before error") { + t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) + } + if !strings.Contains(err.Error(), "after error") { + t.Errorf("expected text of error from After method, but got none in \"%v\"", err) + } +} + +func TestApp_OnUsageError_WithWrongFlagValue(t *testing.T) { + app := NewApp() + app.Flags = []Flag{ + IntFlag{Name: "flag"}, + } + app.OnUsageError = func(c *Context, err error, isSubcommand bool) error { + if isSubcommand { + t.Errorf("Expect no subcommand") + } + if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { + t.Errorf("Expect an invalid value error, but got \"%v\"", err) + } + return errors.New("intercepted: " + err.Error()) + } + app.Commands = []Command{ + Command{ + Name: "bar", + }, + } + + err := app.Run([]string{"foo", "--flag=wrong"}) + if err == nil { + t.Fatalf("expected to receive error from Run, got none") + } + + if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { + t.Errorf("Expect an intercepted error, but got \"%v\"", err) + } +} + +func TestApp_OnUsageError_WithWrongFlagValue_ForSubcommand(t *testing.T) { + app := NewApp() + app.Flags = []Flag{ + IntFlag{Name: "flag"}, + } + app.OnUsageError = func(c *Context, err error, isSubcommand bool) error { + if isSubcommand { + t.Errorf("Expect subcommand") + } + if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { + t.Errorf("Expect an invalid value error, but got \"%v\"", err) + } + return errors.New("intercepted: " + err.Error()) + } + app.Commands = []Command{ + Command{ + Name: "bar", + }, + } + + err := app.Run([]string{"foo", "--flag=wrong", "bar"}) + if err == nil { + t.Fatalf("expected to receive error from Run, got none") + } + + if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { + t.Errorf("Expect an intercepted error, but got \"%v\"", err) + } +} diff --git a/vendor/github.com/codegangsta/cli/appveyor.yml b/vendor/github.com/codegangsta/cli/appveyor.yml new file mode 100644 index 0000000..3ca7afa --- /dev/null +++ b/vendor/github.com/codegangsta/cli/appveyor.yml @@ -0,0 +1,16 @@ +version: "{build}" + +os: Windows Server 2012 R2 + +install: + - go version + - go env + +build_script: + - cd %APPVEYOR_BUILD_FOLDER% + - go vet ./... + - go test -v ./... + +test: off + +deploy: off diff --git a/vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete b/vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete new file mode 100644 index 0000000..21a232f --- /dev/null +++ b/vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete @@ -0,0 +1,14 @@ +#! /bin/bash + +: ${PROG:=$(basename ${BASH_SOURCE})} + +_cli_bash_autocomplete() { + local cur opts base + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + } + + complete -F _cli_bash_autocomplete $PROG diff --git a/vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete b/vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete new file mode 100644 index 0000000..5430a18 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete @@ -0,0 +1,5 @@ +autoload -U compinit && compinit +autoload -U bashcompinit && bashcompinit + +script_dir=$(dirname $0) +source ${script_dir}/bash_autocomplete diff --git a/vendor/github.com/codegangsta/cli/cli.go b/vendor/github.com/codegangsta/cli/cli.go new file mode 100644 index 0000000..31dc912 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/cli.go @@ -0,0 +1,40 @@ +// Package cli provides a minimal framework for creating and organizing command line +// Go applications. cli is designed to be easy to understand and write, the most simple +// cli application can be written as follows: +// func main() { +// cli.NewApp().Run(os.Args) +// } +// +// Of course this application does not do much, so let's make this an actual application: +// func main() { +// app := cli.NewApp() +// app.Name = "greet" +// app.Usage = "say a greeting" +// app.Action = func(c *cli.Context) { +// println("Greetings") +// } +// +// app.Run(os.Args) +// } +package cli + +import ( + "strings" +) + +type MultiError struct { + Errors []error +} + +func NewMultiError(err ...error) MultiError { + return MultiError{Errors: err} +} + +func (m MultiError) Error() string { + errs := make([]string, len(m.Errors)) + for i, err := range m.Errors { + errs[i] = err.Error() + } + + return strings.Join(errs, "\n") +} diff --git a/vendor/github.com/codegangsta/cli/command.go b/vendor/github.com/codegangsta/cli/command.go new file mode 100644 index 0000000..0153713 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/command.go @@ -0,0 +1,250 @@ +package cli + +import ( + "fmt" + "io/ioutil" + "strings" +) + +// Command is a subcommand for a cli.App. +type Command struct { + // The name of the command + Name string + // short name of the command. Typically one character (deprecated, use `Aliases`) + ShortName string + // A list of aliases for the command + Aliases []string + // A short description of the usage of this command + Usage string + // Custom text to show on USAGE section of help + UsageText string + // A longer explanation of how the command works + Description string + // A short description of the arguments of this command + ArgsUsage string + // The function to call when checking for bash command completions + BashComplete func(context *Context) + // An action to execute before any sub-subcommands are run, but after the context is ready + // If a non-nil error is returned, no sub-subcommands are run + Before func(context *Context) error + // An action to execute after any subcommands are run, but before the subcommand has finished + // It is run even if Action() panics + After func(context *Context) error + // The function to call when this command is invoked + Action func(context *Context) + // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. + // This function is able to replace the original error messages. + // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. + OnUsageError func(context *Context, err error) error + // List of child commands + Subcommands []Command + // List of flags to parse + Flags []Flag + // Treat all flags as normal arguments if true + SkipFlagParsing bool + // Boolean to hide built-in help command + HideHelp bool + + // Full name of command for help, defaults to full command name, including parent commands. + HelpName string + commandNamePath []string +} + +// Returns the full name of the command. +// For subcommands this ensures that parent commands are part of the command path +func (c Command) FullName() string { + if c.commandNamePath == nil { + return c.Name + } + return strings.Join(c.commandNamePath, " ") +} + +// Invokes the command given the context, parses ctx.Args() to generate command-specific flags +func (c Command) Run(ctx *Context) (err error) { + if len(c.Subcommands) > 0 { + return c.startApp(ctx) + } + + if !c.HideHelp && (HelpFlag != BoolFlag{}) { + // append help to flags + c.Flags = append( + c.Flags, + HelpFlag, + ) + } + + if ctx.App.EnableBashCompletion { + c.Flags = append(c.Flags, BashCompletionFlag) + } + + set := flagSet(c.Name, c.Flags) + set.SetOutput(ioutil.Discard) + + if !c.SkipFlagParsing { + firstFlagIndex := -1 + terminatorIndex := -1 + for index, arg := range ctx.Args() { + if arg == "--" { + terminatorIndex = index + break + } else if arg == "-" { + // Do nothing. A dash alone is not really a flag. + continue + } else if strings.HasPrefix(arg, "-") && firstFlagIndex == -1 { + firstFlagIndex = index + } + } + + if firstFlagIndex > -1 { + args := ctx.Args() + regularArgs := make([]string, len(args[1:firstFlagIndex])) + copy(regularArgs, args[1:firstFlagIndex]) + + var flagArgs []string + if terminatorIndex > -1 { + flagArgs = args[firstFlagIndex:terminatorIndex] + regularArgs = append(regularArgs, args[terminatorIndex:]...) + } else { + flagArgs = args[firstFlagIndex:] + } + + err = set.Parse(append(flagArgs, regularArgs...)) + } else { + err = set.Parse(ctx.Args().Tail()) + } + } else { + if c.SkipFlagParsing { + err = set.Parse(append([]string{"--"}, ctx.Args().Tail()...)) + } + } + + if err != nil { + if c.OnUsageError != nil { + err := c.OnUsageError(ctx, err) + return err + } else { + fmt.Fprintln(ctx.App.Writer, "Incorrect Usage.") + fmt.Fprintln(ctx.App.Writer) + ShowCommandHelp(ctx, c.Name) + return err + } + } + + nerr := normalizeFlags(c.Flags, set) + if nerr != nil { + fmt.Fprintln(ctx.App.Writer, nerr) + fmt.Fprintln(ctx.App.Writer) + ShowCommandHelp(ctx, c.Name) + return nerr + } + context := NewContext(ctx.App, set, ctx) + + if checkCommandCompletions(context, c.Name) { + return nil + } + + if checkCommandHelp(context, c.Name) { + return nil + } + + if c.After != nil { + defer func() { + afterErr := c.After(context) + if afterErr != nil { + if err != nil { + err = NewMultiError(err, afterErr) + } else { + err = afterErr + } + } + }() + } + + if c.Before != nil { + err := c.Before(context) + if err != nil { + fmt.Fprintln(ctx.App.Writer, err) + fmt.Fprintln(ctx.App.Writer) + ShowCommandHelp(ctx, c.Name) + return err + } + } + + context.Command = c + c.Action(context) + return nil +} + +func (c Command) Names() []string { + names := []string{c.Name} + + if c.ShortName != "" { + names = append(names, c.ShortName) + } + + return append(names, c.Aliases...) +} + +// Returns true if Command.Name or Command.ShortName matches given name +func (c Command) HasName(name string) bool { + for _, n := range c.Names() { + if n == name { + return true + } + } + return false +} + +func (c Command) startApp(ctx *Context) error { + app := NewApp() + + // set the name and usage + app.Name = fmt.Sprintf("%s %s", ctx.App.Name, c.Name) + if c.HelpName == "" { + app.HelpName = c.HelpName + } else { + app.HelpName = app.Name + } + + if c.Description != "" { + app.Usage = c.Description + } else { + app.Usage = c.Usage + } + + // set CommandNotFound + app.CommandNotFound = ctx.App.CommandNotFound + + // set the flags and commands + app.Commands = c.Subcommands + app.Flags = c.Flags + app.HideHelp = c.HideHelp + + app.Version = ctx.App.Version + app.HideVersion = ctx.App.HideVersion + app.Compiled = ctx.App.Compiled + app.Author = ctx.App.Author + app.Email = ctx.App.Email + app.Writer = ctx.App.Writer + + // bash completion + app.EnableBashCompletion = ctx.App.EnableBashCompletion + if c.BashComplete != nil { + app.BashComplete = c.BashComplete + } + + // set the actions + app.Before = c.Before + app.After = c.After + if c.Action != nil { + app.Action = c.Action + } else { + app.Action = helpSubcommand.Action + } + + for index, cc := range app.Commands { + app.Commands[index].commandNamePath = []string{c.Name, cc.Name} + } + + return app.RunAsSubcommand(ctx) +} diff --git a/vendor/github.com/codegangsta/cli/command_test.go b/vendor/github.com/codegangsta/cli/command_test.go new file mode 100644 index 0000000..827da1d --- /dev/null +++ b/vendor/github.com/codegangsta/cli/command_test.go @@ -0,0 +1,97 @@ +package cli + +import ( + "errors" + "flag" + "fmt" + "io/ioutil" + "strings" + "testing" +) + +func TestCommandFlagParsing(t *testing.T) { + cases := []struct { + testArgs []string + skipFlagParsing bool + expectedErr error + }{ + {[]string{"blah", "blah", "-break"}, false, errors.New("flag provided but not defined: -break")}, // Test normal "not ignoring flags" flow + {[]string{"blah", "blah"}, true, nil}, // Test SkipFlagParsing without any args that look like flags + {[]string{"blah", "-break"}, true, nil}, // Test SkipFlagParsing with random flag arg + {[]string{"blah", "-help"}, true, nil}, // Test SkipFlagParsing with "special" help flag arg + } + + for _, c := range cases { + app := NewApp() + app.Writer = ioutil.Discard + set := flag.NewFlagSet("test", 0) + set.Parse(c.testArgs) + + context := NewContext(app, set, nil) + + command := Command{ + Name: "test-cmd", + Aliases: []string{"tc"}, + Usage: "this is for testing", + Description: "testing", + Action: func(_ *Context) {}, + } + + command.SkipFlagParsing = c.skipFlagParsing + + err := command.Run(context) + + expect(t, err, c.expectedErr) + expect(t, []string(context.Args()), c.testArgs) + } +} + +func TestCommand_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) { + app := NewApp() + app.Commands = []Command{ + Command{ + Name: "bar", + Before: func(c *Context) error { return fmt.Errorf("before error") }, + After: func(c *Context) error { return fmt.Errorf("after error") }, + }, + } + + err := app.Run([]string{"foo", "bar"}) + if err == nil { + t.Fatalf("expected to receive error from Run, got none") + } + + if !strings.Contains(err.Error(), "before error") { + t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) + } + if !strings.Contains(err.Error(), "after error") { + t.Errorf("expected text of error from After method, but got none in \"%v\"", err) + } +} + +func TestCommand_OnUsageError_WithWrongFlagValue(t *testing.T) { + app := NewApp() + app.Commands = []Command{ + Command{ + Name: "bar", + Flags: []Flag{ + IntFlag{Name: "flag"}, + }, + OnUsageError: func(c *Context, err error) error { + if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { + t.Errorf("Expect an invalid value error, but got \"%v\"", err) + } + return errors.New("intercepted: " + err.Error()) + }, + }, + } + + err := app.Run([]string{"foo", "bar", "--flag=wrong"}) + if err == nil { + t.Fatalf("expected to receive error from Run, got none") + } + + if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { + t.Errorf("Expect an intercepted error, but got \"%v\"", err) + } +} diff --git a/vendor/github.com/codegangsta/cli/context.go b/vendor/github.com/codegangsta/cli/context.go new file mode 100644 index 0000000..b66d278 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/context.go @@ -0,0 +1,393 @@ +package cli + +import ( + "errors" + "flag" + "strconv" + "strings" + "time" +) + +// Context is a type that is passed through to +// each Handler action in a cli application. Context +// can be used to retrieve context-specific Args and +// parsed command-line options. +type Context struct { + App *App + Command Command + flagSet *flag.FlagSet + setFlags map[string]bool + globalSetFlags map[string]bool + parentContext *Context +} + +// Creates a new context. For use in when invoking an App or Command action. +func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context { + return &Context{App: app, flagSet: set, parentContext: parentCtx} +} + +// Looks up the value of a local int flag, returns 0 if no int flag exists +func (c *Context) Int(name string) int { + return lookupInt(name, c.flagSet) +} + +// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists +func (c *Context) Duration(name string) time.Duration { + return lookupDuration(name, c.flagSet) +} + +// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists +func (c *Context) Float64(name string) float64 { + return lookupFloat64(name, c.flagSet) +} + +// Looks up the value of a local bool flag, returns false if no bool flag exists +func (c *Context) Bool(name string) bool { + return lookupBool(name, c.flagSet) +} + +// Looks up the value of a local boolT flag, returns false if no bool flag exists +func (c *Context) BoolT(name string) bool { + return lookupBoolT(name, c.flagSet) +} + +// Looks up the value of a local string flag, returns "" if no string flag exists +func (c *Context) String(name string) string { + return lookupString(name, c.flagSet) +} + +// Looks up the value of a local string slice flag, returns nil if no string slice flag exists +func (c *Context) StringSlice(name string) []string { + return lookupStringSlice(name, c.flagSet) +} + +// Looks up the value of a local int slice flag, returns nil if no int slice flag exists +func (c *Context) IntSlice(name string) []int { + return lookupIntSlice(name, c.flagSet) +} + +// Looks up the value of a local generic flag, returns nil if no generic flag exists +func (c *Context) Generic(name string) interface{} { + return lookupGeneric(name, c.flagSet) +} + +// Looks up the value of a global int flag, returns 0 if no int flag exists +func (c *Context) GlobalInt(name string) int { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupInt(name, fs) + } + return 0 +} + +// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists +func (c *Context) GlobalDuration(name string) time.Duration { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupDuration(name, fs) + } + return 0 +} + +// Looks up the value of a global bool flag, returns false if no bool flag exists +func (c *Context) GlobalBool(name string) bool { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupBool(name, fs) + } + return false +} + +// Looks up the value of a global string flag, returns "" if no string flag exists +func (c *Context) GlobalString(name string) string { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupString(name, fs) + } + return "" +} + +// Looks up the value of a global string slice flag, returns nil if no string slice flag exists +func (c *Context) GlobalStringSlice(name string) []string { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupStringSlice(name, fs) + } + return nil +} + +// Looks up the value of a global int slice flag, returns nil if no int slice flag exists +func (c *Context) GlobalIntSlice(name string) []int { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupIntSlice(name, fs) + } + return nil +} + +// Looks up the value of a global generic flag, returns nil if no generic flag exists +func (c *Context) GlobalGeneric(name string) interface{} { + if fs := lookupGlobalFlagSet(name, c); fs != nil { + return lookupGeneric(name, fs) + } + return nil +} + +// Returns the number of flags set +func (c *Context) NumFlags() int { + return c.flagSet.NFlag() +} + +// Determines if the flag was actually set +func (c *Context) IsSet(name string) bool { + if c.setFlags == nil { + c.setFlags = make(map[string]bool) + c.flagSet.Visit(func(f *flag.Flag) { + c.setFlags[f.Name] = true + }) + } + return c.setFlags[name] == true +} + +// Determines if the global flag was actually set +func (c *Context) GlobalIsSet(name string) bool { + if c.globalSetFlags == nil { + c.globalSetFlags = make(map[string]bool) + ctx := c + if ctx.parentContext != nil { + ctx = ctx.parentContext + } + for ; ctx != nil && c.globalSetFlags[name] == false; ctx = ctx.parentContext { + ctx.flagSet.Visit(func(f *flag.Flag) { + c.globalSetFlags[f.Name] = true + }) + } + } + return c.globalSetFlags[name] +} + +// Returns a slice of flag names used in this context. +func (c *Context) FlagNames() (names []string) { + for _, flag := range c.Command.Flags { + name := strings.Split(flag.GetName(), ",")[0] + if name == "help" { + continue + } + names = append(names, name) + } + return +} + +// Returns a slice of global flag names used by the app. +func (c *Context) GlobalFlagNames() (names []string) { + for _, flag := range c.App.Flags { + name := strings.Split(flag.GetName(), ",")[0] + if name == "help" || name == "version" { + continue + } + names = append(names, name) + } + return +} + +// Returns the parent context, if any +func (c *Context) Parent() *Context { + return c.parentContext +} + +type Args []string + +// Returns the command line arguments associated with the context. +func (c *Context) Args() Args { + args := Args(c.flagSet.Args()) + return args +} + +// Returns the number of the command line arguments. +func (c *Context) NArg() int { + return len(c.Args()) +} + +// Returns the nth argument, or else a blank string +func (a Args) Get(n int) string { + if len(a) > n { + return a[n] + } + return "" +} + +// Returns the first argument, or else a blank string +func (a Args) First() string { + return a.Get(0) +} + +// Return the rest of the arguments (not the first one) +// or else an empty string slice +func (a Args) Tail() []string { + if len(a) >= 2 { + return []string(a)[1:] + } + return []string{} +} + +// Checks if there are any arguments present +func (a Args) Present() bool { + return len(a) != 0 +} + +// Swaps arguments at the given indexes +func (a Args) Swap(from, to int) error { + if from >= len(a) || to >= len(a) { + return errors.New("index out of range") + } + a[from], a[to] = a[to], a[from] + return nil +} + +func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet { + if ctx.parentContext != nil { + ctx = ctx.parentContext + } + for ; ctx != nil; ctx = ctx.parentContext { + if f := ctx.flagSet.Lookup(name); f != nil { + return ctx.flagSet + } + } + return nil +} + +func lookupInt(name string, set *flag.FlagSet) int { + f := set.Lookup(name) + if f != nil { + val, err := strconv.Atoi(f.Value.String()) + if err != nil { + return 0 + } + return val + } + + return 0 +} + +func lookupDuration(name string, set *flag.FlagSet) time.Duration { + f := set.Lookup(name) + if f != nil { + val, err := time.ParseDuration(f.Value.String()) + if err == nil { + return val + } + } + + return 0 +} + +func lookupFloat64(name string, set *flag.FlagSet) float64 { + f := set.Lookup(name) + if f != nil { + val, err := strconv.ParseFloat(f.Value.String(), 64) + if err != nil { + return 0 + } + return val + } + + return 0 +} + +func lookupString(name string, set *flag.FlagSet) string { + f := set.Lookup(name) + if f != nil { + return f.Value.String() + } + + return "" +} + +func lookupStringSlice(name string, set *flag.FlagSet) []string { + f := set.Lookup(name) + if f != nil { + return (f.Value.(*StringSlice)).Value() + + } + + return nil +} + +func lookupIntSlice(name string, set *flag.FlagSet) []int { + f := set.Lookup(name) + if f != nil { + return (f.Value.(*IntSlice)).Value() + + } + + return nil +} + +func lookupGeneric(name string, set *flag.FlagSet) interface{} { + f := set.Lookup(name) + if f != nil { + return f.Value + } + return nil +} + +func lookupBool(name string, set *flag.FlagSet) bool { + f := set.Lookup(name) + if f != nil { + val, err := strconv.ParseBool(f.Value.String()) + if err != nil { + return false + } + return val + } + + return false +} + +func lookupBoolT(name string, set *flag.FlagSet) bool { + f := set.Lookup(name) + if f != nil { + val, err := strconv.ParseBool(f.Value.String()) + if err != nil { + return true + } + return val + } + + return false +} + +func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) { + switch ff.Value.(type) { + case *StringSlice: + default: + set.Set(name, ff.Value.String()) + } +} + +func normalizeFlags(flags []Flag, set *flag.FlagSet) error { + visited := make(map[string]bool) + set.Visit(func(f *flag.Flag) { + visited[f.Name] = true + }) + for _, f := range flags { + parts := strings.Split(f.GetName(), ",") + if len(parts) == 1 { + continue + } + var ff *flag.Flag + for _, name := range parts { + name = strings.Trim(name, " ") + if visited[name] { + if ff != nil { + return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name) + } + ff = set.Lookup(name) + } + } + if ff == nil { + continue + } + for _, name := range parts { + name = strings.Trim(name, " ") + if !visited[name] { + copyFlag(name, ff, set) + } + } + } + return nil +} diff --git a/vendor/github.com/codegangsta/cli/context_test.go b/vendor/github.com/codegangsta/cli/context_test.go new file mode 100644 index 0000000..b8ab37d --- /dev/null +++ b/vendor/github.com/codegangsta/cli/context_test.go @@ -0,0 +1,121 @@ +package cli + +import ( + "flag" + "testing" + "time" +) + +func TestNewContext(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Int("myflag", 12, "doc") + globalSet := flag.NewFlagSet("test", 0) + globalSet.Int("myflag", 42, "doc") + globalCtx := NewContext(nil, globalSet, nil) + command := Command{Name: "mycommand"} + c := NewContext(nil, set, globalCtx) + c.Command = command + expect(t, c.Int("myflag"), 12) + expect(t, c.GlobalInt("myflag"), 42) + expect(t, c.Command.Name, "mycommand") +} + +func TestContext_Int(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Int("myflag", 12, "doc") + c := NewContext(nil, set, nil) + expect(t, c.Int("myflag"), 12) +} + +func TestContext_Duration(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Duration("myflag", time.Duration(12*time.Second), "doc") + c := NewContext(nil, set, nil) + expect(t, c.Duration("myflag"), time.Duration(12*time.Second)) +} + +func TestContext_String(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.String("myflag", "hello world", "doc") + c := NewContext(nil, set, nil) + expect(t, c.String("myflag"), "hello world") +} + +func TestContext_Bool(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", false, "doc") + c := NewContext(nil, set, nil) + expect(t, c.Bool("myflag"), false) +} + +func TestContext_BoolT(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", true, "doc") + c := NewContext(nil, set, nil) + expect(t, c.BoolT("myflag"), true) +} + +func TestContext_Args(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", false, "doc") + c := NewContext(nil, set, nil) + set.Parse([]string{"--myflag", "bat", "baz"}) + expect(t, len(c.Args()), 2) + expect(t, c.Bool("myflag"), true) +} + +func TestContext_NArg(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", false, "doc") + c := NewContext(nil, set, nil) + set.Parse([]string{"--myflag", "bat", "baz"}) + expect(t, c.NArg(), 2) +} + +func TestContext_IsSet(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", false, "doc") + set.String("otherflag", "hello world", "doc") + globalSet := flag.NewFlagSet("test", 0) + globalSet.Bool("myflagGlobal", true, "doc") + globalCtx := NewContext(nil, globalSet, nil) + c := NewContext(nil, set, globalCtx) + set.Parse([]string{"--myflag", "bat", "baz"}) + globalSet.Parse([]string{"--myflagGlobal", "bat", "baz"}) + expect(t, c.IsSet("myflag"), true) + expect(t, c.IsSet("otherflag"), false) + expect(t, c.IsSet("bogusflag"), false) + expect(t, c.IsSet("myflagGlobal"), false) +} + +func TestContext_GlobalIsSet(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", false, "doc") + set.String("otherflag", "hello world", "doc") + globalSet := flag.NewFlagSet("test", 0) + globalSet.Bool("myflagGlobal", true, "doc") + globalSet.Bool("myflagGlobalUnset", true, "doc") + globalCtx := NewContext(nil, globalSet, nil) + c := NewContext(nil, set, globalCtx) + set.Parse([]string{"--myflag", "bat", "baz"}) + globalSet.Parse([]string{"--myflagGlobal", "bat", "baz"}) + expect(t, c.GlobalIsSet("myflag"), false) + expect(t, c.GlobalIsSet("otherflag"), false) + expect(t, c.GlobalIsSet("bogusflag"), false) + expect(t, c.GlobalIsSet("myflagGlobal"), true) + expect(t, c.GlobalIsSet("myflagGlobalUnset"), false) + expect(t, c.GlobalIsSet("bogusGlobal"), false) +} + +func TestContext_NumFlags(t *testing.T) { + set := flag.NewFlagSet("test", 0) + set.Bool("myflag", false, "doc") + set.String("otherflag", "hello world", "doc") + globalSet := flag.NewFlagSet("test", 0) + globalSet.Bool("myflagGlobal", true, "doc") + globalCtx := NewContext(nil, globalSet, nil) + c := NewContext(nil, set, globalCtx) + set.Parse([]string{"--myflag", "--otherflag=foo"}) + globalSet.Parse([]string{"--myflagGlobal"}) + expect(t, c.NumFlags(), 2) +} diff --git a/vendor/github.com/codegangsta/cli/flag.go b/vendor/github.com/codegangsta/cli/flag.go new file mode 100644 index 0000000..e951c2d --- /dev/null +++ b/vendor/github.com/codegangsta/cli/flag.go @@ -0,0 +1,546 @@ +package cli + +import ( + "flag" + "fmt" + "os" + "runtime" + "strconv" + "strings" + "time" +) + +// This flag enables bash-completion for all commands and subcommands +var BashCompletionFlag = BoolFlag{ + Name: "generate-bash-completion", +} + +// This flag prints the version for the application +var VersionFlag = BoolFlag{ + Name: "version, v", + Usage: "print the version", +} + +// This flag prints the help for all commands and subcommands +// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand +// unless HideHelp is set to true) +var HelpFlag = BoolFlag{ + Name: "help, h", + Usage: "show help", +} + +// Flag is a common interface related to parsing flags in cli. +// For more advanced flag parsing techniques, it is recommended that +// this interface be implemented. +type Flag interface { + fmt.Stringer + // Apply Flag settings to the given flag set + Apply(*flag.FlagSet) + GetName() string +} + +func flagSet(name string, flags []Flag) *flag.FlagSet { + set := flag.NewFlagSet(name, flag.ContinueOnError) + + for _, f := range flags { + f.Apply(set) + } + return set +} + +func eachName(longName string, fn func(string)) { + parts := strings.Split(longName, ",") + for _, name := range parts { + name = strings.Trim(name, " ") + fn(name) + } +} + +// Generic is a generic parseable type identified by a specific flag +type Generic interface { + Set(value string) error + String() string +} + +// GenericFlag is the flag type for types implementing Generic +type GenericFlag struct { + Name string + Value Generic + Usage string + EnvVar string +} + +// String returns the string representation of the generic flag to display the +// help text to the user (uses the String() method of the generic flag to show +// the value) +func (f GenericFlag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) +} + +func (f GenericFlag) FormatValueHelp() string { + if f.Value == nil { + return "" + } + s := f.Value.String() + if len(s) == 0 { + return "" + } + return fmt.Sprintf("\"%s\"", s) +} + +// Apply takes the flagset and calls Set on the generic flag with the value +// provided by the user for parsing by the flag +func (f GenericFlag) Apply(set *flag.FlagSet) { + val := f.Value + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + val.Set(envVal) + break + } + } + } + + eachName(f.Name, func(name string) { + set.Var(f.Value, name, f.Usage) + }) +} + +func (f GenericFlag) GetName() string { + return f.Name +} + +// StringSlice is an opaque type for []string to satisfy flag.Value +type StringSlice []string + +// Set appends the string value to the list of values +func (f *StringSlice) Set(value string) error { + *f = append(*f, value) + return nil +} + +// String returns a readable representation of this value (for usage defaults) +func (f *StringSlice) String() string { + return fmt.Sprintf("%s", *f) +} + +// Value returns the slice of strings set by this flag +func (f *StringSlice) Value() []string { + return *f +} + +// StringSlice is a string flag that can be specified multiple times on the +// command-line +type StringSliceFlag struct { + Name string + Value *StringSlice + Usage string + EnvVar string +} + +// String returns the usage +func (f StringSliceFlag) String() string { + firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") + pref := prefixFor(firstName) + return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f StringSliceFlag) Apply(set *flag.FlagSet) { + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + newVal := &StringSlice{} + for _, s := range strings.Split(envVal, ",") { + s = strings.TrimSpace(s) + newVal.Set(s) + } + f.Value = newVal + break + } + } + } + + eachName(f.Name, func(name string) { + if f.Value == nil { + f.Value = &StringSlice{} + } + set.Var(f.Value, name, f.Usage) + }) +} + +func (f StringSliceFlag) GetName() string { + return f.Name +} + +// StringSlice is an opaque type for []int to satisfy flag.Value +type IntSlice []int + +// Set parses the value into an integer and appends it to the list of values +func (f *IntSlice) Set(value string) error { + tmp, err := strconv.Atoi(value) + if err != nil { + return err + } else { + *f = append(*f, tmp) + } + return nil +} + +// String returns a readable representation of this value (for usage defaults) +func (f *IntSlice) String() string { + return fmt.Sprintf("%d", *f) +} + +// Value returns the slice of ints set by this flag +func (f *IntSlice) Value() []int { + return *f +} + +// IntSliceFlag is an int flag that can be specified multiple times on the +// command-line +type IntSliceFlag struct { + Name string + Value *IntSlice + Usage string + EnvVar string +} + +// String returns the usage +func (f IntSliceFlag) String() string { + firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") + pref := prefixFor(firstName) + return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f IntSliceFlag) Apply(set *flag.FlagSet) { + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + newVal := &IntSlice{} + for _, s := range strings.Split(envVal, ",") { + s = strings.TrimSpace(s) + err := newVal.Set(s) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + } + } + f.Value = newVal + break + } + } + } + + eachName(f.Name, func(name string) { + if f.Value == nil { + f.Value = &IntSlice{} + } + set.Var(f.Value, name, f.Usage) + }) +} + +func (f IntSliceFlag) GetName() string { + return f.Name +} + +// BoolFlag is a switch that defaults to false +type BoolFlag struct { + Name string + Usage string + EnvVar string + Destination *bool +} + +// String returns a readable representation of this value (for usage defaults) +func (f BoolFlag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f BoolFlag) Apply(set *flag.FlagSet) { + val := false + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + envValBool, err := strconv.ParseBool(envVal) + if err == nil { + val = envValBool + } + break + } + } + } + + eachName(f.Name, func(name string) { + if f.Destination != nil { + set.BoolVar(f.Destination, name, val, f.Usage) + return + } + set.Bool(name, val, f.Usage) + }) +} + +func (f BoolFlag) GetName() string { + return f.Name +} + +// BoolTFlag this represents a boolean flag that is true by default, but can +// still be set to false by --some-flag=false +type BoolTFlag struct { + Name string + Usage string + EnvVar string + Destination *bool +} + +// String returns a readable representation of this value (for usage defaults) +func (f BoolTFlag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f BoolTFlag) Apply(set *flag.FlagSet) { + val := true + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + envValBool, err := strconv.ParseBool(envVal) + if err == nil { + val = envValBool + break + } + } + } + } + + eachName(f.Name, func(name string) { + if f.Destination != nil { + set.BoolVar(f.Destination, name, val, f.Usage) + return + } + set.Bool(name, val, f.Usage) + }) +} + +func (f BoolTFlag) GetName() string { + return f.Name +} + +// StringFlag represents a flag that takes as string value +type StringFlag struct { + Name string + Value string + Usage string + EnvVar string + Destination *string +} + +// String returns the usage +func (f StringFlag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) +} + +func (f StringFlag) FormatValueHelp() string { + s := f.Value + if len(s) == 0 { + return "" + } + return fmt.Sprintf("\"%s\"", s) +} + +// Apply populates the flag given the flag set and environment +func (f StringFlag) Apply(set *flag.FlagSet) { + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + f.Value = envVal + break + } + } + } + + eachName(f.Name, func(name string) { + if f.Destination != nil { + set.StringVar(f.Destination, name, f.Value, f.Usage) + return + } + set.String(name, f.Value, f.Usage) + }) +} + +func (f StringFlag) GetName() string { + return f.Name +} + +// IntFlag is a flag that takes an integer +// Errors if the value provided cannot be parsed +type IntFlag struct { + Name string + Value int + Usage string + EnvVar string + Destination *int +} + +// String returns the usage +func (f IntFlag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f IntFlag) Apply(set *flag.FlagSet) { + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + envValInt, err := strconv.ParseInt(envVal, 0, 64) + if err == nil { + f.Value = int(envValInt) + break + } + } + } + } + + eachName(f.Name, func(name string) { + if f.Destination != nil { + set.IntVar(f.Destination, name, f.Value, f.Usage) + return + } + set.Int(name, f.Value, f.Usage) + }) +} + +func (f IntFlag) GetName() string { + return f.Name +} + +// DurationFlag is a flag that takes a duration specified in Go's duration +// format: https://golang.org/pkg/time/#ParseDuration +type DurationFlag struct { + Name string + Value time.Duration + Usage string + EnvVar string + Destination *time.Duration +} + +// String returns a readable representation of this value (for usage defaults) +func (f DurationFlag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f DurationFlag) Apply(set *flag.FlagSet) { + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + envValDuration, err := time.ParseDuration(envVal) + if err == nil { + f.Value = envValDuration + break + } + } + } + } + + eachName(f.Name, func(name string) { + if f.Destination != nil { + set.DurationVar(f.Destination, name, f.Value, f.Usage) + return + } + set.Duration(name, f.Value, f.Usage) + }) +} + +func (f DurationFlag) GetName() string { + return f.Name +} + +// Float64Flag is a flag that takes an float value +// Errors if the value provided cannot be parsed +type Float64Flag struct { + Name string + Value float64 + Usage string + EnvVar string + Destination *float64 +} + +// String returns the usage +func (f Float64Flag) String() string { + return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) +} + +// Apply populates the flag given the flag set and environment +func (f Float64Flag) Apply(set *flag.FlagSet) { + if f.EnvVar != "" { + for _, envVar := range strings.Split(f.EnvVar, ",") { + envVar = strings.TrimSpace(envVar) + if envVal := os.Getenv(envVar); envVal != "" { + envValFloat, err := strconv.ParseFloat(envVal, 10) + if err == nil { + f.Value = float64(envValFloat) + } + } + } + } + + eachName(f.Name, func(name string) { + if f.Destination != nil { + set.Float64Var(f.Destination, name, f.Value, f.Usage) + return + } + set.Float64(name, f.Value, f.Usage) + }) +} + +func (f Float64Flag) GetName() string { + return f.Name +} + +func prefixFor(name string) (prefix string) { + if len(name) == 1 { + prefix = "-" + } else { + prefix = "--" + } + + return +} + +func prefixedNames(fullName string) (prefixed string) { + parts := strings.Split(fullName, ",") + for i, name := range parts { + name = strings.Trim(name, " ") + prefixed += prefixFor(name) + name + if i < len(parts)-1 { + prefixed += ", " + } + } + return +} + +func withEnvHint(envVar, str string) string { + envText := "" + if envVar != "" { + prefix := "$" + suffix := "" + sep := ", $" + if runtime.GOOS == "windows" { + prefix = "%" + suffix = "%" + sep = "%, %" + } + envText = fmt.Sprintf(" [%s%s%s]", prefix, strings.Join(strings.Split(envVar, ","), sep), suffix) + } + return str + envText +} diff --git a/vendor/github.com/codegangsta/cli/flag_test.go b/vendor/github.com/codegangsta/cli/flag_test.go new file mode 100644 index 0000000..3caa70a --- /dev/null +++ b/vendor/github.com/codegangsta/cli/flag_test.go @@ -0,0 +1,859 @@ +package cli + +import ( + "fmt" + "os" + "reflect" + "strings" + "testing" + "runtime" +) + +var boolFlagTests = []struct { + name string + expected string +}{ + {"help", "--help\t"}, + {"h", "-h\t"}, +} + +func TestBoolFlagHelpOutput(t *testing.T) { + + for _, test := range boolFlagTests { + flag := BoolFlag{Name: test.name} + output := flag.String() + + if output != test.expected { + t.Errorf("%s does not match %s", output, test.expected) + } + } +} + +var stringFlagTests = []struct { + name string + value string + expected string +}{ + {"help", "", "--help \t"}, + {"h", "", "-h \t"}, + {"h", "", "-h \t"}, + {"test", "Something", "--test \"Something\"\t"}, +} + +func TestStringFlagHelpOutput(t *testing.T) { + + for _, test := range stringFlagTests { + flag := StringFlag{Name: test.name, Value: test.value} + output := flag.String() + + if output != test.expected { + t.Errorf("%s does not match %s", output, test.expected) + } + } +} + +func TestStringFlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_FOO", "derp") + for _, test := range stringFlagTests { + flag := StringFlag{Name: test.name, Value: test.value, EnvVar: "APP_FOO"} + output := flag.String() + + expectedSuffix := " [$APP_FOO]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_FOO%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%s does not end with" + expectedSuffix, output) + } + } +} + +var stringSliceFlagTests = []struct { + name string + value *StringSlice + expected string +}{ + {"help", func() *StringSlice { + s := &StringSlice{} + s.Set("") + return s + }(), "--help [--help option --help option]\t"}, + {"h", func() *StringSlice { + s := &StringSlice{} + s.Set("") + return s + }(), "-h [-h option -h option]\t"}, + {"h", func() *StringSlice { + s := &StringSlice{} + s.Set("") + return s + }(), "-h [-h option -h option]\t"}, + {"test", func() *StringSlice { + s := &StringSlice{} + s.Set("Something") + return s + }(), "--test [--test option --test option]\t"}, +} + +func TestStringSliceFlagHelpOutput(t *testing.T) { + + for _, test := range stringSliceFlagTests { + flag := StringSliceFlag{Name: test.name, Value: test.value} + output := flag.String() + + if output != test.expected { + t.Errorf("%q does not match %q", output, test.expected) + } + } +} + +func TestStringSliceFlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_QWWX", "11,4") + for _, test := range stringSliceFlagTests { + flag := StringSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_QWWX"} + output := flag.String() + + expectedSuffix := " [$APP_QWWX]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_QWWX%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%q does not end with" + expectedSuffix, output) + } + } +} + +var intFlagTests = []struct { + name string + expected string +}{ + {"help", "--help \"0\"\t"}, + {"h", "-h \"0\"\t"}, +} + +func TestIntFlagHelpOutput(t *testing.T) { + + for _, test := range intFlagTests { + flag := IntFlag{Name: test.name} + output := flag.String() + + if output != test.expected { + t.Errorf("%s does not match %s", output, test.expected) + } + } +} + +func TestIntFlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_BAR", "2") + for _, test := range intFlagTests { + flag := IntFlag{Name: test.name, EnvVar: "APP_BAR"} + output := flag.String() + + expectedSuffix := " [$APP_BAR]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_BAR%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%s does not end with" + expectedSuffix, output) + } + } +} + +var durationFlagTests = []struct { + name string + expected string +}{ + {"help", "--help \"0\"\t"}, + {"h", "-h \"0\"\t"}, +} + +func TestDurationFlagHelpOutput(t *testing.T) { + + for _, test := range durationFlagTests { + flag := DurationFlag{Name: test.name} + output := flag.String() + + if output != test.expected { + t.Errorf("%s does not match %s", output, test.expected) + } + } +} + +func TestDurationFlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_BAR", "2h3m6s") + for _, test := range durationFlagTests { + flag := DurationFlag{Name: test.name, EnvVar: "APP_BAR"} + output := flag.String() + + expectedSuffix := " [$APP_BAR]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_BAR%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%s does not end with" + expectedSuffix, output) + } + } +} + +var intSliceFlagTests = []struct { + name string + value *IntSlice + expected string +}{ + {"help", &IntSlice{}, "--help [--help option --help option]\t"}, + {"h", &IntSlice{}, "-h [-h option -h option]\t"}, + {"h", &IntSlice{}, "-h [-h option -h option]\t"}, + {"test", func() *IntSlice { + i := &IntSlice{} + i.Set("9") + return i + }(), "--test [--test option --test option]\t"}, +} + +func TestIntSliceFlagHelpOutput(t *testing.T) { + + for _, test := range intSliceFlagTests { + flag := IntSliceFlag{Name: test.name, Value: test.value} + output := flag.String() + + if output != test.expected { + t.Errorf("%q does not match %q", output, test.expected) + } + } +} + +func TestIntSliceFlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_SMURF", "42,3") + for _, test := range intSliceFlagTests { + flag := IntSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_SMURF"} + output := flag.String() + + expectedSuffix := " [$APP_SMURF]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_SMURF%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%q does not end with" + expectedSuffix, output) + } + } +} + +var float64FlagTests = []struct { + name string + expected string +}{ + {"help", "--help \"0\"\t"}, + {"h", "-h \"0\"\t"}, +} + +func TestFloat64FlagHelpOutput(t *testing.T) { + + for _, test := range float64FlagTests { + flag := Float64Flag{Name: test.name} + output := flag.String() + + if output != test.expected { + t.Errorf("%s does not match %s", output, test.expected) + } + } +} + +func TestFloat64FlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_BAZ", "99.4") + for _, test := range float64FlagTests { + flag := Float64Flag{Name: test.name, EnvVar: "APP_BAZ"} + output := flag.String() + + expectedSuffix := " [$APP_BAZ]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_BAZ%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%s does not end with" + expectedSuffix, output) + } + } +} + +var genericFlagTests = []struct { + name string + value Generic + expected string +}{ + {"test", &Parser{"abc", "def"}, "--test \"abc,def\"\ttest flag"}, + {"t", &Parser{"abc", "def"}, "-t \"abc,def\"\ttest flag"}, +} + +func TestGenericFlagHelpOutput(t *testing.T) { + + for _, test := range genericFlagTests { + flag := GenericFlag{Name: test.name, Value: test.value, Usage: "test flag"} + output := flag.String() + + if output != test.expected { + t.Errorf("%q does not match %q", output, test.expected) + } + } +} + +func TestGenericFlagWithEnvVarHelpOutput(t *testing.T) { + os.Clearenv() + os.Setenv("APP_ZAP", "3") + for _, test := range genericFlagTests { + flag := GenericFlag{Name: test.name, EnvVar: "APP_ZAP"} + output := flag.String() + + expectedSuffix := " [$APP_ZAP]" + if runtime.GOOS == "windows" { + expectedSuffix = " [%APP_ZAP%]" + } + if !strings.HasSuffix(output, expectedSuffix) { + t.Errorf("%s does not end with" + expectedSuffix, output) + } + } +} + +func TestParseMultiString(t *testing.T) { + (&App{ + Flags: []Flag{ + StringFlag{Name: "serve, s"}, + }, + Action: func(ctx *Context) { + if ctx.String("serve") != "10" { + t.Errorf("main name not set") + } + if ctx.String("s") != "10" { + t.Errorf("short name not set") + } + }, + }).Run([]string{"run", "-s", "10"}) +} + +func TestParseDestinationString(t *testing.T) { + var dest string + a := App{ + Flags: []Flag{ + StringFlag{ + Name: "dest", + Destination: &dest, + }, + }, + Action: func(ctx *Context) { + if dest != "10" { + t.Errorf("expected destination String 10") + } + }, + } + a.Run([]string{"run", "--dest", "10"}) +} + +func TestParseMultiStringFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_COUNT", "20") + (&App{ + Flags: []Flag{ + StringFlag{Name: "count, c", EnvVar: "APP_COUNT"}, + }, + Action: func(ctx *Context) { + if ctx.String("count") != "20" { + t.Errorf("main name not set") + } + if ctx.String("c") != "20" { + t.Errorf("short name not set") + } + }, + }).Run([]string{"run"}) +} + +func TestParseMultiStringFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_COUNT", "20") + (&App{ + Flags: []Flag{ + StringFlag{Name: "count, c", EnvVar: "COMPAT_COUNT,APP_COUNT"}, + }, + Action: func(ctx *Context) { + if ctx.String("count") != "20" { + t.Errorf("main name not set") + } + if ctx.String("c") != "20" { + t.Errorf("short name not set") + } + }, + }).Run([]string{"run"}) +} + +func TestParseMultiStringSlice(t *testing.T) { + (&App{ + Flags: []Flag{ + StringSliceFlag{Name: "serve, s", Value: &StringSlice{}}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.StringSlice("serve"), []string{"10", "20"}) { + t.Errorf("main name not set") + } + if !reflect.DeepEqual(ctx.StringSlice("s"), []string{"10", "20"}) { + t.Errorf("short name not set") + } + }, + }).Run([]string{"run", "-s", "10", "-s", "20"}) +} + +func TestParseMultiStringSliceFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_INTERVALS", "20,30,40") + + (&App{ + Flags: []Flag{ + StringSliceFlag{Name: "intervals, i", Value: &StringSlice{}, EnvVar: "APP_INTERVALS"}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) { + t.Errorf("main name not set from env") + } + if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) { + t.Errorf("short name not set from env") + } + }, + }).Run([]string{"run"}) +} + +func TestParseMultiStringSliceFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_INTERVALS", "20,30,40") + + (&App{ + Flags: []Flag{ + StringSliceFlag{Name: "intervals, i", Value: &StringSlice{}, EnvVar: "COMPAT_INTERVALS,APP_INTERVALS"}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) { + t.Errorf("main name not set from env") + } + if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) { + t.Errorf("short name not set from env") + } + }, + }).Run([]string{"run"}) +} + +func TestParseMultiInt(t *testing.T) { + a := App{ + Flags: []Flag{ + IntFlag{Name: "serve, s"}, + }, + Action: func(ctx *Context) { + if ctx.Int("serve") != 10 { + t.Errorf("main name not set") + } + if ctx.Int("s") != 10 { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run", "-s", "10"}) +} + +func TestParseDestinationInt(t *testing.T) { + var dest int + a := App{ + Flags: []Flag{ + IntFlag{ + Name: "dest", + Destination: &dest, + }, + }, + Action: func(ctx *Context) { + if dest != 10 { + t.Errorf("expected destination Int 10") + } + }, + } + a.Run([]string{"run", "--dest", "10"}) +} + +func TestParseMultiIntFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_TIMEOUT_SECONDS", "10") + a := App{ + Flags: []Flag{ + IntFlag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"}, + }, + Action: func(ctx *Context) { + if ctx.Int("timeout") != 10 { + t.Errorf("main name not set") + } + if ctx.Int("t") != 10 { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiIntFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_TIMEOUT_SECONDS", "10") + a := App{ + Flags: []Flag{ + IntFlag{Name: "timeout, t", EnvVar: "COMPAT_TIMEOUT_SECONDS,APP_TIMEOUT_SECONDS"}, + }, + Action: func(ctx *Context) { + if ctx.Int("timeout") != 10 { + t.Errorf("main name not set") + } + if ctx.Int("t") != 10 { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiIntSlice(t *testing.T) { + (&App{ + Flags: []Flag{ + IntSliceFlag{Name: "serve, s", Value: &IntSlice{}}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.IntSlice("serve"), []int{10, 20}) { + t.Errorf("main name not set") + } + if !reflect.DeepEqual(ctx.IntSlice("s"), []int{10, 20}) { + t.Errorf("short name not set") + } + }, + }).Run([]string{"run", "-s", "10", "-s", "20"}) +} + +func TestParseMultiIntSliceFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_INTERVALS", "20,30,40") + + (&App{ + Flags: []Flag{ + IntSliceFlag{Name: "intervals, i", Value: &IntSlice{}, EnvVar: "APP_INTERVALS"}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) { + t.Errorf("main name not set from env") + } + if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) { + t.Errorf("short name not set from env") + } + }, + }).Run([]string{"run"}) +} + +func TestParseMultiIntSliceFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_INTERVALS", "20,30,40") + + (&App{ + Flags: []Flag{ + IntSliceFlag{Name: "intervals, i", Value: &IntSlice{}, EnvVar: "COMPAT_INTERVALS,APP_INTERVALS"}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) { + t.Errorf("main name not set from env") + } + if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) { + t.Errorf("short name not set from env") + } + }, + }).Run([]string{"run"}) +} + +func TestParseMultiFloat64(t *testing.T) { + a := App{ + Flags: []Flag{ + Float64Flag{Name: "serve, s"}, + }, + Action: func(ctx *Context) { + if ctx.Float64("serve") != 10.2 { + t.Errorf("main name not set") + } + if ctx.Float64("s") != 10.2 { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run", "-s", "10.2"}) +} + +func TestParseDestinationFloat64(t *testing.T) { + var dest float64 + a := App{ + Flags: []Flag{ + Float64Flag{ + Name: "dest", + Destination: &dest, + }, + }, + Action: func(ctx *Context) { + if dest != 10.2 { + t.Errorf("expected destination Float64 10.2") + } + }, + } + a.Run([]string{"run", "--dest", "10.2"}) +} + +func TestParseMultiFloat64FromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_TIMEOUT_SECONDS", "15.5") + a := App{ + Flags: []Flag{ + Float64Flag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"}, + }, + Action: func(ctx *Context) { + if ctx.Float64("timeout") != 15.5 { + t.Errorf("main name not set") + } + if ctx.Float64("t") != 15.5 { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiFloat64FromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_TIMEOUT_SECONDS", "15.5") + a := App{ + Flags: []Flag{ + Float64Flag{Name: "timeout, t", EnvVar: "COMPAT_TIMEOUT_SECONDS,APP_TIMEOUT_SECONDS"}, + }, + Action: func(ctx *Context) { + if ctx.Float64("timeout") != 15.5 { + t.Errorf("main name not set") + } + if ctx.Float64("t") != 15.5 { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiBool(t *testing.T) { + a := App{ + Flags: []Flag{ + BoolFlag{Name: "serve, s"}, + }, + Action: func(ctx *Context) { + if ctx.Bool("serve") != true { + t.Errorf("main name not set") + } + if ctx.Bool("s") != true { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run", "--serve"}) +} + +func TestParseDestinationBool(t *testing.T) { + var dest bool + a := App{ + Flags: []Flag{ + BoolFlag{ + Name: "dest", + Destination: &dest, + }, + }, + Action: func(ctx *Context) { + if dest != true { + t.Errorf("expected destination Bool true") + } + }, + } + a.Run([]string{"run", "--dest"}) +} + +func TestParseMultiBoolFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_DEBUG", "1") + a := App{ + Flags: []Flag{ + BoolFlag{Name: "debug, d", EnvVar: "APP_DEBUG"}, + }, + Action: func(ctx *Context) { + if ctx.Bool("debug") != true { + t.Errorf("main name not set from env") + } + if ctx.Bool("d") != true { + t.Errorf("short name not set from env") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiBoolFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_DEBUG", "1") + a := App{ + Flags: []Flag{ + BoolFlag{Name: "debug, d", EnvVar: "COMPAT_DEBUG,APP_DEBUG"}, + }, + Action: func(ctx *Context) { + if ctx.Bool("debug") != true { + t.Errorf("main name not set from env") + } + if ctx.Bool("d") != true { + t.Errorf("short name not set from env") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiBoolT(t *testing.T) { + a := App{ + Flags: []Flag{ + BoolTFlag{Name: "serve, s"}, + }, + Action: func(ctx *Context) { + if ctx.BoolT("serve") != true { + t.Errorf("main name not set") + } + if ctx.BoolT("s") != true { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run", "--serve"}) +} + +func TestParseDestinationBoolT(t *testing.T) { + var dest bool + a := App{ + Flags: []Flag{ + BoolTFlag{ + Name: "dest", + Destination: &dest, + }, + }, + Action: func(ctx *Context) { + if dest != true { + t.Errorf("expected destination BoolT true") + } + }, + } + a.Run([]string{"run", "--dest"}) +} + +func TestParseMultiBoolTFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_DEBUG", "0") + a := App{ + Flags: []Flag{ + BoolTFlag{Name: "debug, d", EnvVar: "APP_DEBUG"}, + }, + Action: func(ctx *Context) { + if ctx.BoolT("debug") != false { + t.Errorf("main name not set from env") + } + if ctx.BoolT("d") != false { + t.Errorf("short name not set from env") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseMultiBoolTFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_DEBUG", "0") + a := App{ + Flags: []Flag{ + BoolTFlag{Name: "debug, d", EnvVar: "COMPAT_DEBUG,APP_DEBUG"}, + }, + Action: func(ctx *Context) { + if ctx.BoolT("debug") != false { + t.Errorf("main name not set from env") + } + if ctx.BoolT("d") != false { + t.Errorf("short name not set from env") + } + }, + } + a.Run([]string{"run"}) +} + +type Parser [2]string + +func (p *Parser) Set(value string) error { + parts := strings.Split(value, ",") + if len(parts) != 2 { + return fmt.Errorf("invalid format") + } + + (*p)[0] = parts[0] + (*p)[1] = parts[1] + + return nil +} + +func (p *Parser) String() string { + return fmt.Sprintf("%s,%s", p[0], p[1]) +} + +func TestParseGeneric(t *testing.T) { + a := App{ + Flags: []Flag{ + GenericFlag{Name: "serve, s", Value: &Parser{}}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"10", "20"}) { + t.Errorf("main name not set") + } + if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"10", "20"}) { + t.Errorf("short name not set") + } + }, + } + a.Run([]string{"run", "-s", "10,20"}) +} + +func TestParseGenericFromEnv(t *testing.T) { + os.Clearenv() + os.Setenv("APP_SERVE", "20,30") + a := App{ + Flags: []Flag{ + GenericFlag{Name: "serve, s", Value: &Parser{}, EnvVar: "APP_SERVE"}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"20", "30"}) { + t.Errorf("main name not set from env") + } + if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"20", "30"}) { + t.Errorf("short name not set from env") + } + }, + } + a.Run([]string{"run"}) +} + +func TestParseGenericFromEnvCascade(t *testing.T) { + os.Clearenv() + os.Setenv("APP_FOO", "99,2000") + a := App{ + Flags: []Flag{ + GenericFlag{Name: "foos", Value: &Parser{}, EnvVar: "COMPAT_FOO,APP_FOO"}, + }, + Action: func(ctx *Context) { + if !reflect.DeepEqual(ctx.Generic("foos"), &Parser{"99", "2000"}) { + t.Errorf("value not set from env") + } + }, + } + a.Run([]string{"run"}) +} diff --git a/vendor/github.com/codegangsta/cli/help.go b/vendor/github.com/codegangsta/cli/help.go new file mode 100644 index 0000000..15916f8 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/help.go @@ -0,0 +1,248 @@ +package cli + +import ( + "fmt" + "io" + "strings" + "text/tabwriter" + "text/template" +) + +// The text template for the Default help topic. +// cli.go uses text/template to render templates. You can +// render custom help text by setting this variable. +var AppHelpTemplate = `NAME: + {{.Name}} - {{.Usage}} + +USAGE: + {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .Flags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}} + {{if .Version}} +VERSION: + {{.Version}} + {{end}}{{if len .Authors}} +AUTHOR(S): + {{range .Authors}}{{ . }}{{end}} + {{end}}{{if .Commands}} +COMMANDS: + {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} + {{end}}{{end}}{{if .Flags}} +GLOBAL OPTIONS: + {{range .Flags}}{{.}} + {{end}}{{end}}{{if .Copyright }} +COPYRIGHT: + {{.Copyright}} + {{end}} +` + +// The text template for the command help topic. +// cli.go uses text/template to render templates. You can +// render custom help text by setting this variable. +var CommandHelpTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}}{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{if .Description}} + +DESCRIPTION: + {{.Description}}{{end}}{{if .Flags}} + +OPTIONS: + {{range .Flags}}{{.}} + {{end}}{{ end }} +` + +// The text template for the subcommand help topic. +// cli.go uses text/template to render templates. You can +// render custom help text by setting this variable. +var SubcommandHelpTemplate = `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} command{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}} + +COMMANDS: + {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} + {{end}}{{if .Flags}} +OPTIONS: + {{range .Flags}}{{.}} + {{end}}{{end}} +` + +var helpCommand = Command{ + Name: "help", + Aliases: []string{"h"}, + Usage: "Shows a list of commands or help for one command", + ArgsUsage: "[command]", + Action: func(c *Context) { + args := c.Args() + if args.Present() { + ShowCommandHelp(c, args.First()) + } else { + ShowAppHelp(c) + } + }, +} + +var helpSubcommand = Command{ + Name: "help", + Aliases: []string{"h"}, + Usage: "Shows a list of commands or help for one command", + ArgsUsage: "[command]", + Action: func(c *Context) { + args := c.Args() + if args.Present() { + ShowCommandHelp(c, args.First()) + } else { + ShowSubcommandHelp(c) + } + }, +} + +// Prints help for the App or Command +type helpPrinter func(w io.Writer, templ string, data interface{}) + +var HelpPrinter helpPrinter = printHelp + +// Prints version for the App +var VersionPrinter = printVersion + +func ShowAppHelp(c *Context) { + HelpPrinter(c.App.Writer, AppHelpTemplate, c.App) +} + +// Prints the list of subcommands as the default app completion method +func DefaultAppComplete(c *Context) { + for _, command := range c.App.Commands { + for _, name := range command.Names() { + fmt.Fprintln(c.App.Writer, name) + } + } +} + +// Prints help for the given command +func ShowCommandHelp(ctx *Context, command string) { + // show the subcommand help for a command with subcommands + if command == "" { + HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App) + return + } + + for _, c := range ctx.App.Commands { + if c.HasName(command) { + HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c) + return + } + } + + if ctx.App.CommandNotFound != nil { + ctx.App.CommandNotFound(ctx, command) + } else { + fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command) + } +} + +// Prints help for the given subcommand +func ShowSubcommandHelp(c *Context) { + ShowCommandHelp(c, c.Command.Name) +} + +// Prints the version number of the App +func ShowVersion(c *Context) { + VersionPrinter(c) +} + +func printVersion(c *Context) { + fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version) +} + +// Prints the lists of commands within a given context +func ShowCompletions(c *Context) { + a := c.App + if a != nil && a.BashComplete != nil { + a.BashComplete(c) + } +} + +// Prints the custom completions for a given command +func ShowCommandCompletions(ctx *Context, command string) { + c := ctx.App.Command(command) + if c != nil && c.BashComplete != nil { + c.BashComplete(ctx) + } +} + +func printHelp(out io.Writer, templ string, data interface{}) { + funcMap := template.FuncMap{ + "join": strings.Join, + } + + w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0) + t := template.Must(template.New("help").Funcs(funcMap).Parse(templ)) + err := t.Execute(w, data) + if err != nil { + // If the writer is closed, t.Execute will fail, and there's nothing + // we can do to recover. We could send this to os.Stderr if we need. + return + } + w.Flush() +} + +func checkVersion(c *Context) bool { + found := false + if VersionFlag.Name != "" { + eachName(VersionFlag.Name, func(name string) { + if c.GlobalBool(name) || c.Bool(name) { + found = true + } + }) + } + return found +} + +func checkHelp(c *Context) bool { + found := false + if HelpFlag.Name != "" { + eachName(HelpFlag.Name, func(name string) { + if c.GlobalBool(name) || c.Bool(name) { + found = true + } + }) + } + return found +} + +func checkCommandHelp(c *Context, name string) bool { + if c.Bool("h") || c.Bool("help") { + ShowCommandHelp(c, name) + return true + } + + return false +} + +func checkSubcommandHelp(c *Context) bool { + if c.GlobalBool("h") || c.GlobalBool("help") { + ShowSubcommandHelp(c) + return true + } + + return false +} + +func checkCompletions(c *Context) bool { + if (c.GlobalBool(BashCompletionFlag.Name) || c.Bool(BashCompletionFlag.Name)) && c.App.EnableBashCompletion { + ShowCompletions(c) + return true + } + + return false +} + +func checkCommandCompletions(c *Context, name string) bool { + if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion { + ShowCommandCompletions(c, name) + return true + } + + return false +} diff --git a/vendor/github.com/codegangsta/cli/help_test.go b/vendor/github.com/codegangsta/cli/help_test.go new file mode 100644 index 0000000..350e263 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/help_test.go @@ -0,0 +1,94 @@ +package cli + +import ( + "bytes" + "testing" +) + +func Test_ShowAppHelp_NoAuthor(t *testing.T) { + output := new(bytes.Buffer) + app := NewApp() + app.Writer = output + + c := NewContext(app, nil, nil) + + ShowAppHelp(c) + + if bytes.Index(output.Bytes(), []byte("AUTHOR(S):")) != -1 { + t.Errorf("expected\n%snot to include %s", output.String(), "AUTHOR(S):") + } +} + +func Test_ShowAppHelp_NoVersion(t *testing.T) { + output := new(bytes.Buffer) + app := NewApp() + app.Writer = output + + app.Version = "" + + c := NewContext(app, nil, nil) + + ShowAppHelp(c) + + if bytes.Index(output.Bytes(), []byte("VERSION:")) != -1 { + t.Errorf("expected\n%snot to include %s", output.String(), "VERSION:") + } +} + +func Test_Help_Custom_Flags(t *testing.T) { + oldFlag := HelpFlag + defer func() { + HelpFlag = oldFlag + }() + + HelpFlag = BoolFlag{ + Name: "help, x", + Usage: "show help", + } + + app := App{ + Flags: []Flag{ + BoolFlag{Name: "foo, h"}, + }, + Action: func(ctx *Context) { + if ctx.Bool("h") != true { + t.Errorf("custom help flag not set") + } + }, + } + output := new(bytes.Buffer) + app.Writer = output + app.Run([]string{"test", "-h"}) + if output.Len() > 0 { + t.Errorf("unexpected output: %s", output.String()) + } +} + +func Test_Version_Custom_Flags(t *testing.T) { + oldFlag := VersionFlag + defer func() { + VersionFlag = oldFlag + }() + + VersionFlag = BoolFlag{ + Name: "version, V", + Usage: "show version", + } + + app := App{ + Flags: []Flag{ + BoolFlag{Name: "foo, v"}, + }, + Action: func(ctx *Context) { + if ctx.Bool("v") != true { + t.Errorf("custom version flag not set") + } + }, + } + output := new(bytes.Buffer) + app.Writer = output + app.Run([]string{"test", "-v"}) + if output.Len() > 0 { + t.Errorf("unexpected output: %s", output.String()) + } +} diff --git a/vendor/github.com/codegangsta/cli/helpers_test.go b/vendor/github.com/codegangsta/cli/helpers_test.go new file mode 100644 index 0000000..b1b7339 --- /dev/null +++ b/vendor/github.com/codegangsta/cli/helpers_test.go @@ -0,0 +1,19 @@ +package cli + +import ( + "reflect" + "testing" +) + +/* Test Helpers */ +func expect(t *testing.T, a interface{}, b interface{}) { + if !reflect.DeepEqual(a, b) { + t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) + } +} + +func refute(t *testing.T, a interface{}, b interface{}) { + if reflect.DeepEqual(a, b) { + t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) + } +} diff --git a/vendor/github.com/petar/GoLLRB/llrb/avgvar.go b/vendor/github.com/petar/GoLLRB/llrb/avgvar.go new file mode 100644 index 0000000..2d7e2a3 --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/avgvar.go @@ -0,0 +1,39 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +import "math" + +// avgVar maintains the average and variance of a stream of numbers +// in a space-efficient manner. +type avgVar struct { + count int64 + sum, sumsq float64 +} + +func (av *avgVar) Init() { + av.count = 0 + av.sum = 0.0 + av.sumsq = 0.0 +} + +func (av *avgVar) Add(sample float64) { + av.count++ + av.sum += sample + av.sumsq += sample * sample +} + +func (av *avgVar) GetCount() int64 { return av.count } + +func (av *avgVar) GetAvg() float64 { return av.sum / float64(av.count) } + +func (av *avgVar) GetTotal() float64 { return av.sum } + +func (av *avgVar) GetVar() float64 { + a := av.GetAvg() + return av.sumsq/float64(av.count) - a*a +} + +func (av *avgVar) GetStdDev() float64 { return math.Sqrt(av.GetVar()) } diff --git a/vendor/github.com/petar/GoLLRB/llrb/iterator.go b/vendor/github.com/petar/GoLLRB/llrb/iterator.go new file mode 100644 index 0000000..ee7b27f --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/iterator.go @@ -0,0 +1,93 @@ +package llrb + +type ItemIterator func(i Item) bool + +//func (t *Tree) Ascend(iterator ItemIterator) { +// t.AscendGreaterOrEqual(Inf(-1), iterator) +//} + +func (t *LLRB) AscendRange(greaterOrEqual, lessThan Item, iterator ItemIterator) { + t.ascendRange(t.root, greaterOrEqual, lessThan, iterator) +} + +func (t *LLRB) ascendRange(h *Node, inf, sup Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if !less(h.Item, sup) { + return t.ascendRange(h.Left, inf, sup, iterator) + } + if less(h.Item, inf) { + return t.ascendRange(h.Right, inf, sup, iterator) + } + + if !t.ascendRange(h.Left, inf, sup, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + return t.ascendRange(h.Right, inf, sup, iterator) +} + +// AscendGreaterOrEqual will call iterator once for each element greater or equal to +// pivot in ascending order. It will stop whenever the iterator returns false. +func (t *LLRB) AscendGreaterOrEqual(pivot Item, iterator ItemIterator) { + t.ascendGreaterOrEqual(t.root, pivot, iterator) +} + +func (t *LLRB) ascendGreaterOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if !less(h.Item, pivot) { + if !t.ascendGreaterOrEqual(h.Left, pivot, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + } + return t.ascendGreaterOrEqual(h.Right, pivot, iterator) +} + +func (t *LLRB) AscendLessThan(pivot Item, iterator ItemIterator) { + t.ascendLessThan(t.root, pivot, iterator) +} + +func (t *LLRB) ascendLessThan(h *Node, pivot Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if !t.ascendLessThan(h.Left, pivot, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + if less(h.Item, pivot) { + return t.ascendLessThan(h.Left, pivot, iterator) + } + return true +} + +// DescendLessOrEqual will call iterator once for each element less than the +// pivot in descending order. It will stop whenever the iterator returns false. +func (t *LLRB) DescendLessOrEqual(pivot Item, iterator ItemIterator) { + t.descendLessOrEqual(t.root, pivot, iterator) +} + +func (t *LLRB) descendLessOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { + if h == nil { + return true + } + if less(h.Item, pivot) || !less(pivot, h.Item) { + if !t.descendLessOrEqual(h.Right, pivot, iterator) { + return false + } + if !iterator(h.Item) { + return false + } + } + return t.descendLessOrEqual(h.Left, pivot, iterator) +} diff --git a/vendor/github.com/petar/GoLLRB/llrb/iterator_test.go b/vendor/github.com/petar/GoLLRB/llrb/iterator_test.go new file mode 100644 index 0000000..db5e12c --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/iterator_test.go @@ -0,0 +1,76 @@ +package llrb + +import ( + "reflect" + "testing" +) + +func TestAscendGreaterOrEqual(t *testing.T) { + tree := New() + tree.InsertNoReplace(Int(4)) + tree.InsertNoReplace(Int(6)) + tree.InsertNoReplace(Int(1)) + tree.InsertNoReplace(Int(3)) + var ary []Item + tree.AscendGreaterOrEqual(Int(-1), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected := []Item{Int(1), Int(3), Int(4), Int(6)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.AscendGreaterOrEqual(Int(3), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(3), Int(4), Int(6)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.AscendGreaterOrEqual(Int(2), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(3), Int(4), Int(6)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } +} + +func TestDescendLessOrEqual(t *testing.T) { + tree := New() + tree.InsertNoReplace(Int(4)) + tree.InsertNoReplace(Int(6)) + tree.InsertNoReplace(Int(1)) + tree.InsertNoReplace(Int(3)) + var ary []Item + tree.DescendLessOrEqual(Int(10), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected := []Item{Int(6), Int(4), Int(3), Int(1)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.DescendLessOrEqual(Int(4), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(4), Int(3), Int(1)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } + ary = nil + tree.DescendLessOrEqual(Int(5), func(i Item) bool { + ary = append(ary, i) + return true + }) + expected = []Item{Int(4), Int(3), Int(1)} + if !reflect.DeepEqual(ary, expected) { + t.Errorf("expected %v but got %v", expected, ary) + } +} diff --git a/vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go b/vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go new file mode 100644 index 0000000..47126a3 --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go @@ -0,0 +1,46 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +// GetHeight() returns an item in the tree with key @key, and it's height in the tree +func (t *LLRB) GetHeight(key Item) (result Item, depth int) { + return t.getHeight(t.root, key) +} + +func (t *LLRB) getHeight(h *Node, item Item) (Item, int) { + if h == nil { + return nil, 0 + } + if less(item, h.Item) { + result, depth := t.getHeight(h.Left, item) + return result, depth + 1 + } + if less(h.Item, item) { + result, depth := t.getHeight(h.Right, item) + return result, depth + 1 + } + return h.Item, 0 +} + +// HeightStats() returns the average and standard deviation of the height +// of elements in the tree +func (t *LLRB) HeightStats() (avg, stddev float64) { + av := &avgVar{} + heightStats(t.root, 0, av) + return av.GetAvg(), av.GetStdDev() +} + +func heightStats(h *Node, d int, av *avgVar) { + if h == nil { + return + } + av.Add(float64(d)) + if h.Left != nil { + heightStats(h.Left, d+1, av) + } + if h.Right != nil { + heightStats(h.Right, d+1, av) + } +} diff --git a/vendor/github.com/petar/GoLLRB/llrb/llrb.go b/vendor/github.com/petar/GoLLRB/llrb/llrb.go new file mode 100644 index 0000000..81373fb --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/llrb.go @@ -0,0 +1,456 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// A Left-Leaning Red-Black (LLRB) implementation of 2-3 balanced binary search trees, +// based on the following work: +// +// http://www.cs.princeton.edu/~rs/talks/LLRB/08Penn.pdf +// http://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf +// http://www.cs.princeton.edu/~rs/talks/LLRB/Java/RedBlackBST.java +// +// 2-3 trees (and the run-time equivalent 2-3-4 trees) are the de facto standard BST +// algoritms found in implementations of Python, Java, and other libraries. The LLRB +// implementation of 2-3 trees is a recent improvement on the traditional implementation, +// observed and documented by Robert Sedgewick. +// +package llrb + +// Tree is a Left-Leaning Red-Black (LLRB) implementation of 2-3 trees +type LLRB struct { + count int + root *Node +} + +type Node struct { + Item + Left, Right *Node // Pointers to left and right child nodes + Black bool // If set, the color of the link (incoming from the parent) is black + // In the LLRB, new nodes are always red, hence the zero-value for node +} + +type Item interface { + Less(than Item) bool +} + +// +func less(x, y Item) bool { + if x == pinf { + return false + } + if x == ninf { + return true + } + return x.Less(y) +} + +// Inf returns an Item that is "bigger than" any other item, if sign is positive. +// Otherwise it returns an Item that is "smaller than" any other item. +func Inf(sign int) Item { + if sign == 0 { + panic("sign") + } + if sign > 0 { + return pinf + } + return ninf +} + +var ( + ninf = nInf{} + pinf = pInf{} +) + +type nInf struct{} + +func (nInf) Less(Item) bool { + return true +} + +type pInf struct{} + +func (pInf) Less(Item) bool { + return false +} + +// New() allocates a new tree +func New() *LLRB { + return &LLRB{} +} + +// SetRoot sets the root node of the tree. +// It is intended to be used by functions that deserialize the tree. +func (t *LLRB) SetRoot(r *Node) { + t.root = r +} + +// Root returns the root node of the tree. +// It is intended to be used by functions that serialize the tree. +func (t *LLRB) Root() *Node { + return t.root +} + +// Len returns the number of nodes in the tree. +func (t *LLRB) Len() int { return t.count } + +// Has returns true if the tree contains an element whose order is the same as that of key. +func (t *LLRB) Has(key Item) bool { + return t.Get(key) != nil +} + +// Get retrieves an element from the tree whose order is the same as that of key. +func (t *LLRB) Get(key Item) Item { + h := t.root + for h != nil { + switch { + case less(key, h.Item): + h = h.Left + case less(h.Item, key): + h = h.Right + default: + return h.Item + } + } + return nil +} + +// Min returns the minimum element in the tree. +func (t *LLRB) Min() Item { + h := t.root + if h == nil { + return nil + } + for h.Left != nil { + h = h.Left + } + return h.Item +} + +// Max returns the maximum element in the tree. +func (t *LLRB) Max() Item { + h := t.root + if h == nil { + return nil + } + for h.Right != nil { + h = h.Right + } + return h.Item +} + +func (t *LLRB) ReplaceOrInsertBulk(items ...Item) { + for _, i := range items { + t.ReplaceOrInsert(i) + } +} + +func (t *LLRB) InsertNoReplaceBulk(items ...Item) { + for _, i := range items { + t.InsertNoReplace(i) + } +} + +// ReplaceOrInsert inserts item into the tree. If an existing +// element has the same order, it is removed from the tree and returned. +func (t *LLRB) ReplaceOrInsert(item Item) Item { + if item == nil { + panic("inserting nil item") + } + var replaced Item + t.root, replaced = t.replaceOrInsert(t.root, item) + t.root.Black = true + if replaced == nil { + t.count++ + } + return replaced +} + +func (t *LLRB) replaceOrInsert(h *Node, item Item) (*Node, Item) { + if h == nil { + return newNode(item), nil + } + + h = walkDownRot23(h) + + var replaced Item + if less(item, h.Item) { // BUG + h.Left, replaced = t.replaceOrInsert(h.Left, item) + } else if less(h.Item, item) { + h.Right, replaced = t.replaceOrInsert(h.Right, item) + } else { + replaced, h.Item = h.Item, item + } + + h = walkUpRot23(h) + + return h, replaced +} + +// InsertNoReplace inserts item into the tree. If an existing +// element has the same order, both elements remain in the tree. +func (t *LLRB) InsertNoReplace(item Item) { + if item == nil { + panic("inserting nil item") + } + t.root = t.insertNoReplace(t.root, item) + t.root.Black = true + t.count++ +} + +func (t *LLRB) insertNoReplace(h *Node, item Item) *Node { + if h == nil { + return newNode(item) + } + + h = walkDownRot23(h) + + if less(item, h.Item) { + h.Left = t.insertNoReplace(h.Left, item) + } else { + h.Right = t.insertNoReplace(h.Right, item) + } + + return walkUpRot23(h) +} + +// Rotation driver routines for 2-3 algorithm + +func walkDownRot23(h *Node) *Node { return h } + +func walkUpRot23(h *Node) *Node { + if isRed(h.Right) && !isRed(h.Left) { + h = rotateLeft(h) + } + + if isRed(h.Left) && isRed(h.Left.Left) { + h = rotateRight(h) + } + + if isRed(h.Left) && isRed(h.Right) { + flip(h) + } + + return h +} + +// Rotation driver routines for 2-3-4 algorithm + +func walkDownRot234(h *Node) *Node { + if isRed(h.Left) && isRed(h.Right) { + flip(h) + } + + return h +} + +func walkUpRot234(h *Node) *Node { + if isRed(h.Right) && !isRed(h.Left) { + h = rotateLeft(h) + } + + if isRed(h.Left) && isRed(h.Left.Left) { + h = rotateRight(h) + } + + return h +} + +// DeleteMin deletes the minimum element in the tree and returns the +// deleted item or nil otherwise. +func (t *LLRB) DeleteMin() Item { + var deleted Item + t.root, deleted = deleteMin(t.root) + if t.root != nil { + t.root.Black = true + } + if deleted != nil { + t.count-- + } + return deleted +} + +// deleteMin code for LLRB 2-3 trees +func deleteMin(h *Node) (*Node, Item) { + if h == nil { + return nil, nil + } + if h.Left == nil { + return nil, h.Item + } + + if !isRed(h.Left) && !isRed(h.Left.Left) { + h = moveRedLeft(h) + } + + var deleted Item + h.Left, deleted = deleteMin(h.Left) + + return fixUp(h), deleted +} + +// DeleteMax deletes the maximum element in the tree and returns +// the deleted item or nil otherwise +func (t *LLRB) DeleteMax() Item { + var deleted Item + t.root, deleted = deleteMax(t.root) + if t.root != nil { + t.root.Black = true + } + if deleted != nil { + t.count-- + } + return deleted +} + +func deleteMax(h *Node) (*Node, Item) { + if h == nil { + return nil, nil + } + if isRed(h.Left) { + h = rotateRight(h) + } + if h.Right == nil { + return nil, h.Item + } + if !isRed(h.Right) && !isRed(h.Right.Left) { + h = moveRedRight(h) + } + var deleted Item + h.Right, deleted = deleteMax(h.Right) + + return fixUp(h), deleted +} + +// Delete deletes an item from the tree whose key equals key. +// The deleted item is return, otherwise nil is returned. +func (t *LLRB) Delete(key Item) Item { + var deleted Item + t.root, deleted = t.delete(t.root, key) + if t.root != nil { + t.root.Black = true + } + if deleted != nil { + t.count-- + } + return deleted +} + +func (t *LLRB) delete(h *Node, item Item) (*Node, Item) { + var deleted Item + if h == nil { + return nil, nil + } + if less(item, h.Item) { + if h.Left == nil { // item not present. Nothing to delete + return h, nil + } + if !isRed(h.Left) && !isRed(h.Left.Left) { + h = moveRedLeft(h) + } + h.Left, deleted = t.delete(h.Left, item) + } else { + if isRed(h.Left) { + h = rotateRight(h) + } + // If @item equals @h.Item and no right children at @h + if !less(h.Item, item) && h.Right == nil { + return nil, h.Item + } + // PETAR: Added 'h.Right != nil' below + if h.Right != nil && !isRed(h.Right) && !isRed(h.Right.Left) { + h = moveRedRight(h) + } + // If @item equals @h.Item, and (from above) 'h.Right != nil' + if !less(h.Item, item) { + var subDeleted Item + h.Right, subDeleted = deleteMin(h.Right) + if subDeleted == nil { + panic("logic") + } + deleted, h.Item = h.Item, subDeleted + } else { // Else, @item is bigger than @h.Item + h.Right, deleted = t.delete(h.Right, item) + } + } + + return fixUp(h), deleted +} + +// Internal node manipulation routines + +func newNode(item Item) *Node { return &Node{Item: item} } + +func isRed(h *Node) bool { + if h == nil { + return false + } + return !h.Black +} + +func rotateLeft(h *Node) *Node { + x := h.Right + if x.Black { + panic("rotating a black link") + } + h.Right = x.Left + x.Left = h + x.Black = h.Black + h.Black = false + return x +} + +func rotateRight(h *Node) *Node { + x := h.Left + if x.Black { + panic("rotating a black link") + } + h.Left = x.Right + x.Right = h + x.Black = h.Black + h.Black = false + return x +} + +// REQUIRE: Left and Right children must be present +func flip(h *Node) { + h.Black = !h.Black + h.Left.Black = !h.Left.Black + h.Right.Black = !h.Right.Black +} + +// REQUIRE: Left and Right children must be present +func moveRedLeft(h *Node) *Node { + flip(h) + if isRed(h.Right.Left) { + h.Right = rotateRight(h.Right) + h = rotateLeft(h) + flip(h) + } + return h +} + +// REQUIRE: Left and Right children must be present +func moveRedRight(h *Node) *Node { + flip(h) + if isRed(h.Left.Left) { + h = rotateRight(h) + flip(h) + } + return h +} + +func fixUp(h *Node) *Node { + if isRed(h.Right) { + h = rotateLeft(h) + } + + if isRed(h.Left) && isRed(h.Left.Left) { + h = rotateRight(h) + } + + if isRed(h.Left) && isRed(h.Right) { + flip(h) + } + + return h +} diff --git a/vendor/github.com/petar/GoLLRB/llrb/llrb_test.go b/vendor/github.com/petar/GoLLRB/llrb/llrb_test.go new file mode 100644 index 0000000..b7bc978 --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/llrb_test.go @@ -0,0 +1,239 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +import ( + "math" + "math/rand" + "testing" +) + +func TestCases(t *testing.T) { + tree := New() + tree.ReplaceOrInsert(Int(1)) + tree.ReplaceOrInsert(Int(1)) + if tree.Len() != 1 { + t.Errorf("expecting len 1") + } + if !tree.Has(Int(1)) { + t.Errorf("expecting to find key=1") + } + + tree.Delete(Int(1)) + if tree.Len() != 0 { + t.Errorf("expecting len 0") + } + if tree.Has(Int(1)) { + t.Errorf("not expecting to find key=1") + } + + tree.Delete(Int(1)) + if tree.Len() != 0 { + t.Errorf("expecting len 0") + } + if tree.Has(Int(1)) { + t.Errorf("not expecting to find key=1") + } +} + +func TestReverseInsertOrder(t *testing.T) { + tree := New() + n := 100 + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(n - i)) + } + i := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + i++ + if item.(Int) != Int(i) { + t.Errorf("bad order: got %d, expect %d", item.(Int), i) + } + return true + }) +} + +func TestRange(t *testing.T) { + tree := New() + order := []String{ + "ab", "aba", "abc", "a", "aa", "aaa", "b", "a-", "a!", + } + for _, i := range order { + tree.ReplaceOrInsert(i) + } + k := 0 + tree.AscendRange(String("ab"), String("ac"), func(item Item) bool { + if k > 3 { + t.Fatalf("returned more items than expected") + } + i1 := order[k] + i2 := item.(String) + if i1 != i2 { + t.Errorf("expecting %s, got %s", i1, i2) + } + k++ + return true + }) +} + +func TestRandomInsertOrder(t *testing.T) { + tree := New() + n := 1000 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + j := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + if item.(Int) != Int(j) { + t.Fatalf("bad order") + } + j++ + return true + }) +} + +func TestRandomReplace(t *testing.T) { + tree := New() + n := 100 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + perm = rand.Perm(n) + for i := 0; i < n; i++ { + if replaced := tree.ReplaceOrInsert(Int(perm[i])); replaced == nil || replaced.(Int) != Int(perm[i]) { + t.Errorf("error replacing") + } + } +} + +func TestRandomInsertSequentialDelete(t *testing.T) { + tree := New() + n := 1000 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + for i := 0; i < n; i++ { + tree.Delete(Int(i)) + } +} + +func TestRandomInsertDeleteNonExistent(t *testing.T) { + tree := New() + n := 100 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + if tree.Delete(Int(200)) != nil { + t.Errorf("deleted non-existent item") + } + if tree.Delete(Int(-2)) != nil { + t.Errorf("deleted non-existent item") + } + for i := 0; i < n; i++ { + if u := tree.Delete(Int(i)); u == nil || u.(Int) != Int(i) { + t.Errorf("delete failed") + } + } + if tree.Delete(Int(200)) != nil { + t.Errorf("deleted non-existent item") + } + if tree.Delete(Int(-2)) != nil { + t.Errorf("deleted non-existent item") + } +} + +func TestRandomInsertPartialDeleteOrder(t *testing.T) { + tree := New() + n := 100 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + for i := 1; i < n-1; i++ { + tree.Delete(Int(i)) + } + j := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + switch j { + case 0: + if item.(Int) != Int(0) { + t.Errorf("expecting 0") + } + case 1: + if item.(Int) != Int(n-1) { + t.Errorf("expecting %d", n-1) + } + } + j++ + return true + }) +} + +func TestRandomInsertStats(t *testing.T) { + tree := New() + n := 100000 + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.ReplaceOrInsert(Int(perm[i])) + } + avg, _ := tree.HeightStats() + expAvg := math.Log2(float64(n)) - 1.5 + if math.Abs(avg-expAvg) >= 2.0 { + t.Errorf("too much deviation from expected average height") + } +} + +func BenchmarkInsert(b *testing.B) { + tree := New() + for i := 0; i < b.N; i++ { + tree.ReplaceOrInsert(Int(b.N - i)) + } +} + +func BenchmarkDelete(b *testing.B) { + b.StopTimer() + tree := New() + for i := 0; i < b.N; i++ { + tree.ReplaceOrInsert(Int(b.N - i)) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + tree.Delete(Int(i)) + } +} + +func BenchmarkDeleteMin(b *testing.B) { + b.StopTimer() + tree := New() + for i := 0; i < b.N; i++ { + tree.ReplaceOrInsert(Int(b.N - i)) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + tree.DeleteMin() + } +} + +func TestInsertNoReplace(t *testing.T) { + tree := New() + n := 1000 + for q := 0; q < 2; q++ { + perm := rand.Perm(n) + for i := 0; i < n; i++ { + tree.InsertNoReplace(Int(perm[i])) + } + } + j := 0 + tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { + if item.(Int) != Int(j/2) { + t.Fatalf("bad order") + } + j++ + return true + }) +} diff --git a/vendor/github.com/petar/GoLLRB/llrb/util.go b/vendor/github.com/petar/GoLLRB/llrb/util.go new file mode 100644 index 0000000..63dbdb2 --- /dev/null +++ b/vendor/github.com/petar/GoLLRB/llrb/util.go @@ -0,0 +1,17 @@ +// Copyright 2010 Petar Maymounkov. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package llrb + +type Int int + +func (x Int) Less(than Item) bool { + return x < than.(Int) +} + +type String string + +func (x String) Less(than Item) bool { + return x < than.(String) +} From bd779844555d92cedd05fa7a57f6100746c7476d Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:22:08 +0100 Subject: [PATCH 03/10] remove vendor folder --- vendor/github.com/Redundancy/go-sync/LICENSE | 21 - .../github.com/Redundancy/go-sync/README.md | 119 -- .../go-sync/acceptancetests/nulltest.sh | 1 - .../Redundancy/go-sync/acceptancetests/run.sh | 21 - .../go-sync/acceptancetests/truncated_test.sh | 14 - .../acceptancetests/truncation_test.sh | 14 - .../go-sync/blocksources/blocksourcebase.go | 271 ----- .../blocksources/blocksourcebase_test.go | 303 ----- .../blocksources/fixed_size_block_resolver.go | 73 -- .../fixed_size_block_resolver_test.go | 127 -- .../go-sync/blocksources/helpers.go | 124 -- .../go-sync/blocksources/httpblocksource.go | 110 -- .../blocksources/httpblocksource_test.go | 251 ---- .../blocksources/readseeker_blocksource.go | 54 - .../readseeker_blocksource_test.go | 42 - .../blocksources/sortableBlockResponse.go | 19 - .../Redundancy/go-sync/chunks/chunks.go | 87 -- .../go-sync/circularbuffer/noalloc.go | 170 --- .../go-sync/circularbuffer/noalloc_test.go | 182 --- .../Redundancy/go-sync/comparer/comparer.go | 219 ---- .../go-sync/comparer/comparer_bench_test.go | 76 -- .../go-sync/comparer/comparer_test.go | 445 ------- .../Redundancy/go-sync/comparer/merger.go | 309 ----- .../go-sync/comparer/merger_test.go | 401 ------- .../go-sync/filechecksum/filechecksum.go | 237 ---- .../go-sync/filechecksum/filechecksum_test.go | 309 ----- .../go-sync/filechecksum/verifier.go | 48 - .../go-sync/filechecksum/verifier_test.go | 72 -- .../github.com/Redundancy/go-sync/gosync.go | 16 - .../Redundancy/go-sync/gosync/build.go | 114 -- .../Redundancy/go-sync/gosync/common.go | 277 ----- .../Redundancy/go-sync/gosync/diff.go | 145 --- .../Redundancy/go-sync/gosync/fileformat.md | 21 - .../Redundancy/go-sync/gosync/main.go | 65 - .../Redundancy/go-sync/gosync/patch.go | 105 -- .../Redundancy/go-sync/gosync_test.go | 111 -- .../Redundancy/go-sync/http_test.go | 174 --- .../Redundancy/go-sync/index/index.go | 183 --- .../go-sync/index/index_bench_test.go | 148 --- .../Redundancy/go-sync/index/index_test.go | 206 ---- .../go-sync/indexbuilder/indexbuilder.go | 55 - .../Redundancy/go-sync/patcher/blocksource.go | 58 - .../go-sync/patcher/sequential/sequential.go | 145 --- .../patcher/sequential/sequential_test.go | 188 --- .../Redundancy/go-sync/rollsum/rollsum_32.go | 85 -- .../go-sync/rollsum/rollsum_32_base.go | 86 -- .../go-sync/rollsum/rollsum_32_test.go | 298 ----- vendor/github.com/Redundancy/go-sync/rsync.go | 347 ------ .../go-sync/util/readers/injectedreader.go | 18 - .../go-sync/util/readers/nonrepeating.go | 35 - .../go-sync/util/readers/nonrepeating_test.go | 58 - .../go-sync/util/readers/readers.go | 6 - .../go-sync/util/readers/sequence_test.go | 1 - .../go-sync/util/readers/sequencelimit.go | 12 - .../go-sync/util/readers/uniform_test.go | 90 -- .../go-sync/util/readers/uniformreader.go | 55 - vendor/github.com/codegangsta/cli/LICENSE | 21 - vendor/github.com/codegangsta/cli/README.md | 394 ------- .../github.com/codegangsta/cli/altsrc/flag.go | 439 ------- .../codegangsta/cli/altsrc/flag_test.go | 336 ------ .../codegangsta/cli/altsrc/helpers_test.go | 18 - .../cli/altsrc/input_source_context.go | 21 - .../cli/altsrc/map_input_source.go | 152 --- .../cli/altsrc/yaml_command_test.go | 172 --- .../cli/altsrc/yaml_file_loader.go | 84 -- vendor/github.com/codegangsta/cli/app.go | 349 ------ vendor/github.com/codegangsta/cli/app_test.go | 1047 ----------------- .../github.com/codegangsta/cli/appveyor.yml | 16 - .../cli/autocomplete/bash_autocomplete | 14 - .../cli/autocomplete/zsh_autocomplete | 5 - vendor/github.com/codegangsta/cli/cli.go | 40 - vendor/github.com/codegangsta/cli/command.go | 250 ---- .../codegangsta/cli/command_test.go | 97 -- vendor/github.com/codegangsta/cli/context.go | 393 ------- .../codegangsta/cli/context_test.go | 121 -- vendor/github.com/codegangsta/cli/flag.go | 546 --------- .../github.com/codegangsta/cli/flag_test.go | 859 -------------- vendor/github.com/codegangsta/cli/help.go | 248 ---- .../github.com/codegangsta/cli/help_test.go | 94 -- .../codegangsta/cli/helpers_test.go | 19 - vendor/github.com/petar/GoLLRB/llrb/avgvar.go | 39 - .../github.com/petar/GoLLRB/llrb/iterator.go | 93 -- .../petar/GoLLRB/llrb/iterator_test.go | 76 -- .../petar/GoLLRB/llrb/llrb-stats.go | 46 - vendor/github.com/petar/GoLLRB/llrb/llrb.go | 456 ------- .../github.com/petar/GoLLRB/llrb/llrb_test.go | 239 ---- vendor/github.com/petar/GoLLRB/llrb/util.go | 17 - 87 files changed, 13922 deletions(-) delete mode 100644 vendor/github.com/Redundancy/go-sync/LICENSE delete mode 100644 vendor/github.com/Redundancy/go-sync/README.md delete mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh delete mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh delete mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh delete mode 100644 vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/helpers.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go delete mode 100644 vendor/github.com/Redundancy/go-sync/chunks/chunks.go delete mode 100644 vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go delete mode 100644 vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/comparer/comparer.go delete mode 100644 vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/comparer/merger.go delete mode 100644 vendor/github.com/Redundancy/go-sync/comparer/merger_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go delete mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go delete mode 100644 vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync/build.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync/common.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync/diff.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync/fileformat.md delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync/main.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync/patch.go delete mode 100644 vendor/github.com/Redundancy/go-sync/gosync_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/http_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/index/index.go delete mode 100644 vendor/github.com/Redundancy/go-sync/index/index_bench_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/index/index_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go delete mode 100644 vendor/github.com/Redundancy/go-sync/patcher/blocksource.go delete mode 100644 vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go delete mode 100644 vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go delete mode 100644 vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go delete mode 100644 vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/rsync.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/readers.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go delete mode 100644 vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go delete mode 100644 vendor/github.com/codegangsta/cli/LICENSE delete mode 100644 vendor/github.com/codegangsta/cli/README.md delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/flag.go delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/flag_test.go delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/helpers_test.go delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/input_source_context.go delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/map_input_source.go delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go delete mode 100644 vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go delete mode 100644 vendor/github.com/codegangsta/cli/app.go delete mode 100644 vendor/github.com/codegangsta/cli/app_test.go delete mode 100644 vendor/github.com/codegangsta/cli/appveyor.yml delete mode 100644 vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete delete mode 100644 vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete delete mode 100644 vendor/github.com/codegangsta/cli/cli.go delete mode 100644 vendor/github.com/codegangsta/cli/command.go delete mode 100644 vendor/github.com/codegangsta/cli/command_test.go delete mode 100644 vendor/github.com/codegangsta/cli/context.go delete mode 100644 vendor/github.com/codegangsta/cli/context_test.go delete mode 100644 vendor/github.com/codegangsta/cli/flag.go delete mode 100644 vendor/github.com/codegangsta/cli/flag_test.go delete mode 100644 vendor/github.com/codegangsta/cli/help.go delete mode 100644 vendor/github.com/codegangsta/cli/help_test.go delete mode 100644 vendor/github.com/codegangsta/cli/helpers_test.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/avgvar.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/iterator.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/iterator_test.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/llrb.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/llrb_test.go delete mode 100644 vendor/github.com/petar/GoLLRB/llrb/util.go diff --git a/vendor/github.com/Redundancy/go-sync/LICENSE b/vendor/github.com/Redundancy/go-sync/LICENSE deleted file mode 100644 index 1e4b9c6..0000000 --- a/vendor/github.com/Redundancy/go-sync/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Daniel Speed - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/Redundancy/go-sync/README.md b/vendor/github.com/Redundancy/go-sync/README.md deleted file mode 100644 index 584e77d..0000000 --- a/vendor/github.com/Redundancy/go-sync/README.md +++ /dev/null @@ -1,119 +0,0 @@ -Go-Sync ------- -[![Build Status](https://travis-ci.org/Redundancy/go-sync.svg?branch=master)](https://travis-ci.org/Redundancy/go-sync) -[![GoDoc](https://godoc.org/github.com/Redundancy/go-sync?status.svg)](https://godoc.org/github.com/Redundancy/go-sync) - -gosync is a library inspired by zsync and rsync. -Here are the goals: - -### Fast -Using the concurrency and performance features of Golang, Go-sync is designed to take advantage of multiple processors and multiple HTTP connections to make the most of modern hardware and minimize the impact of the bandwidth latency product. - -### Cross Platform -Works on Windows and Linux, without cygwin or fuss. - -### Easy - -A new high-level interface designed to reduce the work of implementing block transfer in your application: -```golang -fs := &BasicSummary{...} - -rsync, err := MakeRSync( - localFilename, - referencePath, - outFilename, - fs, -) - -if err != nil { - return err -} - -err = rsync.Patch() - -if err != nil { - return err -} - -return rsync.Close() -``` - -### Extensible -All functionality is based on interfaces, allowing customization of behavior: - -```golang -// Here, the input version is a local string -inputFile := bytes.NewReader(localVersionAsBytes) - -// And the output is a buffer -patchedFile := bytes.NewBuffer(nil) - -// This information is meta-data on the file that should be loaded / provided -// You can also provide your own implementation of the FileSummary interface -summary := &BasicSummary{ - ChecksumIndex: referenceFileIndex, - // Disable verification of hashes for downloaded data (not really a good idea!) - ChecksumLookup: nil, - BlockCount: uint(blockCount), - BlockSize: blockSize, - FileSize: int64(len(referenceAsBytes)), -} - -rsync := &RSync{ - Input: inputFile, - Output: patchedFile, - // An in-memory block source - Source: blocksources.NewReadSeekerBlockSource( - bytes.NewReader(referenceAsBytes), - blocksources.MakeNullFixedSizeResolver(uint64(blockSize)), - ), - Index: summary, - Summary: summary, - OnClose: nil, -} -``` - -Reuse low level objects to build a new high level library, or implement a new lower-level object to add a new transfer protocol (for example). - -### Tested -GoSync has been built from the ground up with unit tests. -The GoSync command-line tool has acceptance tests, although not everything is covered. - -## Current State -Go-Sync is still probably not ready for production. - -The most obvious areas that still need improvement are the acceptance tests, the error messages, -compression on the blocks that are retrieved from the source and handling of file flags. - -### TODO -- [ ] gzip source blocks (this involves writing out a version of the file that's compressed in block-increments) -- [ ] Clean up naming consistency and clarity: Block / Chunk etc -- [ ] Flesh out full directory build / sync -- [ ] Implement 'patch' payloads from a known start point to a desired end state -- [ ] Validate full file checksum after patching -- [ ] Provide bandwidth limiting / monitoring as part of http blocksource -- [ ] Think about turning the filechecksum into an interface -- [ ] Avoid marshalling / un-marshalling blocks during checksum generation -- [ ] Sequential patcher to resume after error? - -### Testing - -All tests are run by Travis-CI - -#### Unit tests - - go test github.com/Redundancy/go-sync/... - -#### Acceptance Tests -See the "acceptancetests" folder. This is currently difficult to run locally and relies on several linux utilities. - -#### Commandline & files - - go build github.com/Redundancy/go-sync/gosync - gosync build filenameToPatchTo - gosync patch filenameToPatchFrom filenameToPatchTo.gosync filenameToPatchTo - -Note that normally, patching would rely on a remote http/https file source. - -#### Command line tool reference - gosync --help diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh deleted file mode 100644 index 8d8b5a7..0000000 --- a/vendor/github.com/Redundancy/go-sync/acceptancetests/nulltest.sh +++ /dev/null @@ -1 +0,0 @@ -echo "ok!" \ No newline at end of file diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh deleted file mode 100644 index f0e7ca3..0000000 --- a/vendor/github.com/Redundancy/go-sync/acceptancetests/run.sh +++ /dev/null @@ -1,21 +0,0 @@ -counter=1 -for f in $TRAVIS_BUILD_DIR/acceptancetests/*test.sh -do - [ ! -f "$f" ] && continue - - echo "Running acceptance test $f" - echo 'travis_fold:start:test_output$counter' - sh $f - rc=$? - echo 'travis_fold:end:test_output$counter' - - if [ $rc != 0 ]; then - echo "Test Failed" - exit $rc - fi - - echo "Test Passed" - counter=`expr $counter + 1` -done - - diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh deleted file mode 100644 index 9f1ce7a..0000000 --- a/vendor/github.com/Redundancy/go-sync/acceptancetests/truncated_test.sh +++ /dev/null @@ -1,14 +0,0 @@ -echo "Testing filling in a truncated file" -wget -q https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -O test.exe -cp test.exe compare.exe -gosync b test.exe -truncate -s 5000000 test.exe -gosync p test.exe test.gosync https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -diff -q test.exe compare.exe -rc=$? -if [ $rc != 0 ]; then - gosync -version - ls -l compare.exe - ls -l test.exe - exit $rc -fi diff --git a/vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh b/vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh deleted file mode 100644 index f1903cb..0000000 --- a/vendor/github.com/Redundancy/go-sync/acceptancetests/truncation_test.sh +++ /dev/null @@ -1,14 +0,0 @@ -echo "Testing truncating a file too long" -wget -q https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -O test.exe -cp test.exe compare.exe -gosync b test.exe -truncate -s 10000000 test.exe -gosync p test.exe test.gosync https://s3-eu-west-1.amazonaws.com/gosync-test/0.1.2/gosync.exe -diff -q test.exe compare.exe -rc=$? -if [ $rc != 0 ]; then - gosync -version - ls -l compare.exe - ls -l test.exe - exit $rc -fi \ No newline at end of file diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go b/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go deleted file mode 100644 index 95e236a..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase.go +++ /dev/null @@ -1,271 +0,0 @@ -package blocksources - -import ( - "errors" - "fmt" - "sort" - - "github.com/Redundancy/go-sync/patcher" -) - -// BlockSourceRequester does synchronous requests on a remote source of blocks -// concurrency is handled by the BlockSourceBase. This provides a simple way -// of implementing a particular -type BlockSourceRequester interface { - // This method is called on multiple goroutines, and must - // support simultaneous requests - DoRequest(startOffset int64, endOffset int64) (data []byte, err error) - - // If an error raised by DoRequest should cause BlockSourceBase - // to give up, return true - IsFatal(err error) bool -} - -// A BlockSourceOffsetResolver resolves a blockID to a start offset and and end offset in a file -// it also handles splitting up ranges of blocks into multiple requests, allowing requests to be split down to the -// block size, and handling of compressed blocks (given a resolver that can work out the correct range to query for, -// and a BlockSourceRequester that will decompress the result into a full sized block) -type BlockSourceOffsetResolver interface { - GetBlockStartOffset(blockID uint) int64 - GetBlockEndOffset(blockID uint) int64 - SplitBlockRangeToDesiredSize(startBlockID, endBlockID uint) []QueuedRequest -} - -// Checks blocks against their expected checksum -type BlockVerifier interface { - VerifyBlockRange(startBlockID uint, data []byte) bool -} - -func NewBlockSourceBase( - requester BlockSourceRequester, - resolver BlockSourceOffsetResolver, - verifier BlockVerifier, - concurrentRequestCount int, - concurrentBytes int64, -) *BlockSourceBase { - - b := &BlockSourceBase{ - Requester: requester, - BlockSourceResolver: resolver, - Verifier: verifier, - ConcurrentRequests: concurrentRequestCount, - ConcurrentBytes: concurrentBytes, - exitChannel: make(chan bool), - errorChannel: make(chan error), - responseChannel: make(chan patcher.BlockReponse), - requestChannel: make(chan patcher.MissingBlockSpan), - } - - go b.loop() - - return b -} - -// BlockSourceBase provides an implementation of blocksource -// that takes care of everything except for the actual asyncronous request -// this makes blocksources easier and faster to build reliably -// BlockSourceBase implements patcher.BlockSource, and if it's good enough, -// perhaps nobody else ever will have to. -type BlockSourceBase struct { - Requester BlockSourceRequester - BlockSourceResolver BlockSourceOffsetResolver - Verifier BlockVerifier - - // The number of requests that BlockSourceBase may service at once - ConcurrentRequests int - - // The number of bytes that BlockSourceBase may have in-flight - // (requested + pending delivery) - ConcurrentBytes int64 - - hasQuit bool - exitChannel chan bool - errorChannel chan error - responseChannel chan patcher.BlockReponse - requestChannel chan patcher.MissingBlockSpan - - bytesRequested int64 -} - -const ( - STATE_RUNNING = iota - STATE_EXITING -) - -func (s *BlockSourceBase) ReadBytes() int64 { - return s.bytesRequested -} - -func (s *BlockSourceBase) RequestBlocks(block patcher.MissingBlockSpan) error { - s.requestChannel <- block - return nil -} - -func (s *BlockSourceBase) GetResultChannel() <-chan patcher.BlockReponse { - return s.responseChannel -} - -// If the block source encounters an unsurmountable problem -func (s *BlockSourceBase) EncounteredError() <-chan error { - return s.errorChannel -} - -var BlockSourceAlreadyClosedError = errors.New("Block source was already closed") - -func (s *BlockSourceBase) Close() (err error) { - // if it has already been closed, just recover - // however, let the caller know - defer func() { - if recover() != nil { - err = BlockSourceAlreadyClosedError - } - }() - - if !s.hasQuit { - s.exitChannel <- true - } - - return -} - -func (s *BlockSourceBase) loop() { - defer func() { - s.hasQuit = true - close(s.exitChannel) - close(s.errorChannel) - close(s.requestChannel) - close(s.responseChannel) - }() - - state := STATE_RUNNING - inflightRequests := 0 - //inflightBytes := int64(0) - pendingErrors := &errorWatcher{errorChannel: s.errorChannel} - pendingResponse := &pendingResponseHelper{responseChannel: s.responseChannel} - resultChan := make(chan asyncResult) - defer close(resultChan) - - requestQueue := make(QueuedRequestList, 0, s.ConcurrentRequests*2) - - // enable us to order responses for the active requests, lowest to highest - requestOrdering := make(UintSlice, 0, s.ConcurrentRequests) - responseOrdering := make(PendingResponses, 0, s.ConcurrentRequests) - - for state == STATE_RUNNING || inflightRequests > 0 || pendingErrors.Err() != nil { - - // Start any pending work that we can - for inflightRequests < s.ConcurrentRequests && len(requestQueue) > 0 { - inflightRequests += 1 - - nextRequest := requestQueue[len(requestQueue)-1] - - requestOrdering = append(requestOrdering, nextRequest.StartBlockID) - sort.Sort(sort.Reverse(requestOrdering)) - go func() { - resolver := s.BlockSourceResolver - - startOffset := resolver.GetBlockStartOffset( - nextRequest.StartBlockID, - ) - - endOffset := resolver.GetBlockEndOffset( - nextRequest.EndBlockID, - ) - - result, err := s.Requester.DoRequest( - startOffset, - endOffset, - ) - - resultChan <- asyncResult{ - startBlockID: nextRequest.StartBlockID, - endBlockID: nextRequest.EndBlockID, - data: result, - err: err, - } - }() - - // remove dispatched request - requestQueue = requestQueue[:len(requestQueue)-1] - } - - select { - case newRequest := <-s.requestChannel: - requestQueue = append( - requestQueue, - s.BlockSourceResolver.SplitBlockRangeToDesiredSize( - newRequest.StartBlock, - newRequest.EndBlock, - )..., - ) - - sort.Sort(sort.Reverse(requestQueue)) - - case result := <-resultChan: - inflightRequests -= 1 - - if result.err != nil { - pendingErrors.setError(result.err) - pendingResponse.clear() - state = STATE_EXITING - break - } - - s.bytesRequested += int64(len(result.data)) - - if s.Verifier != nil && !s.Verifier.VerifyBlockRange(result.startBlockID, result.data) { - pendingErrors.setError( - fmt.Errorf( - "The returned block range (%v-%v) did not match the expected checksum for the blocks", - result.startBlockID, result.endBlockID, - ), - ) - pendingResponse.clear() - state = STATE_EXITING - break - } - - responseOrdering = append(responseOrdering, - patcher.BlockReponse{ - StartBlock: result.startBlockID, - Data: result.data, - }, - ) - - // sort high to low - sort.Sort(sort.Reverse(responseOrdering)) - - // if we just got the lowest requested block, we can set - // the response. Otherwise, wait. - lowestRequest := requestOrdering[len(requestOrdering)-1] - - if lowestRequest == result.startBlockID { - lowestResponse := responseOrdering[len(responseOrdering)-1] - pendingResponse.clear() - pendingResponse.setResponse(&lowestResponse) - } - - case pendingResponse.sendIfPending() <- pendingResponse.Response(): - pendingResponse.clear() - responseOrdering = responseOrdering[:len(responseOrdering)-1] - requestOrdering = requestOrdering[:len(requestOrdering)-1] - - // check if there's another response to enqueue - if len(responseOrdering) > 0 { - lowestResponse := responseOrdering[len(responseOrdering)-1] - lowestRequest := requestOrdering[len(requestOrdering)-1] - - if lowestRequest == lowestResponse.StartBlock { - pendingResponse.setResponse(&lowestResponse) - - } - } - - case pendingErrors.sendIfSet() <- pendingErrors.Err(): - pendingErrors.clear() - - case <-s.exitChannel: - state = STATE_EXITING - } - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go deleted file mode 100644 index 1aa91b9..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/blocksourcebase_test.go +++ /dev/null @@ -1,303 +0,0 @@ -package blocksources - -import ( - "bytes" - "github.com/Redundancy/go-sync/patcher" - - //"runtime" - "testing" - "time" -) - -//----------------------------------------------------------------------------- -type erroringRequester struct{} -type testError struct{} - -func (e *testError) Error() string { - return "test" -} - -func (e *erroringRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - return nil, &testError{} -} - -func (e *erroringRequester) IsFatal(err error) bool { - return true -} - -//----------------------------------------------------------------------------- -type FunctionRequester func(a, b int64) ([]byte, error) - -func (f FunctionRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - return f(startOffset, endOffset) -} - -func (f FunctionRequester) IsFatal(err error) bool { - return true -} - -//----------------------------------------------------------------------------- - -func init() { - //if runtime.GOMAXPROCS(0) == 1 { - //runtime.GOMAXPROCS(4) - //} -} - -func TestRangeSlice(t *testing.T) { - a := []int{0, 1, 2, 3, 4} - b := a[:len(a)-1] - - if len(b) != len(a)-1 { - t.Errorf("b is wrong length, only supposed to remove one item: %v %v", a, b) - } -} - -func TestCreateAndCloseBlockSourceBase(t *testing.T) { - b := NewBlockSourceBase(nil, nil, nil, 1, 1024) - b.Close() - - // TODO: Race condition here. Can Close() block? - if !b.hasQuit { - t.Fatal("Block source base did not exit") - } -} - -func TestErrorWatcher(t *testing.T) { - e := errorWatcher{errorChannel: make(chan error)} - - if e.sendIfSet() != nil { - t.Errorf("Channel should be nil when created") - } - - e.setError(&testError{}) - - if e.sendIfSet() == nil { - t.Errorf("Channel should be non-nil when error is set") - } - if e.Err() == nil { - t.Errorf("Error should not be nil when set") - } -} - -func TestBlockSourceBaseError(t *testing.T) { - b := NewBlockSourceBase( - &erroringRequester{}, - MakeNullFixedSizeResolver(4), - nil, - 1, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 1, - }) - - select { - case <-time.After(time.Second): - t.Fatal("Timed out waiting for error") - case <-b.EncounteredError(): - } - -} - -func TestBlockSourceRequest(t *testing.T) { - expected := []byte("test") - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - return expected, nil - }), - MakeNullFixedSizeResolver(4), - nil, - 1, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 1, - }) - - result := <-b.GetResultChannel() - - if result.StartBlock != 1 { - t.Errorf("Unexpected start block in result: %v", result.StartBlock) - } - if bytes.Compare(result.Data, expected) != 0 { - t.Errorf("Unexpected data in result: %v", result.Data) - } -} - -func TestConcurrentBlockRequests(t *testing.T) { - content := []byte("test") - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - return content[start:end], nil - }), - MakeNullFixedSizeResolver(2), - nil, - 2, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 2, - StartBlock: 0, - EndBlock: 0, - }) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 2, - StartBlock: 1, - EndBlock: 1, - }) - - for i := uint(0); i < 2; i++ { - select { - case r := <-b.GetResultChannel(): - if r.StartBlock != i { - t.Errorf("Wrong start block: %v", r.StartBlock) - } - if bytes.Compare(r.Data, content[i*2:(i+1)*2]) != 0 { - t.Errorf("Unexpected result content for result %v: %v", i+1, string(r.Data)) - } - case <-time.After(time.Second): - t.Fatal("Timed out on request", i+1) - } - } -} - -func TestOutOfOrderRequestCompletion(t *testing.T) { - content := []byte("test") - - channeler := []chan bool{ - make(chan bool), - make(chan bool), - } - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - // read from the channel based on the start - <-(channeler[start]) - return content[start:end], nil - }), - MakeNullFixedSizeResolver(1), - nil, - 2, - 1024, - ) - defer b.Close() - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 1, - StartBlock: 0, - EndBlock: 0, - }) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 1, - StartBlock: 1, - EndBlock: 1, - }) - - // finish the second request - channeler[1] <- true - - select { - case <-b.GetResultChannel(): - t.Error("Should not deliver any blocks yet") - case <-time.After(time.Second): - } - - // once the first block completes, we're ready to send both - channeler[0] <- true - - for i := uint(0); i < 2; i++ { - select { - case r := <-b.GetResultChannel(): - if r.StartBlock != i { - t.Errorf( - "Wrong start block: %v on result %v", - r.StartBlock, - i+1, - ) - } - case <-time.After(time.Second): - t.Fatal("Timed out on request", i+1) - } - } -} - -func TestRequestCountLimiting(t *testing.T) { - counter := make(chan int) - waiter := make(chan bool) - const ( - MAX_CONCURRENCY = 2 - REQUESTS = 4 - ) - call_counter := 0 - - b := NewBlockSourceBase( - FunctionRequester(func(start, end int64) (data []byte, err error) { - counter <- 1 - call_counter += 1 - <-waiter - counter <- -1 - return []byte{0, 0}, nil - }), - MakeNullFixedSizeResolver(1), - nil, - MAX_CONCURRENCY, - 1024, - ) - defer b.Close() - - count := 0 - max := 0 - - go func() { - for { - change, ok := <-counter - - if !ok { - break - } - - count += change - - if count > max { - max = count - } - } - }() - - for i := 0; i < REQUESTS; i++ { - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 1, - StartBlock: uint(i), - EndBlock: uint(i), - }) - } - - for i := 0; i < REQUESTS; i++ { - waiter <- true - } - - close(counter) - close(waiter) - - if max > MAX_CONCURRENCY { - t.Errorf("Maximum requests in flight was greater than the requested concurrency: %v", max) - } - if call_counter != REQUESTS { - t.Errorf("Total number of requests is not expected: %v", call_counter) - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go b/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go deleted file mode 100644 index d81573e..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver.go +++ /dev/null @@ -1,73 +0,0 @@ -package blocksources - -type FixedSizeBlockResolver struct { - BlockSize uint64 - FileSize int64 - MaxDesiredRequestSize uint64 -} - -func (r *FixedSizeBlockResolver) GetBlockStartOffset(blockID uint) int64 { - if off := int64(uint64(blockID) * r.BlockSize); r.FileSize != 0 && off > r.FileSize { - return r.FileSize - } else { - return off - } -} - -func (r *FixedSizeBlockResolver) GetBlockEndOffset(blockID uint) int64 { - if off := int64(uint64(blockID+1) * r.BlockSize); r.FileSize != 0 && off > r.FileSize { - return r.FileSize - } else { - return off - } -} - -// Split blocks into chunks of the desired size, or less. This implementation assumes a fixed block size at the source. -func (r *FixedSizeBlockResolver) SplitBlockRangeToDesiredSize(startBlockID, endBlockID uint) []QueuedRequest { - - if r.MaxDesiredRequestSize == 0 { - return []QueuedRequest{ - QueuedRequest{ - StartBlockID: startBlockID, - EndBlockID: endBlockID, - }, - } - } - - maxSize := r.MaxDesiredRequestSize - if r.MaxDesiredRequestSize < r.BlockSize { - maxSize = r.BlockSize - } - - // how many blocks is the desired size? - blockCountPerRequest := uint(maxSize / r.BlockSize) - - requests := make([]QueuedRequest, 0, (endBlockID-startBlockID)/blockCountPerRequest+1) - currentBlockID := startBlockID - - for { - maxEndBlock := currentBlockID + blockCountPerRequest - - if maxEndBlock > endBlockID { - requests = append( - requests, - QueuedRequest{ - StartBlockID: currentBlockID, - EndBlockID: endBlockID, - }, - ) - - return requests - } else { - requests = append( - requests, - QueuedRequest{ - StartBlockID: currentBlockID, - EndBlockID: maxEndBlock - 1, - }, - ) - - currentBlockID = maxEndBlock - } - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go deleted file mode 100644 index f484bc5..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/fixed_size_block_resolver_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package blocksources - -import ( - "testing" -) - -func TestNullResolverGivesBackTheSameBlocks(t *testing.T) { - n := MakeNullFixedSizeResolver(5) - result := n.SplitBlockRangeToDesiredSize(0, 10000) - - if len(result) != 1 { - t.Fatalf("Unexpected result length (expected 1): %v", result) - } - - r := result[0] - - if r.StartBlockID != 0 { - t.Errorf("Unexpected start block ID: %v", r) - } - - if r.EndBlockID != 10000 { - t.Errorf("Unexpected end block ID: %v", r) - } -} - -func TestFixedSizeResolverSplitsBlocksOfDesiredSize(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 5, - FileSize: 20000, - } - - // Should split two blocks, each of the desired request size - // into two requests - result := res.SplitBlockRangeToDesiredSize(0, 1) - - if len(result) != 2 { - t.Fatalf("Unexpected result length (expected 2): %v", result) - } - - if result[0].StartBlockID != 0 { - t.Errorf("Unexpected start blockID: %v", result[0]) - } - if result[0].EndBlockID != 0 { - t.Errorf("Unexpected end blockID: %v", result[0]) - } - - if result[1].StartBlockID != 1 { - t.Errorf("Unexpected start blockID: %v", result[1]) - } - if result[1].EndBlockID != 1 { - t.Errorf("Unexpected end blockID: %v", result[1]) - } -} - -func TestThatMultipleBlocksAreSplitByRoundingDown(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 12, - FileSize: 20000, - } - - // 0,1 (10) - 2-3 (10) - result := res.SplitBlockRangeToDesiredSize(0, 3) - - if len(result) != 2 { - t.Fatalf("Unexpected result length (expected 2): %v", result) - } - - if result[0].StartBlockID != 0 { - t.Errorf("Unexpected start blockID: %v", result[0]) - } - if result[0].EndBlockID != 1 { - t.Errorf("Unexpected end blockID: %v", result[0]) - } - - if result[1].StartBlockID != 2 { - t.Errorf("Unexpected start blockID: %v", result[1]) - } - if result[1].EndBlockID != 3 { - t.Errorf("Unexpected end blockID: %v", result[1]) - } -} - -func TestThatADesiredSizeSmallerThanABlockResultsInSingleBlocks(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 4, - FileSize: 20000, - } - - // Should split two blocks - result := res.SplitBlockRangeToDesiredSize(0, 1) - - if len(result) != 2 { - t.Fatalf("Unexpected result length (expected 2): %v", result) - } - - if result[0].StartBlockID != 0 { - t.Errorf("Unexpected start blockID: %v", result[0]) - } - if result[0].EndBlockID != 0 { - t.Errorf("Unexpected end blockID: %v", result[0]) - } - - if result[1].StartBlockID != 1 { - t.Errorf("Unexpected start blockID: %v", result[1]) - } - if result[1].EndBlockID != 1 { - t.Errorf("Unexpected end blockID: %v", result[1]) - } -} - -func TestThatFileSizeTruncatesBlockEnds(t *testing.T) { - res := &FixedSizeBlockResolver{ - BlockSize: 5, - MaxDesiredRequestSize: 100, - FileSize: 13, - } - - // Should split two blocks - result := res.GetBlockEndOffset(3) - - if result != 13 { - t.Errorf("Unexpected BlockEnd Offset:", result) - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/helpers.go b/vendor/github.com/Redundancy/go-sync/blocksources/helpers.go deleted file mode 100644 index 204c484..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/helpers.go +++ /dev/null @@ -1,124 +0,0 @@ -package blocksources - -import ( - "fmt" - "github.com/Redundancy/go-sync/patcher" -) - -// errorWatcher is a small helper object -// sendIfSet will only return a channel if there is an error set -// so w.sendIfSet() <- w.Err() is always safe in a select statement -// even if there is no error set -type errorWatcher struct { - errorChannel chan error - lastError error -} - -func (w *errorWatcher) setError(e error) { - if w.lastError != nil { - panic("cannot set a new error when one is already set!") - } - w.lastError = e -} - -func (w *errorWatcher) clear() { - w.lastError = nil -} - -func (w *errorWatcher) Err() error { - return w.lastError -} - -func (w *errorWatcher) sendIfSet() chan<- error { - if w.lastError != nil { - return w.errorChannel - } else { - return nil - } -} - -type pendingResponseHelper struct { - responseChannel chan patcher.BlockReponse - pendingResponse *patcher.BlockReponse -} - -func (w *pendingResponseHelper) setResponse(r *patcher.BlockReponse) { - if w.pendingResponse != nil { - p := fmt.Sprintf("Setting a response when one is already set! Had startblock %v, got %v", r.StartBlock, w.pendingResponse.StartBlock) - panic(p) - } - w.pendingResponse = r -} - -func (w *pendingResponseHelper) clear() { - w.pendingResponse = nil -} - -func (w *pendingResponseHelper) Response() patcher.BlockReponse { - if w.pendingResponse == nil { - return patcher.BlockReponse{} - } - return *w.pendingResponse -} - -func (w *pendingResponseHelper) sendIfPending() chan<- patcher.BlockReponse { - if w.pendingResponse != nil { - return w.responseChannel - } else { - return nil - } - -} - -type UintSlice []uint - -func (r UintSlice) Len() int { - return len(r) -} - -func (r UintSlice) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r UintSlice) Less(i, j int) bool { - return r[i] < r[j] -} - -type asyncResult struct { - startBlockID uint - endBlockID uint - data []byte - err error -} - -type QueuedRequest struct { - StartBlockID uint - EndBlockID uint -} - -type QueuedRequestList []QueuedRequest - -func (r QueuedRequestList) Len() int { - return len(r) -} - -func (r QueuedRequestList) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r QueuedRequestList) Less(i, j int) bool { - return r[i].StartBlockID < r[j].StartBlockID -} - -func MakeNullFixedSizeResolver(blockSize uint64) BlockSourceOffsetResolver { - return &FixedSizeBlockResolver{ - BlockSize: blockSize, - } -} - -func MakeFileSizedBlockResolver(blockSize uint64, filesize int64) BlockSourceOffsetResolver { - return &FixedSizeBlockResolver{ - BlockSize: blockSize, - FileSize: filesize, - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go b/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go deleted file mode 100644 index 0e93278..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource.go +++ /dev/null @@ -1,110 +0,0 @@ -package blocksources - -import ( - "bytes" - "errors" - "fmt" - "net/http" - "strings" -) - -const MB = 1024 * 1024 - -var RangedRequestNotSupportedError = errors.New("Ranged request not supported (Server did not respond with 206 Status)") -var ResponseFromServerWasGZiped = errors.New("HTTP response was gzip encoded. Ranges may not match those requested.") - -var ClientNoCompression = &http.Client{ - Transport: &http.Transport{}, -} - -func NewHttpBlockSource( - url string, - concurrentRequests int, - resolver BlockSourceOffsetResolver, - verifier BlockVerifier, -) *BlockSourceBase { - return NewBlockSourceBase( - &HttpRequester{ - url: url, - client: http.DefaultClient, - }, - resolver, - verifier, - concurrentRequests, - 4*MB, - ) -} - -type URLNotFoundError string - -func (url URLNotFoundError) Error() string { - return "404 Error on URL: " + string(url) -} - -// This class provides the implementation of BlockSourceRequester for BlockSourceBase -// this simplifies creating new BlockSources that satisfy the requirements down to -// writing a request function -type HttpRequester struct { - client *http.Client - url string -} - -func (r *HttpRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - rangedRequest, err := http.NewRequest("GET", r.url, nil) - - if err != nil { - return nil, fmt.Errorf("Error creating request for \"%v\": %v", r.url, err) - } - - rangeSpecifier := fmt.Sprintf("bytes=%v-%v", startOffset, endOffset-1) - rangedRequest.ProtoAtLeast(1, 1) - rangedRequest.Header.Add("Range", rangeSpecifier) - rangedRequest.Header.Add("Accept-Encoding", "identity") - rangedResponse, err := r.client.Do(rangedRequest) - - if err != nil { - return nil, fmt.Errorf("Error executing request for \"%v\": %v", r.url, err) - } - - defer rangedResponse.Body.Close() - - if rangedResponse.StatusCode == 404 { - return nil, URLNotFoundError(r.url) - } else if rangedResponse.StatusCode != 206 { - return nil, RangedRequestNotSupportedError - } else if strings.Contains( - rangedResponse.Header.Get("Content-Encoding"), - "gzip", - ) { - return nil, ResponseFromServerWasGZiped - } else { - buf := bytes.NewBuffer(make([]byte, 0, endOffset-startOffset)) - _, err = buf.ReadFrom(rangedResponse.Body) - - if err != nil { - err = fmt.Errorf( - "Failed to read response body for %v (%v-%v): %v", - r.url, - startOffset, endOffset-1, - err, - ) - } - - data = buf.Bytes() - - if int64(len(data)) != endOffset-startOffset { - err = fmt.Errorf( - "Unexpected response length %v (%v): %v", - r.url, - endOffset-startOffset+1, - len(data), - ) - } - - return - } -} - -func (r *HttpRequester) IsFatal(err error) bool { - return true -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go deleted file mode 100644 index d7a411c..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/httpblocksource_test.go +++ /dev/null @@ -1,251 +0,0 @@ -package blocksources - -import ( - "bytes" - "crypto/md5" - "fmt" - "net" - "net/http" - "testing" - "time" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/patcher" -) - -var PORT = 8000 - -var TEST_CONTENT = []byte("This is test content used for evaluation of the unit tests") -var content = bytes.NewReader(TEST_CONTENT) -var LOCAL_URL = "" - -func handler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), content) -} - -var PARTIAL_CONTENT = []byte("abcdef") -var partialContent = bytes.NewReader(PARTIAL_CONTENT) - -func partialContentHandler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), partialContent) -} - -var CORRUPT_CONTENT = []byte("sfdfsfhhrtertert sffsfsdfsdfsdf") -var corruptContent = bytes.NewReader(CORRUPT_CONTENT) - -func corruptContentHandler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), corruptContent) -} - -// set up a http server locally that will respond predictably to ranged requests -// NB: Doing this will prevent deadlocks from being caught! -func init() { - s := http.NewServeMux() - s.HandleFunc("/", handler) - s.HandleFunc("/partial", partialContentHandler) - s.HandleFunc("/corrupt", corruptContentHandler) - s.Handle("/404", http.NotFoundHandler()) - - portChan := make(chan int) - - go func() { - var listener net.Listener - var err error - - for { - PORT++ - p := fmt.Sprintf(":%v", PORT) - listener, err = net.Listen("tcp", p) - - if err == nil { - break - } - } - portChan <- PORT - http.Serve(listener, s) - }() - - p := fmt.Sprintf(":%v", <-portChan) - LOCAL_URL = "http://localhost" + p - -} - -func TestHandler(t *testing.T) { - resp, err := http.Get(LOCAL_URL) - - if err != nil { - t.Fatal(err) - } - - if resp.StatusCode != 200 { - t.Fatal(resp.Status) - } -} - -func TestHttpBlockSource(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/", - 2, - MakeNullFixedSizeResolver(4), - nil, - ) - - err := b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 0, - EndBlock: 0, - }) - - if err != nil { - t.Fatal(err) - } - - results := b.GetResultChannel() - - select { - case r := <-results: - if bytes.Compare(r.Data, TEST_CONTENT[:4]) != 0 { - t.Errorf("Data differed from expected content: \"%v\"", string(r.Data)) - } - case e := <-b.EncounteredError(): - t.Fatal(e) - case <-time.After(time.Second): - t.Fatal("Waited a second for the response, timeout.") - } -} - -func TestHttpBlockSource404(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/404", - 2, - MakeNullFixedSizeResolver(4), - nil, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 0, - EndBlock: 0, - }) - - results := b.GetResultChannel() - - select { - case <-results: - t.Fatal("Should not have gotten a result") - case e := <-b.EncounteredError(): - if e == nil { - t.Fatal("Error was nil!") - } else if _, ok := e.(URLNotFoundError); !ok { - t.Errorf("Unexpected error type: %v", e) - } - case <-time.After(time.Second): - t.Fatal("Waited a second for the response, timeout.") - } -} - -func TestHttpBlockSourceOffsetBlockRequest(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/", - 2, - MakeNullFixedSizeResolver(4), - nil, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 3, - }) - - select { - case result := <-b.GetResultChannel(): - if result.StartBlock != 1 { - t.Errorf( - "Unexpected result start block: %v", - result.StartBlock, - ) - } - case <-time.After(time.Second): - t.Fatalf("Timeout waiting for result") - } -} - -func TestHttpBlockSourcePartialContentRequest(t *testing.T) { - b := NewHttpBlockSource( - LOCAL_URL+"/partial", - 2, - MakeFileSizedBlockResolver(4, int64(len(PARTIAL_CONTENT))), - nil, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 1, - EndBlock: 1, - }) - - select { - case result := <-b.GetResultChannel(): - if result.StartBlock != 1 { - t.Errorf( - "Unexpected result start block: %v", - result.StartBlock, - ) - } - if len(result.Data) != 2 { - t.Errorf( - "Unexpected data length: \"%v\"", - string(result.Data), - ) - } - if string(result.Data) != "ef" { - t.Errorf( - "Unexpected result \"%v\"", - string(result.Data), - ) - } - case err := <-b.EncounteredError(): - t.Fatal(err) - case <-time.After(time.Second): - t.Fatalf("Timeout waiting for result") - } -} - -type SingleBlockSource []byte - -func (d SingleBlockSource) GetStrongChecksumForBlock(blockID int) []byte { - m := md5.New() - return m.Sum(d) -} - -func TestHttpBlockSourceVerification(t *testing.T) { - const BLOCK_SIZE = 4 - - b := NewHttpBlockSource( - LOCAL_URL+"/corrupt", - 2, - MakeNullFixedSizeResolver(BLOCK_SIZE), - &filechecksum.HashVerifier{ - Hash: md5.New(), - BlockSize: BLOCK_SIZE, - BlockChecksumGetter: SingleBlockSource(TEST_CONTENT[0:BLOCK_SIZE]), - }, - ) - - b.RequestBlocks(patcher.MissingBlockSpan{ - BlockSize: BLOCK_SIZE, - StartBlock: 0, - EndBlock: 0, - }) - - select { - case result := <-b.GetResultChannel(): - t.Fatalf("Should have thrown an error, got %v", result) - case e := <-b.EncounteredError(): - t.Logf("Encountered expected error: %v", e) - return - case <-time.After(time.Second): - t.Fatalf("Timeout waiting for result") - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go b/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go deleted file mode 100644 index 67958f6..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource.go +++ /dev/null @@ -1,54 +0,0 @@ -package blocksources - -import ( - "io" -) - -const ( - from_start = 0 -) - -type ReadSeeker interface { - Read(b []byte) (n int, err error) - Seek(offset int64, whence int) (int64, error) -} - -func NewReadSeekerBlockSource( - r ReadSeeker, - resolver BlockSourceOffsetResolver, -) *BlockSourceBase { - return NewBlockSourceBase( - &ReadSeekerRequester{ - rs: r, - }, - resolver, - nil, // TODO: No verifier! - 1, - 8*MB, - ) -} - -type ReadSeekerRequester struct { - rs ReadSeeker -} - -func (r *ReadSeekerRequester) DoRequest(startOffset int64, endOffset int64) (data []byte, err error) { - read_length := endOffset - startOffset - buffer := make([]byte, read_length) - - if _, err = r.rs.Seek(startOffset, from_start); err != nil { - return - } - - n, err := io.ReadFull(r.rs, buffer) - - if err != nil && err != io.ErrUnexpectedEOF { - return - } - - return buffer[:n], nil -} - -func (r *ReadSeekerRequester) IsFatal(err error) bool { - return true -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go b/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go deleted file mode 100644 index 37d1970..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/readseeker_blocksource_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package blocksources - -import ( - "bytes" - "github.com/Redundancy/go-sync/patcher" - "testing" -) - -const STRING_DATA = "abcdefghijklmnopqrst" - -var BYTE_BLOCK_SOURCE = NewReadSeekerBlockSource( - bytes.NewReader( - []byte(STRING_DATA), - ), - MakeNullFixedSizeResolver(4), -) - -func TestReadFirstBlock(t *testing.T) { - BYTE_BLOCK_SOURCE.RequestBlocks( - patcher.MissingBlockSpan{ - BlockSize: 4, - StartBlock: 0, - EndBlock: 0, - }, - ) - - result := <-BYTE_BLOCK_SOURCE.GetResultChannel() - - if result.StartBlock != 0 { - t.Errorf("Wrong start block: %v", result.StartBlock) - } - - EXPECTED := STRING_DATA[:4] - - if bytes.Compare(result.Data, []byte(EXPECTED)) != 0 { - t.Errorf( - "Unexpected result data: \"%v\" expected: \"%v\"", - string(result.Data), - EXPECTED, - ) - } -} diff --git a/vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go b/vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go deleted file mode 100644 index 551d75f..0000000 --- a/vendor/github.com/Redundancy/go-sync/blocksources/sortableBlockResponse.go +++ /dev/null @@ -1,19 +0,0 @@ -package blocksources - -import ( - "github.com/Redundancy/go-sync/patcher" -) - -type PendingResponses []patcher.BlockReponse - -func (r PendingResponses) Len() int { - return len(r) -} - -func (r PendingResponses) Swap(i, j int) { - r[i], r[j] = r[j], r[i] -} - -func (r PendingResponses) Less(i, j int) bool { - return r[i].StartBlock < r[j].StartBlock -} diff --git a/vendor/github.com/Redundancy/go-sync/chunks/chunks.go b/vendor/github.com/Redundancy/go-sync/chunks/chunks.go deleted file mode 100644 index aac4143..0000000 --- a/vendor/github.com/Redundancy/go-sync/chunks/chunks.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Package chunks provides the basic structure for a pair of the weak and strong checksums. -Since this is fairly widely used, splitting this out breaks a number of possible circular dependencies -*/ -package chunks - -import ( - "bytes" - "errors" - "io" -) - -// For a given Block, the Weak & Strong hashes, and the offset -// This structure is only used to generate the index of reference files, since -// computing the strong checksum is not done when comparing unless the weak checksum matches -type ChunkChecksum struct { - // an offset in terms of chunk count - ChunkOffset uint - // the size of the block - Size int64 - WeakChecksum []byte - StrongChecksum []byte -} - -// compares a checksum to another based on the checksums, not the offset -func (chunk ChunkChecksum) Match(other ChunkChecksum) bool { - weakEqual := bytes.Compare(chunk.WeakChecksum, other.WeakChecksum) == 0 - strongEqual := false - if weakEqual { - strongEqual = bytes.Compare(chunk.StrongChecksum, other.StrongChecksum) == 0 - } - return weakEqual && strongEqual -} - -var ErrPartialChecksum = errors.New("Reader length was not a multiple of the checksums") - -// Loads chunks from a reader, assuming alternating weak then strong hashes -func LoadChecksumsFromReader( - r io.Reader, - weakHashSize int, - strongHashSize int, -) ([]ChunkChecksum, error) { - - result := make([]ChunkChecksum, 0, 20) - offset := uint(0) - - temp := ChunkChecksum{} - - for { - weakBuffer := make([]byte, weakHashSize) - n, err := io.ReadFull(r, weakBuffer) - - if n == weakHashSize { - temp.ChunkOffset = offset - temp.WeakChecksum = weakBuffer - } else if n == 0 && err == io.EOF { - break - } else { - return nil, ErrPartialChecksum - } - - strongBuffer := make([]byte, strongHashSize) - n, err = io.ReadFull(r, strongBuffer) - - if n == strongHashSize { - temp.StrongChecksum = strongBuffer - result = append(result, temp) - - if err == io.EOF { - break - } - } else { - return nil, ErrPartialChecksum - } - - offset += 1 - } - - return result, nil -} - -// satisfies filechecksum.ChecksumLookup -type StrongChecksumGetter []ChunkChecksum - -func (s StrongChecksumGetter) GetStrongChecksumForBlock(blockID int) []byte { - return s[blockID].StrongChecksum -} diff --git a/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go b/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go deleted file mode 100644 index 0449739..0000000 --- a/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc.go +++ /dev/null @@ -1,170 +0,0 @@ -package circularbuffer - -/* -C2 is a circular buffer intended to allow you to write a block of data of up to 'blocksize', and retrieve the -data evicted by that operation, without allocating any extra slice storage - -This requires that it keep at least blocksize*2 data around. In fact, it doubles that again in order to -guarantee that both of these bits of information can always be obtained in a single contiguous block of memory. - -Other than the cost of the extra memory (4xblocksize), this means that it requires 2 writes for every byte stored. -*/ -type C2 struct { - // used to know how much was evicted - lastWritten int - - // total number of written bytes - // used to track if the buffer has been filled, but goes above blocksize - totalWritten int - - // quick access to the circular buffer size - blocksize int - - // double internal buffer storage - a, b doubleSizeBuffer -} - -type doubleSizeBuffer struct { - // used to reset the head pointer - baseOffset int - - // index of the next byte to be written - head int - - // buffer - buffer []byte -} - -func MakeC2Buffer(blockSize int) *C2 { - return &C2{ - blocksize: blockSize, - a: doubleSizeBuffer{ - baseOffset: 0, - buffer: make([]byte, blockSize*2), - }, - b: doubleSizeBuffer{ - baseOffset: blockSize, - head: blockSize, - buffer: make([]byte, blockSize*2), - }, - } -} - -func (c *C2) Reset() { - c.a.Reset() - c.b.Reset() - c.lastWritten = 0 - c.totalWritten = 0 -} - -// Write new data -func (c *C2) Write(b []byte) { - c.a.Write(b) - c.b.Write(b) - c.lastWritten = len(b) - c.totalWritten += c.lastWritten -} - -func (c *C2) getBlockBuffer() *doubleSizeBuffer { - bufferToRead := &c.a - if c.b.head > c.a.head { - bufferToRead = &c.b - } - - return bufferToRead -} - -// the total written, up to the blocksize -func (c *C2) maxWritten() int { - if c.totalWritten < c.blocksize { - return c.totalWritten - } - - return c.blocksize -} - -func (c *C2) Len() int { - return c.maxWritten() -} - -func (c *C2) Empty() bool { - return c.totalWritten == 0 -} - -// Shortens the content of the circular buffer -// and returns the content removed -func (c *C2) Truncate(byteCount int) (evicted []byte) { - max := c.maxWritten() - - if byteCount > max { - byteCount = max - } - - bufferToRead := c.getBlockBuffer() - start := bufferToRead.head - max - - c.totalWritten = c.maxWritten() - byteCount - return bufferToRead.buffer[start : start+byteCount] -} - -// get the current buffer contents of block -func (c *C2) GetBlock() []byte { - // figure out which buffer has it stored contiguously - bufferToRead := c.getBlockBuffer() - start := bufferToRead.head - c.maxWritten() - - return bufferToRead.buffer[start:bufferToRead.head] -} - -// get the data that was evicted by the last write -func (c *C2) Evicted() []byte { - if c.totalWritten <= c.blocksize { - return nil - } - - bufferToRead := c.a - if c.b.head < c.a.head { - bufferToRead = c.b - } - - bufferStart := bufferToRead.head + c.blocksize - readLength := c.lastWritten - - // if the buffer wasn't full, we don't read the full length - if c.totalWritten-c.lastWritten < c.blocksize { - readLength -= c.lastWritten - c.totalWritten + c.blocksize - } - - return bufferToRead.buffer[bufferStart-readLength : bufferStart] -} - -func (buff *doubleSizeBuffer) Reset() { - buff.head = buff.baseOffset -} - -func (buff *doubleSizeBuffer) Write(by []byte) { - remaining := by - - for len(remaining) > 0 { - remaining_len := len(remaining) - availableSpace := len(buff.buffer) - buff.head - writeThisTime := remaining_len - - if writeThisTime > availableSpace { - writeThisTime = availableSpace - } - - copy( - buff.buffer[buff.head:buff.head+writeThisTime], // to - by, - ) - - buff.head += writeThisTime - - if buff.head == len(buff.buffer) { - buff.head = 0 - } - - remaining = remaining[writeThisTime:] - } -} diff --git a/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go b/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go deleted file mode 100644 index 6315989..0000000 --- a/vendor/github.com/Redundancy/go-sync/circularbuffer/noalloc_test.go +++ /dev/null @@ -1,182 +0,0 @@ -package circularbuffer - -import ( - "bytes" - "testing" -) - -const BLOCK_SIZE = 10 - -var incrementBlock = make([]byte, BLOCK_SIZE) -var incrementBlock2 = make([]byte, BLOCK_SIZE) - -func init() { - for i, _ := range incrementBlock { - incrementBlock[i] = byte(i) - incrementBlock2[i] = byte(i + BLOCK_SIZE) - } -} - -func TestCreateC2Buffer(t *testing.T) { - MakeC2Buffer(BLOCK_SIZE) -} - -func TestWriteBlock(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) -} - -func TestWritingUnderCapacityGivesEmptyEvicted(t *testing.T) { - b := MakeC2Buffer(2) - b.Write([]byte{1, 2}) - - if len(b.Evicted()) != 0 { - t.Fatal("Evicted should have been empty:", b.Evicted()) - } -} - -func TestWritingMultipleBytesWhenBufferIsNotFull(t *testing.T) { - b := MakeC2Buffer(3) - b.Write([]byte{1, 2}) - b.Write([]byte{3, 4}) - - ev := b.Evicted() - - if len(ev) != 1 || ev[0] != 1 { - t.Fatal("Evicted should have been [1,]:", ev) - } -} - -func TestEvictedRegession1(t *testing.T) { - b := MakeC2Buffer(4) - - b.Write([]byte{7, 6}) - b.Write([]byte{5, 1, 2}) - b.Write([]byte{3, 4}) - - ev := b.Evicted() - if len(ev) != 2 || ev[0] != 6 || ev[1] != 5 { - t.Fatalf("Unexpected evicted [6,5]: %v", ev) - } -} - -func TestGetBlock(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - - block := b.GetBlock() - - if len(block) != BLOCK_SIZE { - t.Fatal("Wrong block size returned") - } - - for i, by := range block { - if byte(i) != by { - t.Errorf("byte %v does not match", i) - } - } -} - -func TestWriteTwoBlocksGet(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - b.Write(incrementBlock2) - - if bytes.Compare(b.GetBlock(), incrementBlock2) != 0 { - t.Errorf("Get block did not return the right value: %s", b.GetBlock()) - } -} - -func TestWriteSingleByteGetSingleByte(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - singleByte := []byte{0} - b.Write(singleByte) - - if bytes.Compare(b.GetBlock(), singleByte) != 0 { - t.Errorf("Get block did not return the right value: %s", b.GetBlock()) - } -} - -func TestWriteTwoBlocksGetEvicted(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - b.Write(incrementBlock2) - - if bytes.Compare(b.Evicted(), incrementBlock) != 0 { - t.Errorf("Evicted did not return the right value: %s", b.Evicted()) - } -} - -func TestWriteSingleByteReturnsSingleEvictedByte(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock2) - singleByte := []byte{0} - - b.Write(singleByte) - e := b.Evicted() - - if len(e) != 1 { - t.Fatalf("Evicted length is not correct: %s", e) - } - - if e[0] != byte(10) { - t.Errorf("Evicted content is not correct: %s", e) - } -} - -func TestTruncatingAfterWriting(t *testing.T) { - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - - evicted := b.Truncate(2) - - if len(evicted) != 2 { - t.Fatalf("Truncate did not return expected evicted length: %v", evicted) - } - - if evicted[0] != 0 || evicted[1] != 1 { - t.Errorf("Unexpected content in evicted: %v", evicted) - } -} - -func TestWritingAfterTruncating(t *testing.T) { - // test that after we truncate some content, the next operations - // on the buffer give us the expected results - b := MakeC2Buffer(BLOCK_SIZE) - b.Write(incrementBlock) - b.Truncate(4) - - b.Write([]byte{34, 46}) - - block := b.GetBlock() - - if len(block) != BLOCK_SIZE-2 { - t.Fatalf( - "Unexpected block length after truncation: %v (%v)", - block, - len(block), - ) - } - - if bytes.Compare(block, []byte{4, 5, 6, 7, 8, 9, 34, 46}) != 0 { - t.Errorf( - "Unexpected block content after truncation: %v (%v)", - block, - len(block)) - } -} - -// This should have no allocations! -func BenchmarkSingleWrites(b *testing.B) { - buffer := MakeC2Buffer(BLOCK_SIZE) - buffer.Write(incrementBlock) - b.ReportAllocs() - - singleByte := []byte{0} - b.StartTimer() - for i := 0; i < b.N; i++ { - buffer.Write(singleByte) - buffer.Evicted() - } - b.StopTimer() -} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/comparer.go b/vendor/github.com/Redundancy/go-sync/comparer/comparer.go deleted file mode 100644 index b793818..0000000 --- a/vendor/github.com/Redundancy/go-sync/comparer/comparer.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -package comparer is responsible for using a FileChecksumGenerator (filechecksum) and an index -to move through a file and compare it to the index, producing a FileDiffSummary -*/ -package comparer - -import ( - "fmt" - "io" - "sync/atomic" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/circularbuffer" - "github.com/Redundancy/go-sync/filechecksum" -) - -const ( - READ_NEXT_BYTE = iota - READ_NEXT_BLOCK - READ_NONE -) - -// If the weak Hash object satisfies this interface, then -// StartFindMatchingBlocks will not allocate a circular buffer -type BlockBuffer interface { - Write([]byte) (int, error) - // the last set of bytes of the size of the circular buffer - // oldest to newest - GetLastBlock() []byte -} - -type BlockMatchResult struct { - // In case of error - Err error - - // The offset the comparison + baseOffset - ComparisonOffset int64 - - // The block from the index that it matched - BlockIdx uint -} - -type Index interface { - FindWeakChecksum2(chk []byte) interface{} - FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum -} - -/* -Iterates though comparison looking for blocks that match ones from the index -it emits each block to be read from the returned channel. Callers should check for -.Err != nil on the results, in which case reading will end immediately. - -StartFindMatchingBlocks is capable of running asyncronously -on sub-sections of a larger file. When doing this, you must overlap -by the block size, and use seperate checksum generators. -*/ - -type Comparer struct { - Comparisons int64 - WeakHashHits int64 - StrongHashHits int64 -} - -func (c *Comparer) StartFindMatchingBlocks( - comparison io.Reader, - baseOffset int64, - generator *filechecksum.FileChecksumGenerator, - referenceIndex Index, -) <-chan BlockMatchResult { - - resultStream := make(chan BlockMatchResult) - - go c.startFindMatchingBlocks_int( - resultStream, - comparison, - baseOffset, - generator, - referenceIndex, - ) - - return resultStream -} - -/* -TODO: When matching duplicated blocks, a channel of BlockMatchResult slices would be more efficient -*/ -func (c *Comparer) startFindMatchingBlocks_int( - results chan<- BlockMatchResult, - comparison io.Reader, - baseOffset int64, - generator *filechecksum.FileChecksumGenerator, - reference Index, -) { - defer close(results) - - block := make([]byte, generator.BlockSize) - var err error - - ReportErr := func(err error) { - results <- BlockMatchResult{ - Err: err, - } - } - - _, err = io.ReadFull(comparison, block) - - if err != nil { - ReportErr( - fmt.Errorf("Error reading first block in comparison: %v", err), - ) - return - } - - generator.WeakRollingHash.SetBlock(block) - singleByte := make([]byte, 1) - weaksum := make([]byte, generator.WeakRollingHash.Size()) - strongSum := make([]byte, 0, generator.GetStrongHash().Size()) - - blockMemory := circularbuffer.MakeC2Buffer(int(generator.BlockSize)) - blockMemory.Write(block) - - strong := generator.GetStrongHash() - // All the bytes - i := int64(0) - next := READ_NEXT_BYTE - - //ReadLoop: - for { - - atomic.AddInt64(&c.Comparisons, 1) - - // look for a weak match - generator.WeakRollingHash.GetSum(weaksum) - if weakMatchList := reference.FindWeakChecksum2(weaksum); weakMatchList != nil { - atomic.AddInt64(&c.WeakHashHits, 1) - - block = blockMemory.GetBlock() - - strong.Reset() - strong.Write(block) - strongSum = strong.Sum(strongSum) - strongList := reference.FindStrongChecksum2(strongSum, weakMatchList) - - // clear the slice - strongSum = strongSum[:0] - - // If there are many matches, it means that this block is - // duplicated in the reference. - // since we care about finding all the blocks in the reference, - // we must report all of them - off := i + baseOffset - for _, strongMatch := range strongList { - results <- BlockMatchResult{ - ComparisonOffset: off, - BlockIdx: strongMatch.ChunkOffset, - } - } - - if len(strongList) > 0 { - atomic.AddInt64(&c.StrongHashHits, 1) - if next == READ_NONE { - // found the match at the end, so exit - break - } - // No point looking for a match that overlaps this block - next = READ_NEXT_BLOCK - } - } - - var n int - var readBytes []byte - - switch next { - case READ_NEXT_BYTE: - n, err = comparison.Read(singleByte) - readBytes = singleByte - case READ_NEXT_BLOCK: - n, err = io.ReadFull(comparison, block) - readBytes = block[:n] - next = READ_NEXT_BYTE - } - - if uint(n) == generator.BlockSize { - generator.WeakRollingHash.SetBlock(block) - blockMemory.Write(block) - i += int64(n) - } else if n > 0 { - b_len := blockMemory.Len() - blockMemory.Write(readBytes) - generator.WeakRollingHash.AddAndRemoveBytes( - readBytes, - blockMemory.Evicted(), - b_len, - ) - i += int64(n) - } - - if next != READ_NONE && (err == io.EOF || err == io.ErrUnexpectedEOF) { - err = io.EOF - next = READ_NONE - } - - if next == READ_NONE { - if blockMemory.Empty() { - break - } - - b_len := blockMemory.Len() - removedByte := blockMemory.Truncate(1) - generator.WeakRollingHash.RemoveBytes(removedByte, b_len) - i += 1 - } - } - - if err != io.EOF { - ReportErr(err) - return - } -} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go b/vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go deleted file mode 100644 index cd3b7b5..0000000 --- a/vendor/github.com/Redundancy/go-sync/comparer/comparer_bench_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package comparer - -import ( - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/util/readers" - "testing" -) - -var test = []byte{0, 1, 2, 3} - -type NegativeWeakIndex struct { -} - -func (i *NegativeWeakIndex) FindWeakChecksum2(chk []byte) interface{} { - return nil -} - -func (i *NegativeWeakIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { - return nil -} - -type NegativeStrongIndex struct { -} - -func (i *NegativeStrongIndex) FindWeakChecksum2(chk []byte) interface{} { - return i -} - -func (i *NegativeStrongIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { - return nil -} - -func BenchmarkWeakComparison(b *testing.B) { - b.ReportAllocs() - b.SetBytes(1) - - const BLOCK_SIZE = 8 - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - - b.StartTimer() - - results := (&Comparer{}).StartFindMatchingBlocks( - readers.OneReader(b.N+BLOCK_SIZE), - 0, - generator, - &NegativeWeakIndex{}, - ) - - for _, ok := <-results; ok; { - } - - b.StopTimer() -} - -func BenchmarkStrongComparison(b *testing.B) { - b.ReportAllocs() - b.SetBytes(1) - - const BLOCK_SIZE = 8 - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - - b.StartTimer() - - results := (&Comparer{}).StartFindMatchingBlocks( - readers.OneReader(b.N+BLOCK_SIZE), - 0, - generator, - &NegativeStrongIndex{}, - ) - - for _, ok := <-results; ok; { - } - - b.StopTimer() -} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go b/vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go deleted file mode 100644 index 9337a5e..0000000 --- a/vendor/github.com/Redundancy/go-sync/comparer/comparer_test.go +++ /dev/null @@ -1,445 +0,0 @@ -package comparer - -import ( - "bufio" - "bytes" - "io" - "reflect" - "testing" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/indexbuilder" -) - -func CheckResults( - t *testing.T, - original, modified string, - results <-chan BlockMatchResult, - block_size uint, - expectedResults []string, -) { - var resultStrings []string - - for i := range results { - - if i.Err != nil { - t.Fatal(i.Err) - } - - os := i.BlockIdx * block_size - original_max := os + block_size - if original_max > uint(len(original)) { - original_max = uint(len(original)) - } - - orignal := original[os:original_max] - - compare_max := i.ComparisonOffset + int64(block_size) - if compare_max > int64(len(modified)) { - compare_max = int64(len(modified)) - } - - compare := modified[i.ComparisonOffset:compare_max] - - if orignal != compare { - t.Errorf( - "Bad match: \"%v\" to \"%v\" : %#v", - orignal, compare, - i, - ) - } - - resultStrings = append(resultStrings, orignal) - } - - if len(resultStrings) != len(expectedResults) { - t.Fatalf( - "%#v blocks should have matched, got: %#v", - len(expectedResults), - resultStrings, - ) - } - - for i, v := range expectedResults { - if resultStrings[i] != v { - t.Errorf("%#v != %#v", resultStrings[i], v) - } - } -} - -func compare( - original string, - modified string, - block_size uint, -) (results <-chan BlockMatchResult, err error) { - - originalFileContent := bytes.NewBufferString(original) - generator := filechecksum.NewFileChecksumGenerator(block_size) - - _, reference, _, err := indexbuilder.BuildChecksumIndex( - generator, - originalFileContent, - ) - - if err != nil { - return - } - - modifiedContent := bytes.NewBufferString(modified) - - results = (&Comparer{}).StartFindMatchingBlocks( - modifiedContent, - 0, - generator, - reference, - ) - - return -} - -// Splits successive strings into blocks of size n -// 2, ABCD -> AB, CD -// 2, ABCD, E, FG -> AB, CD, E, FG -func split(n int, ss ...string) (result []string) { - totalLength := 0 - for _, s := range ss { - totalLength += len(s)/n + 1 - } - result = make([]string, 0, totalLength) - - for _, x := range ss { - i := int(0) - for i+n < len(x) { - result = append( - result, - x[i:i+n], - ) - - i += n - } - - if i < len(x)-1 { - result = append( - result, - x[i:], - ) - } - } - - return -} - -func TestSplit(t *testing.T) { - INPUT := "abcdef" - EXPECTED := []string{"ab", "cd", "ef"} - result := split(2, INPUT) - - if !reflect.DeepEqual(result, EXPECTED) { - t.Errorf( - "Lists differ: %v vs %v", - result, - EXPECTED, - ) - } -} - -func TestSplitWithPartial(t *testing.T) { - INPUT := "abcdef" - EXPECTED := []string{"abcd", "ef"} - result := split(4, INPUT) - - if !reflect.DeepEqual(result, EXPECTED) { - t.Errorf( - "Lists differ: %v vs %v", - result, - EXPECTED, - ) - } -} - -func TestMultiSplit(t *testing.T) { - INPUT := []string{"abcdef", "ghij"} - EXPECTED := []string{"abcd", "ef", "ghij"} - result := split(4, INPUT...) - - if !reflect.DeepEqual(result, EXPECTED) { - t.Errorf( - "Lists differ: %v vs %v", - result, - EXPECTED, - ) - } -} - -func TestDetectsPrependedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - - const ORIGINAL_STRING = "abcdefghijklmnop" - const PREPENDED_STRING = "12" + ORIGINAL_STRING - - results, err := compare(ORIGINAL_STRING, PREPENDED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - PREPENDED_STRING, - results, - BLOCK_SIZE, - split(4, ORIGINAL_STRING), - ) -} - -func TestDetectsInjectedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefgh" - const B = "ijklmnop" - const ORIGINAL_STRING = A + B - const MODIFIED_STRING = A + "23" + B - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A, B), - ) -} - -func TestDetectsAppendedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - - const ORIGINAL_STRING = "abcdefghijklmnop" - const MODIFIED_STRING = ORIGINAL_STRING + "23" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, ORIGINAL_STRING), - ) -} - -func TestDetectsModifiedContent(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefgh" - const B = "ijkl" - const C = "mnop" - const ORIGINAL_STRING = A + B + C - const MODIFIED_STRING = A + "i2kl" + C - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A, C), - ) -} - -func TestDetectsPartialBlockAtEnd(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefghijklmnopqrstuvwxyz" - const ORIGINAL_STRING = A - const MODIFIED_STRING = A - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A), - ) -} - -func TestDetectsModifiedPartialBlockAtEnd(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefghijklmnopqrstuvwx" - const ORIGINAL_STRING = A + "yz" - const MODIFIED_STRING = A + "23" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A), - ) -} - -func TestDetectsUnmodifiedPartialBlockAtEnd(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcdefghijklmnopqrst" - const ORIGINAL_STRING = A + "uvwx" + "yz" - const MODIFIED_STRING = A + "us6x" + "yz" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - split(4, A, "yz"), - ) -} - -func TestMultipleResultsForDuplicatedBlocks(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const A = "abcd" - const ORIGINAL_STRING = A + A - const MODIFIED_STRING = A - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - []string{A, A}, - ) -} - -func TestRegression1(t *testing.T) { - const BLOCK_SIZE = 4 - var err error - const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" - const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" - - results, err := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - if err != nil { - t.Fatal(err) - } - - CheckResults( - t, - ORIGINAL_STRING, - MODIFIED_STRING, - results, - BLOCK_SIZE, - []string{"The ", "k br", "own ", "fox ", "jump", "the ", "lazy"}, - ) -} - -func TestTwoComparisons(t *testing.T) { - const BLOCK_SIZE = 4 - const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" - const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" - - numMatchers := int64(4) - sectionSize := int64(len(ORIGINAL_STRING)) / numMatchers - sectionSize += int64(BLOCK_SIZE) - (sectionSize % int64(BLOCK_SIZE)) - - merger := &MatchMerger{} - - originalFile := bytes.NewReader([]byte(ORIGINAL_STRING)) - modifiedFile := bytes.NewReader([]byte(MODIFIED_STRING)) - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - - _, reference, _, _ := indexbuilder.BuildChecksumIndex( - generator, - originalFile, - ) - - for i := int64(0); i < numMatchers; i++ { - compare := &Comparer{} - offset := sectionSize * i - - t.Logf("Section %v: %v-%v", i, offset, offset+sectionSize) - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(modifiedFile, offset, sectionSize+BLOCK_SIZE), - 100000, // 1 MB buffer - ) - - // Bakes in the assumption about how to generate checksums (extract) - sectionGenerator := filechecksum.NewFileChecksumGenerator( - uint(BLOCK_SIZE), - ) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, reference, - ) - - merger.StartMergeResultStream(matchStream, int64(BLOCK_SIZE)) - } - - merged := merger.GetMergedBlocks() - missing := merged.GetMissingBlocks(uint(len(ORIGINAL_STRING) / BLOCK_SIZE)) - - expected := []string{ - "quic", "ed over ", " dog", - } - - t.Logf("Missing blocks: %v", len(missing)) - - for x, v := range missing { - start := v.StartBlock * BLOCK_SIZE - end := (v.EndBlock + 1) * BLOCK_SIZE - if end > uint(len(ORIGINAL_STRING)) { - end = uint(len(ORIGINAL_STRING)) - } - s := ORIGINAL_STRING[start:end] - - if s != expected[x] { - t.Errorf( - "Wrong block %v (%v-%v): %#v (expected %#v)", - x, v.StartBlock, v.EndBlock, s, expected[x], - ) - } else { - t.Logf( - "Correct block %v (%v-%v): %#v (expected %#v)", - x, v.StartBlock, v.EndBlock, s, expected[x], - ) - } - } -} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/merger.go b/vendor/github.com/Redundancy/go-sync/comparer/merger.go deleted file mode 100644 index 59f108b..0000000 --- a/vendor/github.com/Redundancy/go-sync/comparer/merger.go +++ /dev/null @@ -1,309 +0,0 @@ -package comparer - -import ( - "sort" - "sync" - - "github.com/petar/GoLLRB/llrb" -) - -// The result merger takes many BlockMatchResult -// and combines adjoining results into spans of blocks -type MatchMerger struct { - sync.Mutex - - wait sync.WaitGroup - - // Porting from map to llrb, to enable finding of the next - // largest blockID (What's the first blockID after 5?) - startEndBlockMap *llrb.LLRB - - blockCount uint -} - -// a span of multiple blocks, from start to end, which match the blocks -// starting at an offset of ComparisonStartOffset -type BlockSpan struct { - StartBlock uint - EndBlock uint - - // byte offset in the comparison for the match - ComparisonStartOffset int64 -} - -type BlockSpanIndex interface { - Position() uint -} - -// Wraps a blockspan so that it may easily be used -// in llrb. Corresponds to the start block of the blockspan. -type BlockSpanStart BlockSpan - -func (s BlockSpanStart) Position() uint { - return s.StartBlock -} - -func (s BlockSpanStart) Less(than llrb.Item) bool { - return s.StartBlock < than.(BlockSpanIndex).Position() -} - -// Wraps a blockspan so that it may easily be used -// in llrb. Corresponds to the end of the blockspan. -type BlockSpanEnd BlockSpan - -func (s BlockSpanEnd) Position() uint { - return s.EndBlock -} - -func (s BlockSpanEnd) Less(than llrb.Item) bool { - return s.EndBlock < than.(BlockSpanIndex).Position() -} - -// Wraps a block index, allowing easy use of llrb.Get() -type BlockSpanKey uint - -func (s BlockSpanKey) Position() uint { - return uint(s) -} - -func (k BlockSpanKey) Less(than llrb.Item) bool { - return uint(k) < than.(BlockSpanIndex).Position() -} - -func (b BlockSpan) EndOffset(blockSize int64) int64 { - return b.ComparisonStartOffset + blockSize*int64(b.EndBlock-b.StartBlock+1) -} - -func toBlockSpan(b BlockMatchResult) *BlockSpan { - return &BlockSpan{ - StartBlock: b.BlockIdx, - EndBlock: b.BlockIdx, - ComparisonStartOffset: b.ComparisonOffset, - } -} - -func isBordering(a, b *BlockSpan, blockSize int64) bool { - if a.EndBlock == b.StartBlock-1 && a.EndOffset(blockSize) == b.ComparisonStartOffset { - return true - } else if b.EndBlock == a.StartBlock-1 && b.EndOffset(blockSize) == a.ComparisonStartOffset { - return true - } - - return false -} - -func itemToBlockSpan(in llrb.Item) BlockSpan { - switch i := in.(type) { - case BlockSpanStart: - return BlockSpan(i) - case BlockSpanEnd: - return BlockSpan(i) - } - return BlockSpan{} -} - -// if merged, the block span remaining is the one with the lower start block -func (merger *MatchMerger) merge(block1, block2 *BlockSpan, blockSize int64) { - var a, b *BlockSpan = block1, block2 - - if block1.StartBlock > block2.StartBlock { - a, b = block2, block1 - } - - if isBordering(a, b, blockSize) { - // bordering, merge - // A ------ A B ------ B > A ---------------- A - merger.startEndBlockMap.Delete(BlockSpanKey(a.EndBlock)) - merger.startEndBlockMap.Delete(BlockSpanKey(b.StartBlock)) - a.EndBlock = b.EndBlock - - merger.startEndBlockMap.ReplaceOrInsert(BlockSpanStart(*a)) - merger.startEndBlockMap.ReplaceOrInsert(BlockSpanEnd(*a)) - } -} - -// Can be used on multiple streams of results simultaneously -// starts working asyncronously, call from the initiating goroutine -func (merger *MatchMerger) StartMergeResultStream( - resultStream <-chan BlockMatchResult, - blockSize int64, -) { - // Add should be called on the main goroutine - // to ensure that it has happened before wait is called - merger.wait.Add(1) - - if merger.startEndBlockMap == nil { - merger.startEndBlockMap = llrb.New() - } - - // used by the llrb iterator to signal that it found/didn't find - // an existing key on or spanning the given block - //foundExisting := make(chan bool) - - go func() { - defer merger.wait.Done() - - for result := range resultStream { - if result.Err != nil { - return - } - - merger.Lock() - merger.blockCount += 1 - - blockID := result.BlockIdx - preceeding := merger.startEndBlockMap.Get(BlockSpanKey(blockID - 1)) - following := merger.startEndBlockMap.Get(BlockSpanKey(blockID + 1)) - - asBlockSpan := toBlockSpan(result) - - var foundExisting bool - // Exists, or within an existing span - merger.startEndBlockMap.AscendGreaterOrEqual( - BlockSpanKey(blockID), - // iterator - func(i llrb.Item) bool { - j, ok := i.(BlockSpanIndex) - - if !ok { - foundExisting = true - return false - } - - switch k := j.(type) { - case BlockSpanStart: - // it's only overlapping if its the same blockID - foundExisting = k.StartBlock == blockID - return false - - case BlockSpanEnd: - // we didn't find a start, so there's an end that overlaps - foundExisting = true - return false - default: - foundExisting = true - return false - } - - }, - ) - - if foundExisting { - merger.Unlock() - continue - } - - merger.startEndBlockMap.ReplaceOrInsert( - BlockSpanStart(*toBlockSpan(result)), - ) - - if preceeding != nil && following != nil { - a := itemToBlockSpan(preceeding) - merger.merge( - asBlockSpan, - &a, - blockSize, - ) - - b := itemToBlockSpan(following) - merger.merge( - &a, - &b, - blockSize, - ) - - } else if preceeding != nil { - a := itemToBlockSpan(preceeding) - merger.merge( - asBlockSpan, - &a, - blockSize, - ) - } else if following != nil { - b := itemToBlockSpan(following) - merger.merge( - asBlockSpan, - &b, - blockSize, - ) - } - - merger.Unlock() - } - }() -} - -type BlockSpanList []BlockSpan - -func (l BlockSpanList) Len() int { - return len(l) -} - -func (l BlockSpanList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] -} - -func (l BlockSpanList) Less(i, j int) bool { - return l[i].StartBlock < l[j].StartBlock -} - -// Sorted list of blocks, based on StartBlock -func (merger *MatchMerger) GetMergedBlocks() (sorted BlockSpanList) { - merger.wait.Wait() - var smallestKey uint = 0 - m := merger.startEndBlockMap - - m.AscendGreaterOrEqual(m.Min(), func(item llrb.Item) bool { - switch block := item.(type) { - case BlockSpanStart: - sorted = append(sorted, BlockSpan(block)) - smallestKey = block.StartBlock + 1 - } - return true - }) - - sort.Sort(sorted) - return -} - -// Creates a list of spans that are missing. -// note that maxBlock is blockCount-1 -func (l BlockSpanList) GetMissingBlocks(maxBlock uint) (sorted BlockSpanList) { - // it's difficult to know how many spans we will need - sorted = make(BlockSpanList, 0) - - lastBlockSpanIndex := -1 - for _, blockSpan := range l { - if int(blockSpan.StartBlock) > lastBlockSpanIndex+1 { - sorted = append( - sorted, - BlockSpan{ - StartBlock: uint(lastBlockSpanIndex + 1), - EndBlock: blockSpan.StartBlock - 1, - }, - ) - } - - lastBlockSpanIndex = int(blockSpan.EndBlock) - } - - if lastBlockSpanIndex == -1 { - sorted = append( - sorted, - BlockSpan{ - StartBlock: 0, - EndBlock: maxBlock, - }, - ) - } else if uint(lastBlockSpanIndex) < maxBlock { - sorted = append( - sorted, - BlockSpan{ - StartBlock: uint(lastBlockSpanIndex + 1), - EndBlock: maxBlock, - }, - ) - } - - return sorted -} diff --git a/vendor/github.com/Redundancy/go-sync/comparer/merger_test.go b/vendor/github.com/Redundancy/go-sync/comparer/merger_test.go deleted file mode 100644 index 01ca053..0000000 --- a/vendor/github.com/Redundancy/go-sync/comparer/merger_test.go +++ /dev/null @@ -1,401 +0,0 @@ -package comparer - -import ( - "testing" -) - -func TestMergeAdjacentBlocksAfter(t *testing.T) { - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 1 { - t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0]) - } -} - -func TestMergeAdjacentBlocksBefore(t *testing.T) { - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 1 { - t.Errorf("Wrong EndBlock, expected 1 got %#v", merged[0]) - } - - // start and end - if merger.startEndBlockMap.Len() != 2 { - t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) - } -} - -func TestMergeAdjacentBlocksBetween(t *testing.T) { - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 2 * BLOCK_SIZE, - BlockIdx: 2, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - // match in the center - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 2 { - t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0]) - } - if merged[0].StartBlock != 0 { - t.Errorf("Wrong StartBlock, expected 0, got %#v", merged[0]) - } - if merger.startEndBlockMap.Len() != 2 { - t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) - } -} - -func TestMissingBlocksOffsetStart(t *testing.T) { - b := BlockSpanList{ - { - StartBlock: 2, - EndBlock: 3, - }, - } - - m := b.GetMissingBlocks(3) - - if len(m) != 1 { - t.Fatalf("Wrong number of missing blocks: %v", len(m)) - } - - if m[0].StartBlock != 0 { - t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) - } - if m[0].EndBlock != 1 { - t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) - } -} - -func TestMissingCenterBlock(t *testing.T) { - b := BlockSpanList{ - { - StartBlock: 0, - EndBlock: 0, - }, - { - StartBlock: 2, - EndBlock: 3, - }, - } - - m := b.GetMissingBlocks(3) - - if len(m) != 1 { - t.Fatalf("Wrong number of missing blocks: %v", len(m)) - } - - if m[0].StartBlock != 1 { - t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) - } - if m[0].EndBlock != 1 { - t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) - } -} - -func TestMissingEndBlock(t *testing.T) { - b := BlockSpanList{ - { - StartBlock: 0, - EndBlock: 1, - }, - } - - m := b.GetMissingBlocks(3) - - if len(m) != 1 { - t.Fatalf("Wrong number of missing blocks: %v", len(m)) - } - - if m[0].StartBlock != 2 { - t.Errorf("Missing block has wrong start: %v", m[0].StartBlock) - } - if m[0].EndBlock != 3 { - t.Errorf("Missing block has wrong end: %v", m[0].EndBlock) - } -} - -func TestDuplicatedReferenceBlocks(t *testing.T) { - // Reference = AA - // Local = A - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - // When we find multiple strong matches, we send each of them - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 2 { - t.Errorf("Duplicated blocks cannot be merged: %#v", merged) - } - - missing := merged.GetMissingBlocks(1) - - if len(missing) > 0 { - t.Errorf("There were no missing blocks: %#v", missing) - } -} - -func TestDuplicatedLocalBlocks(t *testing.T) { - // Reference = A - // Local = AA - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - // When we find multiple strong matches, we send each of them - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 0, - } - - close(mergeChan) - - // We only need one of the matches in the resulting file - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Errorf("Duplicated blocks cannot be merged: %#v", merged) - } - - missing := merged.GetMissingBlocks(0) - - if len(missing) > 0 { - t.Errorf("There were no missing blocks: %#v", missing) - } -} - -func TestDoublyDuplicatedBlocks(t *testing.T) { - // Reference = AA - // Local = AA - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - // When we find multiple strong matches, we send each of them - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 1, - } - - // Second local match - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 2 { - t.Errorf("Duplicated blocks cannot be merged: %#v", merged) - } - - missing := merged.GetMissingBlocks(1) - - if len(missing) > 0 { - t.Errorf("There were no missing blocks: %#v", missing) - } -} - -func TestBlockWithinSpan(t *testing.T) { - // catch the case where we're informed about a block, - // after we've merged blocks around it, so that the start and end - // are within a span, not bordering one - const BLOCK_SIZE = 4 - - mergeChan := make(chan BlockMatchResult) - merger := &MatchMerger{} - merger.StartMergeResultStream(mergeChan, BLOCK_SIZE) - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 0, - BlockIdx: 0, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - mergeChan <- BlockMatchResult{ - ComparisonOffset: 2 * BLOCK_SIZE, - BlockIdx: 2, - } - - // This one is a duplicate of an earlier one - mergeChan <- BlockMatchResult{ - ComparisonOffset: BLOCK_SIZE, - BlockIdx: 1, - } - - close(mergeChan) - - merged := merger.GetMergedBlocks() - - if len(merged) != 1 { - t.Fatalf("Wrong number of blocks returned: %#v", merged) - } - - if merged[0].EndBlock != 2 { - t.Errorf("Wrong EndBlock, expected 2 got %#v", merged[0]) - } - - // start and end - if merger.startEndBlockMap.Len() != 2 { - t.Errorf("Wrong number of entries in the map: %v", merger.startEndBlockMap.Len()) - } -} - -func TestNilBlockSpanList(t *testing.T) { - s := BlockSpanList(nil) - - missing := s.GetMissingBlocks(1) - - if missing == nil { - t.Fail() - } - - if len(missing) == 0 { - t.Fatal("missing should not be empty") - } - - missingItem := missing[0] - - if missingItem.StartBlock != 0 { - t.Errorf("Wrong startblock: %v", missingItem.StartBlock) - } - if missingItem.EndBlock != 1 { - t.Errorf("Wrong endblock: %v", missingItem.EndBlock) - } -} - -func TestRegression1Merger(t *testing.T) { - const BLOCK_SIZE = 4 - const ORIGINAL_STRING = "The quick brown fox jumped over the lazy dog" - const MODIFIED_STRING = "The qwik brown fox jumped 0v3r the lazy" - - results, _ := compare(ORIGINAL_STRING, MODIFIED_STRING, BLOCK_SIZE) - merger := &MatchMerger{} - merger.StartMergeResultStream(results, BLOCK_SIZE) - - merged := merger.GetMergedBlocks() - missing := merged.GetMissingBlocks(uint(len(ORIGINAL_STRING) / BLOCK_SIZE)) - - expected := []string{ - "quic", "ed over ", " dog", - } - - for i, v := range missing { - start := v.StartBlock * BLOCK_SIZE - end := (v.EndBlock + 1) * BLOCK_SIZE - if end > uint(len(ORIGINAL_STRING)) { - end = uint(len(ORIGINAL_STRING)) - } - s := ORIGINAL_STRING[start:end] - - if s != expected[i] { - t.Errorf("Wrong block %v: %v (expected %v)", i, expected[i]) - } - } -} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go b/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go deleted file mode 100644 index 4f69701..0000000 --- a/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum.go +++ /dev/null @@ -1,237 +0,0 @@ -/* -package filechecksum provides the FileChecksumGenerator, whose main responsibility is to read a file, -and generate both weak and strong checksums for every block. It is also used by the comparer, which -will generate weak checksums for potential byte ranges that could match the index, and strong checksums -if needed. -*/ -package filechecksum - -import ( - "crypto/md5" - "hash" - "io" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/rollsum" -) - -// Rsync swapped to this after version 30 -// this is a factory function, because we don't actually want to share hash state -var DefaultStrongHashGenerator = func() hash.Hash { - return md5.New() -} - -// We provide an overall hash of individual files -var DefaultFileHashGenerator = func() hash.Hash { - return md5.New() -} - -// Uses all default hashes (MD5 & rollsum16) -func NewFileChecksumGenerator(blocksize uint) *FileChecksumGenerator { - return &FileChecksumGenerator{ - BlockSize: blocksize, - WeakRollingHash: rollsum.NewRollsum32Base(blocksize), - //WeakRollingHash: rollsum.NewRollsum16Base(blocksize), - StrongHash: DefaultStrongHashGenerator(), - FileChecksumHash: DefaultFileHashGenerator(), - } -} - -type RollingHash interface { - // the size of the hash output - Size() int - - AddByte(b byte) - RemoveByte(b byte, length int) - - AddBytes(bs []byte) - RemoveBytes(bs []byte, length int) - - // pairs up bytes to do remove/add in the right order - AddAndRemoveBytes(add []byte, remove []byte, length int) - - SetBlock(block []byte) - - GetSum(b []byte) - Reset() -} - -/* -FileChecksumGenerator provides a description of what hashing functions to use to -evaluate a file. Since the hashes store state, it is NOT safe to use a generator concurrently -for different things. -*/ -type FileChecksumGenerator struct { - // See BlockBuffer - WeakRollingHash RollingHash - StrongHash hash.Hash - FileChecksumHash hash.Hash - BlockSize uint -} - -// Reset all hashes to initial state -func (check *FileChecksumGenerator) Reset() { - check.WeakRollingHash.Reset() - check.StrongHash.Reset() - check.FileChecksumHash.Reset() -} - -func (check *FileChecksumGenerator) ChecksumSize() int { - return check.WeakRollingHash.Size() + check.GetStrongHash().Size() -} - -func (check *FileChecksumGenerator) GetChecksumSizes() (int, int) { - return check.WeakRollingHash.Size(), check.GetStrongHash().Size() -} - -// Gets the Hash function for the overall file used on each block -// defaults to md5 -func (check *FileChecksumGenerator) GetFileHash() hash.Hash { - return check.FileChecksumHash -} - -// Gets the Hash function for the strong hash used on each block -// defaults to md5, but can be overriden by the generator -func (check *FileChecksumGenerator) GetStrongHash() hash.Hash { - return check.StrongHash -} - -// GenerateChecksums reads each block of the input file, and outputs first the weak, then the strong checksum -// to the output writer. It will return a checksum for the whole file. -// Potentially speaking, this might be better producing a channel of blocks, which would remove the need for io from -// a number of other places. -func (check *FileChecksumGenerator) GenerateChecksums(inputFile io.Reader, output io.Writer) (fileChecksum []byte, err error) { - for chunkResult := range check.StartChecksumGeneration(inputFile, 64, nil) { - if chunkResult.Err != nil { - return nil, chunkResult.Err - } else if chunkResult.Filechecksum != nil { - return chunkResult.Filechecksum, nil - } - - for _, chunk := range chunkResult.Checksums { - output.Write(chunk.WeakChecksum) - output.Write(chunk.StrongChecksum) - } - } - - return nil, nil -} - -type ChecksumResults struct { - // Return multiple chunks at once for performance - Checksums []chunks.ChunkChecksum - // only used for the last item - Filechecksum []byte - // signals that this is the last item - Err error -} - -// A function or object that can compress blocks -// the compression function must also write out the compressed blocks somewhere! -// Compressed blocks should be independently inflatable -type CompressionFunction func([]byte) (compressedSize int64, err error) - -func (check *FileChecksumGenerator) StartChecksumGeneration( - inputFile io.Reader, - blocksPerResult uint, - compressionFunction CompressionFunction, -) <-chan ChecksumResults { - resultChan := make(chan ChecksumResults) - go check.generate(resultChan, blocksPerResult, compressionFunction, inputFile) - return resultChan -} - -func (check *FileChecksumGenerator) generate( - resultChan chan ChecksumResults, - blocksPerResult uint, - compressionFunction CompressionFunction, - inputFile io.Reader, -) { - defer close(resultChan) - - fullChecksum := check.GetFileHash() - strongHash := check.GetStrongHash() - - buffer := make([]byte, check.BlockSize) - - // ensure that the hashes are clean - strongHash.Reset() - fullChecksum.Reset() - - // We reset the hashes when done do we can reuse the generator - defer check.WeakRollingHash.Reset() - defer strongHash.Reset() - defer fullChecksum.Reset() - - results := make([]chunks.ChunkChecksum, 0, blocksPerResult) - - i := uint(0) - for { - n, err := io.ReadFull(inputFile, buffer) - section := buffer[:n] - - if n == 0 { - break - } - - // As hashes, the assumption is that they never error - // additionally, we assume that the only reason not - // to write a full block would be reaching the end of the file - fullChecksum.Write(section) - check.WeakRollingHash.SetBlock(section) - strongHash.Write(section) - - strongChecksumValue := make([]byte, 0, strongHash.Size()) - weakChecksumValue := make([]byte, check.WeakRollingHash.Size()) - - check.WeakRollingHash.GetSum(weakChecksumValue) - strongChecksumValue = strongHash.Sum(strongChecksumValue) - - blockSize := int64(check.BlockSize) - - if compressionFunction != nil { - blockSize, err = compressionFunction(section) - } - - results = append( - results, - chunks.ChunkChecksum{ - ChunkOffset: i, - Size: blockSize, - WeakChecksum: weakChecksumValue, - StrongChecksum: strongChecksumValue, - }, - ) - - i++ - - if len(results) == cap(results) { - resultChan <- ChecksumResults{ - Checksums: results, - } - results = make([]chunks.ChunkChecksum, 0, blocksPerResult) - } - - // clear it again - strongChecksumValue = strongChecksumValue[:0] - - // Reset the strong - strongHash.Reset() - - if n != len(buffer) || err == io.EOF { - break - } - } - - if len(results) > 0 { - resultChan <- ChecksumResults{ - Checksums: results, - } - } - - resultChan <- ChecksumResults{ - Filechecksum: fullChecksum.Sum(nil), - } - - return -} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go b/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go deleted file mode 100644 index 76d1e74..0000000 --- a/vendor/github.com/Redundancy/go-sync/filechecksum/filechecksum_test.go +++ /dev/null @@ -1,309 +0,0 @@ -package filechecksum - -import ( - "bytes" - "io" - "os" - "testing" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/util/readers" -) - -func TestChecksumGenerationEndsWithFilechecksum(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - lastResult := ChecksumResults{} - - for lastResult = range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { - } - - if lastResult.Checksums != nil { - t.Errorf("Last result had checksums: %#v", lastResult) - } - - if lastResult.Filechecksum == nil { - t.Errorf("Last result did not contain the filechecksum: %#v", lastResult) - } -} - -func TestChecksumGenerationReturnsCorrectChecksumCount(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - resultCount := 0 - - for r := range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { - resultCount += len(r.Checksums) - } - - if resultCount != BLOCK_COUNT { - t.Errorf("Unexpected block count returned: %v", resultCount) - } -} - -func TestChecksumGenerationContainsHashes(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - for r := range checksum.StartChecksumGeneration(emptybuffer, 10, nil) { - for _, r2 := range r.Checksums { - if len(r2.WeakChecksum) != checksum.WeakRollingHash.Size() { - t.Fatalf( - "Wrong length weak checksum: %v vs %v", - len(r2.WeakChecksum), - checksum.WeakRollingHash.Size(), - ) - } - - if len(r2.StrongChecksum) != checksum.StrongHash.Size() { - t.Fatalf( - "Wrong length strong checksum: %v vs %v", - len(r2.StrongChecksum), - checksum.StrongHash.Size(), - ) - } - } - } -} - -func TestRollsumLength(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - emptybuffer := bytes.NewBuffer(make([]byte, BLOCK_COUNT*BLOCKSIZE)) - output := bytes.NewBuffer(nil) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - // output length is expected to be 20 blocks - expectedLength := (BLOCK_COUNT * checksum.GetStrongHash().Size()) + - (BLOCK_COUNT * checksum.WeakRollingHash.Size()) - - _, err := checksum.GenerateChecksums(emptybuffer, output) - - if err != nil { - t.Fatal(err) - } - - if output.Len() != expectedLength { - t.Errorf( - "output length (%v) did not match expected length (%v)", - output.Len(), - expectedLength, - ) - } -} - -func TestRollsumLengthWithPartialBlockAtEnd(t *testing.T) { - const BLOCKSIZE = 100 - const FULL_BLOCK_COUNT = 20 - const BLOCK_COUNT = FULL_BLOCK_COUNT + 1 - - emptybuffer := bytes.NewBuffer(make([]byte, FULL_BLOCK_COUNT*BLOCKSIZE+50)) - output := bytes.NewBuffer(nil) - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - // output length is expected to be 20 blocks - expectedLength := (BLOCK_COUNT * checksum.GetStrongHash().Size()) + - (BLOCK_COUNT * checksum.WeakRollingHash.Size()) - - _, err := checksum.GenerateChecksums(emptybuffer, output) - - if err != nil { - t.Fatal(err) - } - - if output.Len() != expectedLength { - t.Errorf( - "output length (%v) did not match expected length (%v)", - output.Len(), - expectedLength, - ) - } -} - -// Each of the data blocks is the same, so the checksums for the blocks should be the same -func TestChecksumBlocksTheSame(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - output := bytes.NewBuffer(nil) - - _, err := checksum.GenerateChecksums( - readers.OneReader(BLOCKSIZE*BLOCK_COUNT), - output, - ) - - if err != nil { - t.Fatal(err) - } - - weakSize, strongSize := checksum.GetChecksumSizes() - - if output.Len() != BLOCK_COUNT*(strongSize+weakSize) { - t.Errorf( - "Unexpected output length: %v, expected %v", - output.Len(), - BLOCK_COUNT*(strongSize+weakSize), - ) - } - - results, err := chunks.LoadChecksumsFromReader(output, weakSize, strongSize) - - if err != nil { - t.Fatal(err) - } - - if len(results) != BLOCK_COUNT { - t.Fatalf("Results too short! %v", len(results)) - } - - first := results[0] - - for i, chk := range results { - if chk.ChunkOffset != uint(i) { - t.Errorf("Unexpected offset %v on chunk %v", chk.ChunkOffset, i) - } - if !first.Match(chk) { - t.Fatalf("Chunks have different checksums on %v", i) - } - } -} - -func TestPrependedBlocks(t *testing.T) { - const BLOCKSIZE = 100 - const BLOCK_COUNT = 20 - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - file1 := io.LimitReader( - readers.NewNonRepeatingSequence(0), - BLOCKSIZE*BLOCK_COUNT, - ) - - file2 := io.LimitReader( - io.MultiReader( - readers.OneReader(BLOCKSIZE), // Off by one block - readers.NewNonRepeatingSequence(0), - ), - BLOCKSIZE*BLOCK_COUNT, - ) - - output1 := bytes.NewBuffer(nil) - chksum1, _ := checksum.GenerateChecksums(file1, output1) - - output2 := bytes.NewBuffer(nil) - chksum2, _ := checksum.GenerateChecksums(file2, output2) - - if bytes.Compare(chksum1, chksum2) == 0 { - t.Fatal("Checksums should be different") - } - - weaksize, strongSize := checksum.GetChecksumSizes() - sums1, _ := chunks.LoadChecksumsFromReader(output1, weaksize, strongSize) - sums2, _ := chunks.LoadChecksumsFromReader(output2, weaksize, strongSize) - - if len(sums1) != len(sums2) { - t.Fatalf("Checksum lengths differ %v vs %v", len(sums1), len(sums2)) - } - - if sums1[0].Match(sums2[0]) { - t.Error("Chunk sums1[0] should differ from sums2[0]") - } - - for i, _ := range sums2 { - if i == 0 { - continue - } - - if !sums1[i-1].Match(sums2[i]) { - t.Errorf("Chunk sums1[%v] equal sums2[%v]", i-1, i) - } - - } -} - -// Test that partial content that ends in the middle of a weak -// hash is caught correctly -func TestInvalidReaderLength_Weak(t *testing.T) { - const BLOCKSIZE = 100 - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - file1 := io.LimitReader( - readers.NewNonRepeatingSequence(0), - int64(checksum.ChecksumSize())+2, - ) - - ws, ss := checksum.GetChecksumSizes() - r, err := chunks.LoadChecksumsFromReader(file1, ws, ss) - - if r != nil || err != chunks.ErrPartialChecksum { - t.Error("Expected partial checksum error") - } -} - -// Test that partial content that ends in the middle of a strong -// hash is caught correctly -func TestInvalidReaderLength_Strong(t *testing.T) { - const BLOCKSIZE = 100 - - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - file1 := io.LimitReader( - readers.NewNonRepeatingSequence(0), - int64(checksum.ChecksumSize())+int64(checksum.WeakRollingHash.Size())+2, - ) - - ws, ss := checksum.GetChecksumSizes() - r, err := chunks.LoadChecksumsFromReader(file1, ws, ss) - - if r != nil || err != chunks.ErrPartialChecksum { - t.Error("Expected partial checksum error") - } -} - -func ExampleFileChecksumGenerator_LoadChecksumsFromReader() { - const BLOCKSIZE = 8096 - checksum := NewFileChecksumGenerator(BLOCKSIZE) - - // This could be any source that conforms to io.Reader - // sections of a file, or the body of an http response - file1, err := os.Open("fileChecksums.chk") - - if err != nil { - return - } - - defer file1.Close() - - ws, ss := checksum.GetChecksumSizes() - checksums, err := chunks.LoadChecksumsFromReader(file1, ws, ss) - - if err != nil { - return - } - - // Make an index that we can use against our local - // checksums - i := index.MakeChecksumIndex(checksums) - - // example checksum from a local file - // look for the chunk in the index - i.FindWeakChecksumInIndex([]byte("a")) - -} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go b/vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go deleted file mode 100644 index 9d604a5..0000000 --- a/vendor/github.com/Redundancy/go-sync/filechecksum/verifier.go +++ /dev/null @@ -1,48 +0,0 @@ -package filechecksum - -import ( - "bytes" - "hash" -) - -type ChecksumLookup interface { - GetStrongChecksumForBlock(blockID int) []byte -} - -type HashVerifier struct { - BlockSize uint - Hash hash.Hash - BlockChecksumGetter ChecksumLookup -} - -func (v *HashVerifier) VerifyBlockRange(startBlockID uint, data []byte) bool { - for i := 0; i*int(v.BlockSize) < len(data); i++ { - start := i * int(v.BlockSize) - end := start + int(v.BlockSize) - - if end > len(data) { - end = len(data) - } - - blockData := data[start:end] - - expectedChecksum := v.BlockChecksumGetter.GetStrongChecksumForBlock( - int(startBlockID) + i, - ) - - if expectedChecksum == nil { - return true - } - - v.Hash.Write(blockData) - hashedData := v.Hash.Sum(nil) - - if bytes.Compare(expectedChecksum, hashedData) != 0 { - return false - } - - v.Hash.Reset() - } - - return true -} diff --git a/vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go b/vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go deleted file mode 100644 index ce3938f..0000000 --- a/vendor/github.com/Redundancy/go-sync/filechecksum/verifier_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package filechecksum - -import ( - "crypto/md5" - "testing" -) - -type SingleBlockSource []byte - -func (d SingleBlockSource) GetStrongChecksumForBlock(blockID int) []byte { - m := md5.New() - m.Write(d) - return m.Sum(nil) -} - -func TestBlockEqualsItself(t *testing.T) { - data := []byte("fooooo") - - h := HashVerifier{ - Hash: md5.New(), - BlockSize: uint(len(data)), - BlockChecksumGetter: SingleBlockSource(data), - } - - if !h.VerifyBlockRange(0, data) { - t.Error("data did not verify") - } -} - -type FourByteBlockSource []byte - -func (d FourByteBlockSource) GetStrongChecksumForBlock(blockID int) []byte { - m := md5.New() - - start := blockID * 4 - end := start + 4 - - if end >= len(d) { - end = len(d) - } - - m.Write(d[start:end]) - return m.Sum(nil) -} - -func TestSplitBlocksEqualThemselves(t *testing.T) { - data := []byte("foooBaar") - - h := HashVerifier{ - Hash: md5.New(), - BlockSize: uint(4), - BlockChecksumGetter: FourByteBlockSource(data), - } - - if !h.VerifyBlockRange(0, data) { - t.Error("data did not verify") - } -} - -func TestPartialBlock(t *testing.T) { - data := []byte("fo") - - h := HashVerifier{ - Hash: md5.New(), - BlockSize: uint(4), - BlockChecksumGetter: SingleBlockSource(data), - } - - if !h.VerifyBlockRange(0, data) { - t.Error("data did not verify") - } -} diff --git a/vendor/github.com/Redundancy/go-sync/gosync.go b/vendor/github.com/Redundancy/go-sync/gosync.go deleted file mode 100644 index ea07157..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync.go +++ /dev/null @@ -1,16 +0,0 @@ -/* -Package gosync is inspired by zsync, and rsync. It aims to take the fundamentals and create a very flexible library that can be adapted -to work in many ways. - -We rely heavily on built in Go abstractions like io.Reader, hash.Hash and our own interfaces - this makes the code easier to change, and to test. -In particular, no part of the core library should know anything about the transport or layout of the reference data. If you want -to do rsync and do http/https range requests, that's just as good as zsync client-server over an SSH tunnel. The goal is also to allow -support for multiple concurrent connections, so that you can make the best use of your line in the face of the bandwidth latency product -(or other concerns that require concurrency to solve). - -The following optimizations are possible: -* Generate hashes with multiple threads (both during reference generation and local file interrogation) -* Multiple ranged requests (can even be used to get the hashes) - -*/ -package gosync diff --git a/vendor/github.com/Redundancy/go-sync/gosync/build.go b/vendor/github.com/Redundancy/go-sync/gosync/build.go deleted file mode 100644 index 7dfb000..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync/build.go +++ /dev/null @@ -1,114 +0,0 @@ -package main - -import ( - "fmt" - "os" - "path/filepath" - "time" - - "github.com/Redundancy/go-sync/filechecksum" - "github.com/codegangsta/cli" -) - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "build", - ShortName: "b", - Usage: "build a .gosync file for a file", - Action: Build, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "blocksize", - Value: DefaultBlockSize, - Usage: "The block size to use for the gosync file", - }, - }, - }, - ) -} - -func Build(c *cli.Context) { - filename := c.Args()[0] - blocksize := uint32(c.Int("blocksize")) - generator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) - inputFile, err := os.Open(filename) - - if err != nil { - absInputPath, err2 := filepath.Abs(filename) - if err2 == nil { - handleFileError(absInputPath, err) - } else { - handleFileError(filename, err) - } - - os.Exit(1) - } - - s, _ := inputFile.Stat() - // TODO: Error? - file_size := s.Size() - - defer inputFile.Close() - - ext := filepath.Ext(filename) - outfilePath := filename[:len(filename)-len(ext)] + ".gosync" - outputFile, err := os.Create(outfilePath) - - if err != nil { - handleFileError(outfilePath, err) - os.Exit(1) - } - - defer outputFile.Close() - - if err = writeHeaders( - outputFile, - magicString, - blocksize, - file_size, - []uint16{majorVersion, minorVersion, patchVersion}, - ); err != nil { - fmt.Fprintf( - os.Stderr, - "Error getting file info: %v\n", - filename, - err, - ) - os.Exit(2) - } - - start := time.Now() - _, err = generator.GenerateChecksums(inputFile, outputFile) - end := time.Now() - - if err != nil { - fmt.Fprintf( - os.Stderr, - "Error generating checksum: %v\n", - filename, - err, - ) - os.Exit(2) - } - - inputFileInfo, err := os.Stat(filename) - if err != nil { - fmt.Fprintf( - os.Stderr, - "Error getting file info: %v\n", - filename, - err, - ) - os.Exit(2) - } - - fmt.Fprintf( - os.Stderr, - "Index for %v file generated in %v (%v bytes/S)\n", - inputFileInfo.Size(), - end.Sub(start), - float64(inputFileInfo.Size())/end.Sub(start).Seconds(), - ) -} diff --git a/vendor/github.com/Redundancy/go-sync/gosync/common.go b/vendor/github.com/Redundancy/go-sync/gosync/common.go deleted file mode 100644 index f8091a9..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync/common.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "bufio" - "encoding/binary" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "os" - - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/patcher" - "github.com/codegangsta/cli" -) - -const ( - // KB - One Kilobyte - KB = 1024 - // MB - One Megabyte - MB = 1000000 -) - -func errorWrapper(c *cli.Context, f func(*cli.Context) error) { - defer func() { - if p := recover(); p != nil { - fmt.Fprintln(os.Stderr, p) - os.Exit(1) - } - }() - - if err := f(c); err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) - } - - return -} - -func openFileAndHandleError(filename string) (f *os.File) { - var err error - f, err = os.Open(filename) - - if err != nil { - f = nil - handleFileError(filename, err) - } - - return -} - -func formatFileError(filename string, err error) error { - switch { - case os.IsExist(err): - return fmt.Errorf( - "Could not open %v (already exists): %v", - filename, - err, - ) - case os.IsNotExist(err): - return fmt.Errorf( - "Could not find %v: %v\n", - filename, - err, - ) - case os.IsPermission(err): - return fmt.Errorf( - "Could not open %v (permission denied): %v\n", - filename, - err, - ) - default: - return fmt.Errorf( - "Unknown error opening %v: %v\n", - filename, - err, - ) - } -} - -func handleFileError(filename string, err error) { - e := formatFileError(filename, err) - fmt.Fprintln(os.Stderr, e) -} - -func getLocalOrRemoteFile(path string) (io.ReadCloser, error) { - url, err := url.Parse(path) - - switch { - case err != nil: - return os.Open(path) - case url.Scheme == "": - return os.Open(path) - default: - response, err := http.Get(path) - - if err != nil { - return nil, err - } - - if response.StatusCode < 200 || response.StatusCode > 299 { - return nil, fmt.Errorf("Request to %v returned status: %v", path, response.Status) - } - - return response.Body, nil - } -} - -func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} - -func writeHeaders( - f *os.File, - magic string, - blocksize uint32, - filesize int64, - versions []uint16, -) (err error) { - if _, err = f.WriteString(magicString); err != nil { - return - } - - for _, v := range versions { - if err = binary.Write(f, binary.LittleEndian, v); err != nil { - return - } - } - - if err = binary.Write(f, binary.LittleEndian, filesize); err != nil { - return - } - - err = binary.Write(f, binary.LittleEndian, blocksize) - return -} - -// reads the file headers and checks the magic string, then the semantic versioning -func readHeadersAndCheck( - r io.Reader, - magic string, - requiredMajorVersion uint16, -) ( - major, minor, patch uint16, - filesize int64, - blocksize uint32, - err error, -) { - b := make([]byte, len(magicString)) - - if _, err = r.Read(b); err != nil { - return - } else if string(b) != magicString { - err = errors.New( - "file header does not match magic string. Not a valid gosync file", - ) - return - } - - for _, v := range []*uint16{&major, &minor, &patch} { - err = binary.Read(r, binary.LittleEndian, v) - if err != nil { - return - } - } - - if requiredMajorVersion != major { - err = fmt.Errorf( - "The major version of the gosync file (%v.%v.%v) does not match the tool (%v.%v.%v).", - major, minor, patch, - majorVersion, minorVersion, patchVersion, - ) - - return - } - - err = binary.Read(r, binary.LittleEndian, &filesize) - if err != nil { - return - } - - err = binary.Read(r, binary.LittleEndian, &blocksize) - return -} - -func readIndex(r io.Reader, blocksize uint) ( - i *index.ChecksumIndex, - checksumLookup filechecksum.ChecksumLookup, - blockCount uint, - err error, -) { - generator := filechecksum.NewFileChecksumGenerator(blocksize) - - readChunks, e := chunks.LoadChecksumsFromReader( - r, - generator.WeakRollingHash.Size(), - generator.StrongHash.Size(), - ) - - err = e - - if err != nil { - return - } - - checksumLookup = chunks.StrongChecksumGetter(readChunks) - i = index.MakeChecksumIndex(readChunks) - blockCount = uint(len(readChunks)) - - return -} - -func multithreadedMatching( - localFile *os.File, - idx *index.ChecksumIndex, - localFileSize, - matcherCount int64, - blocksize uint, -) (*comparer.MatchMerger, *comparer.Comparer) { - // Note: Since not all sections of the file are equal in work - // it would be better to divide things up into more sections and - // pull work from a queue channel as each finish - sectionSize := localFileSize / matcherCount - sectionSize += int64(blocksize) - (sectionSize % int64(blocksize)) - merger := &comparer.MatchMerger{} - compare := &comparer.Comparer{} - - for i := int64(0); i < matcherCount; i++ { - offset := sectionSize * i - - // Sections must overlap by blocksize (strictly blocksize - 1?) - if i > 0 { - offset -= int64(blocksize) - } - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(localFile, offset, sectionSize), - MB, - ) - - sectionGenerator := filechecksum.NewFileChecksumGenerator(uint(blocksize)) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, idx) - - merger.StartMergeResultStream(matchStream, int64(blocksize)) - } - - return merger, compare -} - -// better way to do this? diff --git a/vendor/github.com/Redundancy/go-sync/gosync/diff.go b/vendor/github.com/Redundancy/go-sync/gosync/diff.go deleted file mode 100644 index 14a4acf..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync/diff.go +++ /dev/null @@ -1,145 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - "time" - - "github.com/codegangsta/cli" -) - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "diff", - ShortName: "d", - Usage: "gosync diff ", - Description: `Compare a file with a reference index, and print statistics on the comparison and performance.`, - Action: Diff, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "p", - Value: runtime.NumCPU(), - Usage: "The number of streams to use concurrently", - }, - }, - }, - ) -} - -func Diff(c *cli.Context) { - localFilename := c.Args()[0] - referenceFilename := c.Args()[1] - startTime := time.Now() - - localFile := openFileAndHandleError(localFilename) - - if localFile == nil { - os.Exit(1) - } - - defer localFile.Close() - - var blocksize uint32 - referenceFile := openFileAndHandleError(referenceFilename) - - if referenceFile == nil { - os.Exit(1) - } - - defer referenceFile.Close() - - _, _, _, _, blocksize, e := readHeadersAndCheck( - referenceFile, - magicString, - majorVersion, - ) - - if e != nil { - fmt.Printf("Error loading index: %v", e) - os.Exit(1) - } - - fmt.Println("Blocksize: ", blocksize) - - index, _, _, err := readIndex(referenceFile, uint(blocksize)) - referenceFile.Close() - - if err != nil { - return - } - - fmt.Println("Weak hash count:", index.WeakCount()) - - fi, err := localFile.Stat() - - if err != nil { - fmt.Println("Could not get info on file:", err) - os.Exit(1) - } - - num_matchers := int64(c.Int("p")) - - localFile_size := fi.Size() - - // Don't split up small files - if localFile_size < 1024*1024 { - num_matchers = 1 - } - - merger, compare := multithreadedMatching( - localFile, - index, - localFile_size, - num_matchers, - uint(blocksize), - ) - - mergedBlocks := merger.GetMergedBlocks() - - fmt.Println("\nMatched:") - totalMatchingSize := uint64(0) - matchedBlockCountAfterMerging := uint(0) - - for _, b := range mergedBlocks { - totalMatchingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) - matchedBlockCountAfterMerging += b.EndBlock - b.StartBlock + 1 - } - - fmt.Println("Comparisons:", compare.Comparisons) - fmt.Println("Weak hash hits:", compare.WeakHashHits) - - if compare.Comparisons > 0 { - fmt.Printf( - "Weak hit rate: %.2f%%\n", - 100.0*float64(compare.WeakHashHits)/float64(compare.Comparisons), - ) - } - - fmt.Println("Strong hash hits:", compare.StrongHashHits) - if compare.WeakHashHits > 0 { - fmt.Printf( - "Weak hash error rate: %.2f%%\n", - 100.0*float64(compare.WeakHashHits-compare.StrongHashHits)/float64(compare.WeakHashHits), - ) - } - - fmt.Println("Total matched bytes:", totalMatchingSize) - fmt.Println("Total matched blocks:", matchedBlockCountAfterMerging) - - // TODO: GetMissingBlocks uses the highest index, not the count, this can be pretty confusing - // Should clean up this interface to avoid that - missing := mergedBlocks.GetMissingBlocks(uint(index.BlockCount) - 1) - fmt.Println("Index blocks:", index.BlockCount) - - totalMissingSize := uint64(0) - for _, b := range missing { - //fmt.Printf("%#v\n", b) - totalMissingSize += uint64(b.EndBlock-b.StartBlock+1) * uint64(blocksize) - } - - fmt.Println("Approximate missing bytes:", totalMissingSize) - fmt.Println("Time taken:", time.Now().Sub(startTime)) -} diff --git a/vendor/github.com/Redundancy/go-sync/gosync/fileformat.md b/vendor/github.com/Redundancy/go-sync/gosync/fileformat.md deleted file mode 100644 index 0fd5522..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync/fileformat.md +++ /dev/null @@ -1,21 +0,0 @@ -*NB: I'm documenting the current format of the gosync files merely as a point in time -reference of format that is in use in the tool that is meant to serve as a practical reference and acceptance testing tool. The gosync tool is not intended as a production-worthy, well supported, tested tool.* - -*The format used exists entirely in service of being able to test the implementation of the gosync library as a cohesive whole in the real world, and therefore backwards and forwards compatibility (or even efficiency) are not primary concerns.* - -# Version 0.2.0 -### The header -(LE = little endian) -* The string "G0S9NC" in UTF-8 -* versions*3 (eg. 0.1.2), uint16 LE -* filesize, int64 LE -* blocksize uint32 LE - -### The body -Repeating: -* WeakChecksum -* StrongChecksum - -each referring to blocks, starting at 0 (file start) and going upwards. - -In the current implementation of the FileChecksumGenerator used the WeakChecksum is the rolling checksum (4 bytes), and StrongChecksum is MD5 (16 bytes). diff --git a/vendor/github.com/Redundancy/go-sync/gosync/main.go b/vendor/github.com/Redundancy/go-sync/gosync/main.go deleted file mode 100644 index 3164a8d..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync/main.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -gosync is a command-line implementation of the gosync package functionality, primarily as a demonstration of usage -but supposed to be functional in itself. -*/ -package main - -import ( - "fmt" - "log" - "net/http" - _ "net/http/pprof" - "os" - "runtime" - - "github.com/codegangsta/cli" -) - -const ( - DefaultBlockSize = 8192 - magicString = "G0S9NC" // just to confirm the file type is used correctly - majorVersion = uint16(0) - minorVersion = uint16(2) - patchVersion = uint16(1) -) - -var app = cli.NewApp() - -func main() { - app.Name = "gosync" - app.Usage = "Build indexes, patches, patch files" - app.Flags = []cli.Flag{ - cli.BoolFlag{ - Name: "profile", - Usage: "enable HTTP profiling", - }, - cli.IntFlag{ - Name: "profilePort", - Value: 6060, - Usage: "The number of streams to use concurrently", - }, - } - - app.Version = fmt.Sprintf( - "%v.%v.%v", - majorVersion, - minorVersion, - patchVersion, - ) - - runtime.GOMAXPROCS(runtime.NumCPU()) - - app.Before = func(c *cli.Context) error { - if c.Bool("profile") { - port := fmt.Sprint(c.Int("profilePort")) - - go func() { - log.Println(http.ListenAndServe("localhost:"+port, nil)) - }() - } - - return nil - } - - app.Run(os.Args) -} diff --git a/vendor/github.com/Redundancy/go-sync/gosync/patch.go b/vendor/github.com/Redundancy/go-sync/gosync/patch.go deleted file mode 100644 index 16730d3..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync/patch.go +++ /dev/null @@ -1,105 +0,0 @@ -package main - -import ( - "fmt" - "os" - "runtime" - - gosync_main "github.com/Redundancy/go-sync" - "github.com/codegangsta/cli" -) - -const usage = "gosync patch []" - -func init() { - app.Commands = append( - app.Commands, - cli.Command{ - Name: "patch", - ShortName: "p", - Usage: usage, - Description: `Recreate the reference source file, using an index and a local file that is believed to be similar. -The index should be produced by "gosync build". - - is a .gosync file and may be a local, unc network path or http/https url - is corresponding target and may be a local, unc network path or http/https url - is optional. If not specified, the local file will be overwritten when done.`, - Action: Patch, - Flags: []cli.Flag{ - cli.IntFlag{ - Name: "p", - Value: runtime.NumCPU(), - Usage: "The number of streams to use concurrently", - }, - }, - }, - ) -} - -// Patch a file -func Patch(c *cli.Context) { - errorWrapper(c, func(c *cli.Context) error { - - fmt.Fprintln(os.Stderr, "Starting patching process") - - if l := len(c.Args()); l < 3 || l > 4 { - return fmt.Errorf( - "Usage is \"%v\" (invalid number of arguments)", - usage, - ) - } - - localFilename := c.Args()[0] - summaryFile := c.Args()[1] - referencePath := c.Args()[2] - - outFilename := localFilename - if len(c.Args()) == 4 { - outFilename = c.Args()[3] - } - - indexReader, e := os.Open(summaryFile) - if e != nil { - return e - } - defer indexReader.Close() - - _, _, _, filesize, blocksize, e := readHeadersAndCheck( - indexReader, - magicString, - majorVersion, - ) - - index, checksumLookup, blockCount, err := readIndex( - indexReader, - uint(blocksize), - ) - - fs := &gosync_main.BasicSummary{ - ChecksumIndex: index, - ChecksumLookup: checksumLookup, - BlockCount: blockCount, - BlockSize: uint(blocksize), - FileSize: filesize, - } - - rsync, err := gosync_main.MakeRSync( - localFilename, - referencePath, - outFilename, - fs, - ) - - if err != nil { - return err - } - - err = rsync.Patch() - - if err != nil { - return err - } - - return rsync.Close() - }) -} diff --git a/vendor/github.com/Redundancy/go-sync/gosync_test.go b/vendor/github.com/Redundancy/go-sync/gosync_test.go deleted file mode 100644 index 72195db..0000000 --- a/vendor/github.com/Redundancy/go-sync/gosync_test.go +++ /dev/null @@ -1,111 +0,0 @@ -package gosync - -import ( - "fmt" - "testing" - - "bytes" - - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/indexbuilder" - "github.com/Redundancy/go-sync/util/readers" -) - -func Example() { - // due to short example strings, use a very small block size - // using one this small in practice would increase your file transfer! - const blockSize = 4 - - // This is the "file" as described by the authoritive version - const reference = "The quick brown fox jumped over the lazy dog" - - // This is what we have locally. Not too far off, but not correct. - const localVersion = "The qwik brown fox jumped 0v3r the lazy" - - generator := filechecksum.NewFileChecksumGenerator(blockSize) - _, referenceFileIndex, _, err := indexbuilder.BuildIndexFromString( - generator, - reference, - ) - - if err != nil { - return - } - - referenceAsBytes := []byte(reference) - localVersionAsBytes := []byte(localVersion) - - blockCount := len(referenceAsBytes) / blockSize - if len(referenceAsBytes)%blockSize != 0 { - blockCount++ - } - - inputFile := bytes.NewReader(localVersionAsBytes) - patchedFile := bytes.NewBuffer(nil) - - // This is more complicated than usual, because we're using in-memory - // "files" and sources. Normally you would use MakeRSync - summary := &BasicSummary{ - ChecksumIndex: referenceFileIndex, - ChecksumLookup: nil, - BlockCount: uint(blockCount), - BlockSize: blockSize, - FileSize: int64(len(referenceAsBytes)), - } - - rsync := &RSync{ - Input: inputFile, - Output: patchedFile, - Source: blocksources.NewReadSeekerBlockSource( - bytes.NewReader(referenceAsBytes), - blocksources.MakeNullFixedSizeResolver(uint64(blockSize)), - ), - Summary: summary, - OnClose: nil, - } - - if err := rsync.Patch(); err != nil { - fmt.Printf("Error: %v", err) - return - } - - fmt.Printf("Patched result: \"%s\"\n", patchedFile.Bytes()) - // Output: - // Patched result: "The quick brown fox jumped over the lazy dog" -} - -const ( - BYTE = 1 - KB = 1024 * BYTE - MB = 1024 * KB -) - -func BenchmarkIndexComparisons(b *testing.B) { - b.ReportAllocs() - - const SIZE = 200 * KB - b.SetBytes(SIZE) - - file := readers.NewSizedNonRepeatingSequence(6, SIZE) - generator := filechecksum.NewFileChecksumGenerator(8 * KB) - _, index, _, err := indexbuilder.BuildChecksumIndex(generator, file) - - if err != nil { - b.Fatal(err) - } - - b.StartTimer() - for i := 0; i < b.N; i++ { - // must reinitialize the file for each comparison - otherFile := readers.NewSizedNonRepeatingSequence(745656, SIZE) - compare := &comparer.Comparer{} - m := compare.StartFindMatchingBlocks(otherFile, 0, generator, index) - - for _, ok := <-m; ok; { - } - } - - b.StopTimer() -} diff --git a/vendor/github.com/Redundancy/go-sync/http_test.go b/vendor/github.com/Redundancy/go-sync/http_test.go deleted file mode 100644 index 6d7c047..0000000 --- a/vendor/github.com/Redundancy/go-sync/http_test.go +++ /dev/null @@ -1,174 +0,0 @@ -package gosync - -import ( - "bytes" - "crypto/md5" - "fmt" - "net" - "net/http" - "time" - - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/indexbuilder" - "github.com/Redundancy/go-sync/patcher" -) - -// due to short example strings, use a very small block size -// using one this small in practice would increase your file transfer! -const BLOCK_SIZE = 4 - -// This is the "file" as described by the authoritive version -const REFERENCE = "The quick brown fox jumped over the lazy dog" - -// This is what we have locally. Not too far off, but not correct. -const LOCAL_VERSION = "The qwik brown fox jumped 0v3r the lazy" - -var content = bytes.NewReader([]byte(REFERENCE)) - -func handler(w http.ResponseWriter, req *http.Request) { - http.ServeContent(w, req, "", time.Now(), content) -} - -// set up a http server locally that will respond predictably to ranged requests -func setupServer() <-chan int { - var PORT = 8000 - s := http.NewServeMux() - s.HandleFunc("/content", handler) - - portChan := make(chan int) - - go func() { - var listener net.Listener - var err error - - for { - PORT++ - p := fmt.Sprintf(":%v", PORT) - listener, err = net.Listen("tcp", p) - - if err == nil { - break - } - } - portChan <- PORT - http.Serve(listener, s) - }() - - return portChan -} - -// This is exceedingly similar to the module Example, but uses the http blocksource and a local http server -func Example_httpBlockSource() { - PORT := <-setupServer() - LOCAL_URL := fmt.Sprintf("http://localhost:%v/content", PORT) - - generator := filechecksum.NewFileChecksumGenerator(BLOCK_SIZE) - _, referenceFileIndex, checksumLookup, err := indexbuilder.BuildIndexFromString(generator, REFERENCE) - - if err != nil { - return - } - - fileSize := int64(len([]byte(REFERENCE))) - - // This would normally be saved in a file - - blockCount := fileSize / BLOCK_SIZE - if fileSize%BLOCK_SIZE != 0 { - blockCount++ - } - - fs := &BasicSummary{ - ChecksumIndex: referenceFileIndex, - ChecksumLookup: checksumLookup, - BlockCount: uint(blockCount), - BlockSize: uint(BLOCK_SIZE), - FileSize: fileSize, - } - - /* - // Normally, this would be: - rsync, err := MakeRSync( - "toPatch.file", - "http://localhost/content", - "out.file", - fs, - ) - */ - // Need to replace the output and the input - inputFile := bytes.NewReader([]byte(LOCAL_VERSION)) - patchedFile := bytes.NewBuffer(nil) - - resolver := blocksources.MakeFileSizedBlockResolver( - uint64(fs.GetBlockSize()), - fs.GetFileSize(), - ) - - rsync := &RSync{ - Input: inputFile, - Output: patchedFile, - Source: blocksources.NewHttpBlockSource( - LOCAL_URL, - 1, - resolver, - &filechecksum.HashVerifier{ - Hash: md5.New(), - BlockSize: fs.GetBlockSize(), - BlockChecksumGetter: fs, - }, - ), - Summary: fs, - OnClose: nil, - } - - err = rsync.Patch() - - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - err = rsync.Close() - - if err != nil { - fmt.Printf("Error: %v\n", err) - return - } - - fmt.Printf("Patched content: \"%v\"\n", patchedFile.String()) - - // Just for inspection - remoteReferenceSource := rsync.Source.(*blocksources.BlockSourceBase) - fmt.Printf("Downloaded Bytes: %v\n", remoteReferenceSource.ReadBytes()) - - // Output: - // Patched content: "The quick brown fox jumped over the lazy dog" - // Downloaded Bytes: 16 -} - -func ToPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func ToPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} diff --git a/vendor/github.com/Redundancy/go-sync/index/index.go b/vendor/github.com/Redundancy/go-sync/index/index.go deleted file mode 100644 index 6711024..0000000 --- a/vendor/github.com/Redundancy/go-sync/index/index.go +++ /dev/null @@ -1,183 +0,0 @@ -/* -Package index provides the functionality to describe a reference 'file' and its contents in terms of -the weak and strong checksums, in such a way that you can check if a weak checksum is present, -then check if there is a strong checksum that matches. - -It also allows lookups in terms of block offsets, so that upon finding a match, you can more efficiently -check if the next block follows it. - -The index structure does not lend itself to being an interface - the pattern of taking the result of looking for -the weak checksum and looking up the strong checksum in that requires us to return an object matching an interface which -both packages must know about. - -Here's the interface: - -type Index interface { - FindWeakChecksum(chk []byte) interface{} - FindStrongChecksum(chk []byte, weak interface{}) []chunks.ChunkChecksum -} - -This allows the implementation to rely on a previously generated value, without the users knowing what it is. -This breaks the dependency that requires so many packages to import index. -*/ -package index - -import ( - "bytes" - "encoding/binary" - "github.com/Redundancy/go-sync/chunks" - "sort" -) - -type ChecksumIndex struct { - BlockCount int - - /* - This datastructure is based on some benchmarking that indicates that it outperforms - a basic map 70ns vs 130ns for ~8192 checksums (which is reasonably large - - say 64 MB with no weak collisions @8192 bytes per block). - - We use a 256 element slice, and the value of the least significant byte to determine - which map to look up into. - */ - weakChecksumLookup []map[uint32]StrongChecksumList - - MaxStrongLength int - AverageStrongLength float32 - Count int -} - -// Builds an index in which chunks can be found, with their corresponding offsets -// We use this for the -func MakeChecksumIndex(checksums []chunks.ChunkChecksum) *ChecksumIndex { - n := &ChecksumIndex{ - BlockCount: len(checksums), - weakChecksumLookup: make([]map[uint32]StrongChecksumList, 256), - } - - for _, chunk := range checksums { - weakChecksumAsInt := binary.LittleEndian.Uint32(chunk.WeakChecksum) - arrayOffset := weakChecksumAsInt & 255 - - if n.weakChecksumLookup[arrayOffset] == nil { - n.weakChecksumLookup[arrayOffset] = make(map[uint32]StrongChecksumList) - } - - n.weakChecksumLookup[arrayOffset][weakChecksumAsInt] = append( - n.weakChecksumLookup[arrayOffset][weakChecksumAsInt], - chunk, - ) - - } - - sum := 0 - count := 0 - - for _, a := range n.weakChecksumLookup { - for _, c := range a { - sort.Sort(c) - if len(c) > n.MaxStrongLength { - n.MaxStrongLength = len(c) - } - sum += len(c) - count += 1 - n.Count += len(c) - } - } - - n.AverageStrongLength = float32(sum) / float32(count) - - return n -} - -func (index *ChecksumIndex) WeakCount() int { - return index.Count -} - -func (index *ChecksumIndex) FindWeakChecksumInIndex(weak []byte) StrongChecksumList { - x := binary.LittleEndian.Uint32(weak) - if index.weakChecksumLookup[x&255] != nil { - if v, ok := index.weakChecksumLookup[x&255][x]; ok { - return v - } - } - return nil -} - -func (index *ChecksumIndex) FindWeakChecksum2(chk []byte) interface{} { - w := index.FindWeakChecksumInIndex(chk) - - if len(w) == 0 { - return nil - } else { - return w - } -} - -func (index *ChecksumIndex) FindStrongChecksum2(chk []byte, weak interface{}) []chunks.ChunkChecksum { - if strongList, ok := weak.(StrongChecksumList); ok { - return strongList.FindStrongChecksum(chk) - } else { - return nil - } -} - -type StrongChecksumList []chunks.ChunkChecksum - -// Sortable interface -func (s StrongChecksumList) Len() int { - return len(s) -} - -// Sortable interface -func (s StrongChecksumList) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} - -// Sortable interface -func (s StrongChecksumList) Less(i, j int) bool { - return bytes.Compare(s[i].StrongChecksum, s[j].StrongChecksum) == -1 -} - -func (s StrongChecksumList) FindStrongChecksum(strong []byte) (result []chunks.ChunkChecksum) { - n := len(s) - - // average length is 1, so fast path comparison - if n == 1 { - if bytes.Compare(s[0].StrongChecksum, strong) == 0 { - return s - } else { - return nil - } - } - - // find the first possible occurance - first_gte_checksum := sort.Search( - n, - func(i int) bool { - return bytes.Compare(s[i].StrongChecksum, strong) >= 0 - }, - ) - - // out of bounds - if first_gte_checksum == -1 || first_gte_checksum == n { - return nil - } - - // Somewhere in the middle, but the next one didn't match - if bytes.Compare(s[first_gte_checksum].StrongChecksum, strong) != 0 { - return nil - } - - end := first_gte_checksum + 1 - for end < n { - if bytes.Compare(s[end].StrongChecksum, strong) == 0 { - end += 1 - } else { - break - } - - } - - return s[first_gte_checksum:end] -} diff --git a/vendor/github.com/Redundancy/go-sync/index/index_bench_test.go b/vendor/github.com/Redundancy/go-sync/index/index_bench_test.go deleted file mode 100644 index aace8a1..0000000 --- a/vendor/github.com/Redundancy/go-sync/index/index_bench_test.go +++ /dev/null @@ -1,148 +0,0 @@ -package index - -import ( - "github.com/Redundancy/go-sync/chunks" - "math/rand" - "sort" - "testing" -) - -var T = []byte{1, 2, 3, 4} - -func BenchmarkIndex1024(b *testing.B) { - i := ChecksumIndex{} - i.weakChecksumLookup = make([]map[uint32]StrongChecksumList, 256) - - for x := 0; x < 1024; x++ { - w := rand.Uint32() - - if i.weakChecksumLookup[w&255] == nil { - i.weakChecksumLookup[w&255] = make(map[uint32]StrongChecksumList) - } - - i.weakChecksumLookup[w&255][w] = append( - i.weakChecksumLookup[w&255][w], - chunks.ChunkChecksum{}, - ) - } - - b.SetBytes(1) - b.StartTimer() - for x := 0; x < b.N; x++ { - i.FindWeakChecksum2(T) - } - b.StopTimer() - -} - -func BenchmarkIndex8192(b *testing.B) { - i := ChecksumIndex{} - i.weakChecksumLookup = make([]map[uint32]StrongChecksumList, 256) - - for x := 0; x < 8192; x++ { - w := rand.Uint32() - - if i.weakChecksumLookup[w&255] == nil { - i.weakChecksumLookup[w&255] = make(map[uint32]StrongChecksumList) - } - - i.weakChecksumLookup[w&255][w] = append( - i.weakChecksumLookup[w&255][w], - chunks.ChunkChecksum{}, - ) - } - - b.SetBytes(1) - b.StartTimer() - for x := 0; x < b.N; x++ { - i.FindWeakChecksum2(T) - } - b.StopTimer() -} - -// Check how fast a sorted list of 8192 items would be -func BenchmarkIndexAsListBinarySearch8192(b *testing.B) { - b.SkipNow() - - s := make([]int, 8192) - for x := 0; x < 8192; x++ { - s[x] = rand.Int() - } - - sort.Ints(s) - - b.StartTimer() - for x := 0; x < b.N; x++ { - sort.SearchInts(s, rand.Int()) - } - b.StopTimer() -} - -// Check how fast a sorted list of 8192 items would be -// Checking for cache coherency gains -func BenchmarkIndexAsListLinearSearch8192(b *testing.B) { - s := make([]int, 8192) - for x := 0; x < 8192; x++ { - s[x] = rand.Int() - } - - sort.Ints(s) - - l := len(s) - b.StartTimer() - for x := 0; x < b.N; x++ { - v := rand.Int() - for i := 0; i < l; i++ { - if v < s[i] { - break - } - } - } - b.StopTimer() -} - -func Benchmark_256SplitBinarySearch(b *testing.B) { - a := make([][]int, 256) - for x := 0; x < 8192; x++ { - i := rand.Int() - a[i&255] = append( - a[i&255], - i, - ) - } - - for x := 0; x < 256; x++ { - sort.Ints(a[x]) - } - - b.StartTimer() - for x := 0; x < b.N; x++ { - v := rand.Int() - sort.SearchInts(a[v&255], v) - } - b.StopTimer() -} - -/* -This is currently the best performing contender for the index data structure for -weak checksum lookups. -*/ -func Benchmark_256Split_Map(b *testing.B) { - a := make([]map[int]interface{}, 256) - for x := 0; x < 8192; x++ { - i := rand.Int() - if a[i&255] == nil { - a[i&255] = make(map[int]interface{}) - } - a[i&255][i] = nil - } - - b.StartTimer() - for x := 0; x < b.N; x++ { - v := rand.Int() - if _, ok := a[v&255][v]; ok { - - } - } - b.StopTimer() -} diff --git a/vendor/github.com/Redundancy/go-sync/index/index_test.go b/vendor/github.com/Redundancy/go-sync/index/index_test.go deleted file mode 100644 index 3336134..0000000 --- a/vendor/github.com/Redundancy/go-sync/index/index_test.go +++ /dev/null @@ -1,206 +0,0 @@ -package index - -import ( - "testing" - - "github.com/Redundancy/go-sync/chunks" -) - -// Weak checksums must be 4 bytes -var WEAK_A = []byte("aaaa") -var WEAK_B = []byte("bbbb") - -/* -ChunkOffset uint -// the size of the block -Size int64 -WeakChecksum []byte -StrongChecksum []byte -*/ - -func TestMakeIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - }, - ) - - if i.Count != 2 { - t.Fatalf("Wrong count on index %v", i.Count) - } -} - -func TestFindWeakInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - result := i.FindWeakChecksumInIndex(WEAK_B) - - if result == nil { - t.Error("Did not find lookfor in the index") - } else if len(result) != 2 { - t.Errorf("Wrong number of possible matches found: %v", len(result)) - } else if result[0].ChunkOffset != 1 { - t.Errorf("Found chunk had offset %v expected 1", result[0].ChunkOffset) - } -} - -func TestWeakNotInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - result := i.FindWeakChecksumInIndex([]byte("afgh")) - - if result != nil { - t.Error("Result from FindWeakChecksumInIndex should be nil") - } - - result2 := i.FindWeakChecksum2([]byte("afgh")) - - if result2 != nil { - t.Errorf("Result from FindWeakChecksum2 should be nil: %#v", result2) - } -} - -func TestWeakNotInIndex2(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - result := i.FindWeakChecksumInIndex([]byte("llll")) - - if result != nil { - t.Error("Result should be nil") - } -} - -func TestFindStrongInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("c")) - - if len(strongs) != 1 { - t.Errorf("Incorrect number of strong checksums found: %v", len(strongs)) - } else if strongs[0].ChunkOffset != 1 { - t.Errorf("Wrong chunk found, had offset %v", strongs[0].ChunkOffset) - } -} - -func TestNotFoundStrongInIndexAtEnd(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("e")) - - if len(strongs) != 0 { - t.Errorf("Incorrect number of strong checksums found: %v", strongs) - } -} - -func TestNotFoundStrongInIndexInCenter(t *testing.T) { - // The strong checksum we're looking for is not found - // but is < another checksum in the strong list - - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - {ChunkOffset: 3, WeakChecksum: WEAK_B, StrongChecksum: []byte("f")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("e")) - - if len(strongs) != 0 { - t.Errorf("Incorrect number of strong checksums found: %v", strongs) - } -} - -func TestFindDuplicatedBlocksInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 0, WeakChecksum: WEAK_A, StrongChecksum: []byte("b")}, - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 3, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("d")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("c")) - - if len(strongs) != 2 { - t.Fatalf("Incorrect number of strong checksums found: %v", strongs) - } - - first := strongs[0] - if first.ChunkOffset != 1 { - t.Errorf("Wrong chunk found, had offset %v", first.ChunkOffset) - } - - second := strongs[1] - if second.ChunkOffset != 3 { - t.Errorf("Wrong chunk found, had offset %v", second.ChunkOffset) - } -} - -func TestFindTwoDuplicatedBlocksInIndex(t *testing.T) { - i := MakeChecksumIndex( - []chunks.ChunkChecksum{ - {ChunkOffset: 1, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - {ChunkOffset: 2, WeakChecksum: WEAK_B, StrongChecksum: []byte("c")}, - }, - ) - - // builds upon TestFindWeakInIndex - result := i.FindWeakChecksumInIndex(WEAK_B) - strongs := result.FindStrongChecksum([]byte("c")) - - if len(strongs) != 2 { - t.Fatalf("Incorrect number of strong checksums found: %v", strongs) - } - - first := strongs[0] - if first.ChunkOffset != 1 { - t.Errorf("Wrong chunk found, had offset %v", first.ChunkOffset) - } - - second := strongs[1] - if second.ChunkOffset != 2 { - t.Errorf("Wrong chunk found, had offset %v", second.ChunkOffset) - } -} diff --git a/vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go b/vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go deleted file mode 100644 index 8d2ebef..0000000 --- a/vendor/github.com/Redundancy/go-sync/indexbuilder/indexbuilder.go +++ /dev/null @@ -1,55 +0,0 @@ -/* -Package indexbuilder provides a few shortbuts to building a checksum index by generating and then loading -the checksums, and building an index from that. It's potentially a sign that the responsibilities here need refactoring. -*/ -package indexbuilder - -import ( - "bytes" - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "io" -) - -// Generates an index from a reader -// This is mostly a utility function to avoid being overly verbose in tests that need -// an index to work, but don't want to construct one by hand in order to avoid the dependencies -// obviously this means that those tests are likely to fail if there are issues with any of the other -// modules, which is not ideal. -// TODO: move to util? -func BuildChecksumIndex(check *filechecksum.FileChecksumGenerator, r io.Reader) ( - fcheck []byte, - i *index.ChecksumIndex, - lookup filechecksum.ChecksumLookup, - err error, -) { - b := bytes.NewBuffer(nil) - fcheck, err = check.GenerateChecksums(r, b) - - if err != nil { - return - } - - weakSize := check.WeakRollingHash.Size() - strongSize := check.GetStrongHash().Size() - readChunks, err := chunks.LoadChecksumsFromReader(b, weakSize, strongSize) - - if err != nil { - return - } - - i = index.MakeChecksumIndex(readChunks) - lookup = chunks.StrongChecksumGetter(readChunks) - - return -} - -func BuildIndexFromString(generator *filechecksum.FileChecksumGenerator, reference string) ( - fileCheckSum []byte, - referenceIndex *index.ChecksumIndex, - lookup filechecksum.ChecksumLookup, - err error, -) { - return BuildChecksumIndex(generator, bytes.NewBufferString(reference)) -} diff --git a/vendor/github.com/Redundancy/go-sync/patcher/blocksource.go b/vendor/github.com/Redundancy/go-sync/patcher/blocksource.go deleted file mode 100644 index c50241d..0000000 --- a/vendor/github.com/Redundancy/go-sync/patcher/blocksource.go +++ /dev/null @@ -1,58 +0,0 @@ -/* -Package patcher follows a pattern established by hash, which defines the interface in the top level package, and then provides implementations -below it. -*/ -package patcher - -import ( - "hash" -) - -/* -BlockSource is an interface used by the patchers to obtain blocks from the reference -It does not stipulate where the reference data might be (it could be local, in a pre-built patch file, on S3 or somewhere else) - -It is assumed that the BlockSource may be slow, and may benefit from request pipelining & concurrency. -Therefore patchers should feel free to request as many block spans as they can handle. - -A BlockSource may be a view onto a larger transport concept, so that multiple files can be handled with wider -knowledge of the number of simultaneous requests allowed, etc. The BlockSource may decide to split BlockSpans -into smaller sizes if it wants. - -It is up to the patcher to receive blocks in a timely manner, and decide what to do with them, rather than -bother the BlockSource with more memory management and buffering logic. - -Since these interfaces require specific structs to satisfy, it's expected that implementers will import this module. - -*/ -type BlockSource interface { - RequestBlocks(MissingBlockSpan) error - - GetResultChannel() <-chan BlockReponse - - // If the block source encounters an unsurmountable problem - EncounteredError() <-chan error -} - -type FoundBlockSpan struct { - StartBlock uint - EndBlock uint - BlockSize int64 - MatchOffset int64 -} - -type MissingBlockSpan struct { - StartBlock uint - EndBlock uint - - BlockSize int64 - // a hasher to use to ensure that the block response matches - Hasher hash.Hash - // the hash values that the blocks should have - ExpectedSums [][]byte -} - -type BlockReponse struct { - StartBlock uint - Data []byte -} diff --git a/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go b/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go deleted file mode 100644 index b8a3018..0000000 --- a/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential.go +++ /dev/null @@ -1,145 +0,0 @@ -/* -Sequential Patcher will stream the patched version of the file to output, -since it works strictly in order, it cannot patch the local file directly -(since it might overwrite a block needed later), -so there would have to be a final copy once the patching was done. -*/ -package sequential - -import ( - "fmt" - "github.com/Redundancy/go-sync/patcher" - "io" -) - -const ( - ABSOLUTE_POSITION = 0 -) - -/* -This simple example currently doesn't do any pipelining of needed blocks, nor does it deal with -blocks being delivered out of order. -*/ -func SequentialPatcher( - localFile io.ReadSeeker, - reference patcher.BlockSource, - requiredRemoteBlocks []patcher.MissingBlockSpan, - locallyAvailableBlocks []patcher.FoundBlockSpan, - maxBlockStorage uint64, // the amount of memory we're allowed to use for temporary data storage - output io.Writer, -) error { - - maxBlockMissing := uint(0) - if len(requiredRemoteBlocks) > 0 { - maxBlockMissing = requiredRemoteBlocks[len(requiredRemoteBlocks)-1].EndBlock - } - - maxBlockFound := uint(0) - if len(locallyAvailableBlocks) > 0 { - maxBlockFound = locallyAvailableBlocks[len(locallyAvailableBlocks)-1].EndBlock - } - - if reference == nil { - return fmt.Errorf("No BlockSource set for obtaining reference blocks") - } - - maxBlock := maxBlockMissing - if maxBlockFound > maxBlock { - maxBlock = maxBlockFound - } - - currentBlock := uint(0) - - // TODO: find a way to test this, since it seemed to be the cause of an issue - for currentBlock <= maxBlock { - // where is the next block supposed to come from? - if withinFirstBlockOfLocalBlocks(currentBlock, locallyAvailableBlocks) { - firstMatched := locallyAvailableBlocks[0] - - // we have the current block range in the local file - localFile.Seek(firstMatched.MatchOffset, ABSOLUTE_POSITION) - blockSizeToRead := int64(firstMatched.EndBlock-firstMatched.StartBlock+1) * firstMatched.BlockSize - - if _, err := io.Copy(output, io.LimitReader(localFile, blockSizeToRead)); err != nil { - return fmt.Errorf("Could not copy %v bytes to output: %v", blockSizeToRead, err) - } - - currentBlock = firstMatched.EndBlock + 1 - locallyAvailableBlocks = locallyAvailableBlocks[1:] - - } else if withinFirstBlockOfRemoteBlocks(currentBlock, requiredRemoteBlocks) { - firstMissing := requiredRemoteBlocks[0] - reference.RequestBlocks(firstMissing) - - select { - case result := <-reference.GetResultChannel(): - if result.StartBlock == currentBlock { - if _, err := output.Write(result.Data); err != nil { - return fmt.Errorf( - "Could not write data to output: %v", - err, - ) - } else { - - completed := calculateNumberOfCompletedBlocks( - len(result.Data), - firstMissing.BlockSize, - ) - - if completed != (firstMissing.EndBlock-firstMissing.StartBlock) + 1 { - return fmt.Errorf( - "Unexpected reponse length from remote source: blocks %v-%v (got %v blocks)", - firstMissing.StartBlock, - firstMissing.EndBlock, - completed, - ) - } - - currentBlock += completed - requiredRemoteBlocks = requiredRemoteBlocks[1:] - } - } else { - return fmt.Errorf( - "Received unexpected block: %v", - result.StartBlock, - ) - } - case err := <-reference.EncounteredError(): - return fmt.Errorf( - "Failed to read from reference file: %v", - err, - ) - } - - } else { - return fmt.Errorf( - "Could not find block in missing or matched list: %v\nRemote: %v\nLocal: %v\n", - currentBlock, - requiredRemoteBlocks, - locallyAvailableBlocks, - ) - } - } - - return nil -} - -func withinFirstBlockOfRemoteBlocks(currentBlock uint, remoteBlocks []patcher.MissingBlockSpan) bool { - return len(remoteBlocks) > 0 && remoteBlocks[0].StartBlock <= currentBlock && remoteBlocks[0].EndBlock >= currentBlock -} - -func withinFirstBlockOfLocalBlocks(currentBlock uint, localBlocks []patcher.FoundBlockSpan) bool { - return len(localBlocks) > 0 && localBlocks[0].StartBlock <= currentBlock && localBlocks[0].EndBlock >= currentBlock -} - -func calculateNumberOfCompletedBlocks(resultLength int, blockSize int64) (completedBlockCount uint) { - // TODO: lots of casting to uint here, is it safe? - completedBlockCount = uint(resultLength) / uint(blockSize) - - // round up in the case of a partial block (last block may not be full sized) - if uint(resultLength)%uint(blockSize) != 0 { - completedBlockCount += 1 - } - - return -} diff --git a/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go b/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go deleted file mode 100644 index 7608f8b..0000000 --- a/vendor/github.com/Redundancy/go-sync/patcher/sequential/sequential_test.go +++ /dev/null @@ -1,188 +0,0 @@ -package sequential - -import ( - "bytes" - "crypto/md5" - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/patcher" - "io" - "io/ioutil" - "strings" - "testing" -) - -const ( - BLOCKSIZE = 4 - REFERENCE_STRING = "The quick brown fox jumped over the lazy dog" -) - -var ( - REFERENCE_BUFFER = bytes.NewBufferString(REFERENCE_STRING) - REFERENCE_BLOCKS []string - BLOCK_COUNT int - REFERENCE_HASHES [][]byte -) - -func init() { - maxLen := len(REFERENCE_STRING) - m := md5.New() - for i := 0; i < maxLen; i += BLOCKSIZE { - last := i + 4 - - if last >= maxLen { - last = maxLen - 1 - } - - block := REFERENCE_STRING[i:last] - - REFERENCE_BLOCKS = append(REFERENCE_BLOCKS, block) - m.Write([]byte(block)) - REFERENCE_HASHES = append(REFERENCE_HASHES, m.Sum(nil)) - m.Reset() - } - - BLOCK_COUNT = len(REFERENCE_BLOCKS) -} - -func stringToReadSeeker(input string) io.ReadSeeker { - return bytes.NewReader([]byte(input)) -} - -func TestPatchingStart(t *testing.T) { - LOCAL := bytes.NewReader([]byte("48 brown fox jumped over the lazy dog")) - out := bytes.NewBuffer(nil) - - missing := []patcher.MissingBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 0, - EndBlock: 2, - Hasher: md5.New(), - ExpectedSums: REFERENCE_HASHES[0:3], - }, - } - - matched := []patcher.FoundBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 3, - EndBlock: 11, - MatchOffset: 5, - }, - } - - err := SequentialPatcher( - LOCAL, - blocksources.NewReadSeekerBlockSource( - stringToReadSeeker(REFERENCE_STRING), - blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), - ), - missing, - matched, - 1024, - out, - ) - - if err != nil { - t.Fatal(err) - } - - if result, err := ioutil.ReadAll(out); err == nil { - t.Logf("String split is: \"%v\"", strings.Join(REFERENCE_BLOCKS, "\", \"")) - if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { - t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) - } - } else { - t.Fatal(err) - } - // -} - -func TestPatchingEnd(t *testing.T) { - LOCAL := bytes.NewReader([]byte("The quick brown fox jumped over the l4zy d0g")) - out := bytes.NewBuffer(nil) - - missing := []patcher.MissingBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 9, - EndBlock: 10, - Hasher: md5.New(), - ExpectedSums: REFERENCE_HASHES[0:3], - }, - } - - matched := []patcher.FoundBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 0, - EndBlock: 8, - MatchOffset: 0, - }, - } - - err := SequentialPatcher( - LOCAL, - blocksources.NewReadSeekerBlockSource( - stringToReadSeeker(REFERENCE_STRING), - blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), - ), - missing, - matched, - 1024, - out, - ) - - if err != nil { - t.Fatal(err) - } - - if result, err := ioutil.ReadAll(out); err == nil { - if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { - t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) - } - } else { - t.Fatal(err) - } -} - -func TestPatchingEntirelyMissing(t *testing.T) { - LOCAL := bytes.NewReader([]byte("")) - out := bytes.NewBuffer(nil) - - missing := []patcher.MissingBlockSpan{ - { - BlockSize: BLOCKSIZE, - StartBlock: 0, - EndBlock: 10, - Hasher: md5.New(), - ExpectedSums: REFERENCE_HASHES[0:10], - }, - } - - matched := []patcher.FoundBlockSpan{} - - err := SequentialPatcher( - LOCAL, - blocksources.NewReadSeekerBlockSource( - stringToReadSeeker(REFERENCE_STRING), - blocksources.MakeNullFixedSizeResolver(BLOCKSIZE), - ), - missing, - matched, - 1024, - out, - ) - - if err != nil { - t.Fatal(err) - } - - if result, err := ioutil.ReadAll(out); err == nil { - if bytes.Compare(result, []byte(REFERENCE_STRING)) != 0 { - t.Errorf("Result does not equal reference: \"%s\" vs \"%v\"", result, REFERENCE_STRING) - } - } else { - t.Fatal(err) - } -} diff --git a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go deleted file mode 100644 index 1fbe7b8..0000000 --- a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32.go +++ /dev/null @@ -1,85 +0,0 @@ -/* -rollsum provides an implementation of a rolling checksum - a checksum that's efficient to advance a byte -or more at a time. It is inspired by the rollsum in rsync, but differs in that the internal values used -are 32bit integers - to make a conformant implementation, a find a replace on "32" should be almost sufficient -(although it would be highly recommended to test against known values from the original implementation). - -Rollsum32 supports the hash.Hash implementation, but is not used much in go-sync, mostly in order to -share and access the underlying circular buffer storage, and use the implementation as efficiently as possible. -*/ -package rollsum - -import ( - "github.com/Redundancy/go-sync/circularbuffer" -) - -func NewRollsum32(blocksize uint) *Rollsum32 { - return &Rollsum32{ - Rollsum32Base: Rollsum32Base{ - blockSize: blocksize, - }, - buffer: circularbuffer.MakeC2Buffer(int(blocksize)), - } -} - -// Rollsum32 is a rolling checksum implemenation -// inspired by rsync, but with 32bit internal values -// Create one using NewRollsum32 -type Rollsum32 struct { - Rollsum32Base - buffer *circularbuffer.C2 -} - -// cannot be called concurrently -func (r *Rollsum32) Write(p []byte) (n int, err error) { - ulen_p := uint(len(p)) - - if ulen_p >= r.blockSize { - // if it's really long, we can just ignore a load of it - remaining := p[ulen_p-r.blockSize:] - r.buffer.Write(remaining) - r.Rollsum32Base.SetBlock(remaining) - } else { - b_len := r.buffer.Len() - r.buffer.Write(p) - evicted := r.buffer.Evicted() - r.Rollsum32Base.AddAndRemoveBytes(p, evicted, b_len) - } - - return len(p), nil -} - -// The most efficient byte length to call Write with -func (r *Rollsum32) BlockSize() int { - return int(r.blockSize) -} - -// the number of bytes -func (r *Rollsum32) Size() int { - return 4 -} - -func (r *Rollsum32) Reset() { - r.Rollsum32Base.Reset() - r.buffer.Reset() -} - -// Sum appends the current hash to b and returns the resulting slice. -// It does not change the underlying hash state. -// Note that this is to allow Sum() to reuse a preallocated buffer -func (r *Rollsum32) Sum(b []byte) []byte { - if b != nil && cap(b)-len(b) >= 4 { - p := len(b) - b = b[:len(b)+4] - r.Rollsum32Base.GetSum(b[p:]) - return b - } else { - result := []byte{0, 0, 0, 0} - r.Rollsum32Base.GetSum(result) - return append(b, result...) - } -} - -func (r *Rollsum32) GetLastBlock() []byte { - return r.buffer.GetBlock() -} diff --git a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go deleted file mode 100644 index 8153e72..0000000 --- a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_base.go +++ /dev/null @@ -1,86 +0,0 @@ -package rollsum - -import ( - "encoding/binary" -) - -const FULL_BYTES_16 = (1 << 16) - 1 - -// Rollsum32Base decouples the rollsum algorithm from the implementation of -// hash.Hash and the storage the rolling checksum window -// this allows us to write different versions of the storage for the distinctly different -// use-cases and optimize the storage with the usage pattern. -func NewRollsum32Base(blockSize uint) *Rollsum32Base { - return &Rollsum32Base{blockSize: blockSize} -} - -// The specification of hash.Hash is such that it cannot be implemented without implementing storage -// but the most optimal storage scheme depends on usage of the circular buffer & hash -type Rollsum32Base struct { - blockSize uint - a, b uint32 -} - -// Add a single byte into the rollsum -func (r *Rollsum32Base) AddByte(b byte) { - r.a += uint32(b) - r.b += r.a -} - -func (r *Rollsum32Base) AddBytes(bs []byte) { - for _, b := range bs { - r.a += uint32(b) - r.b += r.a - } -} - -// Remove a byte from the end of the rollsum -// Use the previous length (before removal) -func (r *Rollsum32Base) RemoveByte(b byte, length int) { - r.a -= uint32(b) - r.b -= uint32(uint(length) * uint(b)) -} - -func (r *Rollsum32Base) RemoveBytes(bs []byte, length int) { - for _, b := range bs { - r.a -= uint32(b) - r.b -= uint32(uint(length) * uint(b)) - length -= 1 - } -} - -func (r *Rollsum32Base) AddAndRemoveBytes(add []byte, remove []byte, length int) { - len_added := len(add) - len_removed := len(remove) - - startEvicted := len_added - len_removed - r.AddBytes(add[:startEvicted]) - length += startEvicted - - for i := startEvicted; i < len_added; i++ { - r.RemoveByte(remove[i-startEvicted], length) - r.AddByte(add[i]) - } -} - -// Set a whole block of blockSize -func (r *Rollsum32Base) SetBlock(block []byte) { - r.Reset() - r.AddBytes(block) -} - -// Reset the hash to the initial state -func (r *Rollsum32Base) Reset() { - r.a, r.b = 0, 0 -} - -// size of the hash in bytes -func (r *Rollsum32Base) Size() int { - return 4 -} - -// Puts the sum into b. Avoids allocation. b must have length >= 4 -func (r *Rollsum32Base) GetSum(b []byte) { - value := uint32((r.a & FULL_BYTES_16) + ((r.b & FULL_BYTES_16) << 16)) - binary.LittleEndian.PutUint32(b, value) -} diff --git a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go b/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go deleted file mode 100644 index cbf4c93..0000000 --- a/vendor/github.com/Redundancy/go-sync/rollsum/rollsum_32_test.go +++ /dev/null @@ -1,298 +0,0 @@ -package rollsum - -import ( - "bytes" - "github.com/Redundancy/go-sync/circularbuffer" - "hash" - "io" - "testing" -) - -func TestThatRollsum32SatisfiesHashInterface(t *testing.T) { - var i hash.Hash = NewRollsum32(10) - i.Reset() -} - -func TestThatRollsum32SatisfiedWriterInterface(t *testing.T) { - var i io.Writer = NewRollsum32(10) - n, err := i.Write([]byte{1, 2, 3, 4}) - - if n != 4 { - t.Error("Did not report writing 4 bytes") - } - - if err != nil { - t.Error(err) - } -} - -func TestThatRollsum32IsTheSameAfterBlockSizeBytes(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3, 4}) - - r2.Write([]byte{7, 6}) - r2.Write([]byte{5, 1, 2}) - r2.Write([]byte{3, 4}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums are different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThatRollsum32IsTheSameAfterBlockSizeBytesWithPartialEviction(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3, 4}) - - r2.Write([]byte{7, 5}) - r2.Write([]byte{1, 2, 3, 4}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums are different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestRegression2(t *testing.T) { - const A = "The quick br" - const B = "The qwik br" - - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte(A[:4])) - r1.Reset() - r1.Write([]byte(A[4:8])) - r1.Reset() - r1.Write([]byte(A[8:12])) - - r2.Write([]byte(B[:4])) - r2.Write([]byte(B[4:8])) - for _, c := range B[8:] { - r2.Write([]byte{byte(c)}) - } - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums are different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThatRollsum32RemovesBytesCorrectly(t *testing.T) { - r1 := NewRollsum32Base(2) - - r1.AddByte(255) - r1.AddByte(10) - r1.RemoveByte(255, 2) - r1.AddByte(0) - r1.RemoveByte(10, 2) - r1.AddByte(0) - - if r1.a != 0 || r1.b != 0 { - t.Errorf("Values are not reset: %v %v", r1.a, r1.b) - } -} - -func TestThatRollsum32IsDifferentForDifferentInput(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3, 4}) - r2.Write([]byte{7, 6, 5, 1}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) == 0 { - t.Errorf( - "Rollsums should be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestResettingTheRollsum32(t *testing.T) { - r1 := NewRollsum32(4) - r2 := NewRollsum32(4) - - r1.Write([]byte{1, 2, 3}) - - r2.Write([]byte{7, 6}) - r2.Reset() - r2.Write([]byte{1, 2, 3}) - - sum1 := r1.Sum(nil) - sum2 := r2.Sum(nil) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums should not be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestTruncatingPartiallyFilledBufferResultsInSameState(t *testing.T) { - r1 := NewRollsum32Base(4) - r2 := NewRollsum32Base(4) - - r1.AddByte(2) - sum1 := make([]byte, 4) - r1.GetSum(sum1) - - r2.AddByte(1) - r2.AddByte(2) - // Removal works from the left - r2.RemoveByte(1, 2) - sum2 := make([]byte, 4) - r2.GetSum(sum2) - - if bytes.Compare(sum1, sum2) != 0 { - t.Errorf( - "Rollsums should not be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThat32SumDoesNotChangeTheHashState(t *testing.T) { - r1 := NewRollsum32(4) - - sum1 := r1.Sum([]byte{1, 2, 3}) - sum2 := r1.Sum([]byte{3, 4, 5}) - - if bytes.Compare(sum1[3:], sum2[3:]) != 0 { - t.Errorf( - "Rollsums should not be different \"%v\" vs \"%v\"", - sum1, - sum2, - ) - } -} - -func TestThat32OutputLengthMatchesSize(t *testing.T) { - r1 := NewRollsum32(4) - sumLength := len(r1.Sum(nil)) - - if sumLength != r1.Size() { - t.Errorf("Unexpected length: %v vs expected %v", sumLength, r1.Size()) - } -} - -func BenchmarkRollsum32(b *testing.B) { - r := NewRollsum32(100) - buffer := make([]byte, 100) - b.ReportAllocs() - b.SetBytes(int64(len(buffer))) - checksum := make([]byte, 16) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.Write(buffer) - r.Sum(checksum) - checksum = checksum[:0] - } - b.StopTimer() -} - -func BenchmarkRollsum32_8096(b *testing.B) { - r := NewRollsum32(8096) - buffer := make([]byte, 8096) - b.ReportAllocs() - b.SetBytes(int64(len(buffer))) - checksum := make([]byte, 16) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.Write(buffer) - r.Sum(checksum) - checksum = checksum[:0] - } - b.StopTimer() -} - -func BenchmarkRollsum32Base(b *testing.B) { - r := Rollsum32Base{blockSize: 100} - buffer := make([]byte, 100) - checksum := make([]byte, 16) - b.ReportAllocs() - b.SetBytes(int64(len(buffer))) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.SetBlock(buffer) - r.GetSum(checksum) - } - b.StopTimer() - -} - -// This is the benchmark where Rollsum should beat a full MD5 for each blocksize -func BenchmarkIncrementalRollsum32(b *testing.B) { - r := NewRollsum32(100) - buffer := make([]byte, 100) - r.Write(buffer) - b.SetBytes(1) - - b.ReportAllocs() - checksum := make([]byte, 16) - increment := make([]byte, 1) - - b.StartTimer() - for i := 0; i < b.N; i++ { - r.Write(increment) - r.Sum(checksum) - checksum = checksum[:0] - } - b.StopTimer() -} - -// The C2 veersion should avoid all allocations in the main loop, and beat the pants off the -// other versions -func BenchmarkIncrementalRollsum32WithC2(b *testing.B) { - const BLOCK_SIZE = 100 - r := NewRollsum32Base(BLOCK_SIZE) - buffer := make([]byte, BLOCK_SIZE) - b.SetBytes(1) - cbuffer := circularbuffer.MakeC2Buffer(BLOCK_SIZE) - - r.AddBytes(buffer) - cbuffer.Write(buffer) - - b.ReportAllocs() - checksum := make([]byte, 16) - increment := make([]byte, 1) - - b.StartTimer() - for i := 0; i < b.N; i++ { - cbuffer.Write(increment) - r.AddAndRemoveBytes(increment, cbuffer.Evicted(), BLOCK_SIZE) - r.GetSum(checksum) - } - b.StopTimer() -} diff --git a/vendor/github.com/Redundancy/go-sync/rsync.go b/vendor/github.com/Redundancy/go-sync/rsync.go deleted file mode 100644 index 59db7c1..0000000 --- a/vendor/github.com/Redundancy/go-sync/rsync.go +++ /dev/null @@ -1,347 +0,0 @@ -package gosync - -import ( - "bufio" - "crypto/md5" - "fmt" - "io" - "io/ioutil" - "os" - "runtime" - - "github.com/Redundancy/go-sync/blocksources" - "github.com/Redundancy/go-sync/chunks" - "github.com/Redundancy/go-sync/comparer" - "github.com/Redundancy/go-sync/filechecksum" - "github.com/Redundancy/go-sync/index" - "github.com/Redundancy/go-sync/patcher" - "github.com/Redundancy/go-sync/patcher/sequential" -) - -const ( - megabyte = 1000000 -) - -var ( - // DefaultConcurrency is the default concurrency level used by patching and downloading - DefaultConcurrency = runtime.NumCPU() -) - -// ReadSeekerAt is the combinaton of ReadSeeker and ReaderAt interfaces -type ReadSeekerAt interface { - io.ReadSeeker - io.ReaderAt -} - -/* -RSync is an object designed to make the standard use-case for gosync as -easy as possible. - -To this end, it hides away many low level choices by default, and makes some -assumptions. -*/ -type RSync struct { - Input ReadSeekerAt - Source patcher.BlockSource - Output io.Writer - - Summary FileSummary - - OnClose []closer -} - -type closer interface { - Close() error -} - -// FileSummary combines many of the interfaces that are needed -// It is expected that you might implement it by embedding existing structs -type FileSummary interface { - GetBlockSize() uint - GetBlockCount() uint - GetFileSize() int64 - FindWeakChecksum2(bytes []byte) interface{} - FindStrongChecksum2(bytes []byte, weak interface{}) []chunks.ChunkChecksum - GetStrongChecksumForBlock(blockID int) []byte -} - -// BasicSummary implements a version of the FileSummary interface -type BasicSummary struct { - BlockSize uint - BlockCount uint - FileSize int64 - *index.ChecksumIndex - filechecksum.ChecksumLookup -} - -// GetBlockSize gets the size of each block -func (fs *BasicSummary) GetBlockSize() uint { - return fs.BlockSize -} - -// GetBlockCount gets the number of blocks -func (fs *BasicSummary) GetBlockCount() uint { - return fs.BlockCount -} - -// GetFileSize gets the file size of the file -func (fs *BasicSummary) GetFileSize() int64 { - return fs.FileSize -} - -// MakeRSync creates an RSync object using string paths, -// inferring most of the configuration -func MakeRSync( - InputFile, - Source, - OutFile string, - Summary FileSummary, -) (r *RSync, err error) { - useTempFile := false - if useTempFile, err = IsSameFile(InputFile, OutFile); err != nil { - return nil, err - } - - inputFile, err := os.Open(InputFile) - - if err != nil { - return - } - - var out io.WriteCloser - var outFilename = OutFile - var copier closer - - if useTempFile { - out, outFilename, err = getTempFile() - - if err != nil { - return - } - - copier = &fileCopyCloser{ - from: outFilename, - to: OutFile, - } - } else { - out, err = getOutFile(OutFile) - - if err != nil { - return - } - - copier = nullCloser{} - } - - // blocksource - var source *blocksources.BlockSourceBase - - resolver := blocksources.MakeFileSizedBlockResolver( - uint64(Summary.GetBlockSize()), - Summary.GetFileSize(), - ) - - source = blocksources.NewHttpBlockSource( - Source, - DefaultConcurrency, - resolver, - &filechecksum.HashVerifier{ - Hash: md5.New(), - BlockSize: Summary.GetBlockSize(), - BlockChecksumGetter: Summary, - }, - ) - - r = &RSync{ - Input: inputFile, - Output: out, - Source: source, - Summary: Summary, - OnClose: []closer{ - &fileCloser{inputFile, InputFile}, - &fileCloser{out, outFilename}, - copier, - }, - } - - return -} - -// Patch the files -func (rsync *RSync) Patch() (err error) { - numMatchers := int64(DefaultConcurrency) - blockSize := rsync.Summary.GetBlockSize() - sectionSize := rsync.Summary.GetFileSize() / numMatchers - sectionSize += int64(blockSize) - (sectionSize % int64(blockSize)) - - merger := &comparer.MatchMerger{} - - for i := int64(0); i < numMatchers; i++ { - compare := &comparer.Comparer{} - offset := sectionSize * i - - sectionReader := bufio.NewReaderSize( - io.NewSectionReader(rsync.Input, offset, sectionSize+int64(blockSize)), - megabyte, // 1 MB buffer - ) - - // Bakes in the assumption about how to generate checksums (extract) - sectionGenerator := filechecksum.NewFileChecksumGenerator( - uint(blockSize), - ) - - matchStream := compare.StartFindMatchingBlocks( - sectionReader, offset, sectionGenerator, rsync.Summary, - ) - - merger.StartMergeResultStream(matchStream, int64(blockSize)) - } - - mergedBlocks := merger.GetMergedBlocks() - missing := mergedBlocks.GetMissingBlocks(rsync.Summary.GetBlockCount() - 1) - - return sequential.SequentialPatcher( - rsync.Input, - rsync.Source, - toPatcherMissingSpan(missing, int64(blockSize)), - toPatcherFoundSpan(mergedBlocks, int64(blockSize)), - 20*megabyte, - rsync.Output, - ) -} - -func getOutFile(filename string) (f io.WriteCloser, err error) { - if _, err = os.Stat(filename); os.IsNotExist(err) { - return os.Create(filename) - } - - return os.OpenFile(filename, os.O_WRONLY, 0) -} - -func getTempFile() (f io.WriteCloser, filename string, err error) { - ft, err := ioutil.TempFile(".", "tmp_") - filename = ft.Name() - f = ft - return -} - -// IsSameFile checks if two file paths are the same file -func IsSameFile(path1, path2 string) (same bool, err error) { - - fi1, err := os.Stat(path1) - - switch { - case os.IsNotExist(err): - return false, nil - case err != nil: - return - } - - fi2, err := os.Stat(path2) - - switch { - case os.IsNotExist(err): - return false, nil - case err != nil: - return - } - - return os.SameFile(fi1, fi2), nil -} - -// Close - close open files, copy to the final location from -// a temporary one if neede -func (rsync *RSync) Close() error { - for _, f := range rsync.OnClose { - if err := f.Close(); err != nil { - return err - } - } - return nil -} - -type fileCloser struct { - f io.Closer - path string -} - -// Close - add file path information to closing a file -func (f *fileCloser) Close() error { - err := f.f.Close() - if err != nil { - return fmt.Errorf( - "Could not close file %v: %v", - f.path, - err, - ) - } - return nil -} - -type nullCloser struct{} - -func (n nullCloser) Close() error { - return nil -} - -type fileCopyCloser struct { - from string - to string -} - -func (f *fileCopyCloser) Close() (err error) { - from, err := os.OpenFile(f.from, os.O_RDONLY, 0) - - if err != nil { - return err - } - - defer func() { - e := from.Close() - if err != nil { - err = e - } - }() - - to, err := os.OpenFile(f.to, os.O_TRUNC|os.O_WRONLY, 0) - - if err != nil { - return err - } - - defer func() { - e := to.Close() - if err != nil { - err = e - } - }() - - bufferedReader := bufio.NewReaderSize(from, megabyte) - _, err = io.Copy(to, bufferedReader) - return -} - -func toPatcherFoundSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.FoundBlockSpan { - result := make([]patcher.FoundBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].MatchOffset = v.ComparisonStartOffset - result[i].BlockSize = blockSize - } - - return result -} - -func toPatcherMissingSpan(sl comparer.BlockSpanList, blockSize int64) []patcher.MissingBlockSpan { - result := make([]patcher.MissingBlockSpan, len(sl)) - - for i, v := range sl { - result[i].StartBlock = v.StartBlock - result[i].EndBlock = v.EndBlock - result[i].BlockSize = blockSize - } - - return result -} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go b/vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go deleted file mode 100644 index 5da00af..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/injectedreader.go +++ /dev/null @@ -1,18 +0,0 @@ -package readers - -import ( - "io" -) - -// Injects the second reader into the first at an offset -func InjectedReader( - offsetFromStart int64, - base io.Reader, - inject io.Reader, -) io.Reader { - return io.MultiReader( - io.LimitReader(base, offsetFromStart), - inject, - base, - ) -} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go b/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go deleted file mode 100644 index f1edc0d..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating.go +++ /dev/null @@ -1,35 +0,0 @@ -package readers - -import ( - "encoding/binary" - "io" -) - -const nonRepeatingModulo = 87178291199 -const nonRepeatingIncrement = 17180131327 - -// *should* produce a non-repeating sequence of bytes in a deterministic fashion -// use io.LimitReader to limit it to a specific length -type nonRepeatingSequenceReader struct { - value int -} - -func NewNonRepeatingSequence(i int) io.Reader { - return &nonRepeatingSequenceReader{i} -} - -func NewSizedNonRepeatingSequence(i int, s int64) io.Reader { - return io.LimitReader(NewNonRepeatingSequence(i), s) -} - -func (r *nonRepeatingSequenceReader) Read(p []byte) (n int, err error) { - lenp := len(p) - b := []byte{1, 2, 3, 4} - - for i := 0; i < lenp; i++ { - binary.LittleEndian.PutUint32(b, uint32(r.value)) - p[i] = b[0] - r.value = (r.value + nonRepeatingIncrement) % nonRepeatingModulo - } - return lenp, nil -} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go b/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go deleted file mode 100644 index 3c65b1c..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/nonrepeating_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package readers - -import ( - "io" - "io/ioutil" - "testing" -) - -// This is only a very basic test -func TestNonRepeatingSequenceReader(t *testing.T) { - i := NewNonRepeatingSequence(0) - a := []byte{0} - b := []byte{0} - - i.Read(a) - i.Read(b) - - if a[0] == b[0] { - t.Fatalf("Bytes should not be the same! %s vs %s", a, b) - } -} - -func TestNonRepeatingSequenceIsDifferent(t *testing.T) { - i := NewNonRepeatingSequence(0) - i2 := NewNonRepeatingSequence(5) - - a := []byte{0} - b := []byte{0} - - commonalities := 0 - - for x := 0; x < 100; x++ { - i.Read(a) - i2.Read(b) - - if a[0] == b[0] { - commonalities += 1 - } - } - - if commonalities > 5 { - t.Fatal("Sequences are too similar") - } -} - -func BenchmarkNonRepeatingSequence(b *testing.B) { - b.SetBytes(1) - - s := NewSizedNonRepeatingSequence(0, int64(b.N)) - - b.StartTimer() - _, err := io.Copy(ioutil.Discard, s) - b.StopTimer() - - if err != nil { - b.Fatal(err) - } -} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/readers.go b/vendor/github.com/Redundancy/go-sync/util/readers/readers.go deleted file mode 100644 index c9db457..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/readers.go +++ /dev/null @@ -1,6 +0,0 @@ -/* -util/readers exists to provide convenient and composable io.Reader compatible streams to allow testing -without having to check in large binary files. - -*/ -package readers diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go b/vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go deleted file mode 100644 index 523084e..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/sequence_test.go +++ /dev/null @@ -1 +0,0 @@ -package readers diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go b/vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go deleted file mode 100644 index 96ba1bd..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/sequencelimit.go +++ /dev/null @@ -1,12 +0,0 @@ -package readers - -import ( - "io" -) - -// read from 'readers' in sequence up to a limit of 'size' -func SequenceLimit(size int64, readers ...io.Reader) io.Reader { - return io.LimitReader( - io.MultiReader(readers...), - size) -} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go b/vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go deleted file mode 100644 index e24e69b..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/uniform_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package readers - -import ( - "io" - "io/ioutil" - "testing" -) - -func TestUniformReaderLength(t *testing.T) { - r, err := ioutil.ReadAll(OneReader(100)) - - if err != nil { - t.Fatal(err) - } - - if len(r) != 100 { - t.Errorf("Unexpected length: %v", len(r)) - } - - for i, b := range r { - if b != 1 { - t.Errorf("Byte at position %v is not 1: %v", i, b) - } - } -} - -func TestReadIntoLargerBuffer(t *testing.T) { - b := make([]byte, 100) - r := OneReader(10) - - n, err := r.Read(b) - - if n != 10 { - t.Errorf("Wrong read length: %v", n) - } - - if err != io.EOF { - t.Errorf("Did not raise EOF after reading: %v", err) - } -} - -func TestMultiUniformReader(t *testing.T) { - r := io.MultiReader( - OneReader(12), - NewSizedNonRepeatingSequence(0, 88), - ) - - b := make([]byte, 100) - - n, err := r.Read(b) - - if n != 12 { - t.Errorf("Wrong read length: %v", n) - } - - if err == io.EOF { - t.Errorf("Raised EOF after reading! %v", err) - } - - n, err = r.Read(b) - - if n != 88 { - t.Errorf("Wrong read length: %v", n) - } - - n, err = r.Read(b) - - if err != io.EOF { - t.Errorf("Really expected EOF by now: %v %v", err, n) - } -} - -func TestFillBuffer(t *testing.T) { - r := io.MultiReader( - OneReader(12), - NewSizedNonRepeatingSequence(0, 88), - ) - - b := make([]byte, 100) - _, err := io.ReadFull(r, b) - - if err != nil && err != io.EOF { - t.Error(err) - } - - if len(b) != cap(b) { - t.Errorf("Expected to fill b: %v", len(b)) - } - -} diff --git a/vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go b/vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go deleted file mode 100644 index e6045da..0000000 --- a/vendor/github.com/Redundancy/go-sync/util/readers/uniformreader.go +++ /dev/null @@ -1,55 +0,0 @@ -package readers - -import ( - "io" -) - -// Reads a continuous stream of bytes with the same value, up to length -type uniformReader struct { - value byte - length int - read int -} - -func (r *uniformReader) Read(p []byte) (n int, err error) { - destinationLength := len(p) - readable := r.length - r.read - read := destinationLength - - if readable < destinationLength { - read = readable - } - - if read == 0 { - return 0, io.EOF - } - - for i := 0; i < read; i++ { - p[i] = r.value - } - - var result error = nil - if read == readable { - result = io.EOF - } - - r.read += read - - return read, result -} - -func ZeroReader(length int) io.Reader { - return &uniformReader{ - value: 0, - length: length, - read: 0, - } -} - -func OneReader(length int) io.Reader { - return &uniformReader{ - value: 1, - length: length, - read: 0, - } -} diff --git a/vendor/github.com/codegangsta/cli/LICENSE b/vendor/github.com/codegangsta/cli/LICENSE deleted file mode 100644 index 5515ccf..0000000 --- a/vendor/github.com/codegangsta/cli/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -Copyright (C) 2013 Jeremy Saenz -All Rights Reserved. - -MIT LICENSE - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/codegangsta/cli/README.md b/vendor/github.com/codegangsta/cli/README.md deleted file mode 100644 index 364c964..0000000 --- a/vendor/github.com/codegangsta/cli/README.md +++ /dev/null @@ -1,394 +0,0 @@ -[![Coverage](http://gocover.io/_badge/github.com/codegangsta/cli?0)](http://gocover.io/github.com/codegangsta/cli) -[![Build Status](https://travis-ci.org/codegangsta/cli.svg?branch=master)](https://travis-ci.org/codegangsta/cli) -[![GoDoc](https://godoc.org/github.com/codegangsta/cli?status.svg)](https://godoc.org/github.com/codegangsta/cli) - -# cli.go - -`cli.go` is simple, fast, and fun package for building command line apps in Go. The goal is to enable developers to write fast and distributable command line applications in an expressive way. - -## Overview - -Command line apps are usually so tiny that there is absolutely no reason why your code should *not* be self-documenting. Things like generating help text and parsing command flags/options should not hinder productivity when writing a command line app. - -**This is where `cli.go` comes into play.** `cli.go` makes command line programming fun, organized, and expressive! - -## Installation - -Make sure you have a working Go environment (go 1.1+ is *required*). [See the install instructions](http://golang.org/doc/install.html). - -To install `cli.go`, simply run: -``` -$ go get github.com/codegangsta/cli -``` - -Make sure your `PATH` includes to the `$GOPATH/bin` directory so your commands can be easily used: -``` -export PATH=$PATH:$GOPATH/bin -``` - -## Getting Started - -One of the philosophies behind `cli.go` is that an API should be playful and full of discovery. So a `cli.go` app can be as little as one line of code in `main()`. - -``` go -package main - -import ( - "os" - "github.com/codegangsta/cli" -) - -func main() { - cli.NewApp().Run(os.Args) -} -``` - -This app will run and show help text, but is not very useful. Let's give an action to execute and some help documentation: - -``` go -package main - -import ( - "os" - "github.com/codegangsta/cli" -) - -func main() { - app := cli.NewApp() - app.Name = "boom" - app.Usage = "make an explosive entrance" - app.Action = func(c *cli.Context) { - println("boom! I say!") - } - - app.Run(os.Args) -} -``` - -Running this already gives you a ton of functionality, plus support for things like subcommands and flags, which are covered below. - -## Example - -Being a programmer can be a lonely job. Thankfully by the power of automation that is not the case! Let's create a greeter app to fend off our demons of loneliness! - -Start by creating a directory named `greet`, and within it, add a file, `greet.go` with the following code in it: - -``` go -package main - -import ( - "os" - "github.com/codegangsta/cli" -) - -func main() { - app := cli.NewApp() - app.Name = "greet" - app.Usage = "fight the loneliness!" - app.Action = func(c *cli.Context) { - println("Hello friend!") - } - - app.Run(os.Args) -} -``` - -Install our command to the `$GOPATH/bin` directory: - -``` -$ go install -``` - -Finally run our new command: - -``` -$ greet -Hello friend! -``` - -`cli.go` also generates neat help text: - -``` -$ greet help -NAME: - greet - fight the loneliness! - -USAGE: - greet [global options] command [command options] [arguments...] - -VERSION: - 0.0.0 - -COMMANDS: - help, h Shows a list of commands or help for one command - -GLOBAL OPTIONS - --version Shows version information -``` - -### Arguments - -You can lookup arguments by calling the `Args` function on `cli.Context`. - -``` go -... -app.Action = func(c *cli.Context) { - println("Hello", c.Args()[0]) -} -... -``` - -### Flags - -Setting and querying flags is simple. - -``` go -... -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang", - Value: "english", - Usage: "language for the greeting", - }, -} -app.Action = func(c *cli.Context) { - name := "someone" - if c.NArg() > 0 { - name = c.Args()[0] - } - if c.String("lang") == "spanish" { - println("Hola", name) - } else { - println("Hello", name) - } -} -... -``` - -You can also set a destination variable for a flag, to which the content will be scanned. - -``` go -... -var language string -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang", - Value: "english", - Usage: "language for the greeting", - Destination: &language, - }, -} -app.Action = func(c *cli.Context) { - name := "someone" - if c.NArg() > 0 { - name = c.Args()[0] - } - if language == "spanish" { - println("Hola", name) - } else { - println("Hello", name) - } -} -... -``` - -See full list of flags at http://godoc.org/github.com/codegangsta/cli - -#### Alternate Names - -You can set alternate (or short) names for flags by providing a comma-delimited list for the `Name`. e.g. - -``` go -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang, l", - Value: "english", - Usage: "language for the greeting", - }, -} -``` - -That flag can then be set with `--lang spanish` or `-l spanish`. Note that giving two different forms of the same flag in the same command invocation is an error. - -#### Values from the Environment - -You can also have the default value set from the environment via `EnvVar`. e.g. - -``` go -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang, l", - Value: "english", - Usage: "language for the greeting", - EnvVar: "APP_LANG", - }, -} -``` - -The `EnvVar` may also be given as a comma-delimited "cascade", where the first environment variable that resolves is used as the default. - -``` go -app.Flags = []cli.Flag { - cli.StringFlag{ - Name: "lang, l", - Value: "english", - Usage: "language for the greeting", - EnvVar: "LEGACY_COMPAT_LANG,APP_LANG,LANG", - }, -} -``` - -#### Values from alternate input sources (YAML and others) - -There is a separate package altsrc that adds support for getting flag values from other input sources like YAML. - -In order to get values for a flag from an alternate input source the following code would be added to wrap an existing cli.Flag like below: - -``` go - altsrc.NewIntFlag(cli.IntFlag{Name: "test"}) -``` - -Initialization must also occur for these flags. Below is an example initializing getting data from a yaml file below. - -``` go - command.Before = altsrc.InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) -``` - -The code above will use the "load" string as a flag name to get the file name of a yaml file from the cli.Context. -It will then use that file name to initialize the yaml input source for any flags that are defined on that command. -As a note the "load" flag used would also have to be defined on the command flags in order for this code snipped to work. - -Currently only YAML files are supported but developers can add support for other input sources by implementing the -altsrc.InputSourceContext for their given sources. - -Here is a more complete sample of a command using YAML support: - -``` go - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - // Action to run - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - err := command.Run(c) -``` - -### Subcommands - -Subcommands can be defined for a more git-like command line app. - -```go -... -app.Commands = []cli.Command{ - { - Name: "add", - Aliases: []string{"a"}, - Usage: "add a task to the list", - Action: func(c *cli.Context) { - println("added task: ", c.Args().First()) - }, - }, - { - Name: "complete", - Aliases: []string{"c"}, - Usage: "complete a task on the list", - Action: func(c *cli.Context) { - println("completed task: ", c.Args().First()) - }, - }, - { - Name: "template", - Aliases: []string{"r"}, - Usage: "options for task templates", - Subcommands: []cli.Command{ - { - Name: "add", - Usage: "add a new template", - Action: func(c *cli.Context) { - println("new task template: ", c.Args().First()) - }, - }, - { - Name: "remove", - Usage: "remove an existing template", - Action: func(c *cli.Context) { - println("removed task template: ", c.Args().First()) - }, - }, - }, - }, -} -... -``` - -### Bash Completion - -You can enable completion commands by setting the `EnableBashCompletion` -flag on the `App` object. By default, this setting will only auto-complete to -show an app's subcommands, but you can write your own completion methods for -the App or its subcommands. - -```go -... -var tasks = []string{"cook", "clean", "laundry", "eat", "sleep", "code"} -app := cli.NewApp() -app.EnableBashCompletion = true -app.Commands = []cli.Command{ - { - Name: "complete", - Aliases: []string{"c"}, - Usage: "complete a task on the list", - Action: func(c *cli.Context) { - println("completed task: ", c.Args().First()) - }, - BashComplete: func(c *cli.Context) { - // This will complete if no args are passed - if c.NArg() > 0 { - return - } - for _, t := range tasks { - fmt.Println(t) - } - }, - } -} -... -``` - -#### To Enable - -Source the `autocomplete/bash_autocomplete` file in your `.bashrc` file while -setting the `PROG` variable to the name of your program: - -`PROG=myprogram source /.../cli/autocomplete/bash_autocomplete` - -#### To Distribute - -Copy `autocomplete/bash_autocomplete` into `/etc/bash_completion.d/` and rename -it to the name of the program you wish to add autocomplete support for (or -automatically install it there if you are distributing a package). Don't forget -to source the file to make it active in the current shell. - -``` -sudo cp src/bash_autocomplete /etc/bash_completion.d/ -source /etc/bash_completion.d/ -``` - -Alternatively, you can just document that users should source the generic -`autocomplete/bash_autocomplete` in their bash configuration with `$PROG` set -to the name of their program (as above). - -## Contribution Guidelines - -Feel free to put up a pull request to fix a bug or maybe add a feature. I will give it a code review and make sure that it does not break backwards compatibility. If I or any other collaborators agree that it is in line with the vision of the project, we will work with you to get the code into a mergeable state and merge it into the master branch. - -If you have contributed something significant to the project, I will most likely add you as a collaborator. As a collaborator you are given the ability to merge others pull requests. It is very important that new code does not break existing code, so be careful about what code you do choose to merge. If you have any questions feel free to link @codegangsta to the issue in question and we can review it together. - -If you feel like you have contributed to the project but have not yet been added as a collaborator, I probably forgot to add you. Hit @codegangsta up over email and we will get it figured out. diff --git a/vendor/github.com/codegangsta/cli/altsrc/flag.go b/vendor/github.com/codegangsta/cli/altsrc/flag.go deleted file mode 100644 index f13ffb4..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/flag.go +++ /dev/null @@ -1,439 +0,0 @@ -package altsrc - -import ( - "flag" - "fmt" - "os" - "strconv" - "strings" - - "github.com/codegangsta/cli" -) - -// FlagInputSourceExtension is an extension interface of cli.Flag that -// allows a value to be set on the existing parsed flags. -type FlagInputSourceExtension interface { - cli.Flag - ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error -} - -// ApplyInputSourceValues iterates over all provided flags and -// executes ApplyInputSourceValue on flags implementing the -// FlagInputSourceExtension interface to initialize these flags -// to an alternate input source. -func ApplyInputSourceValues(context *cli.Context, inputSourceContext InputSourceContext, flags []cli.Flag) error { - for _, f := range flags { - inputSourceExtendedFlag, isType := f.(FlagInputSourceExtension) - if isType { - err := inputSourceExtendedFlag.ApplyInputSourceValue(context, inputSourceContext) - if err != nil { - return err - } - } - } - - return nil -} - -// InitInputSource is used to to setup an InputSourceContext on a cli.Command Before method. It will create a new -// input source based on the func provided. If there is no error it will then apply the new input source to any flags -// that are supported by the input source -func InitInputSource(flags []cli.Flag, createInputSource func() (InputSourceContext, error)) func(context *cli.Context) error { - return func(context *cli.Context) error { - inputSource, err := createInputSource() - if err != nil { - return fmt.Errorf("Unable to create input source: inner error: \n'%v'", err.Error()) - } - - return ApplyInputSourceValues(context, inputSource, flags) - } -} - -// InitInputSourceWithContext is used to to setup an InputSourceContext on a cli.Command Before method. It will create a new -// input source based on the func provided with potentially using existing cli.Context values to initialize itself. If there is -// no error it will then apply the new input source to any flags that are supported by the input source -func InitInputSourceWithContext(flags []cli.Flag, createInputSource func(context *cli.Context) (InputSourceContext, error)) func(context *cli.Context) error { - return func(context *cli.Context) error { - inputSource, err := createInputSource(context) - if err != nil { - return fmt.Errorf("Unable to create input source with context: inner error: \n'%v'", err.Error()) - } - - return ApplyInputSourceValues(context, inputSource, flags) - } -} - -// GenericFlag is the flag type that wraps cli.GenericFlag to allow -// for other values to be specified -type GenericFlag struct { - cli.GenericFlag - set *flag.FlagSet -} - -// NewGenericFlag creates a new GenericFlag -func NewGenericFlag(flag cli.GenericFlag) *GenericFlag { - return &GenericFlag{GenericFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a generic value to the flagSet if required -func (f *GenericFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.Generic(f.GenericFlag.Name) - if err != nil { - return err - } - if value != nil { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, value.String()) - }) - } - } - } - - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped GenericFlag.Apply -func (f *GenericFlag) Apply(set *flag.FlagSet) { - f.set = set - f.GenericFlag.Apply(set) -} - -// StringSliceFlag is the flag type that wraps cli.StringSliceFlag to allow -// for other values to be specified -type StringSliceFlag struct { - cli.StringSliceFlag - set *flag.FlagSet -} - -// NewStringSliceFlag creates a new StringSliceFlag -func NewStringSliceFlag(flag cli.StringSliceFlag) *StringSliceFlag { - return &StringSliceFlag{StringSliceFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a StringSlice value to the flagSet if required -func (f *StringSliceFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.StringSlice(f.StringSliceFlag.Name) - if err != nil { - return err - } - if value != nil { - var sliceValue cli.StringSlice = value - eachName(f.Name, func(name string) { - underlyingFlag := f.set.Lookup(f.Name) - if underlyingFlag != nil { - underlyingFlag.Value = &sliceValue - } - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped StringSliceFlag.Apply -func (f *StringSliceFlag) Apply(set *flag.FlagSet) { - f.set = set - f.StringSliceFlag.Apply(set) -} - -// IntSliceFlag is the flag type that wraps cli.IntSliceFlag to allow -// for other values to be specified -type IntSliceFlag struct { - cli.IntSliceFlag - set *flag.FlagSet -} - -// NewIntSliceFlag creates a new IntSliceFlag -func NewIntSliceFlag(flag cli.IntSliceFlag) *IntSliceFlag { - return &IntSliceFlag{IntSliceFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a IntSlice value if required -func (f *IntSliceFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.IntSlice(f.IntSliceFlag.Name) - if err != nil { - return err - } - if value != nil { - var sliceValue cli.IntSlice = value - eachName(f.Name, func(name string) { - underlyingFlag := f.set.Lookup(f.Name) - if underlyingFlag != nil { - underlyingFlag.Value = &sliceValue - } - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped IntSliceFlag.Apply -func (f *IntSliceFlag) Apply(set *flag.FlagSet) { - f.set = set - f.IntSliceFlag.Apply(set) -} - -// BoolFlag is the flag type that wraps cli.BoolFlag to allow -// for other values to be specified -type BoolFlag struct { - cli.BoolFlag - set *flag.FlagSet -} - -// NewBoolFlag creates a new BoolFlag -func NewBoolFlag(flag cli.BoolFlag) *BoolFlag { - return &BoolFlag{BoolFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a Bool value to the flagSet if required -func (f *BoolFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.Bool(f.BoolFlag.Name) - if err != nil { - return err - } - if value { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, strconv.FormatBool(value)) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped BoolFlag.Apply -func (f *BoolFlag) Apply(set *flag.FlagSet) { - f.set = set - f.BoolFlag.Apply(set) -} - -// BoolTFlag is the flag type that wraps cli.BoolTFlag to allow -// for other values to be specified -type BoolTFlag struct { - cli.BoolTFlag - set *flag.FlagSet -} - -// NewBoolTFlag creates a new BoolTFlag -func NewBoolTFlag(flag cli.BoolTFlag) *BoolTFlag { - return &BoolTFlag{BoolTFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a BoolT value to the flagSet if required -func (f *BoolTFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !context.IsSet(f.Name) && !isEnvVarSet(f.EnvVar) { - value, err := isc.BoolT(f.BoolTFlag.Name) - if err != nil { - return err - } - if !value { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, strconv.FormatBool(value)) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped BoolTFlag.Apply -func (f *BoolTFlag) Apply(set *flag.FlagSet) { - f.set = set - - f.BoolTFlag.Apply(set) -} - -// StringFlag is the flag type that wraps cli.StringFlag to allow -// for other values to be specified -type StringFlag struct { - cli.StringFlag - set *flag.FlagSet -} - -// NewStringFlag creates a new StringFlag -func NewStringFlag(flag cli.StringFlag) *StringFlag { - return &StringFlag{StringFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a String value to the flagSet if required -func (f *StringFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.String(f.StringFlag.Name) - if err != nil { - return err - } - if value != "" { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, value) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped StringFlag.Apply -func (f *StringFlag) Apply(set *flag.FlagSet) { - f.set = set - - f.StringFlag.Apply(set) -} - -// IntFlag is the flag type that wraps cli.IntFlag to allow -// for other values to be specified -type IntFlag struct { - cli.IntFlag - set *flag.FlagSet -} - -// NewIntFlag creates a new IntFlag -func NewIntFlag(flag cli.IntFlag) *IntFlag { - return &IntFlag{IntFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a int value to the flagSet if required -func (f *IntFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.Int(f.IntFlag.Name) - if err != nil { - return err - } - if value > 0 { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, strconv.FormatInt(int64(value), 10)) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped IntFlag.Apply -func (f *IntFlag) Apply(set *flag.FlagSet) { - f.set = set - f.IntFlag.Apply(set) -} - -// DurationFlag is the flag type that wraps cli.DurationFlag to allow -// for other values to be specified -type DurationFlag struct { - cli.DurationFlag - set *flag.FlagSet -} - -// NewDurationFlag creates a new DurationFlag -func NewDurationFlag(flag cli.DurationFlag) *DurationFlag { - return &DurationFlag{DurationFlag: flag, set: nil} -} - -// ApplyInputSourceValue applies a Duration value to the flagSet if required -func (f *DurationFlag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.Duration(f.DurationFlag.Name) - if err != nil { - return err - } - if value > 0 { - eachName(f.Name, func(name string) { - f.set.Set(f.Name, value.String()) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped DurationFlag.Apply -func (f *DurationFlag) Apply(set *flag.FlagSet) { - f.set = set - - f.DurationFlag.Apply(set) -} - -// Float64Flag is the flag type that wraps cli.Float64Flag to allow -// for other values to be specified -type Float64Flag struct { - cli.Float64Flag - set *flag.FlagSet -} - -// NewFloat64Flag creates a new Float64Flag -func NewFloat64Flag(flag cli.Float64Flag) *Float64Flag { - return &Float64Flag{Float64Flag: flag, set: nil} -} - -// ApplyInputSourceValue applies a Float64 value to the flagSet if required -func (f *Float64Flag) ApplyInputSourceValue(context *cli.Context, isc InputSourceContext) error { - if f.set != nil { - if !(context.IsSet(f.Name) || isEnvVarSet(f.EnvVar)) { - value, err := isc.Float64(f.Float64Flag.Name) - if err != nil { - return err - } - if value > 0 { - floatStr := float64ToString(value) - eachName(f.Name, func(name string) { - f.set.Set(f.Name, floatStr) - }) - } - } - } - return nil -} - -// Apply saves the flagSet for later usage then calls -// the wrapped Float64Flag.Apply -func (f *Float64Flag) Apply(set *flag.FlagSet) { - f.set = set - - f.Float64Flag.Apply(set) -} - -func isEnvVarSet(envVars string) bool { - for _, envVar := range strings.Split(envVars, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - // TODO: Can't use this for bools as - // set means that it was true or false based on - // Bool flag type, should work for other types - if len(envVal) > 0 { - return true - } - } - } - - return false -} - -func float64ToString(f float64) string { - return fmt.Sprintf("%v", f) -} - -func eachName(longName string, fn func(string)) { - parts := strings.Split(longName, ",") - for _, name := range parts { - name = strings.Trim(name, " ") - fn(name) - } -} diff --git a/vendor/github.com/codegangsta/cli/altsrc/flag_test.go b/vendor/github.com/codegangsta/cli/altsrc/flag_test.go deleted file mode 100644 index ac4d1f5..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/flag_test.go +++ /dev/null @@ -1,336 +0,0 @@ -package altsrc - -import ( - "flag" - "fmt" - "os" - "strings" - "testing" - "time" - - "github.com/codegangsta/cli" -) - -type testApplyInputSource struct { - Flag FlagInputSourceExtension - FlagName string - FlagSetName string - Expected string - ContextValueString string - ContextValue flag.Value - EnvVarValue string - EnvVarName string - MapValue interface{} -} - -func TestGenericApplyInputSourceValue(t *testing.T) { - v := &Parser{"abc", "def"} - c := runTest(t, testApplyInputSource{ - Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}}), - FlagName: "test", - MapValue: v, - }) - expect(t, v, c.Generic("test")) -} - -func TestGenericApplyInputSourceMethodContextSet(t *testing.T) { - p := &Parser{"abc", "def"} - c := runTest(t, testApplyInputSource{ - Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}}), - FlagName: "test", - MapValue: &Parser{"efg", "hig"}, - ContextValueString: p.String(), - }) - expect(t, p, c.Generic("test")) -} - -func TestGenericApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewGenericFlag(cli.GenericFlag{Name: "test", Value: &Parser{}, EnvVar: "TEST"}), - FlagName: "test", - MapValue: &Parser{"efg", "hij"}, - EnvVarName: "TEST", - EnvVarValue: "abc,def", - }) - expect(t, &Parser{"abc", "def"}, c.Generic("test")) -} - -func TestStringSliceApplyInputSourceValue(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []string{"hello", "world"}, - }) - expect(t, c.StringSlice("test"), []string{"hello", "world"}) -} - -func TestStringSliceApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []string{"hello", "world"}, - ContextValueString: "ohno", - }) - expect(t, c.StringSlice("test"), []string{"ohno"}) -} - -func TestStringSliceApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringSliceFlag(cli.StringSliceFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: []string{"hello", "world"}, - EnvVarName: "TEST", - EnvVarValue: "oh,no", - }) - expect(t, c.StringSlice("test"), []string{"oh", "no"}) -} - -func TestIntSliceApplyInputSourceValue(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []int{1, 2}, - }) - expect(t, c.IntSlice("test"), []int{1, 2}) -} - -func TestIntSliceApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test"}), - FlagName: "test", - MapValue: []int{1, 2}, - ContextValueString: "3", - }) - expect(t, c.IntSlice("test"), []int{3}) -} - -func TestIntSliceApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntSliceFlag(cli.IntSliceFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: []int{1, 2}, - EnvVarName: "TEST", - EnvVarValue: "3,4", - }) - expect(t, c.IntSlice("test"), []int{3, 4}) -} - -func TestBoolApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolFlag(cli.BoolFlag{Name: "test"}), - FlagName: "test", - MapValue: true, - }) - expect(t, true, c.Bool("test")) -} - -func TestBoolApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolFlag(cli.BoolFlag{Name: "test"}), - FlagName: "test", - MapValue: false, - ContextValueString: "true", - }) - expect(t, true, c.Bool("test")) -} - -func TestBoolApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolFlag(cli.BoolFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: false, - EnvVarName: "TEST", - EnvVarValue: "true", - }) - expect(t, true, c.Bool("test")) -} - -func TestBoolTApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test"}), - FlagName: "test", - MapValue: false, - }) - expect(t, false, c.BoolT("test")) -} - -func TestBoolTApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test"}), - FlagName: "test", - MapValue: true, - ContextValueString: "false", - }) - expect(t, false, c.BoolT("test")) -} - -func TestBoolTApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewBoolTFlag(cli.BoolTFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: true, - EnvVarName: "TEST", - EnvVarValue: "false", - }) - expect(t, false, c.BoolT("test")) -} - -func TestStringApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringFlag(cli.StringFlag{Name: "test"}), - FlagName: "test", - MapValue: "hello", - }) - expect(t, "hello", c.String("test")) -} - -func TestStringApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringFlag(cli.StringFlag{Name: "test"}), - FlagName: "test", - MapValue: "hello", - ContextValueString: "goodbye", - }) - expect(t, "goodbye", c.String("test")) -} - -func TestStringApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewStringFlag(cli.StringFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: "hello", - EnvVarName: "TEST", - EnvVarValue: "goodbye", - }) - expect(t, "goodbye", c.String("test")) -} - -func TestIntApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntFlag(cli.IntFlag{Name: "test"}), - FlagName: "test", - MapValue: 15, - }) - expect(t, 15, c.Int("test")) -} - -func TestIntApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntFlag(cli.IntFlag{Name: "test"}), - FlagName: "test", - MapValue: 15, - ContextValueString: "7", - }) - expect(t, 7, c.Int("test")) -} - -func TestIntApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewIntFlag(cli.IntFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: 15, - EnvVarName: "TEST", - EnvVarValue: "12", - }) - expect(t, 12, c.Int("test")) -} - -func TestDurationApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewDurationFlag(cli.DurationFlag{Name: "test"}), - FlagName: "test", - MapValue: time.Duration(30 * time.Second), - }) - expect(t, time.Duration(30*time.Second), c.Duration("test")) -} - -func TestDurationApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewDurationFlag(cli.DurationFlag{Name: "test"}), - FlagName: "test", - MapValue: time.Duration(30 * time.Second), - ContextValueString: time.Duration(15 * time.Second).String(), - }) - expect(t, time.Duration(15*time.Second), c.Duration("test")) -} - -func TestDurationApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewDurationFlag(cli.DurationFlag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: time.Duration(30 * time.Second), - EnvVarName: "TEST", - EnvVarValue: time.Duration(15 * time.Second).String(), - }) - expect(t, time.Duration(15*time.Second), c.Duration("test")) -} - -func TestFloat64ApplyInputSourceMethodSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewFloat64Flag(cli.Float64Flag{Name: "test"}), - FlagName: "test", - MapValue: 1.3, - }) - expect(t, 1.3, c.Float64("test")) -} - -func TestFloat64ApplyInputSourceMethodContextSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewFloat64Flag(cli.Float64Flag{Name: "test"}), - FlagName: "test", - MapValue: 1.3, - ContextValueString: fmt.Sprintf("%v", 1.4), - }) - expect(t, 1.4, c.Float64("test")) -} - -func TestFloat64ApplyInputSourceMethodEnvVarSet(t *testing.T) { - c := runTest(t, testApplyInputSource{ - Flag: NewFloat64Flag(cli.Float64Flag{Name: "test", EnvVar: "TEST"}), - FlagName: "test", - MapValue: 1.3, - EnvVarName: "TEST", - EnvVarValue: fmt.Sprintf("%v", 1.4), - }) - expect(t, 1.4, c.Float64("test")) -} - -func runTest(t *testing.T, test testApplyInputSource) *cli.Context { - inputSource := &MapInputSource{valueMap: map[string]interface{}{test.FlagName: test.MapValue}} - set := flag.NewFlagSet(test.FlagSetName, flag.ContinueOnError) - c := cli.NewContext(nil, set, nil) - if test.EnvVarName != "" && test.EnvVarValue != "" { - os.Setenv(test.EnvVarName, test.EnvVarValue) - defer os.Setenv(test.EnvVarName, "") - } - - test.Flag.Apply(set) - if test.ContextValue != nil { - flag := set.Lookup(test.FlagName) - flag.Value = test.ContextValue - } - if test.ContextValueString != "" { - set.Set(test.FlagName, test.ContextValueString) - } - test.Flag.ApplyInputSourceValue(c, inputSource) - - return c -} - -type Parser [2]string - -func (p *Parser) Set(value string) error { - parts := strings.Split(value, ",") - if len(parts) != 2 { - return fmt.Errorf("invalid format") - } - - (*p)[0] = parts[0] - (*p)[1] = parts[1] - - return nil -} - -func (p *Parser) String() string { - return fmt.Sprintf("%s,%s", p[0], p[1]) -} diff --git a/vendor/github.com/codegangsta/cli/altsrc/helpers_test.go b/vendor/github.com/codegangsta/cli/altsrc/helpers_test.go deleted file mode 100644 index 3b7f7e9..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/helpers_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package altsrc - -import ( - "reflect" - "testing" -) - -func expect(t *testing.T, a interface{}, b interface{}) { - if !reflect.DeepEqual(b, a) { - t.Errorf("Expected %#v (type %v) - Got %#v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} - -func refute(t *testing.T, a interface{}, b interface{}) { - if a == b { - t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} diff --git a/vendor/github.com/codegangsta/cli/altsrc/input_source_context.go b/vendor/github.com/codegangsta/cli/altsrc/input_source_context.go deleted file mode 100644 index 6d695ff..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/input_source_context.go +++ /dev/null @@ -1,21 +0,0 @@ -package altsrc - -import ( - "time" - - "github.com/codegangsta/cli" -) - -// InputSourceContext is an interface used to allow -// other input sources to be implemented as needed. -type InputSourceContext interface { - Int(name string) (int, error) - Duration(name string) (time.Duration, error) - Float64(name string) (float64, error) - String(name string) (string, error) - StringSlice(name string) ([]string, error) - IntSlice(name string) ([]int, error) - Generic(name string) (cli.Generic, error) - Bool(name string) (bool, error) - BoolT(name string) (bool, error) -} diff --git a/vendor/github.com/codegangsta/cli/altsrc/map_input_source.go b/vendor/github.com/codegangsta/cli/altsrc/map_input_source.go deleted file mode 100644 index f1670fb..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/map_input_source.go +++ /dev/null @@ -1,152 +0,0 @@ -package altsrc - -import ( - "fmt" - "reflect" - "time" - - "github.com/codegangsta/cli" -) - -// MapInputSource implements InputSourceContext to return -// data from the map that is loaded. -type MapInputSource struct { - valueMap map[string]interface{} -} - -// Int returns an int from the map if it exists otherwise returns 0 -func (fsm *MapInputSource) Int(name string) (int, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(int) - if !isType { - return 0, incorrectTypeForFlagError(name, "int", otherGenericValue) - } - - return otherValue, nil - } - - return 0, nil -} - -// Duration returns a duration from the map if it exists otherwise returns 0 -func (fsm *MapInputSource) Duration(name string) (time.Duration, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(time.Duration) - if !isType { - return 0, incorrectTypeForFlagError(name, "duration", otherGenericValue) - } - return otherValue, nil - } - - return 0, nil -} - -// Float64 returns an float64 from the map if it exists otherwise returns 0 -func (fsm *MapInputSource) Float64(name string) (float64, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(float64) - if !isType { - return 0, incorrectTypeForFlagError(name, "float64", otherGenericValue) - } - return otherValue, nil - } - - return 0, nil -} - -// String returns a string from the map if it exists otherwise returns an empty string -func (fsm *MapInputSource) String(name string) (string, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(string) - if !isType { - return "", incorrectTypeForFlagError(name, "string", otherGenericValue) - } - return otherValue, nil - } - - return "", nil -} - -// StringSlice returns an []string from the map if it exists otherwise returns nil -func (fsm *MapInputSource) StringSlice(name string) ([]string, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.([]string) - if !isType { - return nil, incorrectTypeForFlagError(name, "[]string", otherGenericValue) - } - return otherValue, nil - } - - return nil, nil -} - -// IntSlice returns an []int from the map if it exists otherwise returns nil -func (fsm *MapInputSource) IntSlice(name string) ([]int, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.([]int) - if !isType { - return nil, incorrectTypeForFlagError(name, "[]int", otherGenericValue) - } - return otherValue, nil - } - - return nil, nil -} - -// Generic returns an cli.Generic from the map if it exists otherwise returns nil -func (fsm *MapInputSource) Generic(name string) (cli.Generic, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(cli.Generic) - if !isType { - return nil, incorrectTypeForFlagError(name, "cli.Generic", otherGenericValue) - } - return otherValue, nil - } - - return nil, nil -} - -// Bool returns an bool from the map otherwise returns false -func (fsm *MapInputSource) Bool(name string) (bool, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(bool) - if !isType { - return false, incorrectTypeForFlagError(name, "bool", otherGenericValue) - } - return otherValue, nil - } - - return false, nil -} - -// BoolT returns an bool from the map otherwise returns true -func (fsm *MapInputSource) BoolT(name string) (bool, error) { - otherGenericValue, exists := fsm.valueMap[name] - if exists { - otherValue, isType := otherGenericValue.(bool) - if !isType { - return true, incorrectTypeForFlagError(name, "bool", otherGenericValue) - } - return otherValue, nil - } - - return true, nil -} - -func incorrectTypeForFlagError(name, expectedTypeName string, value interface{}) error { - valueType := reflect.TypeOf(value) - valueTypeName := "" - if valueType != nil { - valueTypeName = valueType.Name() - } - - return fmt.Errorf("Mismatched type for flag '%s'. Expected '%s' but actual is '%s'", name, expectedTypeName, valueTypeName) -} diff --git a/vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go b/vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go deleted file mode 100644 index c7ccbf7..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/yaml_command_test.go +++ /dev/null @@ -1,172 +0,0 @@ -// Disabling building of yaml support in cases where golang is 1.0 or 1.1 -// as the encoding library is not implemented or supported. - -// +build !go1,!go1.1 - -package altsrc - -import ( - "flag" - "io/ioutil" - "os" - "testing" - - "github.com/codegangsta/cli" -) - -func TestCommandYamlFileTest(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 15) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileTestGlobalEnvVarWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - os.Setenv("THE_TEST", "10") - defer os.Setenv("THE_TEST", "") - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 10) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test", EnvVar: "THE_TEST"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileTestSpecifiedFlagWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - test := []string{"test-cmd", "--load", "current.yaml", "--test", "7"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 7) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileTestDefaultValueFileWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 15) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test", Value: 7}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - - err := command.Run(c) - - expect(t, err, nil) -} - -func TestCommandYamlFileFlagHasDefaultGlobalEnvYamlSetGlobalEnvWins(t *testing.T) { - app := cli.NewApp() - set := flag.NewFlagSet("test", 0) - ioutil.WriteFile("current.yaml", []byte("test: 15"), 0666) - defer os.Remove("current.yaml") - - os.Setenv("THE_TEST", "11") - defer os.Setenv("THE_TEST", "") - - test := []string{"test-cmd", "--load", "current.yaml"} - set.Parse(test) - - c := cli.NewContext(app, set, nil) - - command := &cli.Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(c *cli.Context) { - val := c.Int("test") - expect(t, val, 11) - }, - Flags: []cli.Flag{ - NewIntFlag(cli.IntFlag{Name: "test", Value: 7, EnvVar: "THE_TEST"}), - cli.StringFlag{Name: "load"}}, - } - command.Before = InitInputSourceWithContext(command.Flags, NewYamlSourceFromFlagFunc("load")) - err := command.Run(c) - - expect(t, err, nil) -} diff --git a/vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go b/vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go deleted file mode 100644 index 1251aeb..0000000 --- a/vendor/github.com/codegangsta/cli/altsrc/yaml_file_loader.go +++ /dev/null @@ -1,84 +0,0 @@ -// Disabling building of yaml support in cases where golang is 1.0 or 1.1 -// as the encoding library is not implemented or supported. - -// +build !go1,!go1.1 - -package altsrc - -import ( - "fmt" - "io/ioutil" - "net/http" - "net/url" - "os" - - "github.com/codegangsta/cli" - - "gopkg.in/yaml.v2" -) - -type yamlSourceContext struct { - FilePath string -} - -// NewYamlSourceFromFile creates a new Yaml InputSourceContext from a filepath. -func NewYamlSourceFromFile(file string) (InputSourceContext, error) { - ymlLoader := &yamlSourceLoader{FilePath: file} - var results map[string]interface{} - err := readCommandYaml(ysl.FilePath, &results) - if err != nil { - return fmt.Errorf("Unable to load Yaml file '%s': inner error: \n'%v'", filePath, err.Error()) - } - - return &MapInputSource{valueMap: results}, nil -} - -// NewYamlSourceFromFlagFunc creates a new Yaml InputSourceContext from a provided flag name and source context. -func NewYamlSourceFromFlagFunc(flagFileName string) func(InputSourceContext, error) { - return func(context cli.Context) { - filePath := context.String(flagFileName) - return NewYamlSourceFromFile(filePath) - } -} - -func readCommandYaml(filePath string, container interface{}) (err error) { - b, err := loadDataFrom(filePath) - if err != nil { - return err - } - - err = yaml.Unmarshal(b, container) - if err != nil { - return err - } - - err = nil - return -} - -func loadDataFrom(filePath string) ([]byte, error) { - u, err := url.Parse(filePath) - if err != nil { - return nil, err - } - - if u.Host != "" { // i have a host, now do i support the scheme? - switch u.Scheme { - case "http", "https": - res, err := http.Get(filePath) - if err != nil { - return nil, err - } - return ioutil.ReadAll(res.Body) - default: - return nil, fmt.Errorf("scheme of %s is unsupported", filePath) - } - } else if u.Path != "" { // i dont have a host, but I have a path. I am a local file. - if _, notFoundFileErr := os.Stat(filePath); notFoundFileErr != nil { - return nil, fmt.Errorf("Cannot read from file: '%s' because it does not exist.", filePath) - } - return ioutil.ReadFile(filePath) - } else { - return nil, fmt.Errorf("unable to determine how to load from path %s", filePath) - } -} diff --git a/vendor/github.com/codegangsta/cli/app.go b/vendor/github.com/codegangsta/cli/app.go deleted file mode 100644 index 1ea3fd0..0000000 --- a/vendor/github.com/codegangsta/cli/app.go +++ /dev/null @@ -1,349 +0,0 @@ -package cli - -import ( - "fmt" - "io" - "io/ioutil" - "os" - "path" - "time" -) - -// App is the main structure of a cli application. It is recommended that -// an app be created with the cli.NewApp() function -type App struct { - // The name of the program. Defaults to path.Base(os.Args[0]) - Name string - // Full name of command for help, defaults to Name - HelpName string - // Description of the program. - Usage string - // Text to override the USAGE section of help - UsageText string - // Description of the program argument format. - ArgsUsage string - // Version of the program - Version string - // List of commands to execute - Commands []Command - // List of flags to parse - Flags []Flag - // Boolean to enable bash completion commands - EnableBashCompletion bool - // Boolean to hide built-in help command - HideHelp bool - // Boolean to hide built-in version flag - HideVersion bool - // An action to execute when the bash-completion flag is set - BashComplete func(context *Context) - // An action to execute before any subcommands are run, but after the context is ready - // If a non-nil error is returned, no subcommands are run - Before func(context *Context) error - // An action to execute after any subcommands are run, but after the subcommand has finished - // It is run even if Action() panics - After func(context *Context) error - // The action to execute when no subcommands are specified - Action func(context *Context) - // Execute this function if the proper command cannot be found - CommandNotFound func(context *Context, command string) - // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. - // This function is able to replace the original error messages. - // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. - OnUsageError func(context *Context, err error, isSubcommand bool) error - // Compilation date - Compiled time.Time - // List of all authors who contributed - Authors []Author - // Copyright of the binary if any - Copyright string - // Name of Author (Note: Use App.Authors, this is deprecated) - Author string - // Email of Author (Note: Use App.Authors, this is deprecated) - Email string - // Writer writer to write output to - Writer io.Writer -} - -// Tries to find out when this binary was compiled. -// Returns the current time if it fails to find it. -func compileTime() time.Time { - info, err := os.Stat(os.Args[0]) - if err != nil { - return time.Now() - } - return info.ModTime() -} - -// Creates a new cli Application with some reasonable defaults for Name, Usage, Version and Action. -func NewApp() *App { - return &App{ - Name: path.Base(os.Args[0]), - HelpName: path.Base(os.Args[0]), - Usage: "A new cli application", - UsageText: "", - Version: "0.0.0", - BashComplete: DefaultAppComplete, - Action: helpCommand.Action, - Compiled: compileTime(), - Writer: os.Stdout, - } -} - -// Entry point to the cli app. Parses the arguments slice and routes to the proper flag/args combination -func (a *App) Run(arguments []string) (err error) { - if a.Author != "" || a.Email != "" { - a.Authors = append(a.Authors, Author{Name: a.Author, Email: a.Email}) - } - - newCmds := []Command{} - for _, c := range a.Commands { - if c.HelpName == "" { - c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name) - } - newCmds = append(newCmds, c) - } - a.Commands = newCmds - - // append help to commands - if a.Command(helpCommand.Name) == nil && !a.HideHelp { - a.Commands = append(a.Commands, helpCommand) - if (HelpFlag != BoolFlag{}) { - a.appendFlag(HelpFlag) - } - } - - //append version/help flags - if a.EnableBashCompletion { - a.appendFlag(BashCompletionFlag) - } - - if !a.HideVersion { - a.appendFlag(VersionFlag) - } - - // parse flags - set := flagSet(a.Name, a.Flags) - set.SetOutput(ioutil.Discard) - err = set.Parse(arguments[1:]) - nerr := normalizeFlags(a.Flags, set) - context := NewContext(a, set, nil) - if nerr != nil { - fmt.Fprintln(a.Writer, nerr) - ShowAppHelp(context) - return nerr - } - - if checkCompletions(context) { - return nil - } - - if err != nil { - if a.OnUsageError != nil { - err := a.OnUsageError(context, err, false) - return err - } else { - fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") - ShowAppHelp(context) - return err - } - } - - if !a.HideHelp && checkHelp(context) { - ShowAppHelp(context) - return nil - } - - if !a.HideVersion && checkVersion(context) { - ShowVersion(context) - return nil - } - - if a.After != nil { - defer func() { - if afterErr := a.After(context); afterErr != nil { - if err != nil { - err = NewMultiError(err, afterErr) - } else { - err = afterErr - } - } - }() - } - - if a.Before != nil { - err = a.Before(context) - if err != nil { - fmt.Fprintf(a.Writer, "%v\n\n", err) - ShowAppHelp(context) - return err - } - } - - args := context.Args() - if args.Present() { - name := args.First() - c := a.Command(name) - if c != nil { - return c.Run(context) - } - } - - // Run default Action - a.Action(context) - return nil -} - -// Another entry point to the cli app, takes care of passing arguments and error handling -func (a *App) RunAndExitOnError() { - if err := a.Run(os.Args); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } -} - -// Invokes the subcommand given the context, parses ctx.Args() to generate command-specific flags -func (a *App) RunAsSubcommand(ctx *Context) (err error) { - // append help to commands - if len(a.Commands) > 0 { - if a.Command(helpCommand.Name) == nil && !a.HideHelp { - a.Commands = append(a.Commands, helpCommand) - if (HelpFlag != BoolFlag{}) { - a.appendFlag(HelpFlag) - } - } - } - - newCmds := []Command{} - for _, c := range a.Commands { - if c.HelpName == "" { - c.HelpName = fmt.Sprintf("%s %s", a.HelpName, c.Name) - } - newCmds = append(newCmds, c) - } - a.Commands = newCmds - - // append flags - if a.EnableBashCompletion { - a.appendFlag(BashCompletionFlag) - } - - // parse flags - set := flagSet(a.Name, a.Flags) - set.SetOutput(ioutil.Discard) - err = set.Parse(ctx.Args().Tail()) - nerr := normalizeFlags(a.Flags, set) - context := NewContext(a, set, ctx) - - if nerr != nil { - fmt.Fprintln(a.Writer, nerr) - fmt.Fprintln(a.Writer) - if len(a.Commands) > 0 { - ShowSubcommandHelp(context) - } else { - ShowCommandHelp(ctx, context.Args().First()) - } - return nerr - } - - if checkCompletions(context) { - return nil - } - - if err != nil { - if a.OnUsageError != nil { - err = a.OnUsageError(context, err, true) - return err - } else { - fmt.Fprintf(a.Writer, "%s\n\n", "Incorrect Usage.") - ShowSubcommandHelp(context) - return err - } - } - - if len(a.Commands) > 0 { - if checkSubcommandHelp(context) { - return nil - } - } else { - if checkCommandHelp(ctx, context.Args().First()) { - return nil - } - } - - if a.After != nil { - defer func() { - afterErr := a.After(context) - if afterErr != nil { - if err != nil { - err = NewMultiError(err, afterErr) - } else { - err = afterErr - } - } - }() - } - - if a.Before != nil { - err := a.Before(context) - if err != nil { - return err - } - } - - args := context.Args() - if args.Present() { - name := args.First() - c := a.Command(name) - if c != nil { - return c.Run(context) - } - } - - // Run default Action - a.Action(context) - - return nil -} - -// Returns the named command on App. Returns nil if the command does not exist -func (a *App) Command(name string) *Command { - for _, c := range a.Commands { - if c.HasName(name) { - return &c - } - } - - return nil -} - -func (a *App) hasFlag(flag Flag) bool { - for _, f := range a.Flags { - if flag == f { - return true - } - } - - return false -} - -func (a *App) appendFlag(flag Flag) { - if !a.hasFlag(flag) { - a.Flags = append(a.Flags, flag) - } -} - -// Author represents someone who has contributed to a cli project. -type Author struct { - Name string // The Authors name - Email string // The Authors email -} - -// String makes Author comply to the Stringer interface, to allow an easy print in the templating process -func (a Author) String() string { - e := "" - if a.Email != "" { - e = "<" + a.Email + "> " - } - - return fmt.Sprintf("%v %v", a.Name, e) -} diff --git a/vendor/github.com/codegangsta/cli/app_test.go b/vendor/github.com/codegangsta/cli/app_test.go deleted file mode 100644 index 7feaf1f..0000000 --- a/vendor/github.com/codegangsta/cli/app_test.go +++ /dev/null @@ -1,1047 +0,0 @@ -package cli - -import ( - "bytes" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "strings" - "testing" -) - -func ExampleApp_Run() { - // set args for examples sake - os.Args = []string{"greet", "--name", "Jeremy"} - - app := NewApp() - app.Name = "greet" - app.Flags = []Flag{ - StringFlag{Name: "name", Value: "bob", Usage: "a name to say"}, - } - app.Action = func(c *Context) { - fmt.Printf("Hello %v\n", c.String("name")) - } - app.UsageText = "app [first_arg] [second_arg]" - app.Author = "Harrison" - app.Email = "harrison@lolwut.com" - app.Authors = []Author{Author{Name: "Oliver Allen", Email: "oliver@toyshop.com"}} - app.Run(os.Args) - // Output: - // Hello Jeremy -} - -func ExampleApp_Run_subcommand() { - // set args for examples sake - os.Args = []string{"say", "hi", "english", "--name", "Jeremy"} - app := NewApp() - app.Name = "say" - app.Commands = []Command{ - { - Name: "hello", - Aliases: []string{"hi"}, - Usage: "use it to see a description", - Description: "This is how we describe hello the function", - Subcommands: []Command{ - { - Name: "english", - Aliases: []string{"en"}, - Usage: "sends a greeting in english", - Description: "greets someone in english", - Flags: []Flag{ - StringFlag{ - Name: "name", - Value: "Bob", - Usage: "Name of the person to greet", - }, - }, - Action: func(c *Context) { - fmt.Println("Hello,", c.String("name")) - }, - }, - }, - }, - } - - app.Run(os.Args) - // Output: - // Hello, Jeremy -} - -func ExampleApp_Run_help() { - // set args for examples sake - os.Args = []string{"greet", "h", "describeit"} - - app := NewApp() - app.Name = "greet" - app.Flags = []Flag{ - StringFlag{Name: "name", Value: "bob", Usage: "a name to say"}, - } - app.Commands = []Command{ - { - Name: "describeit", - Aliases: []string{"d"}, - Usage: "use it to see a description", - Description: "This is how we describe describeit the function", - Action: func(c *Context) { - fmt.Printf("i like to describe things") - }, - }, - } - app.Run(os.Args) - // Output: - // NAME: - // greet describeit - use it to see a description - // - // USAGE: - // greet describeit [arguments...] - // - // DESCRIPTION: - // This is how we describe describeit the function -} - -func ExampleApp_Run_bashComplete() { - // set args for examples sake - os.Args = []string{"greet", "--generate-bash-completion"} - - app := NewApp() - app.Name = "greet" - app.EnableBashCompletion = true - app.Commands = []Command{ - { - Name: "describeit", - Aliases: []string{"d"}, - Usage: "use it to see a description", - Description: "This is how we describe describeit the function", - Action: func(c *Context) { - fmt.Printf("i like to describe things") - }, - }, { - Name: "next", - Usage: "next example", - Description: "more stuff to see when generating bash completion", - Action: func(c *Context) { - fmt.Printf("the next example") - }, - }, - } - - app.Run(os.Args) - // Output: - // describeit - // d - // next - // help - // h -} - -func TestApp_Run(t *testing.T) { - s := "" - - app := NewApp() - app.Action = func(c *Context) { - s = s + c.Args().First() - } - - err := app.Run([]string{"command", "foo"}) - expect(t, err, nil) - err = app.Run([]string{"command", "bar"}) - expect(t, err, nil) - expect(t, s, "foobar") -} - -var commandAppTests = []struct { - name string - expected bool -}{ - {"foobar", true}, - {"batbaz", true}, - {"b", true}, - {"f", true}, - {"bat", false}, - {"nothing", false}, -} - -func TestApp_Command(t *testing.T) { - app := NewApp() - fooCommand := Command{Name: "foobar", Aliases: []string{"f"}} - batCommand := Command{Name: "batbaz", Aliases: []string{"b"}} - app.Commands = []Command{ - fooCommand, - batCommand, - } - - for _, test := range commandAppTests { - expect(t, app.Command(test.name) != nil, test.expected) - } -} - -func TestApp_CommandWithArgBeforeFlags(t *testing.T) { - var parsedOption, firstArg string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - StringFlag{Name: "option", Value: "", Usage: "some option"}, - }, - Action: func(c *Context) { - parsedOption = c.String("option") - firstArg = c.Args().First() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "--option", "my-option"}) - - expect(t, parsedOption, "my-option") - expect(t, firstArg, "my-arg") -} - -func TestApp_RunAsSubcommandParseFlags(t *testing.T) { - var context *Context - - a := NewApp() - a.Commands = []Command{ - { - Name: "foo", - Action: func(c *Context) { - context = c - }, - Flags: []Flag{ - StringFlag{ - Name: "lang", - Value: "english", - Usage: "language for the greeting", - }, - }, - Before: func(_ *Context) error { return nil }, - }, - } - a.Run([]string{"", "foo", "--lang", "spanish", "abcd"}) - - expect(t, context.Args().Get(0), "abcd") - expect(t, context.String("lang"), "spanish") -} - -func TestApp_CommandWithFlagBeforeTerminator(t *testing.T) { - var parsedOption string - var args []string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - StringFlag{Name: "option", Value: "", Usage: "some option"}, - }, - Action: func(c *Context) { - parsedOption = c.String("option") - args = c.Args() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "--option", "my-option", "--", "--notARealFlag"}) - - expect(t, parsedOption, "my-option") - expect(t, args[0], "my-arg") - expect(t, args[1], "--") - expect(t, args[2], "--notARealFlag") -} - -func TestApp_CommandWithDash(t *testing.T) { - var args []string - - app := NewApp() - command := Command{ - Name: "cmd", - Action: func(c *Context) { - args = c.Args() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "-"}) - - expect(t, args[0], "my-arg") - expect(t, args[1], "-") -} - -func TestApp_CommandWithNoFlagBeforeTerminator(t *testing.T) { - var args []string - - app := NewApp() - command := Command{ - Name: "cmd", - Action: func(c *Context) { - args = c.Args() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "--", "notAFlagAtAll"}) - - expect(t, args[0], "my-arg") - expect(t, args[1], "--") - expect(t, args[2], "notAFlagAtAll") -} - -func TestApp_Float64Flag(t *testing.T) { - var meters float64 - - app := NewApp() - app.Flags = []Flag{ - Float64Flag{Name: "height", Value: 1.5, Usage: "Set the height, in meters"}, - } - app.Action = func(c *Context) { - meters = c.Float64("height") - } - - app.Run([]string{"", "--height", "1.93"}) - expect(t, meters, 1.93) -} - -func TestApp_ParseSliceFlags(t *testing.T) { - var parsedOption, firstArg string - var parsedIntSlice []int - var parsedStringSlice []string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - IntSliceFlag{Name: "p", Value: &IntSlice{}, Usage: "set one or more ip addr"}, - StringSliceFlag{Name: "ip", Value: &StringSlice{}, Usage: "set one or more ports to open"}, - }, - Action: func(c *Context) { - parsedIntSlice = c.IntSlice("p") - parsedStringSlice = c.StringSlice("ip") - parsedOption = c.String("option") - firstArg = c.Args().First() - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "-p", "22", "-p", "80", "-ip", "8.8.8.8", "-ip", "8.8.4.4"}) - - IntsEquals := func(a, b []int) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true - } - - StrsEquals := func(a, b []string) bool { - if len(a) != len(b) { - return false - } - for i, v := range a { - if v != b[i] { - return false - } - } - return true - } - var expectedIntSlice = []int{22, 80} - var expectedStringSlice = []string{"8.8.8.8", "8.8.4.4"} - - if !IntsEquals(parsedIntSlice, expectedIntSlice) { - t.Errorf("%v does not match %v", parsedIntSlice, expectedIntSlice) - } - - if !StrsEquals(parsedStringSlice, expectedStringSlice) { - t.Errorf("%v does not match %v", parsedStringSlice, expectedStringSlice) - } -} - -func TestApp_ParseSliceFlagsWithMissingValue(t *testing.T) { - var parsedIntSlice []int - var parsedStringSlice []string - - app := NewApp() - command := Command{ - Name: "cmd", - Flags: []Flag{ - IntSliceFlag{Name: "a", Usage: "set numbers"}, - StringSliceFlag{Name: "str", Usage: "set strings"}, - }, - Action: func(c *Context) { - parsedIntSlice = c.IntSlice("a") - parsedStringSlice = c.StringSlice("str") - }, - } - app.Commands = []Command{command} - - app.Run([]string{"", "cmd", "my-arg", "-a", "2", "-str", "A"}) - - var expectedIntSlice = []int{2} - var expectedStringSlice = []string{"A"} - - if parsedIntSlice[0] != expectedIntSlice[0] { - t.Errorf("%v does not match %v", parsedIntSlice[0], expectedIntSlice[0]) - } - - if parsedStringSlice[0] != expectedStringSlice[0] { - t.Errorf("%v does not match %v", parsedIntSlice[0], expectedIntSlice[0]) - } -} - -func TestApp_DefaultStdout(t *testing.T) { - app := NewApp() - - if app.Writer != os.Stdout { - t.Error("Default output writer not set.") - } -} - -type mockWriter struct { - written []byte -} - -func (fw *mockWriter) Write(p []byte) (n int, err error) { - if fw.written == nil { - fw.written = p - } else { - fw.written = append(fw.written, p...) - } - - return len(p), nil -} - -func (fw *mockWriter) GetWritten() (b []byte) { - return fw.written -} - -func TestApp_SetStdout(t *testing.T) { - w := &mockWriter{} - - app := NewApp() - app.Name = "test" - app.Writer = w - - err := app.Run([]string{"help"}) - - if err != nil { - t.Fatalf("Run error: %s", err) - } - - if len(w.written) == 0 { - t.Error("App did not write output to desired writer.") - } -} - -func TestApp_BeforeFunc(t *testing.T) { - beforeRun, subcommandRun := false, false - beforeError := fmt.Errorf("fail") - var err error - - app := NewApp() - - app.Before = func(c *Context) error { - beforeRun = true - s := c.String("opt") - if s == "fail" { - return beforeError - } - - return nil - } - - app.Commands = []Command{ - Command{ - Name: "sub", - Action: func(c *Context) { - subcommandRun = true - }, - }, - } - - app.Flags = []Flag{ - StringFlag{Name: "opt"}, - } - - // run with the Before() func succeeding - err = app.Run([]string{"command", "--opt", "succeed", "sub"}) - - if err != nil { - t.Fatalf("Run error: %s", err) - } - - if beforeRun == false { - t.Errorf("Before() not executed when expected") - } - - if subcommandRun == false { - t.Errorf("Subcommand not executed when expected") - } - - // reset - beforeRun, subcommandRun = false, false - - // run with the Before() func failing - err = app.Run([]string{"command", "--opt", "fail", "sub"}) - - // should be the same error produced by the Before func - if err != beforeError { - t.Errorf("Run error expected, but not received") - } - - if beforeRun == false { - t.Errorf("Before() not executed when expected") - } - - if subcommandRun == true { - t.Errorf("Subcommand executed when NOT expected") - } - -} - -func TestApp_AfterFunc(t *testing.T) { - afterRun, subcommandRun := false, false - afterError := fmt.Errorf("fail") - var err error - - app := NewApp() - - app.After = func(c *Context) error { - afterRun = true - s := c.String("opt") - if s == "fail" { - return afterError - } - - return nil - } - - app.Commands = []Command{ - Command{ - Name: "sub", - Action: func(c *Context) { - subcommandRun = true - }, - }, - } - - app.Flags = []Flag{ - StringFlag{Name: "opt"}, - } - - // run with the After() func succeeding - err = app.Run([]string{"command", "--opt", "succeed", "sub"}) - - if err != nil { - t.Fatalf("Run error: %s", err) - } - - if afterRun == false { - t.Errorf("After() not executed when expected") - } - - if subcommandRun == false { - t.Errorf("Subcommand not executed when expected") - } - - // reset - afterRun, subcommandRun = false, false - - // run with the Before() func failing - err = app.Run([]string{"command", "--opt", "fail", "sub"}) - - // should be the same error produced by the Before func - if err != afterError { - t.Errorf("Run error expected, but not received") - } - - if afterRun == false { - t.Errorf("After() not executed when expected") - } - - if subcommandRun == false { - t.Errorf("Subcommand not executed when expected") - } -} - -func TestAppNoHelpFlag(t *testing.T) { - oldFlag := HelpFlag - defer func() { - HelpFlag = oldFlag - }() - - HelpFlag = BoolFlag{} - - app := NewApp() - app.Writer = ioutil.Discard - err := app.Run([]string{"test", "-h"}) - - if err != flag.ErrHelp { - t.Errorf("expected error about missing help flag, but got: %s (%T)", err, err) - } -} - -func TestAppHelpPrinter(t *testing.T) { - oldPrinter := HelpPrinter - defer func() { - HelpPrinter = oldPrinter - }() - - var wasCalled = false - HelpPrinter = func(w io.Writer, template string, data interface{}) { - wasCalled = true - } - - app := NewApp() - app.Run([]string{"-h"}) - - if wasCalled == false { - t.Errorf("Help printer expected to be called, but was not") - } -} - -func TestAppVersionPrinter(t *testing.T) { - oldPrinter := VersionPrinter - defer func() { - VersionPrinter = oldPrinter - }() - - var wasCalled = false - VersionPrinter = func(c *Context) { - wasCalled = true - } - - app := NewApp() - ctx := NewContext(app, nil, nil) - ShowVersion(ctx) - - if wasCalled == false { - t.Errorf("Version printer expected to be called, but was not") - } -} - -func TestAppCommandNotFound(t *testing.T) { - beforeRun, subcommandRun := false, false - app := NewApp() - - app.CommandNotFound = func(c *Context, command string) { - beforeRun = true - } - - app.Commands = []Command{ - Command{ - Name: "bar", - Action: func(c *Context) { - subcommandRun = true - }, - }, - } - - app.Run([]string{"command", "foo"}) - - expect(t, beforeRun, true) - expect(t, subcommandRun, false) -} - -func TestGlobalFlag(t *testing.T) { - var globalFlag string - var globalFlagSet bool - app := NewApp() - app.Flags = []Flag{ - StringFlag{Name: "global, g", Usage: "global"}, - } - app.Action = func(c *Context) { - globalFlag = c.GlobalString("global") - globalFlagSet = c.GlobalIsSet("global") - } - app.Run([]string{"command", "-g", "foo"}) - expect(t, globalFlag, "foo") - expect(t, globalFlagSet, true) - -} - -func TestGlobalFlagsInSubcommands(t *testing.T) { - subcommandRun := false - parentFlag := false - app := NewApp() - - app.Flags = []Flag{ - BoolFlag{Name: "debug, d", Usage: "Enable debugging"}, - } - - app.Commands = []Command{ - Command{ - Name: "foo", - Flags: []Flag{ - BoolFlag{Name: "parent, p", Usage: "Parent flag"}, - }, - Subcommands: []Command{ - { - Name: "bar", - Action: func(c *Context) { - if c.GlobalBool("debug") { - subcommandRun = true - } - if c.GlobalBool("parent") { - parentFlag = true - } - }, - }, - }, - }, - } - - app.Run([]string{"command", "-d", "foo", "-p", "bar"}) - - expect(t, subcommandRun, true) - expect(t, parentFlag, true) -} - -func TestApp_Run_CommandWithSubcommandHasHelpTopic(t *testing.T) { - var subcommandHelpTopics = [][]string{ - {"command", "foo", "--help"}, - {"command", "foo", "-h"}, - {"command", "foo", "help"}, - } - - for _, flagSet := range subcommandHelpTopics { - t.Logf("==> checking with flags %v", flagSet) - - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - - subCmdBar := Command{ - Name: "bar", - Usage: "does bar things", - } - subCmdBaz := Command{ - Name: "baz", - Usage: "does baz things", - } - cmd := Command{ - Name: "foo", - Description: "descriptive wall of text about how it does foo things", - Subcommands: []Command{subCmdBar, subCmdBaz}, - } - - app.Commands = []Command{cmd} - err := app.Run(flagSet) - - if err != nil { - t.Error(err) - } - - output := buf.String() - t.Logf("output: %q\n", buf.Bytes()) - - if strings.Contains(output, "No help topic for") { - t.Errorf("expect a help topic, got none: \n%q", output) - } - - for _, shouldContain := range []string{ - cmd.Name, cmd.Description, - subCmdBar.Name, subCmdBar.Usage, - subCmdBaz.Name, subCmdBaz.Usage, - } { - if !strings.Contains(output, shouldContain) { - t.Errorf("want help to contain %q, did not: \n%q", shouldContain, output) - } - } - } -} - -func TestApp_Run_SubcommandFullPath(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "command" - subCmd := Command{ - Name: "bar", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "bar", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "command foo bar - does bar things") { - t.Errorf("expected full path to subcommand: %s", output) - } - if !strings.Contains(output, "command foo bar [arguments...]") { - t.Errorf("expected full path to subcommand: %s", output) - } -} - -func TestApp_Run_SubcommandHelpName(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "command" - subCmd := Command{ - Name: "bar", - HelpName: "custom", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "bar", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "custom - does bar things") { - t.Errorf("expected HelpName for subcommand: %s", output) - } - if !strings.Contains(output, "custom [arguments...]") { - t.Errorf("expected HelpName to subcommand: %s", output) - } -} - -func TestApp_Run_CommandHelpName(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "command" - subCmd := Command{ - Name: "bar", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - HelpName: "custom", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "bar", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "command foo bar - does bar things") { - t.Errorf("expected full path to subcommand: %s", output) - } - if !strings.Contains(output, "command foo bar [arguments...]") { - t.Errorf("expected full path to subcommand: %s", output) - } -} - -func TestApp_Run_CommandSubcommandHelpName(t *testing.T) { - app := NewApp() - buf := new(bytes.Buffer) - app.Writer = buf - app.Name = "base" - subCmd := Command{ - Name: "bar", - HelpName: "custom", - Usage: "does bar things", - } - cmd := Command{ - Name: "foo", - Description: "foo commands", - Subcommands: []Command{subCmd}, - } - app.Commands = []Command{cmd} - - err := app.Run([]string{"command", "foo", "--help"}) - if err != nil { - t.Error(err) - } - - output := buf.String() - if !strings.Contains(output, "base foo - foo commands") { - t.Errorf("expected full path to subcommand: %s", output) - } - if !strings.Contains(output, "base foo command [command options] [arguments...]") { - t.Errorf("expected full path to subcommand: %s", output) - } -} - -func TestApp_Run_Help(t *testing.T) { - var helpArguments = [][]string{{"boom", "--help"}, {"boom", "-h"}, {"boom", "help"}} - - for _, args := range helpArguments { - buf := new(bytes.Buffer) - - t.Logf("==> checking with arguments %v", args) - - app := NewApp() - app.Name = "boom" - app.Usage = "make an explosive entrance" - app.Writer = buf - app.Action = func(c *Context) { - buf.WriteString("boom I say!") - } - - err := app.Run(args) - if err != nil { - t.Error(err) - } - - output := buf.String() - t.Logf("output: %q\n", buf.Bytes()) - - if !strings.Contains(output, "boom - make an explosive entrance") { - t.Errorf("want help to contain %q, did not: \n%q", "boom - make an explosive entrance", output) - } - } -} - -func TestApp_Run_Version(t *testing.T) { - var versionArguments = [][]string{{"boom", "--version"}, {"boom", "-v"}} - - for _, args := range versionArguments { - buf := new(bytes.Buffer) - - t.Logf("==> checking with arguments %v", args) - - app := NewApp() - app.Name = "boom" - app.Usage = "make an explosive entrance" - app.Version = "0.1.0" - app.Writer = buf - app.Action = func(c *Context) { - buf.WriteString("boom I say!") - } - - err := app.Run(args) - if err != nil { - t.Error(err) - } - - output := buf.String() - t.Logf("output: %q\n", buf.Bytes()) - - if !strings.Contains(output, "0.1.0") { - t.Errorf("want version to contain %q, did not: \n%q", "0.1.0", output) - } - } -} - -func TestApp_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) { - app := NewApp() - app.Action = func(c *Context) {} - app.Before = func(c *Context) error { return fmt.Errorf("before error") } - app.After = func(c *Context) error { return fmt.Errorf("after error") } - - err := app.Run([]string{"foo"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.Contains(err.Error(), "before error") { - t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) - } - if !strings.Contains(err.Error(), "after error") { - t.Errorf("expected text of error from After method, but got none in \"%v\"", err) - } -} - -func TestApp_Run_SubcommandDoesNotOverwriteErrorFromBefore(t *testing.T) { - app := NewApp() - app.Commands = []Command{ - Command{ - Subcommands: []Command{ - Command{ - Name: "sub", - }, - }, - Name: "bar", - Before: func(c *Context) error { return fmt.Errorf("before error") }, - After: func(c *Context) error { return fmt.Errorf("after error") }, - }, - } - - err := app.Run([]string{"foo", "bar"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.Contains(err.Error(), "before error") { - t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) - } - if !strings.Contains(err.Error(), "after error") { - t.Errorf("expected text of error from After method, but got none in \"%v\"", err) - } -} - -func TestApp_OnUsageError_WithWrongFlagValue(t *testing.T) { - app := NewApp() - app.Flags = []Flag{ - IntFlag{Name: "flag"}, - } - app.OnUsageError = func(c *Context, err error, isSubcommand bool) error { - if isSubcommand { - t.Errorf("Expect no subcommand") - } - if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { - t.Errorf("Expect an invalid value error, but got \"%v\"", err) - } - return errors.New("intercepted: " + err.Error()) - } - app.Commands = []Command{ - Command{ - Name: "bar", - }, - } - - err := app.Run([]string{"foo", "--flag=wrong"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { - t.Errorf("Expect an intercepted error, but got \"%v\"", err) - } -} - -func TestApp_OnUsageError_WithWrongFlagValue_ForSubcommand(t *testing.T) { - app := NewApp() - app.Flags = []Flag{ - IntFlag{Name: "flag"}, - } - app.OnUsageError = func(c *Context, err error, isSubcommand bool) error { - if isSubcommand { - t.Errorf("Expect subcommand") - } - if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { - t.Errorf("Expect an invalid value error, but got \"%v\"", err) - } - return errors.New("intercepted: " + err.Error()) - } - app.Commands = []Command{ - Command{ - Name: "bar", - }, - } - - err := app.Run([]string{"foo", "--flag=wrong", "bar"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { - t.Errorf("Expect an intercepted error, but got \"%v\"", err) - } -} diff --git a/vendor/github.com/codegangsta/cli/appveyor.yml b/vendor/github.com/codegangsta/cli/appveyor.yml deleted file mode 100644 index 3ca7afa..0000000 --- a/vendor/github.com/codegangsta/cli/appveyor.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: "{build}" - -os: Windows Server 2012 R2 - -install: - - go version - - go env - -build_script: - - cd %APPVEYOR_BUILD_FOLDER% - - go vet ./... - - go test -v ./... - -test: off - -deploy: off diff --git a/vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete b/vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete deleted file mode 100644 index 21a232f..0000000 --- a/vendor/github.com/codegangsta/cli/autocomplete/bash_autocomplete +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash - -: ${PROG:=$(basename ${BASH_SOURCE})} - -_cli_bash_autocomplete() { - local cur opts base - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - opts=$( ${COMP_WORDS[@]:0:$COMP_CWORD} --generate-bash-completion ) - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - return 0 - } - - complete -F _cli_bash_autocomplete $PROG diff --git a/vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete b/vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete deleted file mode 100644 index 5430a18..0000000 --- a/vendor/github.com/codegangsta/cli/autocomplete/zsh_autocomplete +++ /dev/null @@ -1,5 +0,0 @@ -autoload -U compinit && compinit -autoload -U bashcompinit && bashcompinit - -script_dir=$(dirname $0) -source ${script_dir}/bash_autocomplete diff --git a/vendor/github.com/codegangsta/cli/cli.go b/vendor/github.com/codegangsta/cli/cli.go deleted file mode 100644 index 31dc912..0000000 --- a/vendor/github.com/codegangsta/cli/cli.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package cli provides a minimal framework for creating and organizing command line -// Go applications. cli is designed to be easy to understand and write, the most simple -// cli application can be written as follows: -// func main() { -// cli.NewApp().Run(os.Args) -// } -// -// Of course this application does not do much, so let's make this an actual application: -// func main() { -// app := cli.NewApp() -// app.Name = "greet" -// app.Usage = "say a greeting" -// app.Action = func(c *cli.Context) { -// println("Greetings") -// } -// -// app.Run(os.Args) -// } -package cli - -import ( - "strings" -) - -type MultiError struct { - Errors []error -} - -func NewMultiError(err ...error) MultiError { - return MultiError{Errors: err} -} - -func (m MultiError) Error() string { - errs := make([]string, len(m.Errors)) - for i, err := range m.Errors { - errs[i] = err.Error() - } - - return strings.Join(errs, "\n") -} diff --git a/vendor/github.com/codegangsta/cli/command.go b/vendor/github.com/codegangsta/cli/command.go deleted file mode 100644 index 0153713..0000000 --- a/vendor/github.com/codegangsta/cli/command.go +++ /dev/null @@ -1,250 +0,0 @@ -package cli - -import ( - "fmt" - "io/ioutil" - "strings" -) - -// Command is a subcommand for a cli.App. -type Command struct { - // The name of the command - Name string - // short name of the command. Typically one character (deprecated, use `Aliases`) - ShortName string - // A list of aliases for the command - Aliases []string - // A short description of the usage of this command - Usage string - // Custom text to show on USAGE section of help - UsageText string - // A longer explanation of how the command works - Description string - // A short description of the arguments of this command - ArgsUsage string - // The function to call when checking for bash command completions - BashComplete func(context *Context) - // An action to execute before any sub-subcommands are run, but after the context is ready - // If a non-nil error is returned, no sub-subcommands are run - Before func(context *Context) error - // An action to execute after any subcommands are run, but before the subcommand has finished - // It is run even if Action() panics - After func(context *Context) error - // The function to call when this command is invoked - Action func(context *Context) - // Execute this function, if an usage error occurs. This is useful for displaying customized usage error messages. - // This function is able to replace the original error messages. - // If this function is not set, the "Incorrect usage" is displayed and the execution is interrupted. - OnUsageError func(context *Context, err error) error - // List of child commands - Subcommands []Command - // List of flags to parse - Flags []Flag - // Treat all flags as normal arguments if true - SkipFlagParsing bool - // Boolean to hide built-in help command - HideHelp bool - - // Full name of command for help, defaults to full command name, including parent commands. - HelpName string - commandNamePath []string -} - -// Returns the full name of the command. -// For subcommands this ensures that parent commands are part of the command path -func (c Command) FullName() string { - if c.commandNamePath == nil { - return c.Name - } - return strings.Join(c.commandNamePath, " ") -} - -// Invokes the command given the context, parses ctx.Args() to generate command-specific flags -func (c Command) Run(ctx *Context) (err error) { - if len(c.Subcommands) > 0 { - return c.startApp(ctx) - } - - if !c.HideHelp && (HelpFlag != BoolFlag{}) { - // append help to flags - c.Flags = append( - c.Flags, - HelpFlag, - ) - } - - if ctx.App.EnableBashCompletion { - c.Flags = append(c.Flags, BashCompletionFlag) - } - - set := flagSet(c.Name, c.Flags) - set.SetOutput(ioutil.Discard) - - if !c.SkipFlagParsing { - firstFlagIndex := -1 - terminatorIndex := -1 - for index, arg := range ctx.Args() { - if arg == "--" { - terminatorIndex = index - break - } else if arg == "-" { - // Do nothing. A dash alone is not really a flag. - continue - } else if strings.HasPrefix(arg, "-") && firstFlagIndex == -1 { - firstFlagIndex = index - } - } - - if firstFlagIndex > -1 { - args := ctx.Args() - regularArgs := make([]string, len(args[1:firstFlagIndex])) - copy(regularArgs, args[1:firstFlagIndex]) - - var flagArgs []string - if terminatorIndex > -1 { - flagArgs = args[firstFlagIndex:terminatorIndex] - regularArgs = append(regularArgs, args[terminatorIndex:]...) - } else { - flagArgs = args[firstFlagIndex:] - } - - err = set.Parse(append(flagArgs, regularArgs...)) - } else { - err = set.Parse(ctx.Args().Tail()) - } - } else { - if c.SkipFlagParsing { - err = set.Parse(append([]string{"--"}, ctx.Args().Tail()...)) - } - } - - if err != nil { - if c.OnUsageError != nil { - err := c.OnUsageError(ctx, err) - return err - } else { - fmt.Fprintln(ctx.App.Writer, "Incorrect Usage.") - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) - return err - } - } - - nerr := normalizeFlags(c.Flags, set) - if nerr != nil { - fmt.Fprintln(ctx.App.Writer, nerr) - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) - return nerr - } - context := NewContext(ctx.App, set, ctx) - - if checkCommandCompletions(context, c.Name) { - return nil - } - - if checkCommandHelp(context, c.Name) { - return nil - } - - if c.After != nil { - defer func() { - afterErr := c.After(context) - if afterErr != nil { - if err != nil { - err = NewMultiError(err, afterErr) - } else { - err = afterErr - } - } - }() - } - - if c.Before != nil { - err := c.Before(context) - if err != nil { - fmt.Fprintln(ctx.App.Writer, err) - fmt.Fprintln(ctx.App.Writer) - ShowCommandHelp(ctx, c.Name) - return err - } - } - - context.Command = c - c.Action(context) - return nil -} - -func (c Command) Names() []string { - names := []string{c.Name} - - if c.ShortName != "" { - names = append(names, c.ShortName) - } - - return append(names, c.Aliases...) -} - -// Returns true if Command.Name or Command.ShortName matches given name -func (c Command) HasName(name string) bool { - for _, n := range c.Names() { - if n == name { - return true - } - } - return false -} - -func (c Command) startApp(ctx *Context) error { - app := NewApp() - - // set the name and usage - app.Name = fmt.Sprintf("%s %s", ctx.App.Name, c.Name) - if c.HelpName == "" { - app.HelpName = c.HelpName - } else { - app.HelpName = app.Name - } - - if c.Description != "" { - app.Usage = c.Description - } else { - app.Usage = c.Usage - } - - // set CommandNotFound - app.CommandNotFound = ctx.App.CommandNotFound - - // set the flags and commands - app.Commands = c.Subcommands - app.Flags = c.Flags - app.HideHelp = c.HideHelp - - app.Version = ctx.App.Version - app.HideVersion = ctx.App.HideVersion - app.Compiled = ctx.App.Compiled - app.Author = ctx.App.Author - app.Email = ctx.App.Email - app.Writer = ctx.App.Writer - - // bash completion - app.EnableBashCompletion = ctx.App.EnableBashCompletion - if c.BashComplete != nil { - app.BashComplete = c.BashComplete - } - - // set the actions - app.Before = c.Before - app.After = c.After - if c.Action != nil { - app.Action = c.Action - } else { - app.Action = helpSubcommand.Action - } - - for index, cc := range app.Commands { - app.Commands[index].commandNamePath = []string{c.Name, cc.Name} - } - - return app.RunAsSubcommand(ctx) -} diff --git a/vendor/github.com/codegangsta/cli/command_test.go b/vendor/github.com/codegangsta/cli/command_test.go deleted file mode 100644 index 827da1d..0000000 --- a/vendor/github.com/codegangsta/cli/command_test.go +++ /dev/null @@ -1,97 +0,0 @@ -package cli - -import ( - "errors" - "flag" - "fmt" - "io/ioutil" - "strings" - "testing" -) - -func TestCommandFlagParsing(t *testing.T) { - cases := []struct { - testArgs []string - skipFlagParsing bool - expectedErr error - }{ - {[]string{"blah", "blah", "-break"}, false, errors.New("flag provided but not defined: -break")}, // Test normal "not ignoring flags" flow - {[]string{"blah", "blah"}, true, nil}, // Test SkipFlagParsing without any args that look like flags - {[]string{"blah", "-break"}, true, nil}, // Test SkipFlagParsing with random flag arg - {[]string{"blah", "-help"}, true, nil}, // Test SkipFlagParsing with "special" help flag arg - } - - for _, c := range cases { - app := NewApp() - app.Writer = ioutil.Discard - set := flag.NewFlagSet("test", 0) - set.Parse(c.testArgs) - - context := NewContext(app, set, nil) - - command := Command{ - Name: "test-cmd", - Aliases: []string{"tc"}, - Usage: "this is for testing", - Description: "testing", - Action: func(_ *Context) {}, - } - - command.SkipFlagParsing = c.skipFlagParsing - - err := command.Run(context) - - expect(t, err, c.expectedErr) - expect(t, []string(context.Args()), c.testArgs) - } -} - -func TestCommand_Run_DoesNotOverwriteErrorFromBefore(t *testing.T) { - app := NewApp() - app.Commands = []Command{ - Command{ - Name: "bar", - Before: func(c *Context) error { return fmt.Errorf("before error") }, - After: func(c *Context) error { return fmt.Errorf("after error") }, - }, - } - - err := app.Run([]string{"foo", "bar"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.Contains(err.Error(), "before error") { - t.Errorf("expected text of error from Before method, but got none in \"%v\"", err) - } - if !strings.Contains(err.Error(), "after error") { - t.Errorf("expected text of error from After method, but got none in \"%v\"", err) - } -} - -func TestCommand_OnUsageError_WithWrongFlagValue(t *testing.T) { - app := NewApp() - app.Commands = []Command{ - Command{ - Name: "bar", - Flags: []Flag{ - IntFlag{Name: "flag"}, - }, - OnUsageError: func(c *Context, err error) error { - if !strings.HasPrefix(err.Error(), "invalid value \"wrong\"") { - t.Errorf("Expect an invalid value error, but got \"%v\"", err) - } - return errors.New("intercepted: " + err.Error()) - }, - }, - } - - err := app.Run([]string{"foo", "bar", "--flag=wrong"}) - if err == nil { - t.Fatalf("expected to receive error from Run, got none") - } - - if !strings.HasPrefix(err.Error(), "intercepted: invalid value") { - t.Errorf("Expect an intercepted error, but got \"%v\"", err) - } -} diff --git a/vendor/github.com/codegangsta/cli/context.go b/vendor/github.com/codegangsta/cli/context.go deleted file mode 100644 index b66d278..0000000 --- a/vendor/github.com/codegangsta/cli/context.go +++ /dev/null @@ -1,393 +0,0 @@ -package cli - -import ( - "errors" - "flag" - "strconv" - "strings" - "time" -) - -// Context is a type that is passed through to -// each Handler action in a cli application. Context -// can be used to retrieve context-specific Args and -// parsed command-line options. -type Context struct { - App *App - Command Command - flagSet *flag.FlagSet - setFlags map[string]bool - globalSetFlags map[string]bool - parentContext *Context -} - -// Creates a new context. For use in when invoking an App or Command action. -func NewContext(app *App, set *flag.FlagSet, parentCtx *Context) *Context { - return &Context{App: app, flagSet: set, parentContext: parentCtx} -} - -// Looks up the value of a local int flag, returns 0 if no int flag exists -func (c *Context) Int(name string) int { - return lookupInt(name, c.flagSet) -} - -// Looks up the value of a local time.Duration flag, returns 0 if no time.Duration flag exists -func (c *Context) Duration(name string) time.Duration { - return lookupDuration(name, c.flagSet) -} - -// Looks up the value of a local float64 flag, returns 0 if no float64 flag exists -func (c *Context) Float64(name string) float64 { - return lookupFloat64(name, c.flagSet) -} - -// Looks up the value of a local bool flag, returns false if no bool flag exists -func (c *Context) Bool(name string) bool { - return lookupBool(name, c.flagSet) -} - -// Looks up the value of a local boolT flag, returns false if no bool flag exists -func (c *Context) BoolT(name string) bool { - return lookupBoolT(name, c.flagSet) -} - -// Looks up the value of a local string flag, returns "" if no string flag exists -func (c *Context) String(name string) string { - return lookupString(name, c.flagSet) -} - -// Looks up the value of a local string slice flag, returns nil if no string slice flag exists -func (c *Context) StringSlice(name string) []string { - return lookupStringSlice(name, c.flagSet) -} - -// Looks up the value of a local int slice flag, returns nil if no int slice flag exists -func (c *Context) IntSlice(name string) []int { - return lookupIntSlice(name, c.flagSet) -} - -// Looks up the value of a local generic flag, returns nil if no generic flag exists -func (c *Context) Generic(name string) interface{} { - return lookupGeneric(name, c.flagSet) -} - -// Looks up the value of a global int flag, returns 0 if no int flag exists -func (c *Context) GlobalInt(name string) int { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupInt(name, fs) - } - return 0 -} - -// Looks up the value of a global time.Duration flag, returns 0 if no time.Duration flag exists -func (c *Context) GlobalDuration(name string) time.Duration { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupDuration(name, fs) - } - return 0 -} - -// Looks up the value of a global bool flag, returns false if no bool flag exists -func (c *Context) GlobalBool(name string) bool { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupBool(name, fs) - } - return false -} - -// Looks up the value of a global string flag, returns "" if no string flag exists -func (c *Context) GlobalString(name string) string { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupString(name, fs) - } - return "" -} - -// Looks up the value of a global string slice flag, returns nil if no string slice flag exists -func (c *Context) GlobalStringSlice(name string) []string { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupStringSlice(name, fs) - } - return nil -} - -// Looks up the value of a global int slice flag, returns nil if no int slice flag exists -func (c *Context) GlobalIntSlice(name string) []int { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupIntSlice(name, fs) - } - return nil -} - -// Looks up the value of a global generic flag, returns nil if no generic flag exists -func (c *Context) GlobalGeneric(name string) interface{} { - if fs := lookupGlobalFlagSet(name, c); fs != nil { - return lookupGeneric(name, fs) - } - return nil -} - -// Returns the number of flags set -func (c *Context) NumFlags() int { - return c.flagSet.NFlag() -} - -// Determines if the flag was actually set -func (c *Context) IsSet(name string) bool { - if c.setFlags == nil { - c.setFlags = make(map[string]bool) - c.flagSet.Visit(func(f *flag.Flag) { - c.setFlags[f.Name] = true - }) - } - return c.setFlags[name] == true -} - -// Determines if the global flag was actually set -func (c *Context) GlobalIsSet(name string) bool { - if c.globalSetFlags == nil { - c.globalSetFlags = make(map[string]bool) - ctx := c - if ctx.parentContext != nil { - ctx = ctx.parentContext - } - for ; ctx != nil && c.globalSetFlags[name] == false; ctx = ctx.parentContext { - ctx.flagSet.Visit(func(f *flag.Flag) { - c.globalSetFlags[f.Name] = true - }) - } - } - return c.globalSetFlags[name] -} - -// Returns a slice of flag names used in this context. -func (c *Context) FlagNames() (names []string) { - for _, flag := range c.Command.Flags { - name := strings.Split(flag.GetName(), ",")[0] - if name == "help" { - continue - } - names = append(names, name) - } - return -} - -// Returns a slice of global flag names used by the app. -func (c *Context) GlobalFlagNames() (names []string) { - for _, flag := range c.App.Flags { - name := strings.Split(flag.GetName(), ",")[0] - if name == "help" || name == "version" { - continue - } - names = append(names, name) - } - return -} - -// Returns the parent context, if any -func (c *Context) Parent() *Context { - return c.parentContext -} - -type Args []string - -// Returns the command line arguments associated with the context. -func (c *Context) Args() Args { - args := Args(c.flagSet.Args()) - return args -} - -// Returns the number of the command line arguments. -func (c *Context) NArg() int { - return len(c.Args()) -} - -// Returns the nth argument, or else a blank string -func (a Args) Get(n int) string { - if len(a) > n { - return a[n] - } - return "" -} - -// Returns the first argument, or else a blank string -func (a Args) First() string { - return a.Get(0) -} - -// Return the rest of the arguments (not the first one) -// or else an empty string slice -func (a Args) Tail() []string { - if len(a) >= 2 { - return []string(a)[1:] - } - return []string{} -} - -// Checks if there are any arguments present -func (a Args) Present() bool { - return len(a) != 0 -} - -// Swaps arguments at the given indexes -func (a Args) Swap(from, to int) error { - if from >= len(a) || to >= len(a) { - return errors.New("index out of range") - } - a[from], a[to] = a[to], a[from] - return nil -} - -func lookupGlobalFlagSet(name string, ctx *Context) *flag.FlagSet { - if ctx.parentContext != nil { - ctx = ctx.parentContext - } - for ; ctx != nil; ctx = ctx.parentContext { - if f := ctx.flagSet.Lookup(name); f != nil { - return ctx.flagSet - } - } - return nil -} - -func lookupInt(name string, set *flag.FlagSet) int { - f := set.Lookup(name) - if f != nil { - val, err := strconv.Atoi(f.Value.String()) - if err != nil { - return 0 - } - return val - } - - return 0 -} - -func lookupDuration(name string, set *flag.FlagSet) time.Duration { - f := set.Lookup(name) - if f != nil { - val, err := time.ParseDuration(f.Value.String()) - if err == nil { - return val - } - } - - return 0 -} - -func lookupFloat64(name string, set *flag.FlagSet) float64 { - f := set.Lookup(name) - if f != nil { - val, err := strconv.ParseFloat(f.Value.String(), 64) - if err != nil { - return 0 - } - return val - } - - return 0 -} - -func lookupString(name string, set *flag.FlagSet) string { - f := set.Lookup(name) - if f != nil { - return f.Value.String() - } - - return "" -} - -func lookupStringSlice(name string, set *flag.FlagSet) []string { - f := set.Lookup(name) - if f != nil { - return (f.Value.(*StringSlice)).Value() - - } - - return nil -} - -func lookupIntSlice(name string, set *flag.FlagSet) []int { - f := set.Lookup(name) - if f != nil { - return (f.Value.(*IntSlice)).Value() - - } - - return nil -} - -func lookupGeneric(name string, set *flag.FlagSet) interface{} { - f := set.Lookup(name) - if f != nil { - return f.Value - } - return nil -} - -func lookupBool(name string, set *flag.FlagSet) bool { - f := set.Lookup(name) - if f != nil { - val, err := strconv.ParseBool(f.Value.String()) - if err != nil { - return false - } - return val - } - - return false -} - -func lookupBoolT(name string, set *flag.FlagSet) bool { - f := set.Lookup(name) - if f != nil { - val, err := strconv.ParseBool(f.Value.String()) - if err != nil { - return true - } - return val - } - - return false -} - -func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) { - switch ff.Value.(type) { - case *StringSlice: - default: - set.Set(name, ff.Value.String()) - } -} - -func normalizeFlags(flags []Flag, set *flag.FlagSet) error { - visited := make(map[string]bool) - set.Visit(func(f *flag.Flag) { - visited[f.Name] = true - }) - for _, f := range flags { - parts := strings.Split(f.GetName(), ",") - if len(parts) == 1 { - continue - } - var ff *flag.Flag - for _, name := range parts { - name = strings.Trim(name, " ") - if visited[name] { - if ff != nil { - return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name) - } - ff = set.Lookup(name) - } - } - if ff == nil { - continue - } - for _, name := range parts { - name = strings.Trim(name, " ") - if !visited[name] { - copyFlag(name, ff, set) - } - } - } - return nil -} diff --git a/vendor/github.com/codegangsta/cli/context_test.go b/vendor/github.com/codegangsta/cli/context_test.go deleted file mode 100644 index b8ab37d..0000000 --- a/vendor/github.com/codegangsta/cli/context_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package cli - -import ( - "flag" - "testing" - "time" -) - -func TestNewContext(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Int("myflag", 12, "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Int("myflag", 42, "doc") - globalCtx := NewContext(nil, globalSet, nil) - command := Command{Name: "mycommand"} - c := NewContext(nil, set, globalCtx) - c.Command = command - expect(t, c.Int("myflag"), 12) - expect(t, c.GlobalInt("myflag"), 42) - expect(t, c.Command.Name, "mycommand") -} - -func TestContext_Int(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Int("myflag", 12, "doc") - c := NewContext(nil, set, nil) - expect(t, c.Int("myflag"), 12) -} - -func TestContext_Duration(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Duration("myflag", time.Duration(12*time.Second), "doc") - c := NewContext(nil, set, nil) - expect(t, c.Duration("myflag"), time.Duration(12*time.Second)) -} - -func TestContext_String(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.String("myflag", "hello world", "doc") - c := NewContext(nil, set, nil) - expect(t, c.String("myflag"), "hello world") -} - -func TestContext_Bool(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - c := NewContext(nil, set, nil) - expect(t, c.Bool("myflag"), false) -} - -func TestContext_BoolT(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", true, "doc") - c := NewContext(nil, set, nil) - expect(t, c.BoolT("myflag"), true) -} - -func TestContext_Args(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - c := NewContext(nil, set, nil) - set.Parse([]string{"--myflag", "bat", "baz"}) - expect(t, len(c.Args()), 2) - expect(t, c.Bool("myflag"), true) -} - -func TestContext_NArg(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - c := NewContext(nil, set, nil) - set.Parse([]string{"--myflag", "bat", "baz"}) - expect(t, c.NArg(), 2) -} - -func TestContext_IsSet(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - set.String("otherflag", "hello world", "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Bool("myflagGlobal", true, "doc") - globalCtx := NewContext(nil, globalSet, nil) - c := NewContext(nil, set, globalCtx) - set.Parse([]string{"--myflag", "bat", "baz"}) - globalSet.Parse([]string{"--myflagGlobal", "bat", "baz"}) - expect(t, c.IsSet("myflag"), true) - expect(t, c.IsSet("otherflag"), false) - expect(t, c.IsSet("bogusflag"), false) - expect(t, c.IsSet("myflagGlobal"), false) -} - -func TestContext_GlobalIsSet(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - set.String("otherflag", "hello world", "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Bool("myflagGlobal", true, "doc") - globalSet.Bool("myflagGlobalUnset", true, "doc") - globalCtx := NewContext(nil, globalSet, nil) - c := NewContext(nil, set, globalCtx) - set.Parse([]string{"--myflag", "bat", "baz"}) - globalSet.Parse([]string{"--myflagGlobal", "bat", "baz"}) - expect(t, c.GlobalIsSet("myflag"), false) - expect(t, c.GlobalIsSet("otherflag"), false) - expect(t, c.GlobalIsSet("bogusflag"), false) - expect(t, c.GlobalIsSet("myflagGlobal"), true) - expect(t, c.GlobalIsSet("myflagGlobalUnset"), false) - expect(t, c.GlobalIsSet("bogusGlobal"), false) -} - -func TestContext_NumFlags(t *testing.T) { - set := flag.NewFlagSet("test", 0) - set.Bool("myflag", false, "doc") - set.String("otherflag", "hello world", "doc") - globalSet := flag.NewFlagSet("test", 0) - globalSet.Bool("myflagGlobal", true, "doc") - globalCtx := NewContext(nil, globalSet, nil) - c := NewContext(nil, set, globalCtx) - set.Parse([]string{"--myflag", "--otherflag=foo"}) - globalSet.Parse([]string{"--myflagGlobal"}) - expect(t, c.NumFlags(), 2) -} diff --git a/vendor/github.com/codegangsta/cli/flag.go b/vendor/github.com/codegangsta/cli/flag.go deleted file mode 100644 index e951c2d..0000000 --- a/vendor/github.com/codegangsta/cli/flag.go +++ /dev/null @@ -1,546 +0,0 @@ -package cli - -import ( - "flag" - "fmt" - "os" - "runtime" - "strconv" - "strings" - "time" -) - -// This flag enables bash-completion for all commands and subcommands -var BashCompletionFlag = BoolFlag{ - Name: "generate-bash-completion", -} - -// This flag prints the version for the application -var VersionFlag = BoolFlag{ - Name: "version, v", - Usage: "print the version", -} - -// This flag prints the help for all commands and subcommands -// Set to the zero value (BoolFlag{}) to disable flag -- keeps subcommand -// unless HideHelp is set to true) -var HelpFlag = BoolFlag{ - Name: "help, h", - Usage: "show help", -} - -// Flag is a common interface related to parsing flags in cli. -// For more advanced flag parsing techniques, it is recommended that -// this interface be implemented. -type Flag interface { - fmt.Stringer - // Apply Flag settings to the given flag set - Apply(*flag.FlagSet) - GetName() string -} - -func flagSet(name string, flags []Flag) *flag.FlagSet { - set := flag.NewFlagSet(name, flag.ContinueOnError) - - for _, f := range flags { - f.Apply(set) - } - return set -} - -func eachName(longName string, fn func(string)) { - parts := strings.Split(longName, ",") - for _, name := range parts { - name = strings.Trim(name, " ") - fn(name) - } -} - -// Generic is a generic parseable type identified by a specific flag -type Generic interface { - Set(value string) error - String() string -} - -// GenericFlag is the flag type for types implementing Generic -type GenericFlag struct { - Name string - Value Generic - Usage string - EnvVar string -} - -// String returns the string representation of the generic flag to display the -// help text to the user (uses the String() method of the generic flag to show -// the value) -func (f GenericFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) -} - -func (f GenericFlag) FormatValueHelp() string { - if f.Value == nil { - return "" - } - s := f.Value.String() - if len(s) == 0 { - return "" - } - return fmt.Sprintf("\"%s\"", s) -} - -// Apply takes the flagset and calls Set on the generic flag with the value -// provided by the user for parsing by the flag -func (f GenericFlag) Apply(set *flag.FlagSet) { - val := f.Value - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - val.Set(envVal) - break - } - } - } - - eachName(f.Name, func(name string) { - set.Var(f.Value, name, f.Usage) - }) -} - -func (f GenericFlag) GetName() string { - return f.Name -} - -// StringSlice is an opaque type for []string to satisfy flag.Value -type StringSlice []string - -// Set appends the string value to the list of values -func (f *StringSlice) Set(value string) error { - *f = append(*f, value) - return nil -} - -// String returns a readable representation of this value (for usage defaults) -func (f *StringSlice) String() string { - return fmt.Sprintf("%s", *f) -} - -// Value returns the slice of strings set by this flag -func (f *StringSlice) Value() []string { - return *f -} - -// StringSlice is a string flag that can be specified multiple times on the -// command-line -type StringSliceFlag struct { - Name string - Value *StringSlice - Usage string - EnvVar string -} - -// String returns the usage -func (f StringSliceFlag) String() string { - firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") - pref := prefixFor(firstName) - return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f StringSliceFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - newVal := &StringSlice{} - for _, s := range strings.Split(envVal, ",") { - s = strings.TrimSpace(s) - newVal.Set(s) - } - f.Value = newVal - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Value == nil { - f.Value = &StringSlice{} - } - set.Var(f.Value, name, f.Usage) - }) -} - -func (f StringSliceFlag) GetName() string { - return f.Name -} - -// StringSlice is an opaque type for []int to satisfy flag.Value -type IntSlice []int - -// Set parses the value into an integer and appends it to the list of values -func (f *IntSlice) Set(value string) error { - tmp, err := strconv.Atoi(value) - if err != nil { - return err - } else { - *f = append(*f, tmp) - } - return nil -} - -// String returns a readable representation of this value (for usage defaults) -func (f *IntSlice) String() string { - return fmt.Sprintf("%d", *f) -} - -// Value returns the slice of ints set by this flag -func (f *IntSlice) Value() []int { - return *f -} - -// IntSliceFlag is an int flag that can be specified multiple times on the -// command-line -type IntSliceFlag struct { - Name string - Value *IntSlice - Usage string - EnvVar string -} - -// String returns the usage -func (f IntSliceFlag) String() string { - firstName := strings.Trim(strings.Split(f.Name, ",")[0], " ") - pref := prefixFor(firstName) - return withEnvHint(f.EnvVar, fmt.Sprintf("%s [%v]\t%v", prefixedNames(f.Name), pref+firstName+" option "+pref+firstName+" option", f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f IntSliceFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - newVal := &IntSlice{} - for _, s := range strings.Split(envVal, ",") { - s = strings.TrimSpace(s) - err := newVal.Set(s) - if err != nil { - fmt.Fprintf(os.Stderr, err.Error()) - } - } - f.Value = newVal - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Value == nil { - f.Value = &IntSlice{} - } - set.Var(f.Value, name, f.Usage) - }) -} - -func (f IntSliceFlag) GetName() string { - return f.Name -} - -// BoolFlag is a switch that defaults to false -type BoolFlag struct { - Name string - Usage string - EnvVar string - Destination *bool -} - -// String returns a readable representation of this value (for usage defaults) -func (f BoolFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f BoolFlag) Apply(set *flag.FlagSet) { - val := false - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValBool, err := strconv.ParseBool(envVal) - if err == nil { - val = envValBool - } - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.BoolVar(f.Destination, name, val, f.Usage) - return - } - set.Bool(name, val, f.Usage) - }) -} - -func (f BoolFlag) GetName() string { - return f.Name -} - -// BoolTFlag this represents a boolean flag that is true by default, but can -// still be set to false by --some-flag=false -type BoolTFlag struct { - Name string - Usage string - EnvVar string - Destination *bool -} - -// String returns a readable representation of this value (for usage defaults) -func (f BoolTFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s\t%v", prefixedNames(f.Name), f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f BoolTFlag) Apply(set *flag.FlagSet) { - val := true - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValBool, err := strconv.ParseBool(envVal) - if err == nil { - val = envValBool - break - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.BoolVar(f.Destination, name, val, f.Usage) - return - } - set.Bool(name, val, f.Usage) - }) -} - -func (f BoolTFlag) GetName() string { - return f.Name -} - -// StringFlag represents a flag that takes as string value -type StringFlag struct { - Name string - Value string - Usage string - EnvVar string - Destination *string -} - -// String returns the usage -func (f StringFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s %v\t%v", prefixedNames(f.Name), f.FormatValueHelp(), f.Usage)) -} - -func (f StringFlag) FormatValueHelp() string { - s := f.Value - if len(s) == 0 { - return "" - } - return fmt.Sprintf("\"%s\"", s) -} - -// Apply populates the flag given the flag set and environment -func (f StringFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - f.Value = envVal - break - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.StringVar(f.Destination, name, f.Value, f.Usage) - return - } - set.String(name, f.Value, f.Usage) - }) -} - -func (f StringFlag) GetName() string { - return f.Name -} - -// IntFlag is a flag that takes an integer -// Errors if the value provided cannot be parsed -type IntFlag struct { - Name string - Value int - Usage string - EnvVar string - Destination *int -} - -// String returns the usage -func (f IntFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f IntFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValInt, err := strconv.ParseInt(envVal, 0, 64) - if err == nil { - f.Value = int(envValInt) - break - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.IntVar(f.Destination, name, f.Value, f.Usage) - return - } - set.Int(name, f.Value, f.Usage) - }) -} - -func (f IntFlag) GetName() string { - return f.Name -} - -// DurationFlag is a flag that takes a duration specified in Go's duration -// format: https://golang.org/pkg/time/#ParseDuration -type DurationFlag struct { - Name string - Value time.Duration - Usage string - EnvVar string - Destination *time.Duration -} - -// String returns a readable representation of this value (for usage defaults) -func (f DurationFlag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f DurationFlag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValDuration, err := time.ParseDuration(envVal) - if err == nil { - f.Value = envValDuration - break - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.DurationVar(f.Destination, name, f.Value, f.Usage) - return - } - set.Duration(name, f.Value, f.Usage) - }) -} - -func (f DurationFlag) GetName() string { - return f.Name -} - -// Float64Flag is a flag that takes an float value -// Errors if the value provided cannot be parsed -type Float64Flag struct { - Name string - Value float64 - Usage string - EnvVar string - Destination *float64 -} - -// String returns the usage -func (f Float64Flag) String() string { - return withEnvHint(f.EnvVar, fmt.Sprintf("%s \"%v\"\t%v", prefixedNames(f.Name), f.Value, f.Usage)) -} - -// Apply populates the flag given the flag set and environment -func (f Float64Flag) Apply(set *flag.FlagSet) { - if f.EnvVar != "" { - for _, envVar := range strings.Split(f.EnvVar, ",") { - envVar = strings.TrimSpace(envVar) - if envVal := os.Getenv(envVar); envVal != "" { - envValFloat, err := strconv.ParseFloat(envVal, 10) - if err == nil { - f.Value = float64(envValFloat) - } - } - } - } - - eachName(f.Name, func(name string) { - if f.Destination != nil { - set.Float64Var(f.Destination, name, f.Value, f.Usage) - return - } - set.Float64(name, f.Value, f.Usage) - }) -} - -func (f Float64Flag) GetName() string { - return f.Name -} - -func prefixFor(name string) (prefix string) { - if len(name) == 1 { - prefix = "-" - } else { - prefix = "--" - } - - return -} - -func prefixedNames(fullName string) (prefixed string) { - parts := strings.Split(fullName, ",") - for i, name := range parts { - name = strings.Trim(name, " ") - prefixed += prefixFor(name) + name - if i < len(parts)-1 { - prefixed += ", " - } - } - return -} - -func withEnvHint(envVar, str string) string { - envText := "" - if envVar != "" { - prefix := "$" - suffix := "" - sep := ", $" - if runtime.GOOS == "windows" { - prefix = "%" - suffix = "%" - sep = "%, %" - } - envText = fmt.Sprintf(" [%s%s%s]", prefix, strings.Join(strings.Split(envVar, ","), sep), suffix) - } - return str + envText -} diff --git a/vendor/github.com/codegangsta/cli/flag_test.go b/vendor/github.com/codegangsta/cli/flag_test.go deleted file mode 100644 index 3caa70a..0000000 --- a/vendor/github.com/codegangsta/cli/flag_test.go +++ /dev/null @@ -1,859 +0,0 @@ -package cli - -import ( - "fmt" - "os" - "reflect" - "strings" - "testing" - "runtime" -) - -var boolFlagTests = []struct { - name string - expected string -}{ - {"help", "--help\t"}, - {"h", "-h\t"}, -} - -func TestBoolFlagHelpOutput(t *testing.T) { - - for _, test := range boolFlagTests { - flag := BoolFlag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -var stringFlagTests = []struct { - name string - value string - expected string -}{ - {"help", "", "--help \t"}, - {"h", "", "-h \t"}, - {"h", "", "-h \t"}, - {"test", "Something", "--test \"Something\"\t"}, -} - -func TestStringFlagHelpOutput(t *testing.T) { - - for _, test := range stringFlagTests { - flag := StringFlag{Name: test.name, Value: test.value} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestStringFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_FOO", "derp") - for _, test := range stringFlagTests { - flag := StringFlag{Name: test.name, Value: test.value, EnvVar: "APP_FOO"} - output := flag.String() - - expectedSuffix := " [$APP_FOO]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_FOO%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var stringSliceFlagTests = []struct { - name string - value *StringSlice - expected string -}{ - {"help", func() *StringSlice { - s := &StringSlice{} - s.Set("") - return s - }(), "--help [--help option --help option]\t"}, - {"h", func() *StringSlice { - s := &StringSlice{} - s.Set("") - return s - }(), "-h [-h option -h option]\t"}, - {"h", func() *StringSlice { - s := &StringSlice{} - s.Set("") - return s - }(), "-h [-h option -h option]\t"}, - {"test", func() *StringSlice { - s := &StringSlice{} - s.Set("Something") - return s - }(), "--test [--test option --test option]\t"}, -} - -func TestStringSliceFlagHelpOutput(t *testing.T) { - - for _, test := range stringSliceFlagTests { - flag := StringSliceFlag{Name: test.name, Value: test.value} - output := flag.String() - - if output != test.expected { - t.Errorf("%q does not match %q", output, test.expected) - } - } -} - -func TestStringSliceFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_QWWX", "11,4") - for _, test := range stringSliceFlagTests { - flag := StringSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_QWWX"} - output := flag.String() - - expectedSuffix := " [$APP_QWWX]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_QWWX%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%q does not end with" + expectedSuffix, output) - } - } -} - -var intFlagTests = []struct { - name string - expected string -}{ - {"help", "--help \"0\"\t"}, - {"h", "-h \"0\"\t"}, -} - -func TestIntFlagHelpOutput(t *testing.T) { - - for _, test := range intFlagTests { - flag := IntFlag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestIntFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_BAR", "2") - for _, test := range intFlagTests { - flag := IntFlag{Name: test.name, EnvVar: "APP_BAR"} - output := flag.String() - - expectedSuffix := " [$APP_BAR]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_BAR%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var durationFlagTests = []struct { - name string - expected string -}{ - {"help", "--help \"0\"\t"}, - {"h", "-h \"0\"\t"}, -} - -func TestDurationFlagHelpOutput(t *testing.T) { - - for _, test := range durationFlagTests { - flag := DurationFlag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestDurationFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_BAR", "2h3m6s") - for _, test := range durationFlagTests { - flag := DurationFlag{Name: test.name, EnvVar: "APP_BAR"} - output := flag.String() - - expectedSuffix := " [$APP_BAR]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_BAR%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var intSliceFlagTests = []struct { - name string - value *IntSlice - expected string -}{ - {"help", &IntSlice{}, "--help [--help option --help option]\t"}, - {"h", &IntSlice{}, "-h [-h option -h option]\t"}, - {"h", &IntSlice{}, "-h [-h option -h option]\t"}, - {"test", func() *IntSlice { - i := &IntSlice{} - i.Set("9") - return i - }(), "--test [--test option --test option]\t"}, -} - -func TestIntSliceFlagHelpOutput(t *testing.T) { - - for _, test := range intSliceFlagTests { - flag := IntSliceFlag{Name: test.name, Value: test.value} - output := flag.String() - - if output != test.expected { - t.Errorf("%q does not match %q", output, test.expected) - } - } -} - -func TestIntSliceFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_SMURF", "42,3") - for _, test := range intSliceFlagTests { - flag := IntSliceFlag{Name: test.name, Value: test.value, EnvVar: "APP_SMURF"} - output := flag.String() - - expectedSuffix := " [$APP_SMURF]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_SMURF%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%q does not end with" + expectedSuffix, output) - } - } -} - -var float64FlagTests = []struct { - name string - expected string -}{ - {"help", "--help \"0\"\t"}, - {"h", "-h \"0\"\t"}, -} - -func TestFloat64FlagHelpOutput(t *testing.T) { - - for _, test := range float64FlagTests { - flag := Float64Flag{Name: test.name} - output := flag.String() - - if output != test.expected { - t.Errorf("%s does not match %s", output, test.expected) - } - } -} - -func TestFloat64FlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_BAZ", "99.4") - for _, test := range float64FlagTests { - flag := Float64Flag{Name: test.name, EnvVar: "APP_BAZ"} - output := flag.String() - - expectedSuffix := " [$APP_BAZ]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_BAZ%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -var genericFlagTests = []struct { - name string - value Generic - expected string -}{ - {"test", &Parser{"abc", "def"}, "--test \"abc,def\"\ttest flag"}, - {"t", &Parser{"abc", "def"}, "-t \"abc,def\"\ttest flag"}, -} - -func TestGenericFlagHelpOutput(t *testing.T) { - - for _, test := range genericFlagTests { - flag := GenericFlag{Name: test.name, Value: test.value, Usage: "test flag"} - output := flag.String() - - if output != test.expected { - t.Errorf("%q does not match %q", output, test.expected) - } - } -} - -func TestGenericFlagWithEnvVarHelpOutput(t *testing.T) { - os.Clearenv() - os.Setenv("APP_ZAP", "3") - for _, test := range genericFlagTests { - flag := GenericFlag{Name: test.name, EnvVar: "APP_ZAP"} - output := flag.String() - - expectedSuffix := " [$APP_ZAP]" - if runtime.GOOS == "windows" { - expectedSuffix = " [%APP_ZAP%]" - } - if !strings.HasSuffix(output, expectedSuffix) { - t.Errorf("%s does not end with" + expectedSuffix, output) - } - } -} - -func TestParseMultiString(t *testing.T) { - (&App{ - Flags: []Flag{ - StringFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.String("serve") != "10" { - t.Errorf("main name not set") - } - if ctx.String("s") != "10" { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run", "-s", "10"}) -} - -func TestParseDestinationString(t *testing.T) { - var dest string - a := App{ - Flags: []Flag{ - StringFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != "10" { - t.Errorf("expected destination String 10") - } - }, - } - a.Run([]string{"run", "--dest", "10"}) -} - -func TestParseMultiStringFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_COUNT", "20") - (&App{ - Flags: []Flag{ - StringFlag{Name: "count, c", EnvVar: "APP_COUNT"}, - }, - Action: func(ctx *Context) { - if ctx.String("count") != "20" { - t.Errorf("main name not set") - } - if ctx.String("c") != "20" { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiStringFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_COUNT", "20") - (&App{ - Flags: []Flag{ - StringFlag{Name: "count, c", EnvVar: "COMPAT_COUNT,APP_COUNT"}, - }, - Action: func(ctx *Context) { - if ctx.String("count") != "20" { - t.Errorf("main name not set") - } - if ctx.String("c") != "20" { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiStringSlice(t *testing.T) { - (&App{ - Flags: []Flag{ - StringSliceFlag{Name: "serve, s", Value: &StringSlice{}}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.StringSlice("serve"), []string{"10", "20"}) { - t.Errorf("main name not set") - } - if !reflect.DeepEqual(ctx.StringSlice("s"), []string{"10", "20"}) { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run", "-s", "10", "-s", "20"}) -} - -func TestParseMultiStringSliceFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - StringSliceFlag{Name: "intervals, i", Value: &StringSlice{}, EnvVar: "APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiStringSliceFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - StringSliceFlag{Name: "intervals, i", Value: &StringSlice{}, EnvVar: "COMPAT_INTERVALS,APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.StringSlice("intervals"), []string{"20", "30", "40"}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.StringSlice("i"), []string{"20", "30", "40"}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiInt(t *testing.T) { - a := App{ - Flags: []Flag{ - IntFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.Int("serve") != 10 { - t.Errorf("main name not set") - } - if ctx.Int("s") != 10 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "-s", "10"}) -} - -func TestParseDestinationInt(t *testing.T) { - var dest int - a := App{ - Flags: []Flag{ - IntFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != 10 { - t.Errorf("expected destination Int 10") - } - }, - } - a.Run([]string{"run", "--dest", "10"}) -} - -func TestParseMultiIntFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "10") - a := App{ - Flags: []Flag{ - IntFlag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Int("timeout") != 10 { - t.Errorf("main name not set") - } - if ctx.Int("t") != 10 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiIntFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "10") - a := App{ - Flags: []Flag{ - IntFlag{Name: "timeout, t", EnvVar: "COMPAT_TIMEOUT_SECONDS,APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Int("timeout") != 10 { - t.Errorf("main name not set") - } - if ctx.Int("t") != 10 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiIntSlice(t *testing.T) { - (&App{ - Flags: []Flag{ - IntSliceFlag{Name: "serve, s", Value: &IntSlice{}}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.IntSlice("serve"), []int{10, 20}) { - t.Errorf("main name not set") - } - if !reflect.DeepEqual(ctx.IntSlice("s"), []int{10, 20}) { - t.Errorf("short name not set") - } - }, - }).Run([]string{"run", "-s", "10", "-s", "20"}) -} - -func TestParseMultiIntSliceFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - IntSliceFlag{Name: "intervals, i", Value: &IntSlice{}, EnvVar: "APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiIntSliceFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_INTERVALS", "20,30,40") - - (&App{ - Flags: []Flag{ - IntSliceFlag{Name: "intervals, i", Value: &IntSlice{}, EnvVar: "COMPAT_INTERVALS,APP_INTERVALS"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.IntSlice("intervals"), []int{20, 30, 40}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.IntSlice("i"), []int{20, 30, 40}) { - t.Errorf("short name not set from env") - } - }, - }).Run([]string{"run"}) -} - -func TestParseMultiFloat64(t *testing.T) { - a := App{ - Flags: []Flag{ - Float64Flag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.Float64("serve") != 10.2 { - t.Errorf("main name not set") - } - if ctx.Float64("s") != 10.2 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "-s", "10.2"}) -} - -func TestParseDestinationFloat64(t *testing.T) { - var dest float64 - a := App{ - Flags: []Flag{ - Float64Flag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != 10.2 { - t.Errorf("expected destination Float64 10.2") - } - }, - } - a.Run([]string{"run", "--dest", "10.2"}) -} - -func TestParseMultiFloat64FromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "15.5") - a := App{ - Flags: []Flag{ - Float64Flag{Name: "timeout, t", EnvVar: "APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Float64("timeout") != 15.5 { - t.Errorf("main name not set") - } - if ctx.Float64("t") != 15.5 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiFloat64FromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_TIMEOUT_SECONDS", "15.5") - a := App{ - Flags: []Flag{ - Float64Flag{Name: "timeout, t", EnvVar: "COMPAT_TIMEOUT_SECONDS,APP_TIMEOUT_SECONDS"}, - }, - Action: func(ctx *Context) { - if ctx.Float64("timeout") != 15.5 { - t.Errorf("main name not set") - } - if ctx.Float64("t") != 15.5 { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBool(t *testing.T) { - a := App{ - Flags: []Flag{ - BoolFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("serve") != true { - t.Errorf("main name not set") - } - if ctx.Bool("s") != true { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "--serve"}) -} - -func TestParseDestinationBool(t *testing.T) { - var dest bool - a := App{ - Flags: []Flag{ - BoolFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != true { - t.Errorf("expected destination Bool true") - } - }, - } - a.Run([]string{"run", "--dest"}) -} - -func TestParseMultiBoolFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "1") - a := App{ - Flags: []Flag{ - BoolFlag{Name: "debug, d", EnvVar: "APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("debug") != true { - t.Errorf("main name not set from env") - } - if ctx.Bool("d") != true { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBoolFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "1") - a := App{ - Flags: []Flag{ - BoolFlag{Name: "debug, d", EnvVar: "COMPAT_DEBUG,APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("debug") != true { - t.Errorf("main name not set from env") - } - if ctx.Bool("d") != true { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBoolT(t *testing.T) { - a := App{ - Flags: []Flag{ - BoolTFlag{Name: "serve, s"}, - }, - Action: func(ctx *Context) { - if ctx.BoolT("serve") != true { - t.Errorf("main name not set") - } - if ctx.BoolT("s") != true { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "--serve"}) -} - -func TestParseDestinationBoolT(t *testing.T) { - var dest bool - a := App{ - Flags: []Flag{ - BoolTFlag{ - Name: "dest", - Destination: &dest, - }, - }, - Action: func(ctx *Context) { - if dest != true { - t.Errorf("expected destination BoolT true") - } - }, - } - a.Run([]string{"run", "--dest"}) -} - -func TestParseMultiBoolTFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "0") - a := App{ - Flags: []Flag{ - BoolTFlag{Name: "debug, d", EnvVar: "APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.BoolT("debug") != false { - t.Errorf("main name not set from env") - } - if ctx.BoolT("d") != false { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseMultiBoolTFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_DEBUG", "0") - a := App{ - Flags: []Flag{ - BoolTFlag{Name: "debug, d", EnvVar: "COMPAT_DEBUG,APP_DEBUG"}, - }, - Action: func(ctx *Context) { - if ctx.BoolT("debug") != false { - t.Errorf("main name not set from env") - } - if ctx.BoolT("d") != false { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -type Parser [2]string - -func (p *Parser) Set(value string) error { - parts := strings.Split(value, ",") - if len(parts) != 2 { - return fmt.Errorf("invalid format") - } - - (*p)[0] = parts[0] - (*p)[1] = parts[1] - - return nil -} - -func (p *Parser) String() string { - return fmt.Sprintf("%s,%s", p[0], p[1]) -} - -func TestParseGeneric(t *testing.T) { - a := App{ - Flags: []Flag{ - GenericFlag{Name: "serve, s", Value: &Parser{}}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"10", "20"}) { - t.Errorf("main name not set") - } - if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"10", "20"}) { - t.Errorf("short name not set") - } - }, - } - a.Run([]string{"run", "-s", "10,20"}) -} - -func TestParseGenericFromEnv(t *testing.T) { - os.Clearenv() - os.Setenv("APP_SERVE", "20,30") - a := App{ - Flags: []Flag{ - GenericFlag{Name: "serve, s", Value: &Parser{}, EnvVar: "APP_SERVE"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.Generic("serve"), &Parser{"20", "30"}) { - t.Errorf("main name not set from env") - } - if !reflect.DeepEqual(ctx.Generic("s"), &Parser{"20", "30"}) { - t.Errorf("short name not set from env") - } - }, - } - a.Run([]string{"run"}) -} - -func TestParseGenericFromEnvCascade(t *testing.T) { - os.Clearenv() - os.Setenv("APP_FOO", "99,2000") - a := App{ - Flags: []Flag{ - GenericFlag{Name: "foos", Value: &Parser{}, EnvVar: "COMPAT_FOO,APP_FOO"}, - }, - Action: func(ctx *Context) { - if !reflect.DeepEqual(ctx.Generic("foos"), &Parser{"99", "2000"}) { - t.Errorf("value not set from env") - } - }, - } - a.Run([]string{"run"}) -} diff --git a/vendor/github.com/codegangsta/cli/help.go b/vendor/github.com/codegangsta/cli/help.go deleted file mode 100644 index 15916f8..0000000 --- a/vendor/github.com/codegangsta/cli/help.go +++ /dev/null @@ -1,248 +0,0 @@ -package cli - -import ( - "fmt" - "io" - "strings" - "text/tabwriter" - "text/template" -) - -// The text template for the Default help topic. -// cli.go uses text/template to render templates. You can -// render custom help text by setting this variable. -var AppHelpTemplate = `NAME: - {{.Name}} - {{.Usage}} - -USAGE: - {{if .UsageText}}{{.UsageText}}{{else}}{{.HelpName}} {{if .Flags}}[global options]{{end}}{{if .Commands}} command [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{end}} - {{if .Version}} -VERSION: - {{.Version}} - {{end}}{{if len .Authors}} -AUTHOR(S): - {{range .Authors}}{{ . }}{{end}} - {{end}}{{if .Commands}} -COMMANDS: - {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -GLOBAL OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}}{{if .Copyright }} -COPYRIGHT: - {{.Copyright}} - {{end}} -` - -// The text template for the command help topic. -// cli.go uses text/template to render templates. You can -// render custom help text by setting this variable. -var CommandHelpTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}}{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}}{{if .Description}} - -DESCRIPTION: - {{.Description}}{{end}}{{if .Flags}} - -OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{ end }} -` - -// The text template for the subcommand help topic. -// cli.go uses text/template to render templates. You can -// render custom help text by setting this variable. -var SubcommandHelpTemplate = `NAME: - {{.HelpName}} - {{.Usage}} - -USAGE: - {{.HelpName}} command{{if .Flags}} [command options]{{end}} {{if .ArgsUsage}}{{.ArgsUsage}}{{else}}[arguments...]{{end}} - -COMMANDS: - {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} - {{end}}{{if .Flags}} -OPTIONS: - {{range .Flags}}{{.}} - {{end}}{{end}} -` - -var helpCommand = Command{ - Name: "help", - Aliases: []string{"h"}, - Usage: "Shows a list of commands or help for one command", - ArgsUsage: "[command]", - Action: func(c *Context) { - args := c.Args() - if args.Present() { - ShowCommandHelp(c, args.First()) - } else { - ShowAppHelp(c) - } - }, -} - -var helpSubcommand = Command{ - Name: "help", - Aliases: []string{"h"}, - Usage: "Shows a list of commands or help for one command", - ArgsUsage: "[command]", - Action: func(c *Context) { - args := c.Args() - if args.Present() { - ShowCommandHelp(c, args.First()) - } else { - ShowSubcommandHelp(c) - } - }, -} - -// Prints help for the App or Command -type helpPrinter func(w io.Writer, templ string, data interface{}) - -var HelpPrinter helpPrinter = printHelp - -// Prints version for the App -var VersionPrinter = printVersion - -func ShowAppHelp(c *Context) { - HelpPrinter(c.App.Writer, AppHelpTemplate, c.App) -} - -// Prints the list of subcommands as the default app completion method -func DefaultAppComplete(c *Context) { - for _, command := range c.App.Commands { - for _, name := range command.Names() { - fmt.Fprintln(c.App.Writer, name) - } - } -} - -// Prints help for the given command -func ShowCommandHelp(ctx *Context, command string) { - // show the subcommand help for a command with subcommands - if command == "" { - HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App) - return - } - - for _, c := range ctx.App.Commands { - if c.HasName(command) { - HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c) - return - } - } - - if ctx.App.CommandNotFound != nil { - ctx.App.CommandNotFound(ctx, command) - } else { - fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command) - } -} - -// Prints help for the given subcommand -func ShowSubcommandHelp(c *Context) { - ShowCommandHelp(c, c.Command.Name) -} - -// Prints the version number of the App -func ShowVersion(c *Context) { - VersionPrinter(c) -} - -func printVersion(c *Context) { - fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version) -} - -// Prints the lists of commands within a given context -func ShowCompletions(c *Context) { - a := c.App - if a != nil && a.BashComplete != nil { - a.BashComplete(c) - } -} - -// Prints the custom completions for a given command -func ShowCommandCompletions(ctx *Context, command string) { - c := ctx.App.Command(command) - if c != nil && c.BashComplete != nil { - c.BashComplete(ctx) - } -} - -func printHelp(out io.Writer, templ string, data interface{}) { - funcMap := template.FuncMap{ - "join": strings.Join, - } - - w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0) - t := template.Must(template.New("help").Funcs(funcMap).Parse(templ)) - err := t.Execute(w, data) - if err != nil { - // If the writer is closed, t.Execute will fail, and there's nothing - // we can do to recover. We could send this to os.Stderr if we need. - return - } - w.Flush() -} - -func checkVersion(c *Context) bool { - found := false - if VersionFlag.Name != "" { - eachName(VersionFlag.Name, func(name string) { - if c.GlobalBool(name) || c.Bool(name) { - found = true - } - }) - } - return found -} - -func checkHelp(c *Context) bool { - found := false - if HelpFlag.Name != "" { - eachName(HelpFlag.Name, func(name string) { - if c.GlobalBool(name) || c.Bool(name) { - found = true - } - }) - } - return found -} - -func checkCommandHelp(c *Context, name string) bool { - if c.Bool("h") || c.Bool("help") { - ShowCommandHelp(c, name) - return true - } - - return false -} - -func checkSubcommandHelp(c *Context) bool { - if c.GlobalBool("h") || c.GlobalBool("help") { - ShowSubcommandHelp(c) - return true - } - - return false -} - -func checkCompletions(c *Context) bool { - if (c.GlobalBool(BashCompletionFlag.Name) || c.Bool(BashCompletionFlag.Name)) && c.App.EnableBashCompletion { - ShowCompletions(c) - return true - } - - return false -} - -func checkCommandCompletions(c *Context, name string) bool { - if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion { - ShowCommandCompletions(c, name) - return true - } - - return false -} diff --git a/vendor/github.com/codegangsta/cli/help_test.go b/vendor/github.com/codegangsta/cli/help_test.go deleted file mode 100644 index 350e263..0000000 --- a/vendor/github.com/codegangsta/cli/help_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package cli - -import ( - "bytes" - "testing" -) - -func Test_ShowAppHelp_NoAuthor(t *testing.T) { - output := new(bytes.Buffer) - app := NewApp() - app.Writer = output - - c := NewContext(app, nil, nil) - - ShowAppHelp(c) - - if bytes.Index(output.Bytes(), []byte("AUTHOR(S):")) != -1 { - t.Errorf("expected\n%snot to include %s", output.String(), "AUTHOR(S):") - } -} - -func Test_ShowAppHelp_NoVersion(t *testing.T) { - output := new(bytes.Buffer) - app := NewApp() - app.Writer = output - - app.Version = "" - - c := NewContext(app, nil, nil) - - ShowAppHelp(c) - - if bytes.Index(output.Bytes(), []byte("VERSION:")) != -1 { - t.Errorf("expected\n%snot to include %s", output.String(), "VERSION:") - } -} - -func Test_Help_Custom_Flags(t *testing.T) { - oldFlag := HelpFlag - defer func() { - HelpFlag = oldFlag - }() - - HelpFlag = BoolFlag{ - Name: "help, x", - Usage: "show help", - } - - app := App{ - Flags: []Flag{ - BoolFlag{Name: "foo, h"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("h") != true { - t.Errorf("custom help flag not set") - } - }, - } - output := new(bytes.Buffer) - app.Writer = output - app.Run([]string{"test", "-h"}) - if output.Len() > 0 { - t.Errorf("unexpected output: %s", output.String()) - } -} - -func Test_Version_Custom_Flags(t *testing.T) { - oldFlag := VersionFlag - defer func() { - VersionFlag = oldFlag - }() - - VersionFlag = BoolFlag{ - Name: "version, V", - Usage: "show version", - } - - app := App{ - Flags: []Flag{ - BoolFlag{Name: "foo, v"}, - }, - Action: func(ctx *Context) { - if ctx.Bool("v") != true { - t.Errorf("custom version flag not set") - } - }, - } - output := new(bytes.Buffer) - app.Writer = output - app.Run([]string{"test", "-v"}) - if output.Len() > 0 { - t.Errorf("unexpected output: %s", output.String()) - } -} diff --git a/vendor/github.com/codegangsta/cli/helpers_test.go b/vendor/github.com/codegangsta/cli/helpers_test.go deleted file mode 100644 index b1b7339..0000000 --- a/vendor/github.com/codegangsta/cli/helpers_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package cli - -import ( - "reflect" - "testing" -) - -/* Test Helpers */ -func expect(t *testing.T, a interface{}, b interface{}) { - if !reflect.DeepEqual(a, b) { - t.Errorf("Expected %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} - -func refute(t *testing.T, a interface{}, b interface{}) { - if reflect.DeepEqual(a, b) { - t.Errorf("Did not expect %v (type %v) - Got %v (type %v)", b, reflect.TypeOf(b), a, reflect.TypeOf(a)) - } -} diff --git a/vendor/github.com/petar/GoLLRB/llrb/avgvar.go b/vendor/github.com/petar/GoLLRB/llrb/avgvar.go deleted file mode 100644 index 2d7e2a3..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/avgvar.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -import "math" - -// avgVar maintains the average and variance of a stream of numbers -// in a space-efficient manner. -type avgVar struct { - count int64 - sum, sumsq float64 -} - -func (av *avgVar) Init() { - av.count = 0 - av.sum = 0.0 - av.sumsq = 0.0 -} - -func (av *avgVar) Add(sample float64) { - av.count++ - av.sum += sample - av.sumsq += sample * sample -} - -func (av *avgVar) GetCount() int64 { return av.count } - -func (av *avgVar) GetAvg() float64 { return av.sum / float64(av.count) } - -func (av *avgVar) GetTotal() float64 { return av.sum } - -func (av *avgVar) GetVar() float64 { - a := av.GetAvg() - return av.sumsq/float64(av.count) - a*a -} - -func (av *avgVar) GetStdDev() float64 { return math.Sqrt(av.GetVar()) } diff --git a/vendor/github.com/petar/GoLLRB/llrb/iterator.go b/vendor/github.com/petar/GoLLRB/llrb/iterator.go deleted file mode 100644 index ee7b27f..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/iterator.go +++ /dev/null @@ -1,93 +0,0 @@ -package llrb - -type ItemIterator func(i Item) bool - -//func (t *Tree) Ascend(iterator ItemIterator) { -// t.AscendGreaterOrEqual(Inf(-1), iterator) -//} - -func (t *LLRB) AscendRange(greaterOrEqual, lessThan Item, iterator ItemIterator) { - t.ascendRange(t.root, greaterOrEqual, lessThan, iterator) -} - -func (t *LLRB) ascendRange(h *Node, inf, sup Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if !less(h.Item, sup) { - return t.ascendRange(h.Left, inf, sup, iterator) - } - if less(h.Item, inf) { - return t.ascendRange(h.Right, inf, sup, iterator) - } - - if !t.ascendRange(h.Left, inf, sup, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - return t.ascendRange(h.Right, inf, sup, iterator) -} - -// AscendGreaterOrEqual will call iterator once for each element greater or equal to -// pivot in ascending order. It will stop whenever the iterator returns false. -func (t *LLRB) AscendGreaterOrEqual(pivot Item, iterator ItemIterator) { - t.ascendGreaterOrEqual(t.root, pivot, iterator) -} - -func (t *LLRB) ascendGreaterOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if !less(h.Item, pivot) { - if !t.ascendGreaterOrEqual(h.Left, pivot, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - } - return t.ascendGreaterOrEqual(h.Right, pivot, iterator) -} - -func (t *LLRB) AscendLessThan(pivot Item, iterator ItemIterator) { - t.ascendLessThan(t.root, pivot, iterator) -} - -func (t *LLRB) ascendLessThan(h *Node, pivot Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if !t.ascendLessThan(h.Left, pivot, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - if less(h.Item, pivot) { - return t.ascendLessThan(h.Left, pivot, iterator) - } - return true -} - -// DescendLessOrEqual will call iterator once for each element less than the -// pivot in descending order. It will stop whenever the iterator returns false. -func (t *LLRB) DescendLessOrEqual(pivot Item, iterator ItemIterator) { - t.descendLessOrEqual(t.root, pivot, iterator) -} - -func (t *LLRB) descendLessOrEqual(h *Node, pivot Item, iterator ItemIterator) bool { - if h == nil { - return true - } - if less(h.Item, pivot) || !less(pivot, h.Item) { - if !t.descendLessOrEqual(h.Right, pivot, iterator) { - return false - } - if !iterator(h.Item) { - return false - } - } - return t.descendLessOrEqual(h.Left, pivot, iterator) -} diff --git a/vendor/github.com/petar/GoLLRB/llrb/iterator_test.go b/vendor/github.com/petar/GoLLRB/llrb/iterator_test.go deleted file mode 100644 index db5e12c..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/iterator_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package llrb - -import ( - "reflect" - "testing" -) - -func TestAscendGreaterOrEqual(t *testing.T) { - tree := New() - tree.InsertNoReplace(Int(4)) - tree.InsertNoReplace(Int(6)) - tree.InsertNoReplace(Int(1)) - tree.InsertNoReplace(Int(3)) - var ary []Item - tree.AscendGreaterOrEqual(Int(-1), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected := []Item{Int(1), Int(3), Int(4), Int(6)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.AscendGreaterOrEqual(Int(3), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(3), Int(4), Int(6)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.AscendGreaterOrEqual(Int(2), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(3), Int(4), Int(6)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } -} - -func TestDescendLessOrEqual(t *testing.T) { - tree := New() - tree.InsertNoReplace(Int(4)) - tree.InsertNoReplace(Int(6)) - tree.InsertNoReplace(Int(1)) - tree.InsertNoReplace(Int(3)) - var ary []Item - tree.DescendLessOrEqual(Int(10), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected := []Item{Int(6), Int(4), Int(3), Int(1)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.DescendLessOrEqual(Int(4), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(4), Int(3), Int(1)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } - ary = nil - tree.DescendLessOrEqual(Int(5), func(i Item) bool { - ary = append(ary, i) - return true - }) - expected = []Item{Int(4), Int(3), Int(1)} - if !reflect.DeepEqual(ary, expected) { - t.Errorf("expected %v but got %v", expected, ary) - } -} diff --git a/vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go b/vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go deleted file mode 100644 index 47126a3..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/llrb-stats.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -// GetHeight() returns an item in the tree with key @key, and it's height in the tree -func (t *LLRB) GetHeight(key Item) (result Item, depth int) { - return t.getHeight(t.root, key) -} - -func (t *LLRB) getHeight(h *Node, item Item) (Item, int) { - if h == nil { - return nil, 0 - } - if less(item, h.Item) { - result, depth := t.getHeight(h.Left, item) - return result, depth + 1 - } - if less(h.Item, item) { - result, depth := t.getHeight(h.Right, item) - return result, depth + 1 - } - return h.Item, 0 -} - -// HeightStats() returns the average and standard deviation of the height -// of elements in the tree -func (t *LLRB) HeightStats() (avg, stddev float64) { - av := &avgVar{} - heightStats(t.root, 0, av) - return av.GetAvg(), av.GetStdDev() -} - -func heightStats(h *Node, d int, av *avgVar) { - if h == nil { - return - } - av.Add(float64(d)) - if h.Left != nil { - heightStats(h.Left, d+1, av) - } - if h.Right != nil { - heightStats(h.Right, d+1, av) - } -} diff --git a/vendor/github.com/petar/GoLLRB/llrb/llrb.go b/vendor/github.com/petar/GoLLRB/llrb/llrb.go deleted file mode 100644 index 81373fb..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/llrb.go +++ /dev/null @@ -1,456 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// A Left-Leaning Red-Black (LLRB) implementation of 2-3 balanced binary search trees, -// based on the following work: -// -// http://www.cs.princeton.edu/~rs/talks/LLRB/08Penn.pdf -// http://www.cs.princeton.edu/~rs/talks/LLRB/LLRB.pdf -// http://www.cs.princeton.edu/~rs/talks/LLRB/Java/RedBlackBST.java -// -// 2-3 trees (and the run-time equivalent 2-3-4 trees) are the de facto standard BST -// algoritms found in implementations of Python, Java, and other libraries. The LLRB -// implementation of 2-3 trees is a recent improvement on the traditional implementation, -// observed and documented by Robert Sedgewick. -// -package llrb - -// Tree is a Left-Leaning Red-Black (LLRB) implementation of 2-3 trees -type LLRB struct { - count int - root *Node -} - -type Node struct { - Item - Left, Right *Node // Pointers to left and right child nodes - Black bool // If set, the color of the link (incoming from the parent) is black - // In the LLRB, new nodes are always red, hence the zero-value for node -} - -type Item interface { - Less(than Item) bool -} - -// -func less(x, y Item) bool { - if x == pinf { - return false - } - if x == ninf { - return true - } - return x.Less(y) -} - -// Inf returns an Item that is "bigger than" any other item, if sign is positive. -// Otherwise it returns an Item that is "smaller than" any other item. -func Inf(sign int) Item { - if sign == 0 { - panic("sign") - } - if sign > 0 { - return pinf - } - return ninf -} - -var ( - ninf = nInf{} - pinf = pInf{} -) - -type nInf struct{} - -func (nInf) Less(Item) bool { - return true -} - -type pInf struct{} - -func (pInf) Less(Item) bool { - return false -} - -// New() allocates a new tree -func New() *LLRB { - return &LLRB{} -} - -// SetRoot sets the root node of the tree. -// It is intended to be used by functions that deserialize the tree. -func (t *LLRB) SetRoot(r *Node) { - t.root = r -} - -// Root returns the root node of the tree. -// It is intended to be used by functions that serialize the tree. -func (t *LLRB) Root() *Node { - return t.root -} - -// Len returns the number of nodes in the tree. -func (t *LLRB) Len() int { return t.count } - -// Has returns true if the tree contains an element whose order is the same as that of key. -func (t *LLRB) Has(key Item) bool { - return t.Get(key) != nil -} - -// Get retrieves an element from the tree whose order is the same as that of key. -func (t *LLRB) Get(key Item) Item { - h := t.root - for h != nil { - switch { - case less(key, h.Item): - h = h.Left - case less(h.Item, key): - h = h.Right - default: - return h.Item - } - } - return nil -} - -// Min returns the minimum element in the tree. -func (t *LLRB) Min() Item { - h := t.root - if h == nil { - return nil - } - for h.Left != nil { - h = h.Left - } - return h.Item -} - -// Max returns the maximum element in the tree. -func (t *LLRB) Max() Item { - h := t.root - if h == nil { - return nil - } - for h.Right != nil { - h = h.Right - } - return h.Item -} - -func (t *LLRB) ReplaceOrInsertBulk(items ...Item) { - for _, i := range items { - t.ReplaceOrInsert(i) - } -} - -func (t *LLRB) InsertNoReplaceBulk(items ...Item) { - for _, i := range items { - t.InsertNoReplace(i) - } -} - -// ReplaceOrInsert inserts item into the tree. If an existing -// element has the same order, it is removed from the tree and returned. -func (t *LLRB) ReplaceOrInsert(item Item) Item { - if item == nil { - panic("inserting nil item") - } - var replaced Item - t.root, replaced = t.replaceOrInsert(t.root, item) - t.root.Black = true - if replaced == nil { - t.count++ - } - return replaced -} - -func (t *LLRB) replaceOrInsert(h *Node, item Item) (*Node, Item) { - if h == nil { - return newNode(item), nil - } - - h = walkDownRot23(h) - - var replaced Item - if less(item, h.Item) { // BUG - h.Left, replaced = t.replaceOrInsert(h.Left, item) - } else if less(h.Item, item) { - h.Right, replaced = t.replaceOrInsert(h.Right, item) - } else { - replaced, h.Item = h.Item, item - } - - h = walkUpRot23(h) - - return h, replaced -} - -// InsertNoReplace inserts item into the tree. If an existing -// element has the same order, both elements remain in the tree. -func (t *LLRB) InsertNoReplace(item Item) { - if item == nil { - panic("inserting nil item") - } - t.root = t.insertNoReplace(t.root, item) - t.root.Black = true - t.count++ -} - -func (t *LLRB) insertNoReplace(h *Node, item Item) *Node { - if h == nil { - return newNode(item) - } - - h = walkDownRot23(h) - - if less(item, h.Item) { - h.Left = t.insertNoReplace(h.Left, item) - } else { - h.Right = t.insertNoReplace(h.Right, item) - } - - return walkUpRot23(h) -} - -// Rotation driver routines for 2-3 algorithm - -func walkDownRot23(h *Node) *Node { return h } - -func walkUpRot23(h *Node) *Node { - if isRed(h.Right) && !isRed(h.Left) { - h = rotateLeft(h) - } - - if isRed(h.Left) && isRed(h.Left.Left) { - h = rotateRight(h) - } - - if isRed(h.Left) && isRed(h.Right) { - flip(h) - } - - return h -} - -// Rotation driver routines for 2-3-4 algorithm - -func walkDownRot234(h *Node) *Node { - if isRed(h.Left) && isRed(h.Right) { - flip(h) - } - - return h -} - -func walkUpRot234(h *Node) *Node { - if isRed(h.Right) && !isRed(h.Left) { - h = rotateLeft(h) - } - - if isRed(h.Left) && isRed(h.Left.Left) { - h = rotateRight(h) - } - - return h -} - -// DeleteMin deletes the minimum element in the tree and returns the -// deleted item or nil otherwise. -func (t *LLRB) DeleteMin() Item { - var deleted Item - t.root, deleted = deleteMin(t.root) - if t.root != nil { - t.root.Black = true - } - if deleted != nil { - t.count-- - } - return deleted -} - -// deleteMin code for LLRB 2-3 trees -func deleteMin(h *Node) (*Node, Item) { - if h == nil { - return nil, nil - } - if h.Left == nil { - return nil, h.Item - } - - if !isRed(h.Left) && !isRed(h.Left.Left) { - h = moveRedLeft(h) - } - - var deleted Item - h.Left, deleted = deleteMin(h.Left) - - return fixUp(h), deleted -} - -// DeleteMax deletes the maximum element in the tree and returns -// the deleted item or nil otherwise -func (t *LLRB) DeleteMax() Item { - var deleted Item - t.root, deleted = deleteMax(t.root) - if t.root != nil { - t.root.Black = true - } - if deleted != nil { - t.count-- - } - return deleted -} - -func deleteMax(h *Node) (*Node, Item) { - if h == nil { - return nil, nil - } - if isRed(h.Left) { - h = rotateRight(h) - } - if h.Right == nil { - return nil, h.Item - } - if !isRed(h.Right) && !isRed(h.Right.Left) { - h = moveRedRight(h) - } - var deleted Item - h.Right, deleted = deleteMax(h.Right) - - return fixUp(h), deleted -} - -// Delete deletes an item from the tree whose key equals key. -// The deleted item is return, otherwise nil is returned. -func (t *LLRB) Delete(key Item) Item { - var deleted Item - t.root, deleted = t.delete(t.root, key) - if t.root != nil { - t.root.Black = true - } - if deleted != nil { - t.count-- - } - return deleted -} - -func (t *LLRB) delete(h *Node, item Item) (*Node, Item) { - var deleted Item - if h == nil { - return nil, nil - } - if less(item, h.Item) { - if h.Left == nil { // item not present. Nothing to delete - return h, nil - } - if !isRed(h.Left) && !isRed(h.Left.Left) { - h = moveRedLeft(h) - } - h.Left, deleted = t.delete(h.Left, item) - } else { - if isRed(h.Left) { - h = rotateRight(h) - } - // If @item equals @h.Item and no right children at @h - if !less(h.Item, item) && h.Right == nil { - return nil, h.Item - } - // PETAR: Added 'h.Right != nil' below - if h.Right != nil && !isRed(h.Right) && !isRed(h.Right.Left) { - h = moveRedRight(h) - } - // If @item equals @h.Item, and (from above) 'h.Right != nil' - if !less(h.Item, item) { - var subDeleted Item - h.Right, subDeleted = deleteMin(h.Right) - if subDeleted == nil { - panic("logic") - } - deleted, h.Item = h.Item, subDeleted - } else { // Else, @item is bigger than @h.Item - h.Right, deleted = t.delete(h.Right, item) - } - } - - return fixUp(h), deleted -} - -// Internal node manipulation routines - -func newNode(item Item) *Node { return &Node{Item: item} } - -func isRed(h *Node) bool { - if h == nil { - return false - } - return !h.Black -} - -func rotateLeft(h *Node) *Node { - x := h.Right - if x.Black { - panic("rotating a black link") - } - h.Right = x.Left - x.Left = h - x.Black = h.Black - h.Black = false - return x -} - -func rotateRight(h *Node) *Node { - x := h.Left - if x.Black { - panic("rotating a black link") - } - h.Left = x.Right - x.Right = h - x.Black = h.Black - h.Black = false - return x -} - -// REQUIRE: Left and Right children must be present -func flip(h *Node) { - h.Black = !h.Black - h.Left.Black = !h.Left.Black - h.Right.Black = !h.Right.Black -} - -// REQUIRE: Left and Right children must be present -func moveRedLeft(h *Node) *Node { - flip(h) - if isRed(h.Right.Left) { - h.Right = rotateRight(h.Right) - h = rotateLeft(h) - flip(h) - } - return h -} - -// REQUIRE: Left and Right children must be present -func moveRedRight(h *Node) *Node { - flip(h) - if isRed(h.Left.Left) { - h = rotateRight(h) - flip(h) - } - return h -} - -func fixUp(h *Node) *Node { - if isRed(h.Right) { - h = rotateLeft(h) - } - - if isRed(h.Left) && isRed(h.Left.Left) { - h = rotateRight(h) - } - - if isRed(h.Left) && isRed(h.Right) { - flip(h) - } - - return h -} diff --git a/vendor/github.com/petar/GoLLRB/llrb/llrb_test.go b/vendor/github.com/petar/GoLLRB/llrb/llrb_test.go deleted file mode 100644 index b7bc978..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/llrb_test.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -import ( - "math" - "math/rand" - "testing" -) - -func TestCases(t *testing.T) { - tree := New() - tree.ReplaceOrInsert(Int(1)) - tree.ReplaceOrInsert(Int(1)) - if tree.Len() != 1 { - t.Errorf("expecting len 1") - } - if !tree.Has(Int(1)) { - t.Errorf("expecting to find key=1") - } - - tree.Delete(Int(1)) - if tree.Len() != 0 { - t.Errorf("expecting len 0") - } - if tree.Has(Int(1)) { - t.Errorf("not expecting to find key=1") - } - - tree.Delete(Int(1)) - if tree.Len() != 0 { - t.Errorf("expecting len 0") - } - if tree.Has(Int(1)) { - t.Errorf("not expecting to find key=1") - } -} - -func TestReverseInsertOrder(t *testing.T) { - tree := New() - n := 100 - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(n - i)) - } - i := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - i++ - if item.(Int) != Int(i) { - t.Errorf("bad order: got %d, expect %d", item.(Int), i) - } - return true - }) -} - -func TestRange(t *testing.T) { - tree := New() - order := []String{ - "ab", "aba", "abc", "a", "aa", "aaa", "b", "a-", "a!", - } - for _, i := range order { - tree.ReplaceOrInsert(i) - } - k := 0 - tree.AscendRange(String("ab"), String("ac"), func(item Item) bool { - if k > 3 { - t.Fatalf("returned more items than expected") - } - i1 := order[k] - i2 := item.(String) - if i1 != i2 { - t.Errorf("expecting %s, got %s", i1, i2) - } - k++ - return true - }) -} - -func TestRandomInsertOrder(t *testing.T) { - tree := New() - n := 1000 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - j := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - if item.(Int) != Int(j) { - t.Fatalf("bad order") - } - j++ - return true - }) -} - -func TestRandomReplace(t *testing.T) { - tree := New() - n := 100 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - perm = rand.Perm(n) - for i := 0; i < n; i++ { - if replaced := tree.ReplaceOrInsert(Int(perm[i])); replaced == nil || replaced.(Int) != Int(perm[i]) { - t.Errorf("error replacing") - } - } -} - -func TestRandomInsertSequentialDelete(t *testing.T) { - tree := New() - n := 1000 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - for i := 0; i < n; i++ { - tree.Delete(Int(i)) - } -} - -func TestRandomInsertDeleteNonExistent(t *testing.T) { - tree := New() - n := 100 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - if tree.Delete(Int(200)) != nil { - t.Errorf("deleted non-existent item") - } - if tree.Delete(Int(-2)) != nil { - t.Errorf("deleted non-existent item") - } - for i := 0; i < n; i++ { - if u := tree.Delete(Int(i)); u == nil || u.(Int) != Int(i) { - t.Errorf("delete failed") - } - } - if tree.Delete(Int(200)) != nil { - t.Errorf("deleted non-existent item") - } - if tree.Delete(Int(-2)) != nil { - t.Errorf("deleted non-existent item") - } -} - -func TestRandomInsertPartialDeleteOrder(t *testing.T) { - tree := New() - n := 100 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - for i := 1; i < n-1; i++ { - tree.Delete(Int(i)) - } - j := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - switch j { - case 0: - if item.(Int) != Int(0) { - t.Errorf("expecting 0") - } - case 1: - if item.(Int) != Int(n-1) { - t.Errorf("expecting %d", n-1) - } - } - j++ - return true - }) -} - -func TestRandomInsertStats(t *testing.T) { - tree := New() - n := 100000 - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.ReplaceOrInsert(Int(perm[i])) - } - avg, _ := tree.HeightStats() - expAvg := math.Log2(float64(n)) - 1.5 - if math.Abs(avg-expAvg) >= 2.0 { - t.Errorf("too much deviation from expected average height") - } -} - -func BenchmarkInsert(b *testing.B) { - tree := New() - for i := 0; i < b.N; i++ { - tree.ReplaceOrInsert(Int(b.N - i)) - } -} - -func BenchmarkDelete(b *testing.B) { - b.StopTimer() - tree := New() - for i := 0; i < b.N; i++ { - tree.ReplaceOrInsert(Int(b.N - i)) - } - b.StartTimer() - for i := 0; i < b.N; i++ { - tree.Delete(Int(i)) - } -} - -func BenchmarkDeleteMin(b *testing.B) { - b.StopTimer() - tree := New() - for i := 0; i < b.N; i++ { - tree.ReplaceOrInsert(Int(b.N - i)) - } - b.StartTimer() - for i := 0; i < b.N; i++ { - tree.DeleteMin() - } -} - -func TestInsertNoReplace(t *testing.T) { - tree := New() - n := 1000 - for q := 0; q < 2; q++ { - perm := rand.Perm(n) - for i := 0; i < n; i++ { - tree.InsertNoReplace(Int(perm[i])) - } - } - j := 0 - tree.AscendGreaterOrEqual(Int(0), func(item Item) bool { - if item.(Int) != Int(j/2) { - t.Fatalf("bad order") - } - j++ - return true - }) -} diff --git a/vendor/github.com/petar/GoLLRB/llrb/util.go b/vendor/github.com/petar/GoLLRB/llrb/util.go deleted file mode 100644 index 63dbdb2..0000000 --- a/vendor/github.com/petar/GoLLRB/llrb/util.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2010 Petar Maymounkov. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package llrb - -type Int int - -func (x Int) Less(than Item) bool { - return x < than.(Int) -} - -type String string - -func (x String) Less(than Item) bool { - return x < than.(String) -} From 70d7362c94d092ad2a6562b436def994935771ce Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:34:29 +0100 Subject: [PATCH 04/10] lint code --- build.go | 12 +++++++----- diff.go | 14 ++++++++------ main.go | 1 + patch.go | 14 +++++++++++--- 4 files changed, 27 insertions(+), 14 deletions(-) diff --git a/build.go b/build.go index 7dfb000..1d2bcd6 100644 --- a/build.go +++ b/build.go @@ -1,3 +1,4 @@ +// Package gosync-cmd is a command-line tool based on gosync package main import ( @@ -29,6 +30,7 @@ func init() { ) } +// Build builds a .gosync file for a file func Build(c *cli.Context) { filename := c.Args()[0] blocksize := uint32(c.Int("blocksize")) @@ -48,7 +50,7 @@ func Build(c *cli.Context) { s, _ := inputFile.Stat() // TODO: Error? - file_size := s.Size() + fileSize := s.Size() defer inputFile.Close() @@ -67,12 +69,12 @@ func Build(c *cli.Context) { outputFile, magicString, blocksize, - file_size, + fileSize, []uint16{majorVersion, minorVersion, patchVersion}, ); err != nil { fmt.Fprintf( os.Stderr, - "Error getting file info: %v\n", + "Error getting file info: %q %v\n", filename, err, ) @@ -86,7 +88,7 @@ func Build(c *cli.Context) { if err != nil { fmt.Fprintf( os.Stderr, - "Error generating checksum: %v\n", + "Error generating checksum: %q %v\n", filename, err, ) @@ -97,7 +99,7 @@ func Build(c *cli.Context) { if err != nil { fmt.Fprintf( os.Stderr, - "Error getting file info: %v\n", + "Error getting file info: %q %v\n", filename, err, ) diff --git a/diff.go b/diff.go index 14a4acf..4266df8 100644 --- a/diff.go +++ b/diff.go @@ -29,6 +29,8 @@ func init() { ) } +// Diff compares a file with a reference index, +// and print statistics on the comparison and performance func Diff(c *cli.Context) { localFilename := c.Args()[0] referenceFilename := c.Args()[1] @@ -80,20 +82,20 @@ func Diff(c *cli.Context) { os.Exit(1) } - num_matchers := int64(c.Int("p")) + numMatchers := int64(c.Int("p")) - localFile_size := fi.Size() + localFileSize := fi.Size() // Don't split up small files - if localFile_size < 1024*1024 { - num_matchers = 1 + if localFileSize < 1024*1024 { + numMatchers = 1 } merger, compare := multithreadedMatching( localFile, index, - localFile_size, - num_matchers, + localFileSize, + numMatchers, uint(blocksize), ) diff --git a/main.go b/main.go index 3164a8d..206c1fc 100644 --- a/main.go +++ b/main.go @@ -16,6 +16,7 @@ import ( ) const ( + // DefaultBlockSize is ... DefaultBlockSize = 8192 magicString = "G0S9NC" // just to confirm the file type is used correctly majorVersion = uint16(0) diff --git a/patch.go b/patch.go index 16730d3..076885b 100644 --- a/patch.go +++ b/patch.go @@ -36,7 +36,13 @@ The index should be produced by "gosync build". ) } -// Patch a file +// Patch recreates the reference source file, +// using an index and a local file that is believed to be similar. +// The index should be produced by "gosync build". +// +// is a .gosync file and may be a local, unc network path or http/https url +// is corresponding target and may be a local, unc network path or http/https url +// is optional. If not specified, the local file will be overwritten when done. func Patch(c *cli.Context) { errorWrapper(c, func(c *cli.Context) error { @@ -64,13 +70,15 @@ func Patch(c *cli.Context) { } defer indexReader.Close() - _, _, _, filesize, blocksize, e := readHeadersAndCheck( + // TODO why this err is ignored ? + _, _, _, filesize, blocksize, _ := readHeadersAndCheck( indexReader, magicString, majorVersion, ) - index, checksumLookup, blockCount, err := readIndex( + // TODO why this err is ignored ? + index, checksumLookup, blockCount, _ := readIndex( indexReader, uint(blocksize), ) From 88805d4302c69000b40d2e4eb1c3d38f1b2438ab Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:34:41 +0100 Subject: [PATCH 05/10] add vendor folder --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index daf913b..d8ac91b 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ # Folders _obj _test +vendor # Architecture specific extensions/prefixes *.[568vq] From 30443cb5166401d63c40ab3b01475230f9a54e4e Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:35:28 +0100 Subject: [PATCH 06/10] add support for glide --- glide.lock | 22 ++++++++++++++++++++++ glide.yaml | 11 +++++++++++ 2 files changed, 33 insertions(+) create mode 100644 glide.lock create mode 100644 glide.yaml diff --git a/glide.lock b/glide.lock new file mode 100644 index 0000000..6a31e00 --- /dev/null +++ b/glide.lock @@ -0,0 +1,22 @@ +hash: 3197611f80e97df39d2e397018b4e84ffbdb801b7bed5e92dbbfc1fb09beb8d1 +updated: 2017-02-26T19:21:29.180464642+01:00 +imports: +- name: github.com/codegangsta/cli + version: 0bdeddeeb0f650497d603c4ad7b20cfe685682f6 +- name: github.com/petar/GoLLRB + version: 53be0d36a84c2a886ca057d34b6aa4468df9ccb4 + subpackages: + - llrb +- name: github.com/Redundancy/go-sync + version: 8931874cad5cacc627b94fcbcb182460e174aeef + subpackages: + - blocksources + - chunks + - circularbuffer + - comparer + - filechecksum + - index + - patcher + - patcher/sequential + - rollsum +testImports: [] diff --git a/glide.yaml b/glide.yaml new file mode 100644 index 0000000..dbbe6f7 --- /dev/null +++ b/glide.yaml @@ -0,0 +1,11 @@ +package: github.com/mh-cbon/gosync-cmd +import: +- package: github.com/Redundancy/go-sync + subpackages: + - chunks + - comparer + - filechecksum + - index + - patcher +- package: github.com/codegangsta/cli + version: ^1.19.1 From b3de209fb3babd8d40487fda5b68a83a7fe4e42c Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:46:50 +0100 Subject: [PATCH 07/10] lint package documentation --- main.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 206c1fc..58213d7 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ -/* -gosync is a command-line implementation of the gosync package functionality, primarily as a demonstration of usage -but supposed to be functional in itself. -*/ +// Package gosync-cmd is a command-line implementation +// of the gosync package functionality, +// primarily as a demonstration of usage +// but supposed to be functional in itself. package main import ( From f0abc3b84aab16e4e6838eb48030cda273076e37 Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:47:11 +0100 Subject: [PATCH 08/10] improve readme --- .gitignore | 1 + README.e.md | 32 ++++++++++++++++ README.md | 103 +++++++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 README.e.md diff --git a/.gitignore b/.gitignore index d8ac91b..ce25e0f 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ _cgo_gotypes.go _cgo_export.* _testmain.go +gosync-cmd *.exe *.test diff --git a/README.e.md b/README.e.md new file mode 100644 index 0000000..5bfe30d --- /dev/null +++ b/README.e.md @@ -0,0 +1,32 @@ +# {{.Name}} + +{{template "badge/travis" .}}{{template "badge/appveyor" .}}{{template "badge/godoc" .}} + +{{pkgdoc}} + +This forks the CLI code that was originally in https://github.com/Redundancy/go-sync +In order to separate the library from the tool, and to bring proper vendoring to the tool. + +# Install + +### Glide + +{{template "glide/install" .}} + +### go build + +```sh +go run *.go +go build -o gosync-cmd *.go +``` + +# Usage + +{{cli "./go-sync" "-h"}} +{{cli "./go-sync" "build" "-h"}} +{{cli "./go-sync" "diff" "-h"}} +{{cli "./go-sync" "patch" "-h"}} + +## Cli examples + +??? diff --git a/README.md b/README.md index f4156b8..f983228 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,106 @@ # gosync-cmd -Command-line tool based on gosync + +[![travis Status](https://travis-ci.org/mh-cbon/gosync-cmd.svg?branch=master)](https://travis-ci.org/mh-cbon/gosync-cmd)[![appveyor Status](https://ci.appveyor.com/api/projects/status/github/mh-cbon/gosync-cmd?branch=master&svg=true)](https://ci.appveyor.com/project/mh-cbon/gosync-cmd) +[![GoDoc](https://godoc.org/github.com/mh-cbon/gosync-cmd?status.svg)](http://godoc.org/github.com/mh-cbon/gosync-cmd) + + +Package gosync-cmd is a command-line implementation +of the gosync package functionality, +primarily as a demonstration of usage +but supposed to be functional in itself. + This forks the CLI code that was originally in https://github.com/Redundancy/go-sync In order to separate the library from the tool, and to bring proper vendoring to the tool. + +# Install + +### Glide + + +```sh +mkdir -p $GOPATH/src/github.com/mh-cbon/gosync-cmd +cd $GOPATH/src/github.com/mh-cbon/gosync-cmd +git clone https://github.com/mh-cbon/gosync-cmd.git . +glide install +go install +``` + + +# Usage + + +__$ go-sync -h__ +```sh +NAME: + gosync - Build indexes, patches, patch files + +USAGE: + go-sync [global options] command [command options] [arguments...] + +VERSION: + 0.2.1 + +COMMANDS: + build, b build a .gosync file for a file + diff, d gosync diff + patch, p gosync patch [] + help, h Shows a list of commands or help for one command + +GLOBAL OPTIONS: + --profile enable HTTP profiling + --profilePort value The number of streams to use concurrently (default: 6060) + --help, -h show help + --version, -v print the version +``` + +__$ go-sync build -h__ +```sh +NAME: + go-sync build - build a .gosync file for a file + +USAGE: + go-sync build [command options] [arguments...] + +OPTIONS: + --blocksize value The block size to use for the gosync file (default: 8192) +``` + +__$ go-sync diff -h__ +```sh +NAME: + go-sync diff - gosync diff + +USAGE: + go-sync diff [command options] [arguments...] + +DESCRIPTION: + Compare a file with a reference index, and print statistics on the comparison and performance. + +OPTIONS: + -p value The number of streams to use concurrently (default: 4) +``` + +__$ go-sync patch -h__ +```sh +NAME: + go-sync patch - gosync patch [] + +USAGE: + go-sync patch [command options] [arguments...] + +DESCRIPTION: + Recreate the reference source file, using an index and a local file that is believed to be similar. +The index should be produced by "gosync build". + + is a .gosync file and may be a local, unc network path or http/https url + is corresponding target and may be a local, unc network path or http/https url + is optional. If not specified, the local file will be overwritten when done. + +OPTIONS: + -p value The number of streams to use concurrently (default: 4) +``` + +## Cli examples + +??? From 94ef7e424f69dc5c20849b365934739e57d7771c Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:48:41 +0100 Subject: [PATCH 09/10] improve readme --- README.e.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.e.md b/README.e.md index 5bfe30d..5ca751a 100644 --- a/README.e.md +++ b/README.e.md @@ -13,11 +13,12 @@ In order to separate the library from the tool, and to bring proper vendoring to {{template "glide/install" .}} -### go build +### run/build ```sh go run *.go go build -o gosync-cmd *.go +emd gen -out README.md ``` # Usage From 7bb66a4d609058d785581874164bba48cb42c5fa Mon Sep 17 00:00:00 2001 From: mh-cbon Date: Sun, 26 Feb 2017 19:49:20 +0100 Subject: [PATCH 10/10] improve readme --- README.e.md | 8 ++++---- README.md | 30 +++++++++++++++++++----------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/README.e.md b/README.e.md index 5ca751a..67a7fe4 100644 --- a/README.e.md +++ b/README.e.md @@ -23,10 +23,10 @@ emd gen -out README.md # Usage -{{cli "./go-sync" "-h"}} -{{cli "./go-sync" "build" "-h"}} -{{cli "./go-sync" "diff" "-h"}} -{{cli "./go-sync" "patch" "-h"}} +{{cli "./gosync-cmd" "-h"}} +{{cli "./gosync-cmd" "build" "-h"}} +{{cli "./gosync-cmd" "diff" "-h"}} +{{cli "./gosync-cmd" "patch" "-h"}} ## Cli examples diff --git a/README.md b/README.md index f983228..75987ef 100644 --- a/README.md +++ b/README.md @@ -27,16 +27,24 @@ go install ``` +### run/build + +```sh +go run *.go +go build -o gosync-cmd *.go +emd gen -out README.md +``` + # Usage -__$ go-sync -h__ +__$ gosync-cmd -h__ ```sh NAME: gosync - Build indexes, patches, patch files USAGE: - go-sync [global options] command [command options] [arguments...] + gosync-cmd [global options] command [command options] [arguments...] VERSION: 0.2.1 @@ -54,25 +62,25 @@ GLOBAL OPTIONS: --version, -v print the version ``` -__$ go-sync build -h__ +__$ gosync-cmd build -h__ ```sh NAME: - go-sync build - build a .gosync file for a file + gosync-cmd build - build a .gosync file for a file USAGE: - go-sync build [command options] [arguments...] + gosync-cmd build [command options] [arguments...] OPTIONS: --blocksize value The block size to use for the gosync file (default: 8192) ``` -__$ go-sync diff -h__ +__$ gosync-cmd diff -h__ ```sh NAME: - go-sync diff - gosync diff + gosync-cmd diff - gosync diff USAGE: - go-sync diff [command options] [arguments...] + gosync-cmd diff [command options] [arguments...] DESCRIPTION: Compare a file with a reference index, and print statistics on the comparison and performance. @@ -81,13 +89,13 @@ OPTIONS: -p value The number of streams to use concurrently (default: 4) ``` -__$ go-sync patch -h__ +__$ gosync-cmd patch -h__ ```sh NAME: - go-sync patch - gosync patch [] + gosync-cmd patch - gosync patch [] USAGE: - go-sync patch [command options] [arguments...] + gosync-cmd patch [command options] [arguments...] DESCRIPTION: Recreate the reference source file, using an index and a local file that is believed to be similar.