diff --git a/kadai3-2/gosagawa/.gitignore b/kadai3-2/gosagawa/.gitignore new file mode 100644 index 0000000..140fada --- /dev/null +++ b/kadai3-2/gosagawa/.gitignore @@ -0,0 +1 @@ +vendor/* diff --git a/kadai3-2/gosagawa/Gopkg.lock b/kadai3-2/gosagawa/Gopkg.lock new file mode 100644 index 0000000..5c4bb3a --- /dev/null +++ b/kadai3-2/gosagawa/Gopkg.lock @@ -0,0 +1,27 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/pkg/errors" + packages = ["."] + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = ["context"] + revision = "161cd47e91fd58ac17490ef4d742dc98bb4cf60e" + +[[projects]] + branch = "master" + name = "golang.org/x/sync" + packages = ["errgroup"] + revision = "1d60e4601c6fd243af51cc01ddf169918a5407ca" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "addf19f649c9f71072f5d8adea40ec75d9b8c864d1952b47c11ad47c2cf7eefa" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/kadai3-2/gosagawa/Gopkg.toml b/kadai3-2/gosagawa/Gopkg.toml new file mode 100644 index 0000000..e69de29 diff --git a/kadai3-2/gosagawa/README.md b/kadai3-2/gosagawa/README.md new file mode 100644 index 0000000..7815817 --- /dev/null +++ b/kadai3-2/gosagawa/README.md @@ -0,0 +1,21 @@ +# kadai3-2 + +##分割ダウンロードを行う +- Rangeアクセスを用いる +- いくつかのゴルーチンでダウンロードしてマージする +- エラー処理を工夫する + - golang.org/x/sync/errgourpパッケージなどを使ってみる +- キャンセルが発生した場合の実装を行う + +##install + +``` +dep ensure +``` + +##usage + +``` +go run main.go +``` + diff --git a/kadai3-2/gosagawa/main.go b/kadai3-2/gosagawa/main.go new file mode 100644 index 0000000..d065347 --- /dev/null +++ b/kadai3-2/gosagawa/main.go @@ -0,0 +1,163 @@ +package main + +import ( + "fmt" + "io" + "net/http" + "os" + "runtime" + "strings" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" +) + +func main() { + + url := os.Args[1] + fmt.Println(url) + + //TODO check args + + //TODO check RangeAccess available + + d, err := NewDownloader(url) + if err != nil { + fmt.Printf("\ndownload initialize error. %v", err) + os.Exit(1) + } + + err = d.Download() + if err != nil { + fmt.Printf("\ndownload error. %v", err) + os.Exit(1) + } + err = d.Merge() + if err != nil { + fmt.Printf("\ndownload merge error. %v", err) + os.Exit(1) + } +} + +type Downloader struct { + FileName string + MaxProcess uint + Workers []*worker +} + +type worker struct { + processId uint + bytesToStartReading uint + bytesToFinishReading uint + resourceUrl string + partFilePath string +} + +func NewDownloader(url string) (*Downloader, error) { + d := new(Downloader) + d.FileName = getFileName(url) + d.MaxProcess = uint(runtime.NumCPU()) + + res, err := http.Head(url) + if err != nil { + return nil, err + } + + size := uint(res.ContentLength) + + split := size / d.MaxProcess + for i := uint(0); i < d.MaxProcess; i++ { + w, err := NewWorker(d, size, i, split, url) + if err != nil { + return nil, errors.Wrap(err, "initialize worker error") + } + d.Workers = append(d.Workers, w) + } + fmt.Fprintf(os.Stdout, "Download start from %s\n", url) + return d, nil +} + +func NewWorker(d *Downloader, size uint, i uint, split uint, url string) (*worker, error) { + bytesToStartReading := split * i + bytesToFinishReading := bytesToStartReading + split - 1 + partFilePath := fmt.Sprintf("%s.%d", d.FileName, i) + + if i == d.MaxProcess-1 { + bytesToFinishReading = size + } + w := &worker{ + processId: i, + bytesToStartReading: bytesToStartReading, + bytesToFinishReading: bytesToFinishReading, + resourceUrl: url, + partFilePath: partFilePath, + } + return w, nil +} + +func (d *Downloader) Download() error { + eg := errgroup.Group{} + for _, worker := range d.Workers { + w := worker + eg.Go(func() error { + return w.Request() + }) + } + + if err := eg.Wait(); err != nil { + return err + } + return nil +} + +func (d *Downloader) Merge() error { + outputFilePath := fmt.Sprintf("%s", d.FileName) + outputFile, err := os.Create(outputFilePath) + if err != nil { + return errors.Wrap(err, "failed to create merge file") + } + defer outputFile.Close() + for i := uint(0); i < d.MaxProcess; i++ { + partFilePath := fmt.Sprintf("%s.%d", outputFilePath, i) + partFile, err := os.Open(partFilePath) + if err != nil { + return errors.Wrap(err, "failed to open part file") + } + io.Copy(outputFile, partFile) + partFile.Close() + if err := os.Remove(partFilePath); err != nil { + return errors.Wrap(err, "failed to remove a file") + } + } + return nil +} + +func (w *worker) Request() error { + res, err := w.MakeResponse() + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to split get requests: %d", w.processId)) + } + defer res.Body.Close() + output, err := os.Create(w.partFilePath) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to create file %s", w.partFilePath)) + } + defer output.Close() + io.Copy(output, res.Body) + return nil +} + +func (w *worker) MakeResponse() (*http.Response, error) { + req, err := http.NewRequest("GET", w.resourceUrl, nil) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to split NewRequest for get: %d", w.processId)) + } + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", w.bytesToStartReading, w.bytesToFinishReading)) + return http.DefaultClient.Do(req) +} + +func getFileName(resourceUrl string) string { + token := strings.Split(resourceUrl, "/") + filename := token[len(token)-1] + return filename +}