diff --git a/.gitignore b/.gitignore index a18c8f3..e31155c 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ # Temp files .#* +rewards +hits +bandit-server \ No newline at end of file diff --git a/.godir b/.godir deleted file mode 100644 index 435f2eb..0000000 --- a/.godir +++ /dev/null @@ -1 +0,0 @@ -bandit-server diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d254697..0000000 --- a/Dockerfile +++ /dev/null @@ -1,25 +0,0 @@ -FROM alpine:latest -MAINTAINER Jose Peleteiro - -RUN apk update \ - && apk upgrade \ - && apk add \ - s6 bash curl make git \ - go \ - && rm -rf /var/cache/apk/* - -ADD . /tmp/build/src/github.com/peleteiro/bandit-server - -RUN export GOPATH=/tmp/build GO15VENDOREXPERIMENT=1\ - && cd /tmp/build/src/github.com/peleteiro/bandit-server \ - && make generate \ - && go build -o /usr/bin/bandit-server bandit-server.go \ - && rm -rf /tmp/build - -ADD ./root / - -ENV PORT=3321 - -CMD ["/bin/s6-svscan", "/etc/s6"] - -EXPOSE 3321 diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json deleted file mode 100644 index 9b030f1..0000000 --- a/Godeps/Godeps.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "ImportPath": "github.com/peleteiro/bandit-server", - "GoVersion": "go1.6", - "GodepVersion": "v60", - "Deps": [ - { - "ImportPath": "github.com/dropbox/godropbox/container/set", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - }, - { - "ImportPath": "github.com/dropbox/godropbox/errors", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - }, - { - "ImportPath": "github.com/dropbox/godropbox/math2/rand2", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - }, - { - "ImportPath": "github.com/dropbox/godropbox/memcache", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - }, - { - "ImportPath": "github.com/dropbox/godropbox/net2", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - }, - { - "ImportPath": "github.com/dropbox/godropbox/resource_pool", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - }, - { - "ImportPath": "github.com/dropbox/godropbox/sync2", - "Rev": "3c8a9189a8748951b62c6a48c072448b2fd4cac7" - } - ] -} diff --git a/Godeps/Readme b/Godeps/Readme deleted file mode 100644 index 4cdaa53..0000000 --- a/Godeps/Readme +++ /dev/null @@ -1,5 +0,0 @@ -This directory tree is generated automatically by godep. - -Please do not edit. - -See https://github.com/tools/godep for more information. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index cf35bea..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Jose Peleteiro - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/Makefile b/Makefile deleted file mode 100644 index 6efc9ea..0000000 --- a/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -.DEFAULT_GOAL=build - -PKG := $(GOPATH)/pkg - -run: generate - go run bandit-server.go - -rerun: generate - $(GOPATH)/bin/rerun github.com/peleteiro/bandit-server - -get: - go get github.com/skelterjohn/rerun/... - go get github.com/jteeuwen/go-bindata/... - -generate: get - $(GOPATH)/bin/go-bindata -o assets/assets.go -pkg=assets -prefix=assets -ignore=.\*.go assets - -build: generate - @GOOS=darwin GOARCH=amd64 go build -o $(PKG)/darwin_amd64/bandit-server bandit-server.go - @GOOS=linux GOARCH=amd64 go build -o $(PKG)/linux_amd64/bandit-server bandit-server.go - @GOOS=linux GOARCH=386 go build -o $(PKG)/linux_386/bandit-server bandit-server.go - -fmt: - go fmt ./... - -test: get - go test ./... diff --git a/Procfile b/Procfile deleted file mode 100644 index 1509382..0000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: bandit-server --port=$PORT diff --git a/README.md b/README.md index 64a5357..06c51d4 100644 --- a/README.md +++ b/README.md @@ -1,79 +1,137 @@ # Bandit-server -Bandit-server is a [Multi-Armed Bandit](http://en.wikipedia.org/wiki/Multi-armed_bandit) api server which needs no configuration neither persistente store. +Bandit-server is a [Multi-Armed Bandit](http://en.wikipedia.org/wiki/Multi-armed_bandit) api server which needs no configuration -## Multi-armed what?! +## Getting Started -A multi-armed bandit is essentially an online alternative to classical A/B testing. Whereas A/B testing is generally split into extended phases of execution and analysis, Bandit algorithms continually adjust to user feedback and optimize between experimental states. Bandits typically require very little curation and can in fact be left running indefinitely if need be. +1. Install bandit-server. ``go get github.com/recoilme/bandit-server`` +2. Run ```bandit-server --port=3000 --debug=true``` -The curious sounding name is drawn from the "one-armed bandit", an colloquialism for casino slot machines. Bandit algorithms can be thought of along similar lines as a eager slot player: if one were to play many slot machines continuously over many thousands of attempts, one would eventually be able to determine which machines were hotter than others. A multi-armed bandit is merely an algorithm that performs exactly this determination, using your user's interaction as its "arm pulls". Extracting winning patterns becomes a fluid part of interacting with the application. +## Routers -John Myles White has an awesome treatise on Bandit implementations in his book [Bandit Algorithms for Website Optimization](http://shop.oreilly.com/product/0636920027393.do). +``` +GET / --> main.ok (3 handlers) - for check status +GET /stats/:group/:count --> main.stats (3 handlers) - for get stats by count arms +POST /stats/:group/:count --> main.stats (3 handlers) - for get stats by arms +POST /write/:param/:group --> main.write (3 handlers) - for write hits & rewards +``` -## Getting Started +## Hits + +Hits - это количество показов объявлений. Для каждого объявления передаются: + +- название объявления, строка, это рука многорукого бандита (arm) +- количество, (cnt) - целое число + +Запрос идет на url http://localhost:3000/write/hits/domainid42 + +где domainid42 - это группа, по которой считается статистика. В нашем случае domainId. Но может быть любая строка. + +Метод - POST + + +Статистика передается в виде json массива. Заголовок: Content-Type: application/json + + +Пример запроса: +``` +curl -X POST --data '[{"arm":"ads 1","cnt":1},{"arm":"ads 2","cnt":1}]' -H "Content-Type: application/json" http://localhost:3000/write/hits/domainid42 +``` + +На этот запрос сервер ответит: -1. Install bandit-server. ``go get github.com/peleteiro/bandit-server`` -2. Run ```bandit-server --port=3000``` -3. Play ``curl http://localhost:3000/ucb1?downloadButtonColor=black,white,blue\&downloadButtonText=default,now`` -4. Reward ``curl -X PUT --data "downloadButtonColor=blue" http://localhost:3000/ucb1`` -## Determining what to test +HTTP/1.1 200 OK -The first task at hand requires a little planning. What are some of the things in your app you've always been curious about changing, but never had empirical data to back up potential modifications? Bandits are best suited to cases where changes can be "slipped in" without the user noticing, but since the state assigned to a user will be persisted to their client, you can also change things like UI. -For our example case, we'll be changing the text and color of a download button in our website to see if either change increases user interaction with the feature. We'll be representing these states as two separate experiments (so a user will get separate assignments for color and text). +Текст ответа: +ok -## Experiments Setup +Пример "кривого", неправильного запроса: -You keep experiments on your app, there's no experiments setup on the bandit-server. When you call the api you give a set of arms for every experiment and the server uses this configuration. If you call the api changing the configuration it will adapt the data for the new configuration. +``` +curl -v -X POST --data '[{"arm":"ads 1","cnt":1},{"arm":"ads 2","cnt":"2"]' -H "Content-Type: application/json" http://localhost:3000/write/hits/domainid42 +``` +Ответ с ошибкой: +``` +HTTP/1.1 422 Unprocessable Entity +Текст ответа: +{"error":"invalid character '\"' after array element"} +``` +422 - это код ошибки. -In your application you call the api ``http://server/ucb1?downloadButtonColor=black,white,blue`` to get the arm you need to show your user and the server will configure itself and the algorithms considering this configuration. If you give up testing "blue" arm, then your app calls ``http://server/ucb1?downloadButtonColor=black,white`` and, again, bandit-server will reconfigure and not losing any statistical data. +## Rewards -## Storage +Rewards - это награда за клик по объявлению. +Формат точно такой же. Единственное отличие - в url вместо hits передается rewards -By default bandit-server uses global memory to storage statistical data. It's the best option, even for multiple instances, if your bandit-server stays alive and you have enough hits. If your server goes down or you add a new server on your cluster, Multi-Armed Bandit algorithms adjust the result fast enough. +Пример: +``` +curl -X POST --data '[{"arm":"ads 2","cnt":1}]' -H "Content-Type: application/json" http://localhost:3000/write/rewards/domainid42 +``` -If you need to persist, you can use memcached: ``bandit-server --memcached=host:port`` +## Stats -## Writing your own client considerations +Статистика запрашивается в следуещем формате: -Bandit-server keeps no user state. But usually you want to keep the experiment's choices for a while. You don't want your user to see a different button color every reload. +curl -X GET http://localhost:3000/stats/domainid42/2 -It's your client job to keep a local cache on your app for the user session. I personally use bandit-server with one-page apps or native mobile apps, and it's really easy to keep this cache using localstore or filesystem. It should be easy to keep cache in traditional web app using sessions as well. +где domainid42 - это наша группа, 2 - Это количество рук, которые надо вернуть -If you're going to write a client for bandit-server, read the [angular-bandit-client](https://github.com/peleteiro/angular-bandit-client) code and how it handles cache and failure. +Пример ответа: -# Algorithms +``` +[{"arm":"var2","hit":1,"rew":0,"score":1.9727697022487511},{"arm":"ads 2","hit":3,"rew":1,"score":1.4723124519757878}] +``` +Это массив, он отсортирован по параметру score. -There's a couple of algorithms for MAB but we only have UCB1 implemented for now. We would love to have more algorithms. Fell free to send a push request. +Также можно запросить статистику по переданным "рукам", при помощи Post запроса с массивом "рук" -# Clients -## Javascript +Пример: -- [angular-bandit-client](https://github.com/peleteiro/angular-bandit-client) +``` +curl -X POST --data '[{"arm":"ads 2"},{"arm":"1"}]' -H "Content-Type: application/json" http://localhost:3000/stats/domainid42/3 -# Sample Project +Ответ: +[{"arm":"1","hit":0,"rew":0,"score":100},{"arm":"ads 2","hit":3,"rew":2,"score":1.5224751688711062}] +``` -See [https://github.com/peleteiro/angular-bandit-client/tree/master/example](https://github.com/peleteiro/angular-bandit-client/tree/master/example). +## Debug +Для запуска в режиме отладки необходимо передать флаг debug=true -# Contributing to bandit-server +Пример отладчика: -We encourage you to contribute to bandit-server! Please check out the guidelines about how to proceed. +``` +./bandit-server --debug=true +[GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production. + - using env: export GIN_MODE=release + - using code: gin.SetMode(gin.ReleaseMode) -* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet. +[GIN-debug] GET / --> main.ok (3 handlers) +[GIN-debug] GET /stats/:group/:count --> main.stats (3 handlers) +[GIN-debug] POST /:param/:group --> main.write (3 handlers) +[{ads 1 1} {ads 2 1}] +[GIN] 2018/11/20 - 19:06:34 | 200 | 2.767846ms | ::1 | POST /hits/domainid42 +[{ads 1 1} {ads 2 1}] +[GIN] 2018/11/20 - 19:06:56 | 200 | 386.706µs | ::1 | POST /hits/domainid42 +[{ads 2 1}] +[GIN] 2018/11/20 - 19:07:12 | 200 | 2.396616ms | ::1 | POST /rewards/domainid42 +[GIN] 2018/11/20 - 19:07:21 | 200 | 710.533µs | ::1 | GET /stats/domainid42/2 +``` -* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it -* Fork the project +## Backup -* Start a feature/bugfix branch -* Commit and push until you are happy with your contribution -Make sure to add tests for the feature/bugfix. This is important so I don't break it in a future version unintentionally. +Надо выполнить запрос: +``` +curl -X GET http://localhost:3000/backup/backup +``` +последний параметр - backup - директория. -* Please try not to mess with the Makefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate it to its own commit so I can cherry-pick around it. +Ответ - 200 ok # License diff --git a/assets/assets.go b/assets/assets.go deleted file mode 100644 index 8a63f31..0000000 --- a/assets/assets.go +++ /dev/null @@ -1,211 +0,0 @@ -// Code generated by go-bindata. -// sources: -// DO NOT EDIT! - -package assets - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" -) - -func bindataRead(data []byte, name string) ([]byte, error) { - gz, err := gzip.NewReader(bytes.NewBuffer(data)) - if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) - } - - var buf bytes.Buffer - _, err = io.Copy(&buf, gz) - clErr := gz.Close() - - if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) - } - if clErr != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -type asset struct { - bytes []byte - info os.FileInfo -} - -type bindataFileInfo struct { - name string - size int64 - mode os.FileMode - modTime time.Time -} - -func (fi bindataFileInfo) Name() string { - return fi.name -} -func (fi bindataFileInfo) Size() int64 { - return fi.size -} -func (fi bindataFileInfo) Mode() os.FileMode { - return fi.mode -} -func (fi bindataFileInfo) ModTime() time.Time { - return fi.modTime -} -func (fi bindataFileInfo) IsDir() bool { - return false -} -func (fi bindataFileInfo) Sys() interface{} { - return nil -} - -// Asset loads and returns the asset for the given name. -// It returns an error if the asset could not be found or -// could not be loaded. -func Asset(name string) ([]byte, error) { - cannonicalName := strings.Replace(name, "\\", "/", -1) - if f, ok := _bindata[cannonicalName]; ok { - a, err := f() - if err != nil { - return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) - } - return a.bytes, nil - } - return nil, fmt.Errorf("Asset %s not found", name) -} - -// MustAsset is like Asset but panics when Asset would return an error. -// It simplifies safe initialization of global variables. -func MustAsset(name string) []byte { - a, err := Asset(name) - if err != nil { - panic("asset: Asset(" + name + "): " + err.Error()) - } - - return a -} - -// AssetInfo loads and returns the asset info for the given name. -// It returns an error if the asset could not be found or -// could not be loaded. -func AssetInfo(name string) (os.FileInfo, error) { - cannonicalName := strings.Replace(name, "\\", "/", -1) - if f, ok := _bindata[cannonicalName]; ok { - a, err := f() - if err != nil { - return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) - } - return a.info, nil - } - return nil, fmt.Errorf("AssetInfo %s not found", name) -} - -// AssetNames returns the names of the assets. -func AssetNames() []string { - names := make([]string, 0, len(_bindata)) - for name := range _bindata { - names = append(names, name) - } - return names -} - -// _bindata is a table, holding each asset generator, mapped to its name. -var _bindata = map[string]func() (*asset, error){ -} - -// AssetDir returns the file names below a certain -// directory embedded in the file by go-bindata. -// For example if you run go-bindata on data/... and data contains the -// following hierarchy: -// data/ -// foo.txt -// img/ -// a.png -// b.png -// then AssetDir("data") would return []string{"foo.txt", "img"} -// AssetDir("data/img") would return []string{"a.png", "b.png"} -// AssetDir("foo.txt") and AssetDir("notexist") would return an error -// AssetDir("") will return []string{"data"}. -func AssetDir(name string) ([]string, error) { - node := _bintree - if len(name) != 0 { - cannonicalName := strings.Replace(name, "\\", "/", -1) - pathList := strings.Split(cannonicalName, "/") - for _, p := range pathList { - node = node.Children[p] - if node == nil { - return nil, fmt.Errorf("Asset %s not found", name) - } - } - } - if node.Func != nil { - return nil, fmt.Errorf("Asset %s not found", name) - } - rv := make([]string, 0, len(node.Children)) - for childName := range node.Children { - rv = append(rv, childName) - } - return rv, nil -} - -type bintree struct { - Func func() (*asset, error) - Children map[string]*bintree -} -var _bintree = &bintree{nil, map[string]*bintree{}} - -// RestoreAsset restores an asset under the given directory -func RestoreAsset(dir, name string) error { - data, err := Asset(name) - if err != nil { - return err - } - info, err := AssetInfo(name) - if err != nil { - return err - } - err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) - if err != nil { - return err - } - err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) - if err != nil { - return err - } - err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) - if err != nil { - return err - } - return nil -} - -// RestoreAssets restores an asset under the given directory recursively -func RestoreAssets(dir, name string) error { - children, err := AssetDir(name) - // File - if err != nil { - return RestoreAsset(dir, name) - } - // Dir - for _, child := range children { - err = RestoreAssets(dir, filepath.Join(name, child)) - if err != nil { - return err - } - } - return nil -} - -func _filePath(dir, name string) string { - cannonicalName := strings.Replace(name, "\\", "/", -1) - return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) -} - diff --git a/assets/favicon.ico b/assets/favicon.ico deleted file mode 100644 index 5ace5f2..0000000 Binary files a/assets/favicon.ico and /dev/null differ diff --git a/assets/robots.txt b/assets/robots.txt deleted file mode 100644 index fe8ce00..0000000 --- a/assets/robots.txt +++ /dev/null @@ -1,3 +0,0 @@ -# See http://www.robotstxt.org/wc/norobots.html for documentation on how to use the robots.txt file -User-Agent: * -Disallow: / diff --git a/bandit-server.go b/bandit-server.go index 93a21ed..b558d07 100644 --- a/bandit-server.go +++ b/bandit-server.go @@ -1,64 +1,399 @@ package main import ( + "context" + "errors" "flag" "fmt" - "github.com/peleteiro/bandit-server/assets" - "github.com/peleteiro/bandit-server/handlers" - "github.com/peleteiro/bandit-server/repository" - "github.com/peleteiro/bandit-server/strategies" "log" - "math/rand" + "math" "net/http" "os" + "os/signal" + "runtime" + "sort" + "strconv" "time" + + "github.com/gin-gonic/gin" + "github.com/recoilme/pudge" ) -var port int = 3000 -var memcached string = "no" +var ( + port = 3000 + debug = false + koef = 1.0 + //cfg *pudge.Config +) -var logger = log.New(os.Stdout, "bandit-server: ", log.Ldate|log.Ltime) +type Hit struct { + Arm string `json:"arm"` + Cnt int `json:"cnt"` +} + +type Arms struct { + Arm string `json:"arm"` +} + +type Stat struct { + Arm string `json:"arm"` + Hit int `json:"hit"` + Rew int `json:"rew"` + Score float64 `json:"score"` +} func init() { flag.IntVar(&port, "port", 3000, "http port") - flag.StringVar(&memcached, "use-memcached", "no", "Memcached host:port. Usualy you don't need memcache (read README.md).") + flag.BoolVar(&debug, "debug", false, "--debug=true") + flag.Float64Var(&koef, "koef", 1.0, "--koef=1.0") + pudge.Open("hits/relap", nil) + pudge.Open("rewards/relap", nil) + // Workaround for issue #17393. + //signal.Notify(make(chan os.Signal), syscall.SIGPIPE) } func main() { + flag.Parse() + log.Println("port:", port, "debug:", debug, "koef:", koef, "maxproc:", runtime.GOMAXPROCS(0)) + srv := &http.Server{ + Addr: fmt.Sprintf(":%d", port), + Handler: InitRouter(), + } - rand.Seed(time.Now().UnixNano()) + // Wait for interrupt signal to gracefully shutdown the server with + // setup signal catching + quit := make(chan os.Signal, 1) + // catch all signals since not explicitly listing + signal.Notify(quit) - var repo repository.Repository + //signal.Notify(sigs,syscall.SIGQUIT) + // method invoked upon seeing signal + go func() { + q := <-quit + log.Printf("RECEIVED SIGNAL: %s", q) + //if q == syscall.SIGPIPE || q.String() == "broken pipe" { + //return + //} + //log.Println("Shutdown Server ...") + ctx, cancel := context.WithTimeout(context.Background(), 2000*time.Millisecond) + defer cancel() + if err := srv.Shutdown(ctx); err != nil { + log.Println("Server Shutdown:", err) + } + // Close db + log.Println("Close db ...") + if err := pudge.CloseAll(); err != nil { + log.Println("Database Shutdown err:", err) + } + log.Println("Close db") + log.Println("Server exiting") + time.Sleep(2 * time.Second) + os.Exit(1) + }() + + //go func() { + // service connections + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("listen: %s\n", err) + } + //}() + +} + +func globalRecover(c *gin.Context) { + defer func(c *gin.Context) { + + if err := recover(); err != nil { + if err := pudge.CloseAll(); err != nil { + log.Println("Database Shutdown err:", err) + } + log.Printf("Server recovery with err:%+v\n", err) + gin.RecoveryWithWriter(gin.DefaultErrorWriter) + c.AbortWithStatus(500) + return + //time.Sleep(200 * time.Millisecond) + //time.Sleep(2 * time.Second) + } + }(c) + c.Next() +} + +// InitRouter - init router +func InitRouter() *gin.Engine { + if debug { + gin.SetMode(gin.DebugMode) - if memcached == "no" { - repo = repository.NewMemory() } else { - repo = repository.NewMemcached(memcached) + gin.SetMode(gin.ReleaseMode) } - http.HandleFunc("/favicon.ico", func(w http.ResponseWriter, r *http.Request) { - var favicon, _ = assets.Asset("favicon.ico") - w.Header().Set("Content-Type", "image/x-icon") - w.Write(favicon) - }) + r := gin.New() + if debug { + r.Use(gin.Logger()) + } + r.Use(globalRecover) - http.HandleFunc("/robots.txt", func(w http.ResponseWriter, r *http.Request) { - var robots, _ = assets.Asset("robots.txt") - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Write(robots) - }) + r.GET("/", ok) + r.GET("/backup/:dir", backup) + r.GET("/stats/:group/:count", stats) + r.POST("/stats/:group/:count", stats) + r.POST("/write/:param/:group", write) - http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - fmt.Fprintf(w, "ok") - }) + return r +} + +func ok(c *gin.Context) { + c.String(http.StatusOK, "%s", "ok") +} + +func backup(c *gin.Context) { + clean(c) + dir := c.Param("dir") + log.Println("backup") + pudge.BackupAll(dir) + log.Println("end") + c.String(http.StatusOK, "%s", "ok") +} + +func clean(c *gin.Context) { + hits, err := pudge.Keys("hits/relap", nil, 0, 0, true) + log.Println(len(hits)) + if err != nil { + panic(err) + } + rew, err := pudge.Open("rewards/relap", nil) + rew2, err := pudge.Open("rewards/relap.new2", nil) + log.Println(rew.Count()) + for _, k := range hits { + var b []byte + has, e := rew.Has(k) + if has && e == nil { + err := rew.Get(k, &b) + if err != nil { + panic(err) + } + e := rew2.Set(k, b) + if e != nil { + panic(e) + } + } + } + log.Println(rew2.Count()) + rew2.Close() + pudge.Open("rewards/relap.new2", nil) + + /* + if dir == "" { + dir = "backup" + } + dbs.Lock() + stores := dbs.dbs + dbs.Unlock() + //tmp := make(map[string]string) + for _, db := range stores { + backup := dir + "/" + db.name + DeleteFile(backup) + keys, err := db.Keys(nil, 0, 0, true) + if err == nil { + for _, k := range keys { + var b []byte + db.Get(k, &b) + Set(backup, k, b) + } + } + Close(backup) + } + */ +} + +func write(c *gin.Context) { + + defer func() { + if r := recover(); r != nil { + log.Println("Recovered in write", r) + } + }() + + var err error + group := c.Param("group") + param := c.Param("param") + dbPrefix := "" + switch param { + case "hits": + dbPrefix = param + break + case "rewards": + dbPrefix = param + break + default: + renderError(c, errors.New("Unknown param:"+param)) + return + } + var hits = make([]Hit, 0, 0) + err = c.ShouldBind(&hits) + if err != nil { + renderError(c, err) + return + } + /* + db, err := pudge.Open(dbPrefix+"/"+group, cfg) + if err != nil { + renderError(c, err) + return + }*/ + for _, h := range hits { + if dbPrefix == "rewards" { + has, err := pudge.Has("hits/"+group, h.Arm) + if !has || err != nil { + continue + } + } + _, err = pudge.Counter(dbPrefix+"/"+group, h.Arm, h.Cnt) + if err != nil { + log.Println("invalid_argument:", "'"+dbPrefix+"'", "'"+group+"'", "'"+h.Arm+"'", h.Cnt) + break + } + } + if err != nil { + renderError(c, err) + return + } + c.String(http.StatusOK, "%s", "ok") +} + +func renderError(c *gin.Context, err error) { + if err != nil { + log.Println(err) + c.Error(err) + c.JSON(http.StatusUnprocessableEntity, c.Errors) + return + } +} - http.HandleFunc("/ucb1", handlers.NewHttpHandler(strategies.NewUCB1(), repo)) +func stats(c *gin.Context) { + defer func() { + if r := recover(); r != nil { + log.Println("Recovered in stats", r) + } + }() + t1 := time.Now() + var err error + group := c.Param("group") + var arms = make([]Arms, 0, 0) + switch c.Request.Method { + case "POST": + err = c.ShouldBind(&arms) + if err != nil { + renderError(c, err) + return + } + } + + count, _ := strconv.Atoi(c.Param("count")) + + /* + dbhits, err := pudge.Open("hits/"+group, cfg) + if err != nil { + renderError(c, err) + return + } + dbrew, err := pudge.Open("rewards/"+group, cfg) + if err != nil { + renderError(c, err) + return + }*/ + var data = make([][]byte, 0, 0) + if len(arms) > 0 { + for _, arm := range arms { + data = append(data, []byte(arm.Arm)) + } + } else { + //log.Println("Error: stats without arms:", arms) + err = errors.New("Error: stats without arms") + renderError(c, err) + return + //data, err = pudge.Keys("hits/"+group, nil, 0, 0, true) //dbhits.Keys(nil, 0, 0, true) + } - logger.Print("Listening on port ", port) - err := http.ListenAndServe(fmt.Sprintf(":%d", port), nil) if err != nil { - logger.Fatal(err) + renderError(c, err) + return + } + t2 := time.Now() + if debug { + //log.Println("len", len(data)) + //fmt.Printf("The t2 took %v to run.\n", t2.Sub(t1)) + } + var stats = make([]Stat, 0, 0) + //var totalrew = 0 + var totalHits int + + for _, key := range data { + + var hit, rew int + t22 := time.Now() + errGet := pudge.Get("hits/"+group, key, &hit) ///dbhits.Get(key, &hit) + if errGet != nil && errGet != pudge.ErrKeyNotFound { + err = errGet + break + } + pudge.Get("rewards/"+group, key, &rew) + t23 := time.Now() + if t23.Sub(t22) > (20 * time.Millisecond) { + //fmt.Printf("The t23 took %v to run. With %s key \n", t23.Sub(t22), string(key)) + } + //dbrew.Get(key, &rew) + + //totalrew += rew + var stat Stat + stat.Arm = string(key) + stat.Hit = hit + stat.Rew = rew + totalHits += hit + stats = append(stats, stat) + } + + t3 := time.Now() + if debug { + _ = t1 + _ = t2 + _ = t3 + //fmt.Printf("The t3 took %v to run.\n", t3.Sub(t2)) + } + if t3.Sub(t2) > (200 * time.Millisecond) { + //fmt.Printf("The t3 took %v to run. With %d %v %+v req \n", t3.Sub(t2), count, t3.Sub(t1), c.Request.Method) + } + + var scores = make([]Stat, 0, 0) + for _, s := range stats { + s.Score = s.calcScore(totalHits) + scores = append(scores, s) + } + sort.Slice(scores, func(i, j int) bool { + return scores[i].Score > scores[j].Score + }) + //t4 := time.Now() + //if t4.Sub(t2) > (10 * time.Millisecond) { + // fmt.Printf("The t4 took %v to run.\n", t4.Sub(t2)) + //} + + if len(scores) > count { + scores = scores[:count] + } + if err != nil { + renderError(c, err) + return + } + //log.Println("total rew:", totalrew) + //log.Println("Score:", scores, stats[0].calcScore(12)) + + c.JSON(http.StatusOK, scores) +} + +func (st *Stat) calcScore(totalHits int) float64 { + if st.Hit == 0 { + return float64(math.Sqrt((2 * math.Log(float64(totalHits+1))))) } + return (koef*float64(st.Rew))/float64(st.Hit) + math.Sqrt((2*math.Log(float64(totalHits)))/float64(st.Hit)) } diff --git a/bandit-server_test.go b/bandit-server_test.go new file mode 100644 index 0000000..dab1f96 --- /dev/null +++ b/bandit-server_test.go @@ -0,0 +1,18 @@ +package main + +import ( + "fmt" + "log" + "math" + "runtime" + "testing" +) + +func TestZero(t *testing.T) { + log.Println(math.Sqrt((2 * math.Log(float64(1))))) + fmt.Printf("GOMAXPROCS is %d\n", getGOMAXPROCS()) +} + +func getGOMAXPROCS() int { + return runtime.GOMAXPROCS(0) +} diff --git a/handlers/http.go b/handlers/http.go deleted file mode 100644 index a0d8a4b..0000000 --- a/handlers/http.go +++ /dev/null @@ -1,69 +0,0 @@ -package handlers - -import ( - "encoding/json" - "fmt" - "github.com/peleteiro/bandit-server/repository" - "github.com/peleteiro/bandit-server/strategies" - "net/http" - "strings" -) - -func toJson(r map[string]string) string { - b, err := json.Marshal(r) - if err != nil { - return "" - } - return string(b) -} - -func doDefaultHeaders(w http.ResponseWriter, r *http.Request) { - if origin := r.Header.Get("Origin"); origin != "" { - w.Header().Set("Access-Control-Allow-Origin", origin) - } - w.Header().Set("Access-Control-Allow-Methods", "PUT, GET, OPTIONS") - w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding") - w.Header().Set("Access-Control-Allow-Credentials", "true") -} - -func doGet(strategy strategies.Strategy, repo repository.Repository, w http.ResponseWriter, r *http.Request) { - doDefaultHeaders(w, r) - - var result map[string]string = make(map[string]string) - - r.ParseForm() - for context, values := range r.Form { - experiments := strings.Split(values[0], ",") - result[context] = strategy.Choose(repo, context, experiments) - } - - w.Header().Set("Content-Type", "application/json; charset=utf-8") - fmt.Fprint(w, toJson(result)) -} - -func doPut(repo repository.Repository, w http.ResponseWriter, r *http.Request) { - doDefaultHeaders(w, r) - - r.ParseForm() - for context, values := range r.Form { - repo.Reward(context, values[0]) - } - - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - fmt.Fprint(w, "ok") -} - -func NewHttpHandler(strategy strategies.Strategy, repo repository.Repository) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - switch r.Method { - case "GET": - doGet(strategy, repo, w, r) - case "PUT": - doPut(repo, w, r) - case "OPTIONS": - doDefaultHeaders(w, r) - default: - http.Error(w, "Method Not Allowed", 405) - } - } -} diff --git a/repository/interface.go b/repository/interface.go deleted file mode 100644 index b9affd7..0000000 --- a/repository/interface.go +++ /dev/null @@ -1,17 +0,0 @@ -package repository - -type ExperimentData struct { - TotalHits int64 - Arms map[string]ArmData -} - -type ArmData struct { - Hits int64 - Rewards int64 -} - -type Repository interface { - Get(experiment string, arms []string) ExperimentData - Hit(experiment string, arm string) - Reward(experiment string, arm string) -} diff --git a/repository/memcached.go b/repository/memcached.go deleted file mode 100644 index ca16ecc..0000000 --- a/repository/memcached.go +++ /dev/null @@ -1,61 +0,0 @@ -package repository - -import ( - "fmt" - "github.com/dropbox/godropbox/memcache" - "github.com/dropbox/godropbox/net2" - "strconv" - "time" -) - -const HIT = "h" -const REWARD = "r" - -type Memcached struct { - mc memcache.Client -} - -func NewMemcached(address string) Memcached { - var maxIdleTime time.Duration = 30 * time.Second - - options := net2.ConnectionOptions{MaxActiveConnections: 2, MaxIdleConnections: 1, MaxIdleTime: &maxIdleTime} - - manager := memcache.NewStaticShardManager([]string{address}, func(key string, numShard int) int { return 0 }, options) - return Memcached{mc: memcache.NewShardedClient(manager, false)} -} - -func (v Memcached) mcGet(kind string, experiment string, arm string) int64 { - key := fmt.Sprintf("%s:%s:%s", kind, experiment, arm) - val := v.mc.Get(key).Value() - if val == nil { - return 0 - } - count, _ := strconv.ParseInt(string(val), 10, 64) - return count -} - -func (v Memcached) mcIncr(kind string, experiment string, arm string) { - key := fmt.Sprintf("%s:%s:%s", kind, experiment, arm) - v.mc.Increment(key, 1, 1, 0) -} - -func (v Memcached) Get(experiment string, arms []string) ExperimentData { - var expData = ExperimentData{0, make(map[string]ArmData)} - - for _, arm := range arms { - armData := ArmData{Hits: v.mcGet(HIT, experiment, arm), Rewards: v.mcGet(REWARD, experiment, arm)} - - expData.Arms[arm] = armData - expData.TotalHits += armData.Hits - } - - return expData -} - -func (v Memcached) Hit(experiment string, arm string) { - v.mcIncr(HIT, experiment, arm) -} - -func (v Memcached) Reward(experiment string, arm string) { - v.mcIncr(REWARD, experiment, arm) -} diff --git a/repository/memcached_test.go b/repository/memcached_test.go deleted file mode 100644 index 05b531b..0000000 --- a/repository/memcached_test.go +++ /dev/null @@ -1,61 +0,0 @@ -package repository - -import ( - "github.com/stretchr/testify/assert" - "testing" - "github.com/dropbox/godropbox/memcache" -) - -func newMemcachedMocked() Memcached { - return Memcached{mc: memcache.NewMockClient()} -} - -func TestMemcachedBlank(t *testing.T) { - repo := newMemcachedMocked() - - data := repo.Get("experiment", []string{"arm1"}) - assert.Equal(t, 0, data.TotalHits) - assert.Equal(t, 0, data.Arms["arm1"].Hits) - assert.Equal(t, 0, data.Arms["arm1"].Rewards) -} - -func TestMemcachedHitCounter(t *testing.T) { - repo := newMemcachedMocked() - repo.Hit("experiment", "arm1") - repo.Hit("experiment", "arm2") - repo.Hit("experiment", "arm2") - - data := repo.Get("experiment", []string{"arm1", "arm2"}) - assert.Equal(t, 3, data.TotalHits) - assert.Equal(t, 1, data.Arms["arm1"].Hits) - assert.Equal(t, 0, data.Arms["arm1"].Rewards) - assert.Equal(t, 2, data.Arms["arm2"].Hits) - assert.Equal(t, 0, data.Arms["arm2"].Rewards) -} - -func TestMemcachedRewardCounter(t *testing.T) { - repo := newMemcachedMocked() - repo.Reward("experiment", "arm1") - repo.Reward("experiment", "arm2") - repo.Reward("experiment", "arm2") - - data := repo.Get("experiment", []string{"arm1", "arm2"}) - assert.Equal(t, 0, data.TotalHits) - assert.Equal(t, 0, data.Arms["arm1"].Hits) - assert.Equal(t, 1, data.Arms["arm1"].Rewards) - assert.Equal(t, 0, data.Arms["arm2"].Hits) - assert.Equal(t, 2, data.Arms["arm2"].Rewards) -} - -func TestMemcachedTotalHitsWhenSubSet(t *testing.T) { - repo := newMemcachedMocked() - repo.Hit("experiment", "arm1") - repo.Hit("experiment", "arm2") - repo.Hit("experiment", "arm2") - repo.Hit("experiment", "arm3") - repo.Hit("experiment", "arm3") - repo.Hit("experiment", "arm3") - - data := repo.Get("experiment", []string{"arm1", "arm2"}) - assert.Equal(t, 3, data.TotalHits) -} diff --git a/repository/memory.go b/repository/memory.go deleted file mode 100644 index 3fa41fa..0000000 --- a/repository/memory.go +++ /dev/null @@ -1,67 +0,0 @@ -package repository - -import ( - "sync/atomic" -) - -type memoryMap map[string]map[string]*int64 - -func (v memoryMap) Get(experiment string, arm string) int64 { - ctx, ok := v[experiment] - if !ok { - return 0 - } - - count, ok := ctx[arm] - if !ok { - return 0 - } - - return *count -} - -func (v memoryMap) Incr(experiment string, arm string) { - ctx, ok := v[experiment] - if !ok { - ctx = make(map[string]*int64) - v[experiment] = ctx - } - - count, ok := ctx[arm] - if !ok { - var inital int64 = 0 - count = &inital - ctx[arm] = count - } - - atomic.AddInt64(count, 1) -} - -type Memory struct { - hits, rewards memoryMap -} - -func NewMemory() Memory { - return Memory{make(map[string]map[string]*int64), make(map[string]map[string]*int64)} -} - -func (v Memory) Get(experiment string, arms []string) ExperimentData { - var expData = ExperimentData{0, make(map[string]ArmData)} - - for _, arm := range arms { - armData := ArmData{Hits: v.hits.Get(experiment, arm), Rewards: v.rewards.Get(experiment, arm)} - - expData.Arms[arm] = armData - expData.TotalHits += armData.Hits - } - - return expData -} - -func (v Memory) Hit(experiment string, arm string) { - v.hits.Incr(experiment, arm) -} - -func (v Memory) Reward(experiment string, arm string) { - v.rewards.Incr(experiment, arm) -} diff --git a/repository/memory_test.go b/repository/memory_test.go deleted file mode 100644 index ea038de..0000000 --- a/repository/memory_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package repository - -import ( - "github.com/stretchr/testify/assert" - "testing" -) - -func TestMemoryBlank(t *testing.T) { - repo := NewMemory() - - data := repo.Get("experiment", []string{"arm1"}) - assert.Equal(t, 0, data.TotalHits) - assert.Equal(t, 0, data.Arms["arm1"].Hits) - assert.Equal(t, 0, data.Arms["arm1"].Rewards) -} - -func TestMemoryHitCounter(t *testing.T) { - repo := NewMemory() - repo.Hit("experiment", "arm1") - repo.Hit("experiment", "arm2") - repo.Hit("experiment", "arm2") - - data := repo.Get("experiment", []string{"arm1", "arm2"}) - assert.Equal(t, 3, data.TotalHits) - assert.Equal(t, 1, data.Arms["arm1"].Hits) - assert.Equal(t, 0, data.Arms["arm1"].Rewards) - assert.Equal(t, 2, data.Arms["arm2"].Hits) - assert.Equal(t, 0, data.Arms["arm2"].Rewards) -} - -func TestMemoryRewardCounter(t *testing.T) { - repo := NewMemory() - repo.Reward("experiment", "arm1") - repo.Reward("experiment", "arm2") - repo.Reward("experiment", "arm2") - - data := repo.Get("experiment", []string{"arm1", "arm2"}) - assert.Equal(t, 0, data.TotalHits) - assert.Equal(t, 0, data.Arms["arm1"].Hits) - assert.Equal(t, 1, data.Arms["arm1"].Rewards) - assert.Equal(t, 0, data.Arms["arm2"].Hits) - assert.Equal(t, 2, data.Arms["arm2"].Rewards) -} - -func TestMemoryTotalHitsWhenSubSet(t *testing.T) { - repo := NewMemory() - repo.Hit("experiment", "arm1") - repo.Hit("experiment", "arm2") - repo.Hit("experiment", "arm2") - repo.Hit("experiment", "arm3") - repo.Hit("experiment", "arm3") - repo.Hit("experiment", "arm3") - - data := repo.Get("experiment", []string{"arm1", "arm2"}) - assert.Equal(t, 3, data.TotalHits) -} diff --git a/root/etc/s6/.s6-svscan/finish b/root/etc/s6/.s6-svscan/finish deleted file mode 100755 index 0a33051..0000000 --- a/root/etc/s6/.s6-svscan/finish +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -exec /bin/true diff --git a/root/etc/s6/search/finish b/root/etc/s6/search/finish deleted file mode 100755 index 2a8569c..0000000 --- a/root/etc/s6/search/finish +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -exec /bin/s6-svscanctl -t /etc/s6 diff --git a/root/etc/s6/search/run b/root/etc/s6/search/run deleted file mode 100755 index 7a33c39..0000000 --- a/root/etc/s6/search/run +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env sh -exec /usr/bin/bandit-server -port $PORT diff --git a/strategies/interface.go b/strategies/interface.go deleted file mode 100644 index 6ba87ff..0000000 --- a/strategies/interface.go +++ /dev/null @@ -1,7 +0,0 @@ -package strategies - -import "github.com/peleteiro/bandit-server/repository" - -type Strategy interface { - Choose(repo repository.Repository, context string, experiments []string) string -} diff --git a/strategies/random.go b/strategies/random.go deleted file mode 100644 index 7cb307f..0000000 --- a/strategies/random.go +++ /dev/null @@ -1,27 +0,0 @@ -package strategies - -import ( - "github.com/peleteiro/bandit-server/repository" - "math/rand" -) - -type Random struct{} - -func NewRandom() Random { - return Random{} -} - -func (_ Random) Choose(repo repository.Repository, experiment string, arms []string) string { - var arm = getRandomArm(arms) - repo.Hit(experiment, arm) - return arm -} - -func getRandomArm(arms []string) string { - var i = randInt(len(arms)) - return arms[i] -} - -func randInt(len int) int { - return rand.Intn(len) -} diff --git a/strategies/ucb1.go b/strategies/ucb1.go deleted file mode 100644 index 6eafcf7..0000000 --- a/strategies/ucb1.go +++ /dev/null @@ -1,43 +0,0 @@ -package strategies - -import ( - "github.com/peleteiro/bandit-server/repository" - "math" -) - -type UCB1 struct{} - -func NewUCB1() UCB1 { - return UCB1{} -} - -func (_ UCB1) Choose(repo repository.Repository, experiment string, arms []string) (arm string) { - arm = getHighestScoreArm(repo, experiment, arms) - repo.Hit(experiment, arm) - return arm -} - -func getHighestScoreArm(repo repository.Repository, experiment string, arms []string) string { - var highestArm string - var highestScore float64 = 0 - - var expData = repo.Get(experiment, arms) - for arm, armData := range expData.Arms { - if armData.Hits == 0 { - return arm - } - - var score = calcScore(expData.TotalHits, armData.Hits, armData.Rewards) - - if score > highestScore { - highestArm = arm - highestScore = score - } - } - - return highestArm -} - -func calcScore(totalHits int64, hits int64, rewards int64) float64 { - return float64((rewards/4)/hits) + math.Sqrt((2*math.Log(float64(totalHits)))/float64(hits)) -} diff --git a/strategies/ucb1_test.go b/strategies/ucb1_test.go deleted file mode 100644 index f74db6f..0000000 --- a/strategies/ucb1_test.go +++ /dev/null @@ -1,18 +0,0 @@ -package strategies - -import ( - "github.com/peleteiro/bandit-server/repository" - "github.com/stretchr/testify/assert" - "testing" -) - -var strgy = NewUCB1() - -func TestReturnTheOneWithZeroHits(t *testing.T) { - repo := repository.NewMemory() - repo.Hit("exp", "arm1") - - choosenOne := strgy.Choose(repo, "exp", []string{"arm1", "armWithZeroHits"}) - - assert.Equal(t, choosenOne, "armWithZeroHits") -} diff --git a/vendor/github.com/dropbox/godropbox/container/set/doc.go b/vendor/github.com/dropbox/godropbox/container/set/doc.go deleted file mode 100644 index ebb408c..0000000 --- a/vendor/github.com/dropbox/godropbox/container/set/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Implementation of a Set container -package set diff --git a/vendor/github.com/dropbox/godropbox/container/set/set.go b/vendor/github.com/dropbox/godropbox/container/set/set.go deleted file mode 100644 index 14b9a40..0000000 --- a/vendor/github.com/dropbox/godropbox/container/set/set.go +++ /dev/null @@ -1,341 +0,0 @@ -package set - -// An unordered collection of unique elements which supports lookups, insertions, deletions, -// iteration, and common binary set operations. It is not guaranteed to be thread-safe. -type Set interface { - // Returns a new Set that contains exactly the same elements as this set. - Copy() Set - - // Returns the cardinality of this set. - Len() int - - // Returns true if and only if this set contains v (according to Go equality rules). - Contains(v interface{}) bool - // Inserts v into this set. - Add(v interface{}) - // Removes v from this set, if it is present. Returns true if and only if v was present. - Remove(v interface{}) bool - - // Executes f(v) for every element v in this set. If f mutates this set, behavior is undefined. - Do(f func(interface{})) - // Executes f(v) once for every element v in the set, aborting if f ever returns false. If f - // mutates this set, behavior is undefined. - DoWhile(f func(interface{}) bool) - // Returns a channel from which each element in the set can be read exactly once. If this set - // is mutated before the channel is emptied, the exact data read from the channel is undefined. - Iter() <-chan interface{} - - // Adds every element in s into this set. - Union(s Set) - // Removes every element not in s from this set. - Intersect(s Set) - // Removes every element in s from this set. - Subtract(s Set) - // Removes all elements from the set. - Init() - // Returns true if and only if all elements in this set are elements in s. - IsSubset(s Set) bool - // Returns true if and only if all elements in s are elements in this set. - IsSuperset(s Set) bool - // Returns true if and only if this set and s contain exactly the same elements. - IsEqual(s Set) bool - // Removes all elements v from this set that satisfy f(v) == true. - RemoveIf(f func(interface{}) bool) -} - -// Returns a new set which is the union of s1 and s2. s1 and s2 are unmodified. -func Union(s1 Set, s2 Set) Set { - s3 := s1.Copy() - s3.Union(s2) - return s3 -} - -// Returns a new set which is the intersect of s1 and s2. s1 and s2 are -// unmodified. -func Intersect(s1 Set, s2 Set) Set { - s3 := s1.Copy() - s3.Intersect(s2) - return s3 -} - -// Returns a new set which is the difference between s1 and s2. s1 and s2 are -// unmodified. -func Subtract(s1 Set, s2 Set) Set { - s3 := s1.Copy() - s3.Subtract(s2) - return s3 -} - -// Returns a new Set pre-populated with the given items -func NewSet(items ...interface{}) Set { - res := setImpl{ - data: make(map[interface{}]struct{}), - } - for _, item := range items { - res.Add(item) - } - return res -} - -// Returns a new Set pre-populated with the given items -func NewKeyedSet(keyf func(interface{}) interface{}, items ...interface{}) Set { - res := keyedSetImpl{ - data: make(map[interface{}]interface{}), - keyfunc: keyf, - } - for _, item := range items { - res.Add(item) - } - return res -} - -type setImpl struct { - data map[interface{}]struct{} -} - -func (s setImpl) Len() int { - return len(s.data) -} - -func (s setImpl) Copy() Set { - res := NewSet() - res.Union(s) - return res -} - -func (s setImpl) Init() { - s.data = make(map[interface{}]struct{}) -} - -func (s setImpl) Contains(v interface{}) bool { - _, ok := s.data[v] - return ok -} - -func (s setImpl) Add(v interface{}) { - s.data[v] = struct{}{} -} - -func (s setImpl) Remove(v interface{}) bool { - _, ok := s.data[v] - if ok { - delete(s.data, v) - } - return ok -} - -func (s setImpl) Do(f func(interface{})) { - for key := range s.data { - f(key) - } -} - -func (s setImpl) DoWhile(f func(interface{}) bool) { - for key := range s.data { - if !f(key) { - break - } - } -} - -func (s setImpl) Iter() <-chan interface{} { - iter := make(chan interface{}) - go func() { - for key := range s.data { - iter <- key - } - close(iter) - }() - return iter -} - -func (s setImpl) Union(s2 Set) { - union(s, s2) -} - -func (s setImpl) Intersect(s2 Set) { - var toRemove []interface{} - for key := range s.data { - if !s2.Contains(key) { - toRemove = append(toRemove, key) - } - } - - for _, key := range toRemove { - s.Remove(key) - } -} - -func (s setImpl) Subtract(s2 Set) { - subtract(s, s2) -} - -func (s setImpl) IsSubset(s2 Set) (isSubset bool) { - return subset(s, s2) -} - -func (s setImpl) IsSuperset(s2 Set) bool { - return superset(s, s2) -} - -func (s setImpl) IsEqual(s2 Set) bool { - return equal(s, s2) -} - -func (s setImpl) RemoveIf(f func(interface{}) bool) { - var toRemove []interface{} - for item := range s.data { - if f(item) { - toRemove = append(toRemove, item) - } - } - - for _, item := range toRemove { - s.Remove(item) - } -} - -// keyedSetImpl implementation below here - -type keyedSetImpl struct { - data map[interface{}]interface{} - keyfunc (func(interface{}) interface{}) -} - -func (s keyedSetImpl) Len() int { - return len(s.data) -} - -func (s keyedSetImpl) Copy() Set { - res := NewKeyedSet(s.keyfunc) - res.Union(s) - return res -} - -func (s keyedSetImpl) Init() { - s.data = make(map[interface{}]interface{}) -} - -func (s keyedSetImpl) Contains(v interface{}) bool { - _, ok := s.data[s.keyfunc(v)] - return ok -} - -func (s keyedSetImpl) Add(v interface{}) { - s.data[s.keyfunc(v)] = v -} - -func (s keyedSetImpl) Remove(v interface{}) bool { - key := s.keyfunc(v) - _, ok := s.data[key] - if ok { - delete(s.data, key) - } - return ok - -} - -func (s keyedSetImpl) Do(f func(interface{})) { - for _, v := range s.data { - f(v) - } -} - -func (s keyedSetImpl) DoWhile(f func(interface{}) bool) { - for _, v := range s.data { - if !f(v) { - break - } - } -} - -func (s keyedSetImpl) Iter() <-chan interface{} { - iter := make(chan interface{}) - go func() { - for _, v := range s.data { - iter <- v - } - close(iter) - }() - return iter -} - -func (s keyedSetImpl) Union(s2 Set) { - union(s, s2) -} - -func (s keyedSetImpl) Intersect(s2 Set) { - var toRemove []interface{} - for _, v := range s.data { - if !s2.Contains(v) { - toRemove = append(toRemove, v) - } - } - - for _, v := range toRemove { - s.Remove(v) - } -} - -func (s keyedSetImpl) Subtract(s2 Set) { - subtract(s, s2) -} - -func (s keyedSetImpl) IsSubset(s2 Set) (isSubset bool) { - return subset(s, s2) -} - -func (s keyedSetImpl) IsSuperset(s2 Set) bool { - return superset(s, s2) -} - -func (s keyedSetImpl) IsEqual(s2 Set) bool { - return equal(s, s2) -} - -func (s keyedSetImpl) RemoveIf(f func(interface{}) bool) { - var toRemove []interface{} - for _, item := range s.data { - if f(item) { - toRemove = append(toRemove, item) - } - } - - for _, item := range toRemove { - s.Remove(item) - } -} - -// Common functions between the two implementations, since go -// does not allow for any inheritance. - -func equal(s Set, s2 Set) bool { - if s.Len() != s2.Len() { - return false - } - - return s.IsSubset(s2) -} - -func superset(s Set, s2 Set) bool { - return s2.IsSubset(s) -} - -func subset(s Set, s2 Set) (isSubset bool) { - isSubset = true - s.DoWhile(func(item interface{}) bool { - if !s2.Contains(item) { - isSubset = false - } - return isSubset - }) - return -} - -func subtract(s Set, s2 Set) { - s2.Do(func(item interface{}) { s.Remove(item) }) -} - -func union(s Set, s2 Set) { - s2.Do(func(item interface{}) { s.Add(item) }) -} diff --git a/vendor/github.com/dropbox/godropbox/errors/errors.go b/vendor/github.com/dropbox/godropbox/errors/errors.go deleted file mode 100644 index 3047938..0000000 --- a/vendor/github.com/dropbox/godropbox/errors/errors.go +++ /dev/null @@ -1,293 +0,0 @@ -// This module implements functions which manipulate errors and provide stack -// trace information. -// -// NOTE: This package intentionally mirrors the standard "errors" module. -// All dropbox code should use this. -package errors - -import ( - "bytes" - "fmt" - "reflect" - "runtime" - "strings" -) - -// This interface exposes additional information about the error. -type DropboxError interface { - // This returns the error message without the stack trace. - GetMessage() string - - // This returns the stack trace without the error message. - GetStack() string - - // This returns the stack trace's context. - GetContext() string - - // This returns the wrapped error. This returns nil if this does not wrap - // another error. - GetInner() error - - // Implements the built-in error interface. - Error() string -} - -// Standard struct for general types of errors. -// -// For an example of custom error type, look at databaseError/newDatabaseError -// in errors_test.go. -type DropboxBaseError struct { - Msg string - Stack string - Context string - inner error -} - -// This returns the error string without stack trace information. -func GetMessage(err interface{}) string { - switch e := err.(type) { - case DropboxError: - dberr := DropboxError(e) - ret := []string{} - for dberr != nil { - ret = append(ret, dberr.GetMessage()) - d := dberr.GetInner() - if d == nil { - break - } - var ok bool - dberr, ok = d.(DropboxError) - if !ok { - ret = append(ret, d.Error()) - break - } - } - return strings.Join(ret, " ") - case runtime.Error: - return runtime.Error(e).Error() - default: - return "Passed a non-error to GetMessage" - } -} - -// This returns a string with all available error information, including inner -// errors that are wrapped by this errors. -func (e *DropboxBaseError) Error() string { - return DefaultError(e) -} - -// This returns the error message without the stack trace. -func (e *DropboxBaseError) GetMessage() string { - return e.Msg -} - -// This returns the stack trace without the error message. -func (e *DropboxBaseError) GetStack() string { - return e.Stack -} - -// This returns the stack trace's context. -func (e *DropboxBaseError) GetContext() string { - return e.Context -} - -// This returns the wrapped error, if there is one. -func (e *DropboxBaseError) GetInner() error { - return e.inner -} - -// This returns a new DropboxBaseError initialized with the given message and -// the current stack trace. -func New(msg string) DropboxError { - stack, context := StackTrace() - return &DropboxBaseError{ - Msg: msg, - Stack: stack, - Context: context, - } -} - -// Same as New, but with fmt.Printf-style parameters. -func Newf(format string, args ...interface{}) DropboxError { - stack, context := StackTrace() - return &DropboxBaseError{ - Msg: fmt.Sprintf(format, args...), - Stack: stack, - Context: context, - } -} - -// Wraps another error in a new DropboxBaseError. -func Wrap(err error, msg string) DropboxError { - stack, context := StackTrace() - return &DropboxBaseError{ - Msg: msg, - Stack: stack, - Context: context, - inner: err, - } -} - -// Same as Wrap, but with fmt.Printf-style parameters. -func Wrapf(err error, format string, args ...interface{}) DropboxError { - stack, context := StackTrace() - return &DropboxBaseError{ - Msg: fmt.Sprintf(format, args...), - Stack: stack, - Context: context, - inner: err, - } -} - -// A default implementation of the Error method of the error interface. -func DefaultError(e DropboxError) string { - // Find the "original" stack trace, which is probably the most helpful for - // debugging. - errLines := make([]string, 1) - var origStack string - errLines[0] = "ERROR:" - fillErrorInfo(e, &errLines, &origStack) - errLines = append(errLines, "") - errLines = append(errLines, "ORIGINAL STACK TRACE:") - errLines = append(errLines, origStack) - return strings.Join(errLines, "\n") -} - -// Fills errLines with all error messages, and origStack with the inner-most -// stack. -func fillErrorInfo(err error, errLines *[]string, origStack *string) { - if err == nil { - return - } - - derr, ok := err.(DropboxError) - if ok { - *errLines = append(*errLines, derr.GetMessage()) - *origStack = derr.GetStack() - fillErrorInfo(derr.GetInner(), errLines, origStack) - } else { - *errLines = append(*errLines, err.Error()) - } -} - -// Returns a copy of the error with the stack trace field populated and any -// other shared initialization; skips 'skip' levels of the stack trace. -// -// NOTE: This panics on any error. -func stackTrace(skip int) (current, context string) { - // grow buf until it's large enough to store entire stack trace - buf := make([]byte, 128) - for { - n := runtime.Stack(buf, false) - if n < len(buf) { - buf = buf[:n] - break - } - buf = make([]byte, len(buf)*2) - } - - // Returns the index of the first occurrence of '\n' in the buffer 'b' - // starting with index 'start'. - // - // In case no occurrence of '\n' is found, it returns len(b). This - // simplifies the logic on the calling sites. - indexNewline := func(b []byte, start int) int { - if start >= len(b) { - return len(b) - } - searchBuf := b[start:] - index := bytes.IndexByte(searchBuf, '\n') - if index == -1 { - return len(b) - } - return (start + index) - } - - // Strip initial levels of stack trace, but keep header line that - // identifies the current goroutine. - var strippedBuf bytes.Buffer - index := indexNewline(buf, 0) - if index != -1 { - strippedBuf.Write(buf[:index]) - } - - // Skip lines. - for i := 0; i < skip; i++ { - index = indexNewline(buf, index+1) - index = indexNewline(buf, index+1) - } - - isDone := false - startIndex := index - lastIndex := index - for !isDone { - index = indexNewline(buf, index+1) - if (index - lastIndex) <= 1 { - isDone = true - } else { - lastIndex = index - } - } - strippedBuf.Write(buf[startIndex:index]) - return strippedBuf.String(), string(buf[index:]) -} - -// This returns the current stack trace string. NOTE: the stack creation code -// is excluded from the stack trace. -func StackTrace() (current, context string) { - return stackTrace(3) -} - -// Return a wrapped error or nil if there is none. -func unwrapError(ierr error) (nerr error) { - // Internal errors have a well defined bit of context. - if dbxErr, ok := ierr.(DropboxError); ok { - return dbxErr.GetInner() - } - - // At this point, if anything goes wrong, just return nil. - defer func() { - if x := recover(); x != nil { - nerr = nil - } - }() - - // Go system errors have a convention but paradoxically no - // interface. All of these panic on error. - errV := reflect.ValueOf(ierr).Elem() - errV = errV.FieldByName("Err") - return errV.Interface().(error) -} - -// Keep peeling away layers or context until a primitive error is revealed. -func RootError(ierr error) (nerr error) { - nerr = ierr - for i := 0; i < 20; i++ { - terr := unwrapError(nerr) - if terr == nil { - return nerr - } - nerr = terr - } - return fmt.Errorf("too many iterations: %T", nerr) -} - -// Perform a deep check, unwrapping errors as much as possilbe and -// comparing the string version of the error. -func IsError(err, errConst error) bool { - if err == errConst { - return true - } - // Must rely on string equivalence, otherwise a value is not equal - // to its pointer value. - rootErrStr := "" - rootErr := RootError(err) - if rootErr != nil { - rootErrStr = rootErr.Error() - } - errConstStr := "" - if errConst != nil { - errConstStr = errConst.Error() - } - return rootErrStr == errConstStr -} diff --git a/vendor/github.com/dropbox/godropbox/license.txt b/vendor/github.com/dropbox/godropbox/license.txt deleted file mode 100644 index 04d4210..0000000 --- a/vendor/github.com/dropbox/godropbox/license.txt +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2014 Dropbox, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this -list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright notice, -this list of conditions and the following disclaimer in the documentation -and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its contributors -may be used to endorse or promote products derived from this software without -specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/dropbox/godropbox/math2/rand2/rand.go b/vendor/github.com/dropbox/godropbox/math2/rand2/rand.go deleted file mode 100644 index 5b456aa..0000000 --- a/vendor/github.com/dropbox/godropbox/math2/rand2/rand.go +++ /dev/null @@ -1,179 +0,0 @@ -// rand2 is a drop-in replacement for the "math/rand" package. It initializes -// the global random generator with a random seed (instead of 1), and provides -// additional functionality over the standard "math/rand" package. -package rand2 - -import ( - "math/rand" - "os" - "sort" - "sync" - "time" - - "github.com/dropbox/godropbox/container/set" - "github.com/dropbox/godropbox/errors" -) - -type lockedSource struct { - mutex sync.Mutex - src rand.Source -} - -func (r *lockedSource) Int63() int64 { - r.mutex.Lock() - val := r.src.Int63() - r.mutex.Unlock() - return val -} - -func (r *lockedSource) Seed(seed int64) { - r.mutex.Lock() - r.src.Seed(seed) - r.mutex.Unlock() -} - -// This returns a thread-safe random source. -func NewSource(seed int64) rand.Source { - return &lockedSource{ - src: rand.NewSource(seed), - } -} - -func init() { - now := time.Now() - seed := now.Unix() + int64(now.Nanosecond()) + 12345*int64(os.Getpid()) - rand.Seed(seed) -} - -var ( - // See math/rand for documentation. - New = rand.New - - // See math/rand for documentation. - Seed = rand.Seed - - // See math/rand for documentation. - Int63 = rand.Int63 - - // See math/rand for documentation. - Uint32 = rand.Uint32 - - // See math/rand for documentation. - Int31 = rand.Int31 - - // See math/rand for documentation. - Int = rand.Int - - // See math/rand for documentation. - Int63n = rand.Int63n - - // See math/rand for documentation. - Int31n = rand.Int31n - - // See math/rand for documentation. - Intn = rand.Intn - - // See math/rand for documentation. - Float64 = rand.Float64 - - // See math/rand for documentation. - Float32 = rand.Float32 - - // See math/rand for documentation. - Perm = rand.Perm - - // See math/rand for documentation. - NormFloat64 = rand.NormFloat64 - - // See math/rand for documentation. - ExpFloat64 = rand.ExpFloat64 - - // See math/rand for documentation. - NewZipf = rand.NewZipf -) - -// Dur returns a pseudo-random Duration in [0, max) -func Dur(max time.Duration) time.Duration { - return time.Duration(Int63n(int64(max))) -} - -// Uniformly jitters the provided duration by +/- 50%. -func Jitter(period time.Duration) time.Duration { - return JitterFraction(period, .5) -} - -// Uniformly jitters the provided duration by +/- the given fraction. NOTE: -// fraction must be in (0, 1]. -func JitterFraction(period time.Duration, fraction float64) time.Duration { - fixed := time.Duration(float64(period) * (1 - fraction)) - return fixed + Dur(2*(period-fixed)) -} - -// Samples 'k' unique ints from the range [0, n) -func SampleInts(n int, k int) (res []int, err error) { - if k < 0 { - err = errors.Newf("invalid sample size k") - return - } - - if n < k { - err = errors.Newf("sample size k larger than n") - return - } - - picked := set.NewSet() - for picked.Len() < k { - i := Intn(n) - picked.Add(i) - } - - res = make([]int, k) - e := 0 - for i := range picked.Iter() { - res[e] = i.(int) - e++ - } - - return -} - -// Samples 'k' elements from the given slice -func Sample(population []interface{}, k int) (res []interface{}, err error) { - n := len(population) - idxs, err := SampleInts(n, k) - if err != nil { - return - } - - res = []interface{}{} - for _, idx := range idxs { - res = append(res, population[idx]) - } - - return -} - -// Same as 'Sample' except it returns both the 'picked' sample set and the -// 'remaining' elements. -func PickN(population []interface{}, n int) ( - picked []interface{}, remaining []interface{}, err error) { - - total := len(population) - idxs, err := SampleInts(total, n) - if err != nil { - return - } - sort.Ints(idxs) - - picked, remaining = []interface{}{}, []interface{}{} - for x, elem := range population { - if len(idxs) > 0 && x == idxs[0] { - picked = append(picked, elem) - idxs = idxs[1:] - } else { - remaining = append(remaining, elem) - } - } - - return -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/base_shard_manager.go b/vendor/github.com/dropbox/godropbox/memcache/base_shard_manager.go deleted file mode 100644 index 91e57fe..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/base_shard_manager.go +++ /dev/null @@ -1,271 +0,0 @@ -package memcache - -import ( - "expvar" - "sync" - - "github.com/dropbox/godropbox/container/set" - "github.com/dropbox/godropbox/net2" -) - -type MemcachedState int - -const ( - ActiveServer = MemcachedState(0) - WriteOnlyServer = MemcachedState(1) - DownServer = MemcachedState(2) - WarmUpServer = MemcachedState(4) -) - -var ( - // Counters for number of connections that succeeded / errored / were skipped, by address. - connOkByAddr = expvar.NewMap("ShardManagerConnOkByAddrCounter") - connErrByAddr = expvar.NewMap("ShardManagerConnErrByAddrCounter") - connSkippedByAddr = expvar.NewMap("ShardManagerConnSkippedByAddrCounter") -) - -type ShardState struct { - Address string - State MemcachedState -} - -// A base shard manager implementation that can be used to implement other -// shard managers. -type BaseShardManager struct { - getShardId (func(key string, numShard int) (shard int)) - pool net2.ConnectionPool - - rwMutex sync.RWMutex - shardStates []ShardState // guarded by rwMutex - - logError func(err error) - logInfo func(v ...interface{}) -} - -// Initializes the BaseShardManager. -func (m *BaseShardManager) Init( - shardFunc func(key string, numShard int) (shard int), - logError func(err error), - logInfo func(v ...interface{}), - options net2.ConnectionOptions) { - - m.InitWithPool( - shardFunc, - logError, - logInfo, - net2.NewMultiConnectionPool(options)) -} - -func (m *BaseShardManager) InitWithPool( - shardFunc func(key string, numShard int) (shard int), - logError func(err error), - logInfo func(v ...interface{}), - pool net2.ConnectionPool) { - - m.shardStates = make([]ShardState, 0, 0) - m.getShardId = shardFunc - m.pool = pool - - m.logError = logError - m.logInfo = logInfo -} - -// This updates the shard manager to use new shard states. -func (m *BaseShardManager) UpdateShardStates(shardStates []ShardState) { - newAddrs := set.NewSet() - for _, state := range shardStates { - newAddrs.Add(state.Address) - } - - m.rwMutex.Lock() - defer m.rwMutex.Unlock() - - oldAddrs := set.NewSet() - for _, state := range m.shardStates { - oldAddrs.Add(state.Address) - } - - for address := range set.Subtract(newAddrs, oldAddrs).Iter() { - if err := m.pool.Register("tcp", address.(string)); err != nil { - m.logError(err) - } - } - - for address := range set.Subtract(oldAddrs, newAddrs).Iter() { - if err := m.pool.Unregister("tcp", address.(string)); err != nil { - m.logError(err) - } - } - - m.shardStates = shardStates -} - -// See ShardManager interface for documentation. -func (m *BaseShardManager) GetShard( - key string) ( - shardId int, - conn net2.ManagedConn, - err error) { - - m.rwMutex.RLock() - defer m.rwMutex.RUnlock() - - shardId = m.getShardId(key, len(m.shardStates)) - if shardId == -1 { - return - } - - state := m.shardStates[shardId] - if state.State != ActiveServer { - m.logInfo("Memcache shard ", shardId, " is not in active state.") - connSkippedByAddr.Add(state.Address, 1) - return - } - - entry := &ShardMapping{} - m.fillEntryWithConnection(state.Address, entry) - conn, err = entry.Connection, entry.ConnErr - - return -} - -// See ShardManager interface for documentation. -func (m *BaseShardManager) GetShardsForKeys( - keys []string) map[int]*ShardMapping { - - m.rwMutex.RLock() - defer m.rwMutex.RUnlock() - - numShards := len(m.shardStates) - results := make(map[int]*ShardMapping) - - for _, key := range keys { - shardId := m.getShardId(key, numShards) - - entry, inMap := results[shardId] - if !inMap { - entry = &ShardMapping{} - if shardId != -1 { - state := m.shardStates[shardId] - if state.State == ActiveServer { - m.fillEntryWithConnection(state.Address, entry) - } else { - connSkippedByAddr.Add(state.Address, 1) - } - } - entry.Keys = make([]string, 0, 1) - results[shardId] = entry - } - entry.Keys = append(entry.Keys, key) - } - - return results -} - -// See ShardManager interface for documentation. -func (m *BaseShardManager) GetShardsForItems( - items []*Item) map[int]*ShardMapping { - - m.rwMutex.RLock() - defer m.rwMutex.RUnlock() - - numShards := len(m.shardStates) - results := make(map[int]*ShardMapping) - - for _, item := range items { - shardId := m.getShardId(item.Key, numShards) - - entry, inMap := results[shardId] - if !inMap { - entry = &ShardMapping{} - if shardId != -1 { - state := m.shardStates[shardId] - if state.State == ActiveServer { - m.fillEntryWithConnection(state.Address, entry) - } else { - connSkippedByAddr.Add(state.Address, 1) - } - } - entry.Items = make([]*Item, 0, 1) - results[shardId] = entry - } - entry.Items = append(entry.Items, item) - } - - return results -} - -// See ShardManager interface for documentation. -func (m *BaseShardManager) GetShardsForSentinels( - items []*Item) map[int]*ShardMapping { - - m.rwMutex.RLock() - defer m.rwMutex.RUnlock() - - numShards := len(m.shardStates) - results := make(map[int]*ShardMapping) - - for _, item := range items { - shardId := m.getShardId(item.Key, numShards) - - entry, inMap := results[shardId] - if !inMap { - entry = &ShardMapping{} - if shardId != -1 { - state := m.shardStates[shardId] - if state.State == ActiveServer || - state.State == WriteOnlyServer || - state.State == WarmUpServer { - - m.fillEntryWithConnection(state.Address, entry) - - // During WARM_UP state, we do try to write sentinels to - // memcache but any failures are ignored. We run memcache - // server in this mode for sometime to prime our memcache - // and warm up memcache server. - if state.State == WarmUpServer { - entry.WarmingUp = true - } - } else { - connSkippedByAddr.Add(state.Address, 1) - } - } - entry.Items = make([]*Item, 0, 1) - results[shardId] = entry - } - entry.Items = append(entry.Items, item) - } - - return results -} - -// See ShardManager interface for documentation. -func (m *BaseShardManager) GetAllShards() map[int]net2.ManagedConn { - results := make(map[int]net2.ManagedConn) - - m.rwMutex.RLock() - defer m.rwMutex.RUnlock() - - for i, state := range m.shardStates { - conn, err := m.pool.Get("tcp", state.Address) - if err != nil { - m.logError(err) - conn = nil - } - results[i] = conn - } - - return results -} - -func (m *BaseShardManager) fillEntryWithConnection(address string, entry *ShardMapping) { - conn, err := m.pool.Get("tcp", address) - if err != nil { - m.logError(err) - connErrByAddr.Add(address, 1) - entry.ConnErr = err - } else { - connOkByAddr.Add(address, 1) - entry.Connection = conn - } -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/constants.go b/vendor/github.com/dropbox/godropbox/memcache/constants.go deleted file mode 100644 index 060419b..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/constants.go +++ /dev/null @@ -1,107 +0,0 @@ -package memcache - -// -// Magic Byte -// - -const ( - reqMagicByte uint8 = 0x80 - respMagicByte uint8 = 0x81 -) - -// -// Response Status -// - -type ResponseStatus uint16 - -const ( - StatusNoError ResponseStatus = iota - StatusKeyNotFound - StatusKeyExists - StatusValueTooLarge - StatusInvalidArguments - StatusItemNotStored - StatusIncrDecrOnNonNumericValue - StatusVbucketBelongsToAnotherServer // Not used - StatusAuthenticationError // Not used - StatusAuthenticationContinue // Not used -) - -const ( - StatusUnknownCommand ResponseStatus = 0x81 + iota - StatusOutOfMemory - StatusNotSupported - StatusInternalError - StatusBusy - StatusTempFailure -) - -// -// Command Opcodes -// - -type opCode uint8 - -const ( - opGet opCode = iota - opSet - opAdd - opReplace - opDelete - opIncrement - opDecrement - opQuit // Unsupported - opFlush - opGetQ // Unsupported - opNoOp // Unsupported - opVersion - opGetK - opGetKQ // Unsupported - opAppend - opPrepend - opStat - opSetQ // Unsupported - opAddQ // Unsupported - opReplaceQ // Unsupported - opDeleteQ // Unsupported - opIncrementQ // Unsupported - opDecrementQ // Unsupported - opQuitQ // Unsupported - opFlushQ // Unsupported - opAppendQ // Unsupported - opPrependQ // Unsupported - opVerbosity - opTouch // Unsupported - opGAT // Unsupported - opGATQ // Unsupported -) - -// More unsupported opcodes: -// 0x20 SASL list mechs -// 0x21 SASL Auth -// 0x22 SASL Step -// 0x30 RGet -// 0x31 RSet -// 0x32 RSetQ -// 0x33 RAppend -// 0x34 RAppendQ -// 0x35 RPrepend -// 0x36 RPrependQ -// 0x37 RDelete -// 0x38 RDeleteQ -// 0x39 RIncr -// 0x3a RIncrQ -// 0x3b RDecr -// 0x3c RDecrQ -// 0x3d Set VBucket -// 0x3e Get VBucket -// 0x3f Del VBucket -// 0x40 TAP Connect -// 0x41 TAP Mutation -// 0x42 TAP Delete -// 0x43 TAP Flush -// 0x44 TAP Opaque -// 0x45 TAP VBucket Set -// 0x46 TAP Checkpoint Start -// 0x47 TAP Checkpoint End diff --git a/vendor/github.com/dropbox/godropbox/memcache/doc.go b/vendor/github.com/dropbox/godropbox/memcache/doc.go deleted file mode 100644 index 612157e..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// A generic memcache client library which supports connection pooling and -// flexible sharding. -// -// Implementation note: this client uses memcached's binary protocol. See -// https://code.google.com/p/memcached/wiki/BinaryProtocolRevamped for -// additional details. -package memcache diff --git a/vendor/github.com/dropbox/godropbox/memcache/interface.go b/vendor/github.com/dropbox/godropbox/memcache/interface.go deleted file mode 100644 index 70966e2..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/interface.go +++ /dev/null @@ -1,275 +0,0 @@ -package memcache - -import ( - "github.com/dropbox/godropbox/net2" -) - -// An item to be gotten from or stored in a memcache server. -type Item struct { - // The item's key (the key can be up to 250 bytes maximum). - Key string - - // The item's value. - Value []byte - - // Flags are server-opaque flags whose semantics are entirely up to the app. - Flags uint32 - - // aka CAS (check and set) in memcache documentation. - DataVersionId uint64 - - // Expiration is the cache expiration time, in seconds: either a relative - // time from now (up to 1 month), or an absolute Unix epoch time. - // Zero means the Item has no expiration time. - Expiration uint32 -} - -// A generic response to a memcache request. -type Response interface { - // This returns the status returned by the memcache server. When Error() - // is non-nil, this value may not be valid. - // - // NOTE: - // 1. For stat request, this returns the first non-StatusNoError encountered - // (or StatusNoError if there were no errors). - // 2. If the client is sharded, flush/stats/version/verbosity requests - // will return the first non-StatusNoError encountered (or StatusNoError - // if there were no errors). - Status() ResponseStatus - - // This returns nil when no error is encountered by the client, and the - // response status returned by the memcache server is StatusNoError. - // Otherwise, this returns an error. - // - // NOTE: - // 1. For get requests, this also returns nil when the response status - // StatusKeyNotFound. - // 2. For stat request, this returns the first error encountered (or nil - // if there were no errors). - // 3. If the client is sharded, flush/stats/version/verbosity requests - // will return the first error encountered (or nil if there were no - // errors). - Error() error -} - -// Response returned by Get/GetKey/GetAndTouch requests. -type GetResponse interface { - Response - - // This returns the key for the requested value. - Key() string - - // This returns the retreived entry. The value may be nil. - Value() []byte - - // This returns the entry's flags value. The value is only valid when - // the entry is found. - Flags() uint32 - - // This returns the data version id (aka CAS) for the item. The value is - // only valid when the entry is found. - DataVersionId() uint64 -} - -// Response returned by Set/Add/Replace/Delete/Append/Prepend requests. -type MutateResponse interface { - Response - - // This returns the input key (useful for SetMulti where operations may be - // applied out of order). - Key() string - - // This returns the data version id (aka CAS) for the item. For delete - // requests, this always returns zero. - DataVersionId() uint64 -} - -// Response returned by Increment/Decrement requests. -type CountResponse interface { - Response - - // This returns the input key (useful for SetMulti where operations may be - // applied out of order). - Key() string - - // This returns the resulting count value. On error status, this returns - // zero. - Count() uint64 -} - -// Response returned by Version request. -type VersionResponse interface { - Response - - // This returns the memcache version entries. On error status, this - // returns an empty string. The mapping is stored as: - // shard id -> version string - // (If the client is unsharded, the shard id is always zero). - Versions() map[int]string -} - -// Response returned by Stat request. -type StatResponse interface { - Response - - // This returns the retrieved stat entries. On error status, this returns - // nil. The mapping is stored as: - // shard id -> stats key -> stats value - // (If the client is unsharded, the shard id is always zero). - Entries() map[int](map[string]string) -} - -type Client interface { - // This retrieves a single entry from memcache. - Get(key string) GetResponse - - // Batch version of the Get method. - GetMulti(keys []string) map[string]GetResponse - - // This sets a single entry into memcache. If the item's data version id - // (aka CAS) is nonzero, the set operation can only succeed if the item - // exists in memcache and has a same data version id. - Set(item *Item) MutateResponse - - // Batch version of the Set method. Note that the response entries - // ordering is undefined (i.e., may not match the input ordering). - SetMulti(items []*Item) []MutateResponse - - // *** This method is specific to Dropbox zookeeper-managed memcache *** - // This is the same as SetMutli. The only difference is that SetMulti will - // only write to ACTIVE memcache shards, while SetSentinels will write to - // both ACTIVE and WRITE_ONLY memcache shards. - SetSentinels(items []*Item) []MutateResponse - - // This adds a single entry into memcache. Note: Add will fail if the - // item already exist in memcache. - Add(item *Item) MutateResponse - - // Batch version of the Add method. Note that the response entries - // ordering is undefined (i.e., may not match the input ordering). - AddMulti(item []*Item) []MutateResponse - - // This replaces a single entry in memcache. Note: Replace will fail if - // the does not exist in memcache. - Replace(item *Item) MutateResponse - - // This delets a single entry from memcache. - Delete(key string) MutateResponse - - // Batch version of the Delete method. Note that the response entries - // ordering is undefined (i.e., may not match the input ordering) - DeleteMulti(keys []string) []MutateResponse - - // This appends the value bytes to the end of an existing entry. Note that - // this does not allow you to extend past the item limit. - Append(key string, value []byte) MutateResponse - - // This prepends the value bytes to the end of an existing entry. Note that - // this does not allow you to extend past the item limit. - Prepend(key string, value []byte) MutateResponse - - // This increments the key's counter by delta. If the counter does not - // exist, one of two things may happen: - // 1. If the expiration value is all one-bits (0xffffffff), the operation - // will fail with StatusNotFound. - // 2. For all other expiration values, the operation will succeed by - // seeding the value for this key with the provided initValue to expire - // with the provided expiration time. The flags will be set to zero. - // - // NOTE: - // 1. If you want to set the value of the counter with add/set/replace, - // the objects data must be the ascii representation of the value and - // not the byte values of a 64 bit integer. - // 2. Incrementing the counter may cause the counter to wrap. - Increment( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse - - // This decrements the key's counter by delta. If the counter does not - // exist, one of two things may happen: - // 1. If the expiration value is all one-bits (0xffffffff), the operation - // will fail with StatusNotFound. - // 2. For all other expiration values, the operation will succeed by - // seeding the value for this key with the provided initValue to expire - // with the provided expiration time. The flags will be set to zero. - // - // NOTE: - // 1. If you want to set the value of the counter with add/set/replace, - // the objects data must be the ascii representation of the value and - // not the byte values of a 64 bit integer. - // 2. Decrementing a counter will never result in a "negative value" (or - // cause the counter to "wrap"). instead the counter is set to 0. - Decrement( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse - - // This invalidates all existing cache items after expiration number of - // seconds. - Flush(expiration uint32) Response - - // This requests the server statistics. When the key is an empty string, - // the server will respond with a "default" set of statistics information. - Stat(statsKey string) StatResponse - - // This returns the server's version string. - Version() VersionResponse - - // This set the verbosity level of the server. - Verbosity(verbosity uint32) Response -} - -// A memcache client which communicates with a specific memcache shard. -type ClientShard interface { - Client - - // This returns the memcache server's shard id. - ShardId() int - - // This returns true if the client is in a valid state. If the client is - // in invalid state, the user should abandon the current client / channel, - // and create a new client / channel as replacment. - IsValidState() bool -} - -// Used for returning shard mapping results from ShardManager's -// GetShardsForKeys/GetShardsForItems calls. -type ShardMapping struct { - Connection net2.ManagedConn - ConnErr error - Keys []string // Populated for GetShardsForKeys - Items []*Item // Populated for GetShardsForItems - WarmingUp bool // Populated for GetShardsForSentinels -} - -// The ShardManager decides which memcache shard a key/item belongs to, and -// provide connections to the shards. -type ShardManager interface { - // This returns the shard id and a connection to shard for a single key. - // If the key does not belong to any shard, this returns (-1, nil). Note - // that the connection may be nil for valid shard id (for instance, if - // the shard server is temporarily unavailable). - GetShard(key string) (shardId int, conn net2.ManagedConn, err error) - - // This returns a (shard id -> (connection, list of keys)) mapping for - // the requested keys. Keys that do not belong to any shard are mapped - // to shard id -1. - GetShardsForKeys(keys []string) map[int]*ShardMapping - - // This returns a (shard id -> (connection, list of items)) mapping for - // the requested items. Items that do not belong to any shard are mapped - // to shard id -1. - GetShardsForItems(items []*Item) map[int]*ShardMapping - - // *** This method is specific to Dropbox zookeeper-managed memcache *** - // This returns a (shard id -> (connection, list of items)) mapping for - // the requested sentinel items. Sential items that do not belong to any - // shard are mapped to shard id -1. - GetShardsForSentinels(items []*Item) map[int]*ShardMapping - - // This return a (shard id -> connection) mapping for all shards. - GetAllShards() map[int]net2.ManagedConn -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/mock_client.go b/vendor/github.com/dropbox/godropbox/memcache/mock_client.go deleted file mode 100644 index c2c078d..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/mock_client.go +++ /dev/null @@ -1,290 +0,0 @@ -package memcache - -import ( - "sync" - - "github.com/dropbox/godropbox/errors" -) - -type MockClient struct { - data map[string]*Item - version uint64 - mutex sync.Mutex -} - -func NewMockClient() Client { - return &MockClient{data: make(map[string]*Item)} -} - -func (c *MockClient) getHelper(key string) GetResponse { - if v, ok := c.data[key]; ok { - return NewGetResponse( - key, - StatusNoError, - v.Flags, - v.Value, - v.DataVersionId) - } - return NewGetResponse(key, StatusKeyNotFound, 0, nil, 0) -} - -// This retrieves a single entry from memcache. -func (c *MockClient) Get(key string) GetResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - return c.getHelper(key) -} - -// Batch version of the Get method. -func (c *MockClient) GetMulti(keys []string) map[string]GetResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - res := make(map[string]GetResponse) - for _, key := range keys { - res[key] = c.getHelper(key) - } - return res -} - -func (c *MockClient) setHelper(item *Item) MutateResponse { - c.version++ - - newItem := &Item{ - Key: item.Key, - Value: item.Value, - Flags: item.Flags, - Expiration: item.Expiration, - DataVersionId: c.version, - } - - existing, ok := c.data[newItem.Key] - - if item.DataVersionId == 0 || - (ok && item.DataVersionId == existing.DataVersionId) { - - c.data[newItem.Key] = newItem - return NewMutateResponse( - newItem.Key, - StatusNoError, - newItem.DataVersionId, - false) - } else if !ok { - return NewMutateResponse( - newItem.Key, - StatusKeyNotFound, - 0, - false) - } else { - // CAS mismatch - return NewMutateResponse( - newItem.Key, - StatusKeyExists, - 0, - false) - } - -} - -// This sets a single entry into memcache. If the item's data version id -// (aka CAS) is nonzero, the set operation can only succeed if the item -// exists in memcache and has a same data version id. -func (c *MockClient) Set(item *Item) MutateResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - return c.setHelper(item) -} - -// Batch version of the Set method. Note that the response entries -// ordering is undefined (i.e., may not match the input ordering). -func (c *MockClient) SetMulti(items []*Item) []MutateResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - res := make([]MutateResponse, len(items)) - for i, item := range items { - res[i] = c.setHelper(item) - } - return res -} - -func (c *MockClient) SetSentinels(items []*Item) []MutateResponse { - // TODO(patrick): Support state mocking - return c.SetMulti(items) -} - -func (c *MockClient) addHelper(item *Item) MutateResponse { - c.version++ - - newItem := &Item{ - Key: item.Key, - Value: item.Value, - Flags: item.Flags, - Expiration: item.Expiration, - DataVersionId: c.version, - } - - if _, ok := c.data[newItem.Key]; !ok { - c.data[newItem.Key] = newItem - return NewMutateResponse( - newItem.Key, - StatusNoError, - newItem.DataVersionId, - false) - } else { - return NewMutateResponse( - newItem.Key, - StatusKeyExists, - 0, - false) - } -} - -// This adds a single entry into memcache. Note: Add will fail if the -// item already exist in memcache. -func (c *MockClient) Add(item *Item) MutateResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - return c.addHelper(item) -} - -// Batch version of the Add method. Note that the response entries -// ordering is undefined (i.e., may not match the input ordering). -func (c *MockClient) AddMulti(items []*Item) []MutateResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - res := make([]MutateResponse, len(items)) - for i, item := range items { - res[i] = c.addHelper(item) - } - return res -} - -// This replaces a single entry in memcache. Note: Replace will fail if -// the does not exist in memcache. -func (c *MockClient) Replace(item *Item) MutateResponse { - return NewMutateErrorResponse( - item.Key, - errors.Newf("Replace not implemented")) -} - -// This deletes a single entry from memcache. -func (c *MockClient) Delete(key string) MutateResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - _, ok := c.data[key] - if !ok { - return NewMutateResponse( - key, - StatusKeyNotFound, - 0, - false) - } - - delete(c.data, key) - - return NewMutateResponse( - key, - StatusNoError, - 0, - false) -} - -// Batch version of the Delete method. Note that the response entries -// ordering is undefined (i.e., may not match the input ordering) -func (c *MockClient) DeleteMulti(keys []string) []MutateResponse { - res := make([]MutateResponse, len(keys)) - for i, key := range keys { - res[i] = c.Delete(key) - } - return res -} - -// This appends the value bytes to the end of an existing entry. Note that -// this does not allow you to extend past the item limit. -func (c *MockClient) Append(key string, value []byte) MutateResponse { - return NewMutateErrorResponse(key, errors.Newf("Append not implemented")) -} - -// This prepends the value bytes to the end of an existing entry. Note that -// this does not allow you to extend past the item limit. -func (c *MockClient) Prepend(key string, value []byte) MutateResponse { - return NewMutateErrorResponse(key, errors.Newf("Prepend not implemented")) -} - -// This increments the key's counter by delta. If the counter does not -// exist, one of two things may happen: -// 1. If the expiration value is all one-bits (0xffffffff), the operation -// will fail with StatusNotFound. -// 2. For all other expiration values, the operation will succeed by -// seeding the value for this key with the provided initValue to expire -// with the provided expiration time. The flags will be set to zero. -// -// NOTE: -// 1. If you want to set the value of the counter with add/set/replace, -// the objects data must be the ascii representation of the value and -// not the byte values of a 64 bit integer. -// 2. Incrementing the counter may cause the counter to wrap. -func (c *MockClient) Increment( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - return NewCountErrorResponse(key, errors.Newf("Increment not implemented")) -} - -// This decrements the key's counter by delta. If the counter does not -// exist, one of two things may happen: -// 1. If the expiration value is all one-bits (0xffffffff), the operation -// will fail with StatusNotFound. -// 2. For all other expiration values, the operation will succeed by -// seeding the value for this key with the provided initValue to expire -// with the provided expiration time. The flags will be set to zero. -// -// NOTE: -// 1. If you want to set the value of the counter with add/set/replace, -// the objects data must be the ascii representation of the value and -// not the byte values of a 64 bit integer. -// 2. Decrementing a counter will never result in a "negative value" (or -// cause the counter to "wrap"). instead the counter is set to 0. -func (c *MockClient) Decrement( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - return NewCountErrorResponse(key, errors.Newf("Decrement not implemented")) -} - -// This invalidates all existing cache items after expiration number of -// seconds. -func (c *MockClient) Flush(expiration uint32) Response { - c.mutex.Lock() - defer c.mutex.Unlock() - - // TODO(patrick): Use expiration argument - c.data = make(map[string]*Item) - return NewResponse(StatusNoError) -} - -// This requests the server statistics. When the key is an empty string, -// the server will respond with a "default" set of statistics information. -func (c *MockClient) Stat(statsKey string) StatResponse { - return NewStatErrorResponse(errors.Newf("Stat not implemented"), nil) -} - -// This returns the server's version string. -func (c *MockClient) Version() VersionResponse { - return NewVersionResponse(StatusNoError, map[int]string{0: "MockSever"}) -} - -// This set the verbosity level of the server. -func (c *MockClient) Verbosity(verbosity uint32) Response { - return NewResponse(StatusNoError) -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/raw_ascii_client.go b/vendor/github.com/dropbox/godropbox/memcache/raw_ascii_client.go deleted file mode 100644 index 7d8f94f..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/raw_ascii_client.go +++ /dev/null @@ -1,751 +0,0 @@ -package memcache - -import ( - "bufio" - "io" - "strconv" - "strings" - "sync" - - "github.com/dropbox/godropbox/errors" -) - -// An unsharded memcache client implementation which operates on a pre-existing -// io channel (The user must explicitly setup and close down the channel), -// using the ascii memcache protocol. Note that the client assumes nothing -// else is sending or receiving on the network channel. In general, all client -// operations are serialized (Use multiple channels / clients if parallelism -// is needed). -type RawAsciiClient struct { - shard int - channel io.ReadWriter - - mutex sync.Mutex - validState bool - writer *bufio.Writer - reader *bufio.Reader -} - -// This creates a new memcache RawAsciiClient. -func NewRawAsciiClient(shard int, channel io.ReadWriter) ClientShard { - return &RawAsciiClient{ - shard: shard, - channel: channel, - validState: true, - writer: bufio.NewWriter(channel), - reader: bufio.NewReader(channel), - } -} - -func (c *RawAsciiClient) writeStrings(strs ...string) error { - if !c.validState { - return errors.New("Skipping due to previous error") - } - - for _, str := range strs { - _, err := c.writer.WriteString(str) - if err != nil { - c.validState = false - return err - } - } - - return nil -} - -func (c *RawAsciiClient) flushWriter() error { - if !c.validState { - return errors.New("Skipping due to previous error") - } - - err := c.writer.Flush() - if err != nil { - c.validState = false - return err - } - - return nil -} - -func (c *RawAsciiClient) readLine() (string, error) { - line, isPrefix, err := c.reader.ReadLine() - if err != nil { - c.validState = false - return "", err - } - if isPrefix { - c.validState = false - return "", errors.New("Readline truncated") - } - - return string(line), nil -} - -func (c *RawAsciiClient) read(numBytes int) ([]byte, error) { - result := make([]byte, numBytes, numBytes) - - _, err := io.ReadFull(c.reader, result) - if err != nil { - c.validState = false - return nil, err - } - - return result, nil -} - -func (c *RawAsciiClient) checkEmptyBuffers() error { - if c.writer.Buffered() != 0 { - c.validState = false - return errors.New("writer buffer not fully flushed") - } - if c.reader.Buffered() != 0 { - c.validState = false - return errors.New("reader buffer not fully drained") - } - - return nil -} - -func (c *RawAsciiClient) ShardId() int { - return c.shard -} - -func (c *RawAsciiClient) IsValidState() bool { - c.mutex.Lock() - defer c.mutex.Unlock() - - return c.validState -} - -func (c *RawAsciiClient) Get(key string) GetResponse { - return c.GetMulti([]string{key})[key] -} - -func (c *RawAsciiClient) GetMulti(keys []string) map[string]GetResponse { - responses := make(map[string]GetResponse, len(keys)) - neededKeys := []string{} - for _, key := range keys { - if _, ok := responses[key]; ok { - continue - } - - if !isValidKeyString(key) { - responses[key] = NewGetErrorResponse(key, errors.New("Invalid key")) - continue - } - - neededKeys = append(neededKeys, key) - responses[key] = nil - } - - if len(neededKeys) == 0 { - return responses - } - - populateErrorResponses := func(e error) { - for _, key := range neededKeys { - if responses[key] == nil { - responses[key] = NewGetErrorResponse(key, e) - } - } - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - // NOTE: Always use gets instead of get since returning the extra cas id - // info is relatively cheap. - err := c.writeStrings("gets") - if err != nil { - populateErrorResponses(err) - return responses - } - - for _, key := range neededKeys { - err := c.writeStrings(" ", key) - if err != nil { - populateErrorResponses(err) - return responses - } - } - - err = c.writeStrings("\r\n") - if err != nil { - populateErrorResponses(err) - return responses - } - - err = c.flushWriter() - if err != nil { - populateErrorResponses(err) - return responses - } - - // Any error that occurs while reading the results will result in mid - // stream termination, i.e., the channel is no longer in valid state. - for { - line, err := c.readLine() - if err != nil { - populateErrorResponses(err) - return responses - } - - if line == "END" { - break - } - - slice := strings.Split(line, " ") - - // line is of the form: VALUE - if len(slice) != 5 || slice[0] != "VALUE" { - c.validState = false - populateErrorResponses(errors.New(line)) - return responses - } - - key := slice[1] - if v, ok := responses[key]; !ok || v != nil { - c.validState = false - populateErrorResponses(errors.New(line)) - return responses - } - - flags, err := strconv.ParseUint(slice[2], 10, 32) - if err != nil { - c.validState = false - populateErrorResponses(errors.New(line)) - return responses - } - - size, err := strconv.ParseUint(slice[3], 10, 32) - if err != nil { - c.validState = false - populateErrorResponses(errors.New(line)) - return responses - } - - version, err := strconv.ParseUint(slice[4], 10, 64) - if err != nil { - c.validState = false - populateErrorResponses(errors.New(line)) - return responses - } - - value, err := c.read(int(size) + 2) - if err != nil { - populateErrorResponses(err) - return responses - } - - if value[size] != '\r' && value[size+1] != '\n' { - // sanity check - populateErrorResponses(errors.New("Corrupted stream")) - return responses - } - value = value[:size] - - // TODO(patrick): check status - responses[key] = NewGetResponse( - key, - StatusNoError, - uint32(flags), - value, - version) - } - - err = c.checkEmptyBuffers() - if err != nil { - populateErrorResponses(err) - return responses - } - - for _, key := range neededKeys { - if responses[key] == nil { - responses[key] = NewGetResponse(key, StatusKeyNotFound, 0, nil, 0) - } - } - - return responses -} - -func (c *RawAsciiClient) storeRequests( - cmd string, - items []*Item) []MutateResponse { - - var err error - responses := make([]MutateResponse, len(items), len(items)) - needSending := false - for i, item := range items { - if item == nil { - responses[i] = NewMutateErrorResponse("", errors.New("item is nil")) - continue - } - - if item.DataVersionId != 0 && cmd != "set" { - responses[i] = NewMutateErrorResponse( - item.Key, - errors.Newf( - "Ascii protocol does not support %s with cas id", - cmd)) - continue - } - - if !isValidKeyString(item.Key) { - responses[i] = NewMutateErrorResponse( - item.Key, - errors.New("Invalid key")) - continue - } - - err = validateValue(item.Value) - if err != nil { - responses[i] = NewMutateErrorResponse(item.Key, err) - continue - } - - needSending = true - } - - if !needSending { - return responses - } - - populateErrorResponses := func(e error) { - for i, item := range items { - if responses[i] == nil { - responses[i] = NewMutateErrorResponse(item.Key, e) - } - } - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - // NOTE: store requests are pipelined. - for i, item := range items { - if responses[i] != nil { - continue - } - - flags := strconv.FormatUint(uint64(item.Flags), 10) - expiration := strconv.FormatUint(uint64(item.Expiration), 10) - size := strconv.Itoa(len(item.Value)) - - if item.DataVersionId != 0 { - // We have already verified that cmd must be "set" - err = c.writeStrings( - "cas ", - item.Key, " ", - flags, " ", - expiration, " ", - size, " ", - strconv.FormatUint(item.DataVersionId, 10), - "\r\n") - } else { - err = c.writeStrings( - cmd, " ", - item.Key, " ", - flags, " ", - expiration, " ", - size, - "\r\n") - } - - if err != nil { - populateErrorResponses(err) - return responses - } - - err = c.writeStrings(string(item.Value), "\r\n") - if err != nil { - populateErrorResponses(err) - return responses - } - } - - err = c.flushWriter() - if err != nil { - populateErrorResponses(err) - return responses - } - - for i, item := range items { - if responses[i] != nil { - continue - } - - line, err := c.readLine() - if err != nil { - populateErrorResponses(err) - return responses - } - - // NOTE: Unfortunately, the returned response does not include - // cas info. - if line == "STORED" { - responses[i] = NewMutateResponse( - item.Key, - StatusNoError, - 0, - true) - } else if line == "NOT_FOUND" { - responses[i] = NewMutateResponse( - item.Key, - StatusKeyNotFound, - 0, - true) - } else if line == "NOT_STORED" { - responses[i] = NewMutateResponse( - item.Key, - StatusItemNotStored, - 0, - true) - } else if line == "EXISTS" { - responses[i] = NewMutateResponse( - item.Key, - StatusKeyExists, - 0, - true) - } else { - responses[i] = NewMutateErrorResponse(item.Key, errors.New(line)) - } - } - - _ = c.checkEmptyBuffers() - - return responses -} - -func (c *RawAsciiClient) Set(item *Item) MutateResponse { - return c.SetMulti([]*Item{item})[0] -} - -func (c *RawAsciiClient) SetMulti(items []*Item) []MutateResponse { - return c.storeRequests("set", items) -} - -func (c *RawAsciiClient) SetSentinels(items []*Item) []MutateResponse { - // There are no difference between SetMutli and SetSentinels since - // SetMulti issues set / cas commands depending on the items' version ids. - return c.SetMulti(items) -} - -func (c *RawAsciiClient) Add(item *Item) MutateResponse { - return c.AddMulti([]*Item{item})[0] -} - -func (c *RawAsciiClient) AddMulti(items []*Item) []MutateResponse { - return c.storeRequests("add", items) -} - -func (c *RawAsciiClient) Replace(item *Item) MutateResponse { - return c.storeRequests("replace", []*Item{item})[0] -} - -func (c *RawAsciiClient) Append(key string, value []byte) MutateResponse { - items := []*Item{ - &Item{ - Key: key, - Value: value, - }, - } - return c.storeRequests("append", items)[0] -} - -func (c *RawAsciiClient) Prepend(key string, value []byte) MutateResponse { - items := []*Item{ - &Item{ - Key: key, - Value: value, - }, - } - return c.storeRequests("prepend", items)[0] -} - -func (c *RawAsciiClient) Delete(key string) MutateResponse { - return c.DeleteMulti([]string{key})[0] -} - -func (c *RawAsciiClient) DeleteMulti(keys []string) []MutateResponse { - responses := make([]MutateResponse, len(keys), len(keys)) - - c.mutex.Lock() - defer c.mutex.Unlock() - - // NOTE: delete requests are pipelined. - for i, key := range keys { - if !isValidKeyString(key) { - responses[i] = NewMutateErrorResponse( - key, - errors.New("Invalid key")) - continue - } - - err := c.writeStrings("delete ", key, "\r\n") - if err != nil { - responses[i] = NewMutateErrorResponse(key, err) - } - } - - err := c.flushWriter() - if err != nil { - // The delete requests may or may not have successfully reached the - // memcached, just error out. - for i, key := range keys { - if responses[i] == nil { - responses[i] = NewMutateErrorResponse(key, err) - } - } - } - - for i, key := range keys { - if responses[i] != nil { - continue - } - - line, err := c.readLine() - if err != nil { - responses[i] = NewMutateErrorResponse(key, err) - continue - } - - if line == "DELETED" { - responses[i] = NewMutateResponse(key, StatusNoError, 0, true) - } else if line == "NOT_FOUND" { - responses[i] = NewMutateResponse(key, StatusKeyNotFound, 0, true) - } else { // Unexpected error msg - responses[i] = NewMutateErrorResponse(key, errors.New(line)) - } - } - - _ = c.checkEmptyBuffers() - - return responses -} - -func (c *RawAsciiClient) countRequest( - cmd string, - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - if expiration != 0xffffffff { - return NewCountErrorResponse( - key, - errors.New( - "Ascii protocol does not support initial value / "+ - "expiration. expiration must be set to 0xffffffff.")) - } - - if !isValidKeyString(key) { - return NewCountErrorResponse( - key, - errors.New("Invalid key")) - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - err := c.writeStrings( - cmd, " ", - key, " ", - strconv.FormatUint(delta, 10), "\r\n") - if err != nil { - return NewCountErrorResponse(key, err) - } - - err = c.flushWriter() - if err != nil { - return NewCountErrorResponse(key, err) - } - - line, err := c.readLine() - if err != nil { - return NewCountErrorResponse(key, err) - } - - _ = c.checkEmptyBuffers() - - if line == "NOT_FOUND" { - return NewCountResponse(key, StatusKeyNotFound, 0) - } - - val, err := strconv.ParseUint(line, 10, 64) - if err != nil { - return NewCountErrorResponse(key, err) - } - - return NewCountResponse(key, StatusNoError, val) -} - -func (c *RawAsciiClient) Increment( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - return c.countRequest("incr", key, delta, initValue, expiration) -} - -func (c *RawAsciiClient) Decrement( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - return c.countRequest("decr", key, delta, initValue, expiration) -} - -func (c *RawAsciiClient) Flush(expiration uint32) Response { - c.mutex.Lock() - defer c.mutex.Unlock() - - err := c.writeStrings( - "flush_all ", - strconv.FormatUint(uint64(expiration), 10), - "\r\n") - if err != nil { - return NewErrorResponse(err) - } - - err = c.flushWriter() - if err != nil { - return NewErrorResponse(err) - } - - line, err := c.readLine() - if err != nil { - return NewErrorResponse(err) - } - - _ = c.checkEmptyBuffers() - - if line != "OK" { - // memcached returned an error message. This should never happen - // according to the docs. - return NewErrorResponse(errors.New(line)) - } - - return NewResponse(StatusNoError) -} - -func (c *RawAsciiClient) Stat(statsKey string) StatResponse { - shardEntries := make(map[int](map[string]string)) - entries := make(map[string]string) - shardEntries[c.ShardId()] = entries - - if statsKey != "" { - return NewStatErrorResponse( - errors.New("Ascii protocol does not support specific stats lookup"), - shardEntries) - } - - var err error - - c.mutex.Lock() - defer c.mutex.Unlock() - - err = c.writeStrings("stats\r\n") - if err != nil { - return NewStatErrorResponse(err, shardEntries) - } - - err = c.flushWriter() - if err != nil { - return NewStatErrorResponse(err, shardEntries) - } - - for { - line, err := c.readLine() - if err != nil { - NewStatErrorResponse(err, shardEntries) - } - - if line == "END" { - break - } - - // line is of the form: STAT - slice := strings.SplitN(line, " ", 3) - - if len(slice) != 3 || slice[0] != "STAT" { - // The channel is no longer in valid state since we're exiting - // stats mid stream. - c.validState = false - return NewStatErrorResponse(errors.New(line), shardEntries) - } - - entries[slice[1]] = slice[2] - } - - _ = c.checkEmptyBuffers() - - return NewStatResponse(StatusNoError, shardEntries) -} - -func (c *RawAsciiClient) Version() VersionResponse { - versions := make(map[int]string, 1) - - c.mutex.Lock() - defer c.mutex.Unlock() - - err := c.writeStrings("version\r\n") - if err != nil { - return NewVersionErrorResponse(err, versions) - } - - err = c.flushWriter() - if err != nil { - return NewVersionErrorResponse(err, versions) - } - - line, err := c.readLine() - if err != nil { - return NewVersionErrorResponse(err, versions) - } - - _ = c.checkEmptyBuffers() - - if !strings.HasPrefix(line, "VERSION ") { - // memcached returned an error message. - return NewVersionErrorResponse(errors.New(line), versions) - } - - versions[c.ShardId()] = line[len("VERSION "):len(line)] - - return NewVersionResponse(StatusNoError, versions) -} - -func (c *RawAsciiClient) Verbosity(verbosity uint32) Response { - c.mutex.Lock() - defer c.mutex.Unlock() - - err := c.writeStrings( - "verbosity ", - strconv.FormatUint(uint64(verbosity), 10), - "\r\n") - if err != nil { - return NewErrorResponse(err) - } - - err = c.flushWriter() - if err != nil { - return NewErrorResponse(err) - } - - line, err := c.readLine() - if err != nil { - return NewErrorResponse(err) - } - - _ = c.checkEmptyBuffers() - - if line != "OK" { - // memcached returned an error message. This should never happen - // according to the docs. - return NewErrorResponse(errors.New(line)) - } - - return NewResponse(StatusNoError) -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/raw_binary_client.go b/vendor/github.com/dropbox/godropbox/memcache/raw_binary_client.go deleted file mode 100644 index ecf9fb2..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/raw_binary_client.go +++ /dev/null @@ -1,745 +0,0 @@ -package memcache - -import ( - "bytes" - "encoding/binary" - "io" - "sync" - - "github.com/dropbox/godropbox/errors" -) - -const ( - headerLength = 24 - maxKeyLength = 250 - // NOTE: Storing values larger than 1MB requires recompiling memcached. - maxValueLength = 1024 * 1024 -) - -func isValidKeyChar(char byte) bool { - return (0x21 <= char && char <= 0x7e) || (0x80 <= char && char <= 0xff) -} - -func isValidKeyString(key string) bool { - if len(key) > maxKeyLength { - return false - } - - for _, char := range []byte(key) { - if !isValidKeyChar(char) { - return false - } - } - - return true -} - -func validateValue(value []byte) error { - if value == nil { - return errors.New("Invalid value: cannot be nil") - } - - if len(value) > maxValueLength { - return errors.Newf( - "Invalid value: length %d longer than max length %d", - len(value), - maxValueLength) - } - - return nil -} - -type header struct { - Magic uint8 - OpCode uint8 - KeyLength uint16 - ExtrasLength uint8 - DataType uint8 - VBucketIdOrStatus uint16 // vbucket id for request, status for response - TotalBodyLength uint32 - Opaque uint32 // unless value - DataVersionId uint64 // aka CAS -} - -// An unsharded memcache client implementation which operates on a pre-existing -// io channel (The user must explicitly setup and close down the channel), -// using the binary memcached protocol. Note that the client assumes nothing -// else is sending or receiving on the network channel. In general, all client -// operations are serialized (Use multiple channels / clients if parallelism -// is needed). -type RawBinaryClient struct { - shard int - channel io.ReadWriter - mutex sync.Mutex - validState bool -} - -// This creates a new memcache RawBinaryClient. -func NewRawBinaryClient(shard int, channel io.ReadWriter) ClientShard { - return &RawBinaryClient{ - shard: shard, - channel: channel, - validState: true, - } -} - -// See ClientShard interface for documentation. -func (c *RawBinaryClient) ShardId() int { - return c.shard -} - -// See ClientShard interface for documentation. -func (c *RawBinaryClient) IsValidState() bool { - return c.validState -} - -// Sends a memcache request through the connection. NOTE: extras must be -// fix-sized values. -func (c *RawBinaryClient) sendRequest( - code opCode, - dataVersionId uint64, // aka CAS - key []byte, // may be nil - value []byte, // may be nil - extras ...interface{}) (err error) { - - if !c.validState { - // An error has occurred previously. It's not safe to continue sending. - return errors.New("Skipping due to previous error") - } - defer func() { - if err != nil { - c.validState = false - } - }() - - extrasBuffer := new(bytes.Buffer) - for _, extra := range extras { - err := binary.Write(extrasBuffer, binary.BigEndian, extra) - if err != nil { - return errors.Wrap(err, "Failed to write extra") - } - } - - // NOTE: - // - memcache only supports a single dataType (0x0) - // - vbucket id is not used by the library since vbucket related op - // codes are unsupported - hdr := header{ - Magic: reqMagicByte, - OpCode: byte(code), - KeyLength: uint16(len(key)), - ExtrasLength: uint8(extrasBuffer.Len()), - TotalBodyLength: uint32(len(key) + len(value) + extrasBuffer.Len()), - DataVersionId: dataVersionId, - } - - msgBuffer := new(bytes.Buffer) - - if err := binary.Write(msgBuffer, binary.BigEndian, hdr); err != nil { - return errors.Wrap(err, "Failed to write header") - } - if msgBuffer.Len() != headerLength { // sanity check - return errors.Newf("Incorrect header size: %d", msgBuffer.Len()) - } - - bytesWritten, err := extrasBuffer.WriteTo(msgBuffer) - if err != nil { - return errors.Wrap(err, "Failed to add extras to msg") - } - if bytesWritten != int64(hdr.ExtrasLength) { - return errors.New("Failed to write out extras") - } - - if key != nil { - if _, err := msgBuffer.Write(key); err != nil { - return errors.Wrap(err, "Failed to write key") - } - } - - if value != nil { - if _, err := msgBuffer.Write(value); err != nil { - return errors.Wrap(err, "Failed to write value") - } - } - - bytesWritten, err = msgBuffer.WriteTo(c.channel) - if err != nil { - return errors.Wrap(err, "Failed to send msg") - } - if bytesWritten != int64((hdr.TotalBodyLength)+headerLength) { - return errors.New("Failed to sent out message") - } - - return nil -} - -// Receive a memcache response from the connection. The status, -// dataVersionId (aka CAS), key and value are returned, while the extra -// values are stored in the arguments. NOTE: extras must be pointers to -// fix-sized values. -func (c *RawBinaryClient) receiveResponse( - expectedCode opCode, - extras ...interface{}) ( - status ResponseStatus, - dataVersionId uint64, - key []byte, // is nil when key length is zero - value []byte, // is nil when the value length is zero - err error) { - - if !c.validState { - // An error has occurred previously. It's not safe to continue sending. - err = errors.New("Skipping due to previous error") - return - } - defer func() { - if err != nil { - c.validState = false - } - }() - - hdr := header{} - if err = binary.Read(c.channel, binary.BigEndian, &hdr); err != nil { - err = errors.Wrap(err, "Failed to read header") - return - } - if hdr.Magic != respMagicByte { - err = errors.Newf("Invalid response magic byte: %d", hdr.Magic) - return - } - if hdr.OpCode != byte(expectedCode) { - err = errors.Newf("Invalid response op code: %d", hdr.OpCode) - return - } - if hdr.DataType != 0 { - err = errors.Newf("Invalid data type: %d", hdr.DataType) - return - } - - valueLength := int(hdr.TotalBodyLength) - valueLength -= (int(hdr.KeyLength) + int(hdr.ExtrasLength)) - if valueLength < 0 { - err = errors.Newf("Invalid response header. Wrong payload size.") - return - } - - status = ResponseStatus(hdr.VBucketIdOrStatus) - dataVersionId = hdr.DataVersionId - - if hdr.ExtrasLength == 0 { - if status == StatusNoError && len(extras) != 0 { - err = errors.Newf("Expecting extras payload") - return - } - // the response has no extras - } else { - extrasBytes := make([]byte, hdr.ExtrasLength, hdr.ExtrasLength) - if _, err = io.ReadFull(c.channel, extrasBytes); err != nil { - err = errors.Wrap(err, "Failed to read extra") - return - } - - extrasBuffer := bytes.NewBuffer(extrasBytes) - - for _, extra := range extras { - err = binary.Read(extrasBuffer, binary.BigEndian, extra) - if err != nil { - err = errors.Wrap(err, "Failed to deserialize extra") - return - } - } - - if extrasBuffer.Len() != 0 { - err = errors.Newf("Not all bytes are consumed by extras fields") - return - } - } - - if hdr.KeyLength > 0 { - key = make([]byte, hdr.KeyLength, hdr.KeyLength) - if _, err = io.ReadFull(c.channel, key); err != nil { - err = errors.Wrap(err, "Failed to read key") - return - } - } - - if valueLength > 0 { - value = make([]byte, valueLength, valueLength) - if _, err = io.ReadFull(c.channel, value); err != nil { - err = errors.Wrap(err, "Failed to read value") - return - } - } - - return -} - -func (c *RawBinaryClient) sendGetRequest(key string) GetResponse { - if !isValidKeyString(key) { - return NewGetErrorResponse( - key, - errors.New("Invalid key")) - } - - err := c.sendRequest(opGet, 0, []byte(key), nil) - if err != nil { - return NewGetErrorResponse(key, err) - } - - return nil -} - -func (c *RawBinaryClient) receiveGetResponse(key string) GetResponse { - var flags uint32 - status, version, _, value, err := c.receiveResponse(opGet, &flags) - if err != nil { - return NewGetErrorResponse(key, err) - } - return NewGetResponse(key, status, flags, value, version) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Get(key string) GetResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - if resp := c.sendGetRequest(key); resp != nil { - return resp - } - - return c.receiveGetResponse(key) -} - -func (c *RawBinaryClient) removeDuplicateKey(keys []string) []string { - keyMap := make(map[string]interface{}) - for _, key := range keys { - keyMap[key] = nil - } - cacheKeys := make([]string, len(keyMap)) - i := 0 - for key, _ := range keyMap { - cacheKeys[i] = key - i = i + 1 - } - return cacheKeys -} - -// See Client interface for documentation. -func (c *RawBinaryClient) GetMulti(keys []string) map[string]GetResponse { - if keys == nil { - return nil - } - - responses := make(map[string]GetResponse) - cacheKeys := c.removeDuplicateKey(keys) - - c.mutex.Lock() - defer c.mutex.Unlock() - - for _, key := range cacheKeys { - if resp := c.sendGetRequest(key); resp != nil { - responses[key] = resp - } - } - - for _, key := range cacheKeys { - if _, inMap := responses[key]; inMap { // error occurred while sending - continue - } - responses[key] = c.receiveGetResponse(key) - } - - return responses -} - -func (c *RawBinaryClient) sendMutateRequest( - code opCode, - item *Item, - addExtras bool) MutateResponse { - - if item == nil { - return NewMutateErrorResponse("", errors.New("item is nil")) - } - - if !isValidKeyString(item.Key) { - return NewMutateErrorResponse( - item.Key, - errors.New("Invalid key")) - } - - if err := validateValue(item.Value); err != nil { - return NewMutateErrorResponse(item.Key, err) - } - - extras := make([]interface{}, 0, 2) - if addExtras { - extras = append(extras, item.Flags) - extras = append(extras, item.Expiration) - } - - err := c.sendRequest( - code, - item.DataVersionId, - []byte(item.Key), - item.Value, - extras...) - if err != nil { - return NewMutateErrorResponse(item.Key, err) - } - return nil -} - -func (c *RawBinaryClient) receiveMutateResponse( - code opCode, - key string) MutateResponse { - - status, version, _, _, err := c.receiveResponse(code) - if err != nil { - return NewMutateErrorResponse(key, err) - } - return NewMutateResponse(key, status, version, false) -} - -// Perform a mutation operation specified by the given code. -func (c *RawBinaryClient) mutate(code opCode, item *Item) MutateResponse { - if item == nil { - return NewMutateErrorResponse("", errors.New("item is nil")) - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - if resp := c.sendMutateRequest(code, item, true); resp != nil { - return resp - } - - return c.receiveMutateResponse(code, item.Key) -} - -// Batch version of the mutate method. Note that the response entries -// ordering is undefined (i.e., may not match the input ordering) -func (c *RawBinaryClient) mutateMulti( - code opCode, - items []*Item) []MutateResponse { - - if items == nil { - return nil - } - - responses := make([]MutateResponse, len(items), len(items)) - - // Short-circuit function to avoid locking. - if len(items) == 0 { - return responses - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - for i, item := range items { - responses[i] = c.sendMutateRequest(code, item, true) - } - - for i, item := range items { - if responses[i] != nil { // error occurred while sending - continue - } - responses[i] = c.receiveMutateResponse(code, item.Key) - } - - return responses -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Set(item *Item) MutateResponse { - return c.mutate(opSet, item) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) SetMulti(items []*Item) []MutateResponse { - return c.mutateMulti(opSet, items) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) SetSentinels(items []*Item) []MutateResponse { - // For raw clients, there are no difference between SetMulti and - // SetSentinels. - return c.SetMulti(items) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Add(item *Item) MutateResponse { - return c.mutate(opAdd, item) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) AddMulti(items []*Item) []MutateResponse { - return c.mutateMulti(opAdd, items) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Replace(item *Item) MutateResponse { - if item == nil { - return NewMutateErrorResponse("", errors.New("item is nil")) - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - if resp := c.sendMutateRequest(opReplace, item, true); resp != nil { - return resp - } - - return c.receiveMutateResponse(opReplace, item.Key) -} - -func (c *RawBinaryClient) sendDeleteRequest(key string) MutateResponse { - if !isValidKeyString(key) { - return NewMutateErrorResponse( - key, - errors.New("Invalid key")) - } - - if err := c.sendRequest(opDelete, 0, []byte(key), nil); err != nil { - return NewMutateErrorResponse(key, err) - } - return nil -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Delete(key string) MutateResponse { - c.mutex.Lock() - defer c.mutex.Unlock() - - if resp := c.sendDeleteRequest(key); resp != nil { - return resp - } - - return c.receiveMutateResponse(opDelete, key) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) DeleteMulti(keys []string) []MutateResponse { - if keys == nil { - return nil - } - - responses := make([]MutateResponse, len(keys), len(keys)) - - c.mutex.Lock() - defer c.mutex.Unlock() - - for i, key := range keys { - responses[i] = c.sendDeleteRequest(key) - } - - for i, key := range keys { - if responses[i] != nil { // error occurred while sending - continue - } - responses[i] = c.receiveMutateResponse(opDelete, key) - } - - return responses -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Append(key string, value []byte) MutateResponse { - item := &Item{ - Key: key, - Value: value, - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - if resp := c.sendMutateRequest(opAppend, item, false); resp != nil { - return resp - } - - return c.receiveMutateResponse(opAppend, item.Key) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Prepend(key string, value []byte) MutateResponse { - item := &Item{ - Key: key, - Value: value, - } - - c.mutex.Lock() - defer c.mutex.Unlock() - - if resp := c.sendMutateRequest(opPrepend, item, false); resp != nil { - return resp - } - - return c.receiveMutateResponse(opPrepend, item.Key) -} - -func (c *RawBinaryClient) sendCountRequest( - code opCode, - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - if !isValidKeyString(key) { - return NewCountErrorResponse( - key, - errors.New("Invalid key")) - } - - err := c.sendRequest( - code, - 0, - []byte(key), - nil, - delta, - initValue, - expiration) - if err != nil { - return NewCountErrorResponse(key, err) - } - return nil -} - -func (c *RawBinaryClient) receiveCountResponse( - code opCode, - key string) CountResponse { - - status, _, _, value, err := c.receiveResponse(code) - if err != nil { - return NewCountErrorResponse(key, err) - } - - valueBuffer := bytes.NewBuffer(value) - var count uint64 - if err := binary.Read(valueBuffer, binary.BigEndian, &count); err != nil { - return NewCountErrorResponse(key, err) - } - - return NewCountResponse(key, status, count) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Increment( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - c.mutex.Lock() - defer c.mutex.Unlock() - - resp := c.sendCountRequest(opIncrement, key, delta, initValue, expiration) - if resp != nil { - return resp - } - return c.receiveCountResponse(opIncrement, key) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Decrement( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - c.mutex.Lock() - defer c.mutex.Unlock() - - resp := c.sendCountRequest(opDecrement, key, delta, initValue, expiration) - if resp != nil { - return resp - } - return c.receiveCountResponse(opDecrement, key) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Stat(statsKey string) StatResponse { - shardEntries := make(map[int](map[string]string)) - entries := make(map[string]string) - shardEntries[c.ShardId()] = entries - - c.mutex.Lock() - defer c.mutex.Unlock() - - if !isValidKeyString(statsKey) { - return NewStatErrorResponse( - errors.Newf("Invalid key: %s", statsKey), - shardEntries) - } - - err := c.sendRequest(opStat, 0, []byte(statsKey), nil) - if err != nil { - return NewStatErrorResponse(err, shardEntries) - } - - for true { - status, _, key, value, err := c.receiveResponse(opStat) - if err != nil { - return NewStatErrorResponse(err, shardEntries) - } - if status != StatusNoError { - // In theory, this is a valid state, but treating this as valid - // complicates the code even more. - c.validState = false - return NewStatResponse(status, shardEntries) - } - if key == nil && value == nil { // the last entry - break - } - entries[string(key)] = string(value) - } - return NewStatResponse(StatusNoError, shardEntries) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Version() VersionResponse { - versions := make(map[int]string) - - c.mutex.Lock() - defer c.mutex.Unlock() - - err := c.sendRequest(opVersion, 0, nil, nil) - if err != nil { - return NewVersionErrorResponse(err, versions) - } - - status, _, _, value, err := c.receiveResponse(opVersion) - if err != nil { - return NewVersionErrorResponse(err, versions) - } - - versions[c.ShardId()] = string(value) - return NewVersionResponse(status, versions) -} - -func (c *RawBinaryClient) genericOp( - code opCode, - extras ...interface{}) Response { - - c.mutex.Lock() - defer c.mutex.Unlock() - - err := c.sendRequest(code, 0, nil, nil, extras...) - if err != nil { - return NewErrorResponse(err) - } - - status, _, _, _, err := c.receiveResponse(code) - if err != nil { - return NewErrorResponse(err) - } - return NewResponse(status) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Flush(expiration uint32) Response { - return c.genericOp(opFlush, expiration) -} - -// See Client interface for documentation. -func (c *RawBinaryClient) Verbosity(verbosity uint32) Response { - return c.genericOp(opVerbosity, verbosity) -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/responses.go b/vendor/github.com/dropbox/godropbox/memcache/responses.go deleted file mode 100644 index db2688e..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/responses.go +++ /dev/null @@ -1,265 +0,0 @@ -package memcache - -import ( - "github.com/dropbox/godropbox/errors" -) - -func NewStatusCodeError(status ResponseStatus) error { - switch status { - case StatusNoError: - return nil - case StatusKeyNotFound: - return errors.New("Key not found") - case StatusKeyExists: - return errors.New("Key exists") - case StatusValueTooLarge: - return errors.New("Value too large") - case StatusInvalidArguments: - return errors.New("Invalid arguments") - case StatusItemNotStored: - return errors.New("Item not stored") - case StatusIncrDecrOnNonNumericValue: - return errors.New("Incr/decr on non-numeric value") - case StatusVbucketBelongsToAnotherServer: - return errors.New("VBucket belongs to another server") - case StatusAuthenticationError: - return errors.New("Authentication error") - case StatusAuthenticationContinue: - return errors.New("Authentication continue") - case StatusUnknownCommand: - return errors.New("Unknown command") - case StatusOutOfMemory: - return errors.New("Server out of memory") - case StatusNotSupported: - return errors.New("Not supported") - case StatusInternalError: - return errors.New("Server internal error") - case StatusBusy: - return errors.New("Server busy") - case StatusTempFailure: - return errors.New("Temporary server failure") - default: - return errors.Newf("Invalid status: %d", int(status)) - } -} - -// The genericResponse is an union of all response types. Response interfaces -// will cover the fact that there's only one implementation for everything. -type genericResponse struct { - // err and status are used by all responses. - err error - status ResponseStatus - - // key is used by get / mutate / count responses. The rest is used only - // by get response. - item Item - - // set to true only for get response - allowNotFound bool - - // count is used by count response. - count uint64 - - // versions is used by version response. - versions map[int]string - - // statEntries is used by stat response. - statEntries map[int](map[string]string) - - // set to true for mutate operations over the ascii protocol. - asciiMutateResponse bool -} - -func (r *genericResponse) Status() ResponseStatus { - return r.status -} - -func (r *genericResponse) Error() error { - if r.err != nil { - return r.err - } - if r.status == StatusNoError { - return nil - } - if r.allowNotFound && r.status == StatusKeyNotFound { - return nil - } - return NewStatusCodeError(r.status) -} - -func (r *genericResponse) Key() string { - return r.item.Key -} - -func (r *genericResponse) Value() []byte { - return r.item.Value -} - -func (r *genericResponse) Flags() uint32 { - return r.item.Flags -} - -func (r *genericResponse) DataVersionId() uint64 { - if r.asciiMutateResponse { - panic("Ascii protocol does not support version id in MutateResponse.") - } - - return r.item.DataVersionId -} - -func (r *genericResponse) Count() uint64 { - return r.count -} - -func (r *genericResponse) Versions() map[int]string { - return r.versions -} - -func (r *genericResponse) Entries() map[int](map[string]string) { - return r.statEntries -} - -// This creates a Response from an error. -func NewErrorResponse(err error) Response { - return &genericResponse{ - err: err, - } -} - -// This creates a Response from status. -func NewResponse(status ResponseStatus) Response { - return &genericResponse{ - status: status, - } -} - -// This creates a GetResponse from an error. -func NewGetErrorResponse(key string, err error) GetResponse { - resp := &genericResponse{ - err: err, - allowNotFound: true, - } - resp.item.Key = key - return resp -} - -// This creates a normal GetResponse. -func NewGetResponse( - key string, - status ResponseStatus, - flags uint32, - value []byte, - version uint64) GetResponse { - - resp := &genericResponse{ - status: status, - allowNotFound: true, - } - resp.item.Key = key - if status == StatusNoError { - if value == nil { - resp.item.Value = []byte{} - } else { - resp.item.Value = value - } - resp.item.Flags = flags - resp.item.DataVersionId = version - } - return resp -} - -// This creates a MutateResponse from an error. -func NewMutateErrorResponse(key string, err error) MutateResponse { - resp := &genericResponse{ - err: err, - } - resp.item.Key = key - return resp -} - -// This creates a normal MutateResponse. -func NewMutateResponse( - key string, - status ResponseStatus, - version uint64, - asciiMutateResponse bool) MutateResponse { - - resp := &genericResponse{ - status: status, - asciiMutateResponse: asciiMutateResponse, - } - resp.item.Key = key - if status == StatusNoError { - resp.item.DataVersionId = version - } - return resp -} - -// This creates a CountResponse from an error. -func NewCountErrorResponse(key string, err error) CountResponse { - resp := &genericResponse{ - err: err, - } - resp.item.Key = key - return resp -} - -// This creates a normal CountResponse. -func NewCountResponse( - key string, - status ResponseStatus, - count uint64) CountResponse { - - resp := &genericResponse{ - status: status, - } - resp.item.Key = key - if status == StatusNoError { - resp.count = count - } - return resp -} - -// This creates a VersionResponse from an error. -func NewVersionErrorResponse( - err error, - versions map[int]string) VersionResponse { - return &genericResponse{ - err: err, - versions: versions, - } -} - -// This creates a normal VersionResponse. -func NewVersionResponse( - status ResponseStatus, - versions map[int]string) VersionResponse { - - resp := &genericResponse{ - status: status, - versions: versions, - } - return resp -} - -// This creates a StatResponse from an error. -func NewStatErrorResponse( - err error, - entries map[int](map[string]string)) StatResponse { - return &genericResponse{ - err: err, - statEntries: entries, - } -} - -// This creates a normal StatResponse. -func NewStatResponse( - status ResponseStatus, - entries map[int](map[string]string)) StatResponse { - - resp := &genericResponse{ - status: status, - statEntries: entries, - } - return resp -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/sharded_client.go b/vendor/github.com/dropbox/godropbox/memcache/sharded_client.go deleted file mode 100644 index d4abf90..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/sharded_client.go +++ /dev/null @@ -1,671 +0,0 @@ -package memcache - -import ( - "expvar" - "github.com/dropbox/godropbox/errors" - "github.com/dropbox/godropbox/net2" -) - -// A sharded memcache client implementation where sharding management is -// handled by the provided ShardManager. -type ShardedClient struct { - manager ShardManager - - // When true, use ascii protocol. Otherwise, use binary protocol. - useAsciiProtocol bool -} - -var ( - // Counters for number of get requests that successed / errored, by address. - getOkByAddr = expvar.NewMap("ShardedClientGetOkByAddrCounter") - getErrByAddr = expvar.NewMap("ShardedClientGetErrByAddrCounter") -) - -// This creates a new ShardedClient. -func NewShardedClient( - manager ShardManager, - useAsciiProtocol bool) Client { - - return &ShardedClient{ - manager: manager, - useAsciiProtocol: useAsciiProtocol, - } -} - -func (s *ShardedClient) newRawClient( - shard int, - conn net2.ManagedConn) ClientShard { - - if s.useAsciiProtocol { - return NewRawAsciiClient(shard, conn) - } - return NewRawBinaryClient(shard, conn) -} - -func (c *ShardedClient) release(rawClient ClientShard, conn net2.ManagedConn) { - if rawClient.IsValidState() { - conn.ReleaseConnection() - } else { - conn.DiscardConnection() - } -} - -func (c *ShardedClient) unmappedError(key string) error { - return errors.Newf("Key '%s' does not map to any memcache shard", key) -} - -func (c *ShardedClient) connectionError(shard int, err error) error { - if err == nil { - return errors.Newf( - "Connection unavailable for memcache shard %d", shard) - } - return errors.Wrapf( - err, - "Connection unavailable for memcache shard %d", shard) -} - -// See Client interface for documentation. -func (c *ShardedClient) Get(key string) GetResponse { - shard, conn, err := c.manager.GetShard(key) - if shard == -1 { - return NewGetErrorResponse(key, c.unmappedError(key)) - } - if err != nil { - return NewGetErrorResponse(key, c.connectionError(shard, err)) - } - if conn == nil { - // NOTE: zero is an invalid version id. - return NewGetResponse(key, StatusKeyNotFound, 0, nil, 0) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - result := client.Get(key) - if client.IsValidState() { - getOkByAddr.Add(conn.Key().Address, 1) - } else { - getErrByAddr.Add(conn.Key().Address, 1) - } - return result -} - -func (c *ShardedClient) getMultiHelper( - shard int, - conn net2.ManagedConn, - connErr error, - keys []string, - resultsChannel chan map[string]GetResponse) { - - var results map[string]GetResponse - if shard == -1 { - results = make(map[string]GetResponse) - for _, key := range keys { - results[key] = NewGetErrorResponse(key, c.unmappedError(key)) - } - } else if connErr != nil { - results = make(map[string]GetResponse) - for _, key := range keys { - results[key] = NewGetErrorResponse( - key, - c.connectionError(shard, connErr)) - } - } else if conn == nil { - results = make(map[string]GetResponse) - for _, key := range keys { - // NOTE: zero is an invalid version id. - results[key] = NewGetResponse(key, StatusKeyNotFound, 0, nil, 0) - } - } else { - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - results = client.GetMulti(keys) - if client.IsValidState() { - getOkByAddr.Add(conn.Key().Address, 1) - } else { - getErrByAddr.Add(conn.Key().Address, 1) - } - } - resultsChannel <- results -} - -// See Client interface for documentation. -func (c *ShardedClient) GetMulti(keys []string) map[string]GetResponse { - shardMapping := c.manager.GetShardsForKeys(keys) - - resultsChannel := make(chan map[string]GetResponse, len(shardMapping)) - for shard, mapping := range shardMapping { - go c.getMultiHelper( - shard, - mapping.Connection, - mapping.ConnErr, - mapping.Keys, - resultsChannel) - } - - results := make(map[string]GetResponse) - for i := 0; i < len(shardMapping); i++ { - for key, resp := range <-resultsChannel { - results[key] = resp - } - } - return results -} - -func (c *ShardedClient) mutate( - mutateFunc func(Client, *Item) MutateResponse, - item *Item) MutateResponse { - shard, conn, err := c.manager.GetShard(item.Key) - if shard == -1 { - return NewMutateErrorResponse(item.Key, c.unmappedError(item.Key)) - } - if err != nil { - return NewMutateErrorResponse(item.Key, c.connectionError(shard, err)) - } - if conn == nil { - // NOTE: zero is an invalid version id. - return NewMutateResponse(item.Key, StatusNoError, 0, c.useAsciiProtocol) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return mutateFunc(client, item) -} - -// A helper used to specify a set mutation operation on a shard client. -func setMutator(shardClient Client, shardItem *Item) MutateResponse { - return shardClient.Set(shardItem) -} - -// See Client interface for documentation. -func (c *ShardedClient) Set(item *Item) MutateResponse { - return c.mutate(setMutator, item) -} - -func (c *ShardedClient) mutateMultiHelper( - mutateMultiFunc func(Client, []*Item) []MutateResponse, - shard int, - conn net2.ManagedConn, - connErr error, - items []*Item, - warmingUp bool, - resultsChannel chan []MutateResponse) { - - var results []MutateResponse - if shard == -1 { - results = make([]MutateResponse, 0, len(items)) - for _, item := range items { - results = append( - results, - NewMutateErrorResponse(item.Key, c.unmappedError(item.Key))) - } - } else if connErr != nil { - results = make([]MutateResponse, 0, len(items)) - for _, item := range items { - results = append( - results, - NewMutateErrorResponse( - item.Key, - c.connectionError(shard, connErr))) - } - } else if conn == nil { - results = make([]MutateResponse, 0, len(items)) - for _, item := range items { - // NOTE: zero is an invalid version id. - results = append( - results, - NewMutateResponse( - item.Key, - StatusNoError, - 0, - c.useAsciiProtocol)) - } - } else { - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - results = mutateMultiFunc(client, items) - } - - // If server is warming up, we override all failures with success message. - if warmingUp { - for idx, item := range items { - if results[idx].Error() != nil { - results[idx] = NewMutateResponse( - item.Key, - StatusNoError, - 0, - c.useAsciiProtocol) - } - } - } - - resultsChannel <- results -} - -// See Client interface for documentation. -func (c *ShardedClient) mutateMulti( - mutateMultiFunc func(Client, []*Item) []MutateResponse, - items []*Item) []MutateResponse { - shardMapping := c.manager.GetShardsForItems(items) - - resultsChannel := make(chan []MutateResponse, len(shardMapping)) - for shard, mapping := range shardMapping { - go c.mutateMultiHelper( - mutateMultiFunc, - shard, - mapping.Connection, - mapping.ConnErr, - mapping.Items, - false, - resultsChannel) - } - - results := make([]MutateResponse, 0, len(items)) - for i := 0; i < len(shardMapping); i++ { - results = append(results, (<-resultsChannel)...) - } - return results -} - -// A helper used to specify a SetMulti mutation operation on a shard client. -func setMultiMutator(shardClient Client, shardItems []*Item) []MutateResponse { - return shardClient.SetMulti(shardItems) -} - -// See Client interface for documentation. -func (c *ShardedClient) SetMulti(items []*Item) []MutateResponse { - return c.mutateMulti(setMultiMutator, items) -} - -// See Client interface for documentation. -func (c *ShardedClient) SetSentinels(items []*Item) []MutateResponse { - shardMapping := c.manager.GetShardsForSentinels(items) - - resultsChannel := make(chan []MutateResponse, len(shardMapping)) - for shard, mapping := range shardMapping { - go c.mutateMultiHelper( - setMultiMutator, - shard, - mapping.Connection, - mapping.ConnErr, - mapping.Items, - mapping.WarmingUp, - resultsChannel) - } - - results := make([]MutateResponse, 0, len(items)) - for i := 0; i < len(shardMapping); i++ { - results = append(results, (<-resultsChannel)...) - } - return results -} - -// A helper used to specify an Add mutation operation on a shard client. -func addMutator(shardClient Client, shardItem *Item) MutateResponse { - return shardClient.Add(shardItem) -} - -// See Client interface for documentation. -func (c *ShardedClient) Add(item *Item) MutateResponse { - return c.mutate(addMutator, item) -} - -// A helper used to specify a AddMulti mutation operation on a shard client. -func addMultiMutator(shardClient Client, shardItems []*Item) []MutateResponse { - return shardClient.AddMulti(shardItems) -} - -// See Client interface for documentation. -func (c *ShardedClient) AddMulti(items []*Item) []MutateResponse { - return c.mutateMulti(addMultiMutator, items) -} - -// See Client interface for documentation. -func (c *ShardedClient) Replace(item *Item) MutateResponse { - shard, conn, err := c.manager.GetShard(item.Key) - if shard == -1 { - return NewMutateErrorResponse(item.Key, c.unmappedError(item.Key)) - } - if err != nil { - return NewMutateErrorResponse( - item.Key, - c.connectionError(shard, err)) - } - if conn == nil { - // NOTE: zero is an invalid version id. - return NewMutateResponse(item.Key, StatusNoError, 0, c.useAsciiProtocol) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Replace(item) -} - -// See Client interface for documentation. -func (c *ShardedClient) Delete(key string) MutateResponse { - shard, conn, err := c.manager.GetShard(key) - if shard == -1 { - return NewMutateErrorResponse(key, c.unmappedError(key)) - } - if err != nil { - return NewMutateErrorResponse(key, c.connectionError(shard, err)) - } - if conn == nil { - // NOTE: zero is an invalid version id. - return NewMutateResponse(key, StatusNoError, 0, c.useAsciiProtocol) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Delete(key) -} - -func (c *ShardedClient) deleteMultiHelper( - shard int, - conn net2.ManagedConn, - connErr error, - keys []string, - resultsChannel chan []MutateResponse) { - - var results []MutateResponse - if shard == -1 { - results = make([]MutateResponse, 0, len(keys)) - for _, key := range keys { - results = append( - results, - NewMutateErrorResponse(key, c.unmappedError(key))) - } - } else if connErr != nil { - results = make([]MutateResponse, 0, len(keys)) - for _, key := range keys { - results = append( - results, - NewMutateErrorResponse(key, c.connectionError(shard, connErr))) - } - } else if conn == nil { - results = make([]MutateResponse, 0, len(keys)) - for _, key := range keys { - // NOTE: zero is an invalid version id. - results = append( - results, - NewMutateResponse(key, StatusNoError, 0, c.useAsciiProtocol)) - } - } else { - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - results = client.DeleteMulti(keys) - } - resultsChannel <- results -} - -// See Client interface for documentation. -func (c *ShardedClient) DeleteMulti(keys []string) []MutateResponse { - shardMapping := c.manager.GetShardsForKeys(keys) - - resultsChannel := make(chan []MutateResponse, len(shardMapping)) - for shard, mapping := range shardMapping { - go c.deleteMultiHelper( - shard, - mapping.Connection, - mapping.ConnErr, - mapping.Keys, - resultsChannel) - } - - results := make([]MutateResponse, 0, len(keys)) - for i := 0; i < len(shardMapping); i++ { - results = append(results, (<-resultsChannel)...) - } - return results -} - -// See Client interface for documentation. -func (c *ShardedClient) Append(key string, value []byte) MutateResponse { - shard, conn, err := c.manager.GetShard(key) - if shard == -1 { - return NewMutateErrorResponse(key, c.unmappedError(key)) - } - if err != nil { - return NewMutateErrorResponse(key, c.connectionError(shard, err)) - } - if conn == nil { - // NOTE: zero is an invalid version id. - return NewMutateResponse(key, StatusNoError, 0, c.useAsciiProtocol) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Append(key, value) -} - -// See Client interface for documentation. -func (c *ShardedClient) Prepend(key string, value []byte) MutateResponse { - shard, conn, err := c.manager.GetShard(key) - if shard == -1 { - return NewMutateErrorResponse(key, c.unmappedError(key)) - } - if err != nil { - return NewMutateErrorResponse(key, c.connectionError(shard, err)) - } - if conn == nil { - // NOTE: zero is an invalid version id. - return NewMutateResponse(key, StatusNoError, 0, c.useAsciiProtocol) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Prepend(key, value) -} - -// See Client interface for documentation. -func (c *ShardedClient) Increment( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - shard, conn, err := c.manager.GetShard(key) - if shard == -1 { - return NewCountErrorResponse(key, c.unmappedError(key)) - } - if err != nil { - return NewCountErrorResponse(key, c.connectionError(shard, err)) - } - if conn == nil { - return NewCountResponse(key, StatusNoError, 0) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Increment(key, delta, initValue, expiration) -} - -// See Client interface for documentation. -func (c *ShardedClient) Decrement( - key string, - delta uint64, - initValue uint64, - expiration uint32) CountResponse { - - shard, conn, err := c.manager.GetShard(key) - if shard == -1 { - return NewCountErrorResponse(key, c.unmappedError(key)) - } - if err != nil { - return NewCountErrorResponse(key, c.connectionError(shard, err)) - } - if conn == nil { - return NewCountResponse(key, StatusNoError, 0) - } - - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Decrement(key, delta, initValue, expiration) -} - -func (c *ShardedClient) flushHelper( - shard int, - conn net2.ManagedConn, - expiration uint32) Response { - - if conn == nil { - return NewErrorResponse(c.connectionError(shard, nil)) - } - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Flush(expiration) -} - -// See Client interface for documentation. -func (c *ShardedClient) Flush(expiration uint32) Response { - var err error - for shard, conn := range c.manager.GetAllShards() { - response := c.flushHelper(shard, conn, expiration) - if response.Error() != nil { - if err == nil { - err = response.Error() - } else { - err = errors.Wrap(response.Error(), err.Error()) - } - } - } - - if err != nil { - return NewErrorResponse(err) - } - - return NewResponse(StatusNoError) -} - -func (c *ShardedClient) statHelper( - shard int, - conn net2.ManagedConn, - statsKey string) StatResponse { - - if conn == nil { - return NewStatErrorResponse( - c.connectionError(shard, nil), - make(map[int](map[string]string))) - } - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Stat(statsKey) -} - -// See Client interface for documentation. -func (c *ShardedClient) Stat(statsKey string) StatResponse { - statEntries := make(map[int](map[string]string)) - - var err error - for shard, conn := range c.manager.GetAllShards() { - response := c.statHelper(shard, conn, statsKey) - if response.Error() != nil { - if err == nil { - err = response.Error() - } else { - err = errors.Wrap(response.Error(), err.Error()) - } - } - - for shardId, entries := range response.Entries() { - statEntries[shardId] = entries - } - } - - if err != nil { - return NewStatErrorResponse(err, statEntries) - } - - return NewStatResponse(StatusNoError, statEntries) -} - -func (c *ShardedClient) versionHelper( - shard int, - conn net2.ManagedConn) VersionResponse { - - if conn == nil { - return NewVersionErrorResponse( - c.connectionError(shard, nil), - make(map[int]string)) - } - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Version() -} - -// See Client interface for documentation. -func (c *ShardedClient) Version() VersionResponse { - shardConns := c.manager.GetAllShards() - - var err error - versions := make(map[int]string) - for shard, conn := range shardConns { - response := c.versionHelper(shard, conn) - if response.Error() != nil { - if err == nil { - err = response.Error() - } else { - err = errors.Wrap(response.Error(), err.Error()) - } - continue - } - - for shardId, versionString := range response.Versions() { - versions[shardId] = versionString - } - } - - if err != nil { - return NewVersionErrorResponse(err, versions) - } - - return NewVersionResponse(StatusNoError, versions) -} - -func (c *ShardedClient) verbosityHelper( - shard int, - conn net2.ManagedConn, - verbosity uint32) Response { - - if conn == nil { - return NewErrorResponse(c.connectionError(shard, nil)) - } - client := c.newRawClient(shard, conn) - defer c.release(client, conn) - - return client.Verbosity(verbosity) -} - -// See Client interface for documentation. -func (c *ShardedClient) Verbosity(verbosity uint32) Response { - var err error - for shard, conn := range c.manager.GetAllShards() { - response := c.verbosityHelper(shard, conn, verbosity) - if response.Error() != nil { - if err == nil { - err = response.Error() - } else { - err = errors.Wrap(response.Error(), err.Error()) - } - } - } - - if err != nil { - return NewErrorResponse(err) - } - - return NewResponse(StatusNoError) -} diff --git a/vendor/github.com/dropbox/godropbox/memcache/static_shard_manager.go b/vendor/github.com/dropbox/godropbox/memcache/static_shard_manager.go deleted file mode 100644 index 54d54ff..0000000 --- a/vendor/github.com/dropbox/godropbox/memcache/static_shard_manager.go +++ /dev/null @@ -1,43 +0,0 @@ -package memcache - -import ( - "log" - - "github.com/dropbox/godropbox/net2" -) - -// A shard manager that returns connections from a static list of memcache -// shards. NOTE: This is only for illustration purposes. DO NOT USE IN -// PRODUCTION. (Dropbox internally uses a different shard manager which is -// also based on BaseShardManager. Our memcache config is managed by zookeeper. -// When our memcache config changes, zookeeper will notify the shard manager of -// these updates and the shard manager will in turn swap in/out shards via -// UpdateShardStates.) -type StaticShardManager struct { - BaseShardManager -} - -// This creates a StaticShardManager, which returns connections from a static -// list of memcache shards. -func NewStaticShardManager( - serverAddrs []string, - shardFunc func(key string, numShard int) (shard int), - options net2.ConnectionOptions) ShardManager { - - manager := &StaticShardManager{} - manager.Init( - shardFunc, - func(err error) { log.Print(err) }, - log.Print, - options) - - shardStates := make([]ShardState, len(serverAddrs), len(serverAddrs)) - for i, addr := range serverAddrs { - shardStates[i].Address = addr - shardStates[i].State = ActiveServer - } - - manager.UpdateShardStates(shardStates) - - return manager -} diff --git a/vendor/github.com/dropbox/godropbox/net2/base_connection_pool.go b/vendor/github.com/dropbox/godropbox/net2/base_connection_pool.go deleted file mode 100644 index 52e1c03..0000000 --- a/vendor/github.com/dropbox/godropbox/net2/base_connection_pool.go +++ /dev/null @@ -1,157 +0,0 @@ -package net2 - -import ( - "net" - "strings" - "time" - - rp "github.com/dropbox/godropbox/resource_pool" -) - -const defaultDialTimeout = 1 * time.Second - -func parseResourceLocation(resourceLocation string) ( - network string, - address string) { - - idx := strings.Index(resourceLocation, " ") - if idx >= 0 { - return resourceLocation[:idx], resourceLocation[idx+1:] - } - - return "", resourceLocation -} - -// A thin wrapper around the underlying resource pool. -type BaseConnectionPool struct { - options ConnectionOptions - - pool rp.ResourcePool -} - -// This returns a connection pool where all connections are connected -// to the same (network, address) -func newBaseConnectionPool( - options ConnectionOptions, - createPool func(rp.Options) rp.ResourcePool) ConnectionPool { - - dial := options.Dial - if dial == nil { - dial = func(network string, address string) (net.Conn, error) { - return net.DialTimeout(network, address, defaultDialTimeout) - } - } - - openFunc := func(loc string) (interface{}, error) { - network, address := parseResourceLocation(loc) - return dial(network, address) - } - - closeFunc := func(handle interface{}) error { - return handle.(net.Conn).Close() - } - - poolOptions := rp.Options{ - MaxActiveHandles: options.MaxActiveConnections, - MaxIdleHandles: options.MaxIdleConnections, - MaxIdleTime: options.MaxIdleTime, - OpenMaxConcurrency: options.DialMaxConcurrency, - Open: openFunc, - Close: closeFunc, - NowFunc: options.NowFunc, - } - - return &BaseConnectionPool{ - options: options, - pool: createPool(poolOptions), - } -} - -// This returns a connection pool where all connections are connected -// to the same (network, address) -func NewSimpleConnectionPool(options ConnectionOptions) ConnectionPool { - return newBaseConnectionPool(options, rp.NewSimpleResourcePool) -} - -// This returns a connection pool that manages multiple (network, address) -// entries. The connections to each (network, address) entry acts -// independently. For example ("tcp", "localhost:11211") could act as memcache -// shard 0 and ("tcp", "localhost:11212") could act as memcache shard 1. -func NewMultiConnectionPool(options ConnectionOptions) ConnectionPool { - return newBaseConnectionPool( - options, - func(poolOptions rp.Options) rp.ResourcePool { - return rp.NewMultiResourcePool(poolOptions, nil) - }) -} - -// See ConnectionPool for documentation. -func (p *BaseConnectionPool) NumActive() int32 { - return p.pool.NumActive() -} - -// See ConnectionPool for documentation. -func (p *BaseConnectionPool) ActiveHighWaterMark() int32 { - return p.pool.ActiveHighWaterMark() -} - -// This returns the number of alive idle connections. This method is not part -// of ConnectionPool's API. It is used only for testing. -func (p *BaseConnectionPool) NumIdle() int { - return p.pool.NumIdle() -} - -// BaseConnectionPool can only register a single (network, address) entry. -// Register should be call before any Get calls. -func (p *BaseConnectionPool) Register(network string, address string) error { - return p.pool.Register(network + " " + address) -} - -// BaseConnectionPool has nothing to do on Unregister. -func (p *BaseConnectionPool) Unregister(network string, address string) error { - return nil -} - -func (p *BaseConnectionPool) ListRegistered() []NetworkAddress { - result := make([]NetworkAddress, 0, 1) - for _, location := range p.pool.ListRegistered() { - network, address := parseResourceLocation(location) - - result = append( - result, - NetworkAddress{ - Network: network, - Address: address, - }) - } - return result -} - -// This gets an active connection from the connection pool. Note that network -// and address arguments are ignored (The connections with point to the -// network/address provided by the first Register call). -func (p *BaseConnectionPool) Get( - network string, - address string) (ManagedConn, error) { - - handle, err := p.pool.Get(network + " " + address) - if err != nil { - return nil, err - } - return NewManagedConn(network, address, handle, p, p.options), nil -} - -// See ConnectionPool for documentation. -func (p *BaseConnectionPool) Release(conn ManagedConn) error { - return conn.ReleaseConnection() -} - -// See ConnectionPool for documentation. -func (p *BaseConnectionPool) Discard(conn ManagedConn) error { - return conn.DiscardConnection() -} - -// See ConnectionPool for documentation. -func (p *BaseConnectionPool) EnterLameDuckMode() { - p.pool.EnterLameDuckMode() -} diff --git a/vendor/github.com/dropbox/godropbox/net2/connection_pool.go b/vendor/github.com/dropbox/godropbox/net2/connection_pool.go deleted file mode 100644 index 8f65240..0000000 --- a/vendor/github.com/dropbox/godropbox/net2/connection_pool.go +++ /dev/null @@ -1,93 +0,0 @@ -package net2 - -import ( - "net" - "time" -) - -type ConnectionOptions struct { - // The maximum number of connections that can be active per host at any - // given time (A non-positive value indicates the number of connections - // is unbounded). - MaxActiveConnections int32 - - // The maximum number of idle connections per host that are kept alive by - // the connection pool. - MaxIdleConnections uint32 - - // The maximum amount of time an idle connection can alive (if specified). - MaxIdleTime *time.Duration - - // This limits the number of concurrent Dial calls (there's no limit when - // DialMaxConcurrency is non-positive). - DialMaxConcurrency int - - // Dial specifies the dial function for creating network connections. - // If Dial is nil, net.DialTimeout is used, with timeout set to 1 second. - Dial func(network string, address string) (net.Conn, error) - - // This specifies the now time function. When the function is non-nil, the - // connection pool will use the specified function instead of time.Now to - // generate the current time. - NowFunc func() time.Time - - // This specifies the timeout for any Read() operation. - ReadTimeout time.Duration - - // This specifies the timeout for any Write() operation. - WriteTimeout time.Duration -} - -func (o ConnectionOptions) getCurrentTime() time.Time { - if o.NowFunc == nil { - return time.Now() - } else { - return o.NowFunc() - } -} - -// A generic interface for managed connection pool. All connection pool -// implementations must be threadsafe. -type ConnectionPool interface { - // This returns the number of active connections that are on loan. - NumActive() int32 - - // This returns the highest number of active connections for the entire - // lifetime of the pool. - ActiveHighWaterMark() int32 - - // This returns the number of idle connections that are in the pool. - NumIdle() int - - // This associates (network, address) to the connection pool; afterwhich, - // the user can get connections to (network, address). - Register(network string, address string) error - - // This dissociate (network, address) from the connection pool; - // afterwhich, the user can no longer get connections to - // (network, address). - Unregister(network string, address string) error - - // This returns the list of registered (network, address) entries. - ListRegistered() []NetworkAddress - - // This gets an active connection from the connection pool. The connection - // will remain active until one of the following is called: - // 1. conn.ReleaseConnection() - // 2. conn.DiscardConnection() - // 3. pool.Release(conn) - // 4. pool.Discard(conn) - Get(network string, address string) (ManagedConn, error) - - // This releases an active connection back to the connection pool. - Release(conn ManagedConn) error - - // This discards an active connection from the connection pool. - Discard(conn ManagedConn) error - - // Enter the connection pool into lame duck mode. The connection pool - // will no longer return connections, and all idle connections are closed - // immediately (including active connections that are released back to the - // pool afterward). - EnterLameDuckMode() -} diff --git a/vendor/github.com/dropbox/godropbox/net2/doc.go b/vendor/github.com/dropbox/godropbox/net2/doc.go deleted file mode 100644 index 3d8d1c7..0000000 --- a/vendor/github.com/dropbox/godropbox/net2/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// net2 is a collection of functions meant to supplement the capabilities -// provided by the standard "net" package. -package net2 diff --git a/vendor/github.com/dropbox/godropbox/net2/ip.go b/vendor/github.com/dropbox/godropbox/net2/ip.go deleted file mode 100644 index d8fd963..0000000 --- a/vendor/github.com/dropbox/godropbox/net2/ip.go +++ /dev/null @@ -1,105 +0,0 @@ -package net2 - -import ( - "log" - "net" - "os" - "sync" - - "github.com/dropbox/godropbox/errors" -) - -var myHostname string -var myHostnameOnce sync.Once - -// Like os.Hostname but caches first successful result, making it cheap to call it -// over and over. -// It will also crash whole process if fetching Hostname fails! -func MyHostname() string { - myHostnameOnce.Do(func() { - var err error - myHostname, err = os.Hostname() - if err != nil { - log.Fatal(err) - } - }) - return myHostname -} - -var myIp4 *net.IPAddr -var myIp4Once sync.Once - -// Resolves `MyHostname()` to an Ip4 address. Caches first successful result, making it -// cheap to call it over and over. -// It will also crash whole process if resolving the IP fails! -func MyIp4() *net.IPAddr { - myIp4Once.Do(func() { - var err error - myIp4, err = net.ResolveIPAddr("ip4", MyHostname()) - if err != nil { - log.Fatal(err) - } - }) - return myIp4 -} - -var myIp6 *net.IPAddr -var myIp6Once sync.Once - -// Resolves `MyHostname()` to an Ip6 address. Caches first successful result, making it -// cheap to call it over and over. -// It will also crash whole process if resolving the IP fails! -func MyIp6() *net.IPAddr { - myIp6Once.Do(func() { - var err error - myIp6, err = net.ResolveIPAddr("ip6", MyHostname()) - if err != nil { - log.Fatal(err) - } - }) - return myIp6 -} - -// This returns the list of local ip addresses which other hosts can connect -// to (NOTE: Loopback ip is ignored). -// Also resolves Hostname to an address and adds it to the list too, so -// IPs from /etc/hosts can work too. -func GetLocalIPs() ([]*net.IP, error) { - hostname, err := os.Hostname() - if err != nil { - return nil, errors.Wrap(err, "Failed to lookup hostname") - } - // Resolves IP Address from Hostname, this way overrides in /etc/hosts - // can work too for IP resolution. - ipInfo, err := net.ResolveIPAddr("ip4", hostname) - if err != nil { - return nil, errors.Wrap(err, "Failed to resolve ip") - } - ips := []*net.IP{&ipInfo.IP} - - // TODO(zviad): Is rest of the code really necessary? - addrs, err := net.InterfaceAddrs() - if err != nil { - return nil, errors.Wrap(err, "Failed to get interface addresses.") - } - for _, addr := range addrs { - ipnet, ok := addr.(*net.IPNet) - if !ok { - continue - } - - if ipnet.IP.IsLoopback() { - continue - } - - ips = append(ips, &ipnet.IP) - } - return ips, nil -} - -// Given a host string, return true if the host is an ip (v4/v6) localhost. -func IsLocalhost(host string) bool { - return host == "localhost" || - host == "ip6-localhost" || - host == "ipv6-localhost" -} diff --git a/vendor/github.com/dropbox/godropbox/net2/managed_connection.go b/vendor/github.com/dropbox/godropbox/net2/managed_connection.go deleted file mode 100644 index dcc9efe..0000000 --- a/vendor/github.com/dropbox/godropbox/net2/managed_connection.go +++ /dev/null @@ -1,171 +0,0 @@ -package net2 - -import ( - "net" - "time" - - "github.com/dropbox/godropbox/errors" - "github.com/dropbox/godropbox/resource_pool" -) - -// Dial's arguments. -type NetworkAddress struct { - Network string - Address string -} - -// A connection managed by a connection pool. NOTE: SetDeadline, -// SetReadDeadline and SetWriteDeadline are disabled for managed connections. -// (The deadlines are set by the connection pool). -type ManagedConn interface { - net.Conn - - // This returns the original (network, address) entry used for creating - // the connection. - Key() NetworkAddress - - // This returns the underlying net.Conn implementation. - RawConn() net.Conn - - // This returns the connection pool which owns this connection. - Owner() ConnectionPool - - // This indictes a user is done with the connection and releases the - // connection back to the connection pool. - ReleaseConnection() error - - // This indicates the connection is an invalid state, and that the - // connection should be discarded from the connection pool. - DiscardConnection() error -} - -// A physical implementation of ManagedConn -type ManagedConnImpl struct { - addr NetworkAddress - handle resource_pool.ManagedHandle - pool ConnectionPool - options ConnectionOptions -} - -// This creates a managed connection wrapper. -func NewManagedConn( - network string, - address string, - handle resource_pool.ManagedHandle, - pool ConnectionPool, - options ConnectionOptions) ManagedConn { - - addr := NetworkAddress{ - Network: network, - Address: address, - } - - return &ManagedConnImpl{ - addr: addr, - handle: handle, - pool: pool, - options: options, - } -} - -func (c *ManagedConnImpl) rawConn() (net.Conn, error) { - h, err := c.handle.Handle() - return h.(net.Conn), err -} - -// See ManagedConn for documentation. -func (c *ManagedConnImpl) RawConn() net.Conn { - h, _ := c.handle.Handle() - return h.(net.Conn) -} - -// See ManagedConn for documentation. -func (c *ManagedConnImpl) Key() NetworkAddress { - return c.addr -} - -// See ManagedConn for documentation. -func (c *ManagedConnImpl) Owner() ConnectionPool { - return c.pool -} - -// See ManagedConn for documentation. -func (c *ManagedConnImpl) ReleaseConnection() error { - return c.handle.Release() -} - -// See ManagedConn for documentation. -func (c *ManagedConnImpl) DiscardConnection() error { - return c.handle.Discard() -} - -// See net.Conn for documentation -func (c *ManagedConnImpl) Read(b []byte) (n int, err error) { - conn, err := c.rawConn() - if err != nil { - return 0, err - } - - if c.options.ReadTimeout > 0 { - deadline := c.options.getCurrentTime().Add(c.options.ReadTimeout) - _ = conn.SetReadDeadline(deadline) - } - n, err = conn.Read(b) - if err != nil { - err = errors.Wrap(err, "Read error") - } - return -} - -// See net.Conn for documentation -func (c *ManagedConnImpl) Write(b []byte) (n int, err error) { - conn, err := c.rawConn() - if err != nil { - return 0, err - } - - if c.options.WriteTimeout > 0 { - deadline := c.options.getCurrentTime().Add(c.options.WriteTimeout) - _ = conn.SetWriteDeadline(deadline) - } - n, err = conn.Write(b) - if err != nil { - err = errors.Wrap(err, "Write error") - } - return -} - -// See net.Conn for documentation -func (c *ManagedConnImpl) Close() error { - return c.handle.Discard() -} - -// See net.Conn for documentation -func (c *ManagedConnImpl) LocalAddr() net.Addr { - conn, _ := c.rawConn() - return conn.LocalAddr() -} - -// See net.Conn for documentation -func (c *ManagedConnImpl) RemoteAddr() net.Addr { - conn, _ := c.rawConn() - return conn.RemoteAddr() -} - -// SetDeadline is disabled for managed connection (The deadline is set by -// us, with respect to the read/write timeouts specified in ConnectionOptions). -func (c *ManagedConnImpl) SetDeadline(t time.Time) error { - return errors.New("Cannot set deadline for managed connection") -} - -// SetReadDeadline is disabled for managed connection (The deadline is set by -// us with respect to the read timeout specified in ConnectionOptions). -func (c *ManagedConnImpl) SetReadDeadline(t time.Time) error { - return errors.New("Cannot set read deadline for managed connection") -} - -// SetWriteDeadline is disabled for managed connection (The deadline is set by -// us with respect to the write timeout specified in ConnectionOptions). -func (c *ManagedConnImpl) SetWriteDeadline(t time.Time) error { - return errors.New("Cannot set write deadline for managed connection") -} diff --git a/vendor/github.com/dropbox/godropbox/resource_pool/doc.go b/vendor/github.com/dropbox/godropbox/resource_pool/doc.go deleted file mode 100644 index 992fa0b..0000000 --- a/vendor/github.com/dropbox/godropbox/resource_pool/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// A generic resource pool for managing resources such as network connections. -package resource_pool diff --git a/vendor/github.com/dropbox/godropbox/resource_pool/managed_handle.go b/vendor/github.com/dropbox/godropbox/resource_pool/managed_handle.go deleted file mode 100644 index e60820b..0000000 --- a/vendor/github.com/dropbox/godropbox/resource_pool/managed_handle.go +++ /dev/null @@ -1,97 +0,0 @@ -package resource_pool - -import ( - "sync/atomic" - - "github.com/dropbox/godropbox/errors" -) - -// A resource handle managed by a resource pool. -type ManagedHandle interface { - // This returns the handle's resource location. - ResourceLocation() string - - // This returns the underlying resource handle (or error if the handle - // is no longer active). - Handle() (interface{}, error) - - // This returns the resource pool which owns this handle. - Owner() ResourcePool - - // The releases the underlying resource handle to the caller and marks the - // managed handle as inactive. The caller is responsible for cleaning up - // the released handle. This returns nil if the managed handle no longer - // owns the resource. - ReleaseUnderlyingHandle() interface{} - - // This indictes a user is done with the handle and releases the handle - // back to the resource pool. - Release() error - - // This indicates the handle is an invalid state, and that the - // connection should be discarded from the connection pool. - Discard() error -} - -// A physical implementation of ManagedHandle -type ManagedHandleImpl struct { - location string - handle interface{} - pool ResourcePool - isActive int32 // atomic bool - options Options -} - -// This creates a managed handle wrapper. -func NewManagedHandle( - resourceLocation string, - handle interface{}, - pool ResourcePool, - options Options) ManagedHandle { - - h := &ManagedHandleImpl{ - location: resourceLocation, - handle: handle, - pool: pool, - options: options, - } - atomic.StoreInt32(&h.isActive, 1) - - return h -} - -// See ManagedHandle for documentation. -func (c *ManagedHandleImpl) ResourceLocation() string { - return c.location -} - -// See ManagedHandle for documentation. -func (c *ManagedHandleImpl) Handle() (interface{}, error) { - if atomic.LoadInt32(&c.isActive) == 0 { - return c.handle, errors.New("Resource handle is no longer valid") - } - return c.handle, nil -} - -// See ManagedHandle for documentation. -func (c *ManagedHandleImpl) Owner() ResourcePool { - return c.pool -} - -// See ManagedHandle for documentation. -func (c *ManagedHandleImpl) ReleaseUnderlyingHandle() interface{} { - if atomic.CompareAndSwapInt32(&c.isActive, 1, 0) { - return c.handle - } - return nil -} - -// See ManagedHandle for documentation. -func (c *ManagedHandleImpl) Release() error { - return c.pool.Release(c) -} - -// See ManagedHandle for documentation. -func (c *ManagedHandleImpl) Discard() error { - return c.pool.Discard(c) -} diff --git a/vendor/github.com/dropbox/godropbox/resource_pool/multi_resource_pool.go b/vendor/github.com/dropbox/godropbox/resource_pool/multi_resource_pool.go deleted file mode 100644 index 33fb252..0000000 --- a/vendor/github.com/dropbox/godropbox/resource_pool/multi_resource_pool.go +++ /dev/null @@ -1,198 +0,0 @@ -package resource_pool - -import ( - "sync" - - "github.com/dropbox/godropbox/errors" -) - -// A resource pool implementation that manages multiple resource location -// entries. The handles to each resource location entry acts independently. -// For example "tcp localhost:11211" could act as memcache -// shard 0 and "tcp localhost:11212" could act as memcache shard 1. -type MultiResourcePool struct { - options Options - - createPool func(Options) ResourcePool - - rwMutex sync.RWMutex - isLameDuck bool // guarded by rwMutex - // NOTE: the locationPools is guarded by rwMutex, but the pool entries - // are not. - locationPools map[string]ResourcePool -} - -// This returns a MultiResourcePool, which manages multiple -// resource location entries. The handles to each resource location -// entry acts independently. -// -// When createPool is nil, NewSimpleResourcePool is used as default. -func NewMultiResourcePool( - options Options, - createPool func(Options) ResourcePool) ResourcePool { - - if createPool == nil { - createPool = NewSimpleResourcePool - } - - return &MultiResourcePool{ - options: options, - createPool: createPool, - rwMutex: sync.RWMutex{}, - isLameDuck: false, - locationPools: make(map[string]ResourcePool), - } -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) NumActive() int32 { - total := int32(0) - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - for _, pool := range p.locationPools { - total += pool.NumActive() - } - return total -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) ActiveHighWaterMark() int32 { - high := int32(0) - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - for _, pool := range p.locationPools { - val := pool.ActiveHighWaterMark() - if val > high { - high = val - } - } - return high -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) NumIdle() int { - total := 0 - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - for _, pool := range p.locationPools { - total += pool.NumIdle() - } - return total -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) Register(resourceLocation string) error { - if resourceLocation == "" { - return errors.New("Registering invalid resource location") - } - - p.rwMutex.Lock() - defer p.rwMutex.Unlock() - - if p.isLameDuck { - return errors.Newf( - "Cannot register %s to lame duck resource pool", - resourceLocation) - } - - if _, inMap := p.locationPools[resourceLocation]; inMap { - return nil - } - - pool := p.createPool(p.options) - if err := pool.Register(resourceLocation); err != nil { - return err - } - - p.locationPools[resourceLocation] = pool - return nil -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) Unregister(resourceLocation string) error { - p.rwMutex.Lock() - defer p.rwMutex.Unlock() - - if pool, inMap := p.locationPools[resourceLocation]; inMap { - pool.EnterLameDuckMode() - delete(p.locationPools, resourceLocation) - } - return nil -} - -func (p *MultiResourcePool) ListRegistered() []string { - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - result := make([]string, 0, len(p.locationPools)) - for key, _ := range p.locationPools { - result = append(result, key) - } - - return result -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) Get( - resourceLocation string) (ManagedHandle, error) { - - pool := p.getPool(resourceLocation) - if pool == nil { - return nil, errors.Newf( - "%s is not registered in the resource pool", - resourceLocation) - } - return pool.Get(resourceLocation) -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) Release(handle ManagedHandle) error { - pool := p.getPool(handle.ResourceLocation()) - if pool == nil { - return errors.New( - "Resource pool cannot take control of a handle owned " + - "by another resource pool") - } - - return pool.Release(handle) -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) Discard(handle ManagedHandle) error { - pool := p.getPool(handle.ResourceLocation()) - if pool == nil { - return errors.New( - "Resource pool cannot take control of a handle owned " + - "by another resource pool") - } - - return pool.Discard(handle) -} - -// See ResourcePool for documentation. -func (p *MultiResourcePool) EnterLameDuckMode() { - p.rwMutex.Lock() - defer p.rwMutex.Unlock() - - p.isLameDuck = true - - for _, pool := range p.locationPools { - pool.EnterLameDuckMode() - } -} - -func (p *MultiResourcePool) getPool(resourceLocation string) ResourcePool { - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - if pool, inMap := p.locationPools[resourceLocation]; inMap { - return pool - } - return nil -} diff --git a/vendor/github.com/dropbox/godropbox/resource_pool/resource_pool.go b/vendor/github.com/dropbox/godropbox/resource_pool/resource_pool.go deleted file mode 100644 index 5ad8aca..0000000 --- a/vendor/github.com/dropbox/godropbox/resource_pool/resource_pool.go +++ /dev/null @@ -1,94 +0,0 @@ -package resource_pool - -import ( - "time" -) - -type Options struct { - // The maximum number of active resource handles per resource location. (A - // non-positive value indicates the number of active resource handles is - // unbounded). - MaxActiveHandles int32 - - // The maximum number of idle resource handles per resource location that - // are kept alive by the resource pool. - MaxIdleHandles uint32 - - // The maximum amount of time an idle resource handle can remain alive (if - // specified). - MaxIdleTime *time.Duration - - // This limits the number of concurrent Open calls (there's no limit when - // OpenMaxConcurrency is non-positive). - OpenMaxConcurrency int - - // This function creates a resource handle (e.g., a connection) for a - // resource location. The function must be thread-safe. - Open func(resourceLocation string) ( - handle interface{}, - err error) - - // This function destroys a resource handle and performs the necessary - // cleanup to free up resources. The function must be thread-safe. - Close func(handle interface{}) error - - // This specifies the now time function. When the function is non-nil, the - // resource pool will use the specified function instead of time.Now to - // generate the current time. - NowFunc func() time.Time -} - -func (o Options) getCurrentTime() time.Time { - if o.NowFunc == nil { - return time.Now() - } else { - return o.NowFunc() - } -} - -// A generic interface for managed resource pool. All resource pool -// implementations must be threadsafe. -type ResourcePool interface { - // This returns the number of active resource handles. - NumActive() int32 - - // This returns the highest number of actives handles for the entire - // lifetime of the pool. If the pool contains multiple sub-pools, the - // high water mark is the max of the sub-pools' high water marks. - ActiveHighWaterMark() int32 - - // This returns the number of alive idle handles. NOTE: This is only used - // for testing. - NumIdle() int - - // This associates a resource location to the resource pool; afterwhich, - // the user can get resource handles for the resource location. - Register(resourceLocation string) error - - // This dissociates a resource location from the resource pool; afterwhich, - // the user can no longer get resource handles for the resource location. - Unregister(resourceLocation string) error - - // This returns the list of registered resource location entries. - ListRegistered() []string - - // This gets an active resource handle from the resource pool. The - // handle will remain active until one of the following is called: - // 1. handle.Release() - // 2. handle.Discard() - // 3. pool.Release(handle) - // 4. pool.Discard(handle) - Get(key string) (ManagedHandle, error) - - // This releases an active resource handle back to the resource pool. - Release(handle ManagedHandle) error - - // This discards an active resource from the resource pool. - Discard(handle ManagedHandle) error - - // Enter the resource pool into lame duck mode. The resource pool - // will no longer return resource handles, and all idle resource handles - // are closed immediately (including active resource handles that are - // released back to the pool afterward). - EnterLameDuckMode() -} diff --git a/vendor/github.com/dropbox/godropbox/resource_pool/round_robin_resource_pool.go b/vendor/github.com/dropbox/godropbox/resource_pool/round_robin_resource_pool.go deleted file mode 100644 index fc29ad0..0000000 --- a/vendor/github.com/dropbox/godropbox/resource_pool/round_robin_resource_pool.go +++ /dev/null @@ -1,239 +0,0 @@ -package resource_pool - -import ( - "sync" - "sync/atomic" - - "github.com/dropbox/godropbox/errors" - "github.com/dropbox/godropbox/math2/rand2" -) - -type ResourceLocationPool struct { - ResourceLocation string - Pool ResourcePool -} - -func shuffle(pools []*ResourceLocationPool) { - for i := len(pools) - 1; i > 0; i-- { - idx := rand2.Intn(i + 1) - pools[i], pools[idx] = pools[idx], pools[i] - } -} - -// A resource pool implementation which returns handles from the registered -// resource locations in a round robin fashion. -type RoundRobinResourcePool struct { - options Options - - createPool func(Options) ResourcePool - - rwMutex sync.RWMutex - isLameDuck bool - pools []*ResourceLocationPool - - counter *int64 // atomic counter -} - -// This returns a RoundRobinResourcePool. -func NewRoundRobinResourcePool( - options Options, - createPool func(Options) ResourcePool, - pools ...*ResourceLocationPool) (ResourcePool, error) { - - locations := make(map[string]bool) - - for _, pool := range pools { - if pool.ResourceLocation == "" { - return nil, errors.New("Invalid resource location") - } - - if locations[pool.ResourceLocation] { - return nil, errors.Newf( - "Duplication resource location %s", - pool.ResourceLocation) - } - locations[pool.ResourceLocation] = true - - if pool.Pool == nil { - return nil, errors.New("Invalid pool") - } - } - - if createPool == nil { - createPool = NewSimpleResourcePool - } - - counter := new(int64) - atomic.StoreInt64(counter, 0) - - shuffle(pools) - - return &RoundRobinResourcePool{ - options: options, - createPool: createPool, - rwMutex: sync.RWMutex{}, - isLameDuck: false, - pools: pools, - counter: counter, - }, nil -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) NumActive() int32 { - total := int32(0) - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - for _, locPool := range p.pools { - total += locPool.Pool.NumActive() - } - return total -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) ActiveHighWaterMark() int32 { - high := int32(0) - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - for _, locPool := range p.pools { - val := locPool.Pool.ActiveHighWaterMark() - if val > high { - high = val - } - } - return high -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) NumIdle() int { - total := 0 - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - for _, locPool := range p.pools { - total += locPool.Pool.NumIdle() - } - return total -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) Register(resourceLocation string) error { - if resourceLocation == "" { - return errors.New("Registering invalid resource location") - } - - p.rwMutex.Lock() - defer p.rwMutex.Unlock() - - if p.isLameDuck { - return errors.Newf( - "Cannot register %s to lame duck resource pool", - resourceLocation) - } - - for _, locPool := range p.pools { - if locPool.ResourceLocation == resourceLocation { - return nil - } - } - - pool := p.createPool(p.options) - if err := pool.Register(resourceLocation); err != nil { - return err - } - - p.pools = append( - p.pools, - &ResourceLocationPool{ - ResourceLocation: resourceLocation, - Pool: pool, - }) - - shuffle(p.pools) - return nil -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) Unregister(resourceLocation string) error { - p.rwMutex.Lock() - defer p.rwMutex.Unlock() - - idx := -1 - for i, locPool := range p.pools { - if locPool.ResourceLocation == resourceLocation { - idx = i - break - } - } - - if idx >= 0 { - tail := p.pools[idx+1:] - p.pools = p.pools[0:idx] - p.pools = append(p.pools, tail...) - shuffle(p.pools) - } - return nil -} - -func (p *RoundRobinResourcePool) ListRegistered() []string { - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - result := make([]string, 0, len(p.pools)) - for _, locPool := range p.pools { - result = append(result, locPool.ResourceLocation) - } - return result -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) Get(key string) (ManagedHandle, error) { - - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - var err error - var handle ManagedHandle - - for i := 0; i < len(p.pools); i++ { - next := int(atomic.AddInt64(p.counter, 1) % int64(len(p.pools))) - pool := p.pools[next].Pool - - handle, err = pool.Get(key) - if err == nil { - return handle, nil - } - } - - return nil, errors.Wrap(err, "No resource handle available") -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) Release(handle ManagedHandle) error { - // NOTE: check if the handle belongs to this pool is expensive, so we'll - // just skip the check. - return handle.Release() -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) Discard(handle ManagedHandle) error { - // NOTE: check if the handle belongs to this pool is expensive, so we'll - // just skip the check. - return handle.Discard() -} - -// See ResourcePool for documentation. -func (p *RoundRobinResourcePool) EnterLameDuckMode() { - p.rwMutex.RLock() - defer p.rwMutex.RUnlock() - - p.isLameDuck = true - - for _, locPool := range p.pools { - locPool.Pool.EnterLameDuckMode() - } -} diff --git a/vendor/github.com/dropbox/godropbox/resource_pool/simple_resource_pool.go b/vendor/github.com/dropbox/godropbox/resource_pool/simple_resource_pool.go deleted file mode 100644 index 62646b8..0000000 --- a/vendor/github.com/dropbox/godropbox/resource_pool/simple_resource_pool.go +++ /dev/null @@ -1,322 +0,0 @@ -package resource_pool - -import ( - "errors" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/dropbox/godropbox/sync2" -) - -type idleHandle struct { - handle interface{} - keepUntil *time.Time -} - -// A resource pool implementation where all handles are associated to the -// same resource location. -type SimpleResourcePool struct { - options Options - - numActive *int32 // atomic counter - - activeHighWaterMark *int32 // atomic / monotonically increasing value - - openTokens sync2.Semaphore - - mutex sync.Mutex - location string // guard by mutex - idleHandles []*idleHandle // guarded by mutex - isLameDuck bool // guarded by mutex -} - -// This returns a SimpleResourcePool, where all handles are associated to a -// single resource location. -func NewSimpleResourcePool(options Options) ResourcePool { - numActive := new(int32) - atomic.StoreInt32(numActive, 0) - - activeHighWaterMark := new(int32) - atomic.StoreInt32(activeHighWaterMark, 0) - - var tokens sync2.Semaphore - if options.OpenMaxConcurrency > 0 { - tokens = sync2.NewBoundedSemaphore(uint(options.OpenMaxConcurrency)) - } - - return &SimpleResourcePool{ - location: "", - options: options, - numActive: numActive, - activeHighWaterMark: activeHighWaterMark, - openTokens: tokens, - mutex: sync.Mutex{}, - idleHandles: make([]*idleHandle, 0, 0), - isLameDuck: false, - } -} - -// See ResourcePool for documentation. -func (p *SimpleResourcePool) NumActive() int32 { - return atomic.LoadInt32(p.numActive) -} - -// See ResourcePool for documentation. -func (p *SimpleResourcePool) ActiveHighWaterMark() int32 { - return atomic.LoadInt32(p.activeHighWaterMark) -} - -// See ResourcePool for documentation. -func (p *SimpleResourcePool) NumIdle() int { - p.mutex.Lock() - defer p.mutex.Unlock() - return len(p.idleHandles) -} - -// SimpleResourcePool can only register a single (network, address) entry. -// Register should be call before any Get calls. -func (p *SimpleResourcePool) Register(resourceLocation string) error { - if resourceLocation == "" { - return errors.New("Invalid resource location") - } - - p.mutex.Lock() - defer p.mutex.Unlock() - - if p.isLameDuck { - return fmt.Errorf( - "Cannot register %s to lame duck resource pool", - resourceLocation) - } - - if p.location == "" { - p.location = resourceLocation - return nil - } - return errors.New("SimpleResourcePool can only register one location") -} - -// SimpleResourcePool does not support Unregister. -func (p *SimpleResourcePool) Unregister(resourceLocation string) error { - return errors.New("SimpleResourcePool does not support Unregister") -} - -func (p *SimpleResourcePool) ListRegistered() []string { - p.mutex.Lock() - defer p.mutex.Unlock() - - if p.location != "" { - return []string{p.location} - } - return []string{} -} - -func (p *SimpleResourcePool) getLocation() (string, error) { - p.mutex.Lock() - defer p.mutex.Unlock() - - if p.location == "" { - return "", fmt.Errorf( - "Resource location is not set for SimpleResourcePool") - } - - if p.isLameDuck { - return "", fmt.Errorf( - "Lame duck resource pool cannot return handles to %s", - p.location) - } - - return p.location, nil -} - -// This gets an active resource from the resource pool. Note that the -// resourceLocation argument is ignroed (The handles are associated to the -// resource location provided by the first Register call). -func (p *SimpleResourcePool) Get(unused string) (ManagedHandle, error) { - activeCount := atomic.AddInt32(p.numActive, 1) - if p.options.MaxActiveHandles > 0 && - activeCount > p.options.MaxActiveHandles { - - atomic.AddInt32(p.numActive, -1) - return nil, fmt.Errorf( - "Too many handles to %s", - p.location) - } - - highest := atomic.LoadInt32(p.activeHighWaterMark) - for activeCount > highest && - !atomic.CompareAndSwapInt32( - p.activeHighWaterMark, - highest, - activeCount) { - - highest = atomic.LoadInt32(p.activeHighWaterMark) - } - - if h := p.getIdleHandle(); h != nil { - return h, nil - } - - location, err := p.getLocation() - if err != nil { - atomic.AddInt32(p.numActive, -1) - return nil, err - } - - if p.openTokens != nil { - p.openTokens.Acquire() - defer p.openTokens.Release() - } - - handle, err := p.options.Open(location) - if err != nil { - atomic.AddInt32(p.numActive, -1) - return nil, fmt.Errorf( - "Failed to open resource handle: %s (%v)", - p.location, - err) - } - - return NewManagedHandle(p.location, handle, p, p.options), nil -} - -// See ResourcePool for documentation. -func (p *SimpleResourcePool) Release(handle ManagedHandle) error { - if pool, ok := handle.Owner().(*SimpleResourcePool); !ok || pool != p { - return errors.New( - "Resource pool cannot take control of a handle owned " + - "by another resource pool") - } - - h := handle.ReleaseUnderlyingHandle() - if h != nil { - // We can unref either before or after queuing the idle handle. - // The advantage of unref-ing before queuing is that there is - // a higher chance of successful Get when number of active handles - // is close to the limit (but potentially more handle creation). - // The advantage of queuing before unref-ing is that there's a - // higher chance of reusing handle (but potentially more Get failures). - atomic.AddInt32(p.numActive, -1) - p.queueIdleHandles(h) - } - - return nil -} - -// See ResourcePool for documentation. -func (p *SimpleResourcePool) Discard(handle ManagedHandle) error { - if pool, ok := handle.Owner().(*SimpleResourcePool); !ok || pool != p { - return errors.New( - "Resource pool cannot take control of a handle owned " + - "by another resource pool") - } - - h := handle.ReleaseUnderlyingHandle() - if h != nil { - atomic.AddInt32(p.numActive, -1) - if err := p.options.Close(h); err != nil { - return fmt.Errorf("Failed to close resource handle: %v", err) - } - } - return nil -} - -// See ResourcePool for documentation. -func (p *SimpleResourcePool) EnterLameDuckMode() { - p.mutex.Lock() - - toClose := p.idleHandles - p.isLameDuck = true - p.idleHandles = []*idleHandle{} - - p.mutex.Unlock() - - p.closeHandles(toClose) -} - -// This returns an idle resource, if there is one. -func (p *SimpleResourcePool) getIdleHandle() ManagedHandle { - var toClose []*idleHandle - defer func() { - // NOTE: Must keep the closure around to late bind the toClose slice. - p.closeHandles(toClose) - }() - - now := p.options.getCurrentTime() - - p.mutex.Lock() - defer p.mutex.Unlock() - - var i int - for i = 0; i < len(p.idleHandles); i++ { - idle := p.idleHandles[i] - if idle.keepUntil == nil || now.Before(*idle.keepUntil) { - break - } - } - if i > 0 { - toClose = p.idleHandles[0 : i-1] - } - - if i < len(p.idleHandles) { - idle := p.idleHandles[i] - p.idleHandles = p.idleHandles[i+1:] - return NewManagedHandle(p.location, idle.handle, p, p.options) - } - - if len(p.idleHandles) > 0 { - p.idleHandles = []*idleHandle{} - } - return nil -} - -// This adds an idle resource to the pool. -func (p *SimpleResourcePool) queueIdleHandles(handle interface{}) { - var toClose []*idleHandle - defer func() { - // NOTE: Must keep the closure around to late bind the toClose slice. - p.closeHandles(toClose) - }() - - now := p.options.getCurrentTime() - var keepUntil *time.Time - if p.options.MaxIdleTime != nil { - // NOTE: Assign to temp variable first to work around compiler bug - x := now.Add(*p.options.MaxIdleTime) - keepUntil = &x - } - - p.mutex.Lock() - defer p.mutex.Unlock() - - if p.isLameDuck { - toClose = []*idleHandle{ - &idleHandle{handle: handle}, - } - return - } - - p.idleHandles = append( - p.idleHandles, - &idleHandle{ - handle: handle, - keepUntil: keepUntil, - }) - - nIdleHandles := uint32(len(p.idleHandles)) - if nIdleHandles > p.options.MaxIdleHandles { - handlesToClose := nIdleHandles - p.options.MaxIdleHandles - toClose = p.idleHandles[0:handlesToClose] - p.idleHandles = p.idleHandles[handlesToClose:nIdleHandles] - } -} - -// Closes resources, at this point it is assumed that this resources -// are no longer referenced from the main idleHandles slice. -func (p *SimpleResourcePool) closeHandles(handles []*idleHandle) { - for _, handle := range handles { - _ = p.options.Close(handle.handle) - } -} diff --git a/vendor/github.com/dropbox/godropbox/sync2/atomic.go b/vendor/github.com/dropbox/godropbox/sync2/atomic.go deleted file mode 100644 index db39891..0000000 --- a/vendor/github.com/dropbox/godropbox/sync2/atomic.go +++ /dev/null @@ -1,78 +0,0 @@ -package sync2 - -import ( - "sync/atomic" - "time" -) - -type AtomicInt32 int32 - -func (i32 *AtomicInt32) Add(n int32) int32 { - return atomic.AddInt32((*int32)(i32), n) -} - -func (i32 *AtomicInt32) Set(n int32) { - atomic.StoreInt32((*int32)(i32), n) -} - -func (i32 *AtomicInt32) Get() int32 { - return atomic.LoadInt32((*int32)(i32)) -} - -func (i32 *AtomicInt32) CompareAndSwap(oldval, newval int32) (swapped bool) { - return atomic.CompareAndSwapInt32((*int32)(i32), oldval, newval) -} - -type AtomicUint32 uint32 - -func (u32 *AtomicUint32) Add(n uint32) uint32 { - return atomic.AddUint32((*uint32)(u32), n) -} - -func (u32 *AtomicUint32) Set(n uint32) { - atomic.StoreUint32((*uint32)(u32), n) -} - -func (u32 *AtomicUint32) Get() uint32 { - return atomic.LoadUint32((*uint32)(u32)) -} - -func (u32 *AtomicUint32) CompareAndSwap(oldval, newval uint32) (swapped bool) { - return atomic.CompareAndSwapUint32((*uint32)(u32), oldval, newval) -} - -type AtomicInt64 int64 - -func (i64 *AtomicInt64) Add(n int64) int64 { - return atomic.AddInt64((*int64)(i64), n) -} - -func (i64 *AtomicInt64) Set(n int64) { - atomic.StoreInt64((*int64)(i64), n) -} - -func (i64 *AtomicInt64) Get() int64 { - return atomic.LoadInt64((*int64)(i64)) -} - -func (i64 *AtomicInt64) CompareAndSwap(oldval, newval int64) (swapped bool) { - return atomic.CompareAndSwapInt64((*int64)(i64), oldval, newval) -} - -type AtomicDuration int64 - -func (dur *AtomicDuration) Add(dururation time.Duration) time.Duration { - return time.Duration(atomic.AddInt64((*int64)(dur), int64(dururation))) -} - -func (dur *AtomicDuration) Set(dururation time.Duration) { - atomic.StoreInt64((*int64)(dur), int64(dururation)) -} - -func (dur *AtomicDuration) Get() time.Duration { - return time.Duration(atomic.LoadInt64((*int64)(dur))) -} - -func (dur *AtomicDuration) CompareAndSwap(oldval, newval time.Duration) (swapped bool) { - return atomic.CompareAndSwapInt64((*int64)(dur), int64(oldval), int64(newval)) -} diff --git a/vendor/github.com/dropbox/godropbox/sync2/boundedrwlock.go b/vendor/github.com/dropbox/godropbox/sync2/boundedrwlock.go deleted file mode 100644 index 25f6f0a..0000000 --- a/vendor/github.com/dropbox/godropbox/sync2/boundedrwlock.go +++ /dev/null @@ -1,219 +0,0 @@ -package sync2 - -import ( - "sync" - "sync/atomic" - "time" - - "github.com/dropbox/godropbox/errors" -) - -// A fair RWLock with timeouts and a capacity. -// -// Obeys the typical rules about RWLocks -// -// 1. If a writer holds the lock, only a single writer is in the lock. -// 2. If a writer does not hold the lock, any number of readers may hold the -// lock. -// -// The lock favors writers, but readers are not starved, and the next batch of -// readers will be served in before any waiting writers in FIFO order -// (when a writer releases the lock). -type BoundedRWLock struct { - waiters chan *rwwait - control *sync.Mutex - readers int - nextWriter *rwwait -} - -// Create a new BoundedRWLock with the given capacity. -// -// RLocks or WLocks beyond this capacity will fail fast with an error. -func NewBoundedRWLock(capacity int) *BoundedRWLock { - return &BoundedRWLock{ - waiters: make(chan *rwwait, capacity), - control: &sync.Mutex{}, - } -} - -// Wait for a read lock for up to 'timeout'. -// -// Error will be non-nil on timeout or when the wait list is at capacity. -func (rw *BoundedRWLock) RLock(timeout time.Duration) (err error) { - deadline := time.After(timeout) - rw.control.Lock() - if rw.nextWriter != nil { - me := newWait(false) - select { - case rw.waiters <- me: - default: - err = errors.New("Waiter capacity reached in RLock") - } - rw.control.Unlock() - if err != nil { - return - } - - woken := me.WaitAtomic(deadline) - if !woken { - return errors.New("Waiter timeout") - } - } else { - rw.readers++ - rw.control.Unlock() - } - return -} - -// Unlock a read lock. -// -// Should be called only on a goroutine which has gotten a non-error return -// value from RLock(). -func (rw *BoundedRWLock) RUnlock() { - rw.control.Lock() - rw.readers-- - if rw.readers == 0 { - rw.processQueue() - } - rw.control.Unlock() -} - -// Lock for writing, waiting up to 'timeout' for successful exclusive -// acquisition of the lock. -func (rw *BoundedRWLock) WLock(timeout time.Duration) (err error) { - deadline := time.After(timeout) - rw.control.Lock() - if rw.readers != 0 || rw.nextWriter != nil { - me := newWait(true) - if rw.nextWriter == nil { - rw.nextWriter = me - } else { - select { - case rw.waiters <- me: - default: - err = errors.New("Waiter capacity reached in WLock") - } - } - rw.control.Unlock() - if err != nil { - return - } - - woken := me.WaitAtomic(deadline) - if !woken { - return errors.New("Waiter timeout") - } - rw.control.Lock() - if rw.readers != 0 { - panic("readers??") - } - if rw.nextWriter != me { - panic("not me??") - } - } else { - rw.nextWriter = newWait(true) - } - rw.control.Unlock() - return -} - -// Unlock the write lock. -// -// Should be called only on a goroutine which has gotten a non-error return -// value from WLock(). -func (rw *BoundedRWLock) WUnlock() { - rw.control.Lock() - rw.nextWriter = nil - rw.processQueue() - rw.control.Unlock() -} - -// Walks the queue of eligible waiters (if any) and wakes them (if they're not -// timed out). -// -// Any writer "stops" the walk of the queue. -func (rw *BoundedRWLock) processQueue() { - - if rw.readers != 0 { - panic("readers??") - } - - if rw.nextWriter != nil { - if rw.nextWriter.WakeAtomic() { - return - } - rw.nextWriter = nil - } - - for { - var next *rwwait - select { - case next = <-rw.waiters: - default: - return - } - if next.writer { - // No readers scheduled yet? - if rw.readers == 0 { - // If they wake up, no one else gets to go - if next.WakeAtomic() { - rw.nextWriter = next - return - } - } else { - rw.nextWriter = next - return - } - } else { - // Reader? Let them enter now. - if next.WakeAtomic() { - rw.readers++ - } - } - } - return -} - -// A waiting entity, writer or reader. -type rwwait struct { - writer bool - wake chan bool - alive int32 -} - -func newWait(writer bool) *rwwait { - return &rwwait{writer, make(chan bool, 1), 1} -} - -// Wait for a signal on the waiter, with the guarantee that both goroutines -// will agree on whether or not the signal was delivered. -// -// Returns true if the wake occurred, false on timeout. -func (wait *rwwait) WaitAtomic(after <-chan time.Time) bool { - select { - case <-wait.wake: - return true - case <-after: - } - swapped := atomic.CompareAndSwapInt32(&wait.alive, 1, 0) - // They're gonna put it. - if !swapped { - <-wait.wake - return true - } - return false -} - -// Signal the wait to wake. -// -// Returns true of the waiter got the signal, false if the waiter timed out -// before we could deliver the signal. -func (wait *rwwait) WakeAtomic() bool { - swapped := atomic.CompareAndSwapInt32(&wait.alive, 1, 0) - if !swapped { - // They've moved on. - return false - } - wait.wake <- true - return true -} diff --git a/vendor/github.com/dropbox/godropbox/sync2/doc.go b/vendor/github.com/dropbox/godropbox/sync2/doc.go deleted file mode 100644 index ce4a320..0000000 --- a/vendor/github.com/dropbox/godropbox/sync2/doc.go +++ /dev/null @@ -1,3 +0,0 @@ -// sync2 is a collection of functions meant to supplement the capabilities -// provided by the standard "sync" package. -package sync2 diff --git a/vendor/github.com/dropbox/godropbox/sync2/semaphore.go b/vendor/github.com/dropbox/godropbox/sync2/semaphore.go deleted file mode 100644 index 72e7634..0000000 --- a/vendor/github.com/dropbox/godropbox/sync2/semaphore.go +++ /dev/null @@ -1,149 +0,0 @@ -package sync2 - -import ( - "fmt" - "sync" - "sync/atomic" - "time" -) - -type Semaphore interface { - // Increment the semaphore counter by one. - Release() - - // Decrement the semaphore counter by one, and block if counter < 0 - Acquire() - - // Decrement the semaphore counter by one, and block if counter < 0 - // Wait for up to the given duration. Returns true if did not timeout - TryAcquire(timeout time.Duration) bool -} - -// A simple counting Semaphore. -type boundedSemaphore struct { - slots chan struct{} -} - -// Create a bounded semaphore. The count parameter must be a positive number. -// NOTE: The bounded semaphore will panic if the user tries to Release -// beyond the specified count. -func NewBoundedSemaphore(count uint) Semaphore { - sem := &boundedSemaphore{ - slots: make(chan struct{}, int(count)), - } - for i := 0; i < cap(sem.slots); i++ { - sem.slots <- struct{}{} - } - return sem -} - -// Acquire returns on successful acquisition. -func (sem *boundedSemaphore) Acquire() { - <-sem.slots -} - -// TryAcquire returns true if it acquires a resource slot within the -// timeout, false otherwise. -func (sem *boundedSemaphore) TryAcquire(timeout time.Duration) bool { - if timeout > 0 { - tm := time.NewTimer(timeout) - defer tm.Stop() - select { - case <-sem.slots: - return true - case <-tm.C: - return false - } - } else { - select { - case <-sem.slots: - return true - default: - return false - } - } -} - -// Release the acquired semaphore. You must not release more than you -// have acquired. -func (sem *boundedSemaphore) Release() { - select { - case sem.slots <- struct{}{}: - default: - // slots is buffered. If a send blocks, it indicates a programming - // error. - panic(fmt.Errorf("too many releases for boundedSemaphore")) - } -} - -// This returns an unbound counting semaphore with the specified initial count. -// The semaphore counter can be arbitrary large (i.e., Release can be called -// unlimited amount of times). -// -// NOTE: In general, users should use bounded semaphore since it is more -// efficient than unbounded semaphore. -func NewUnboundedSemaphore(initialCount int) Semaphore { - res := &unboundedSemaphore{ - counter: int64(initialCount), - } - res.cond.L = &res.lock - return res -} - -type unboundedSemaphore struct { - lock sync.Mutex - cond sync.Cond - counter int64 -} - -func (s *unboundedSemaphore) Release() { - s.lock.Lock() - s.counter += 1 - if s.counter > 0 { - // Not broadcasting here since it's unlike we can satify all waiting - // goroutines. Instead, we will Signal again if there are left over - // quota after Acquire, in case of lost wakeups. - s.cond.Signal() - } - s.lock.Unlock() -} - -func (s *unboundedSemaphore) Acquire() { - s.lock.Lock() - for s.counter < 1 { - s.cond.Wait() - } - s.counter -= 1 - if s.counter > 0 { - s.cond.Signal() - } - s.lock.Unlock() -} - -func (s *unboundedSemaphore) TryAcquire(timeout time.Duration) bool { - done := make(chan bool, 1) - // Gate used to communicate between the threads and decide what the result - // is. If the main thread decides, we have timed out, otherwise we succeed. - decided := new(int32) - atomic.StoreInt32(decided, 0) - go func() { - s.Acquire() - if atomic.SwapInt32(decided, 1) == 0 { - // Acquire won the race - done <- true - } else { - // If we already decided the result, and this thread did not win - s.Release() - } - }() - select { - case <-done: - return true - case <-time.After(timeout): - if atomic.SwapInt32(decided, 1) == 1 { - // The other thread already decided the result - return true - } - return false - } -} diff --git a/vendor/github.com/dropbox/godropbox/sync2/with.go b/vendor/github.com/dropbox/godropbox/sync2/with.go deleted file mode 100644 index 049a201..0000000 --- a/vendor/github.com/dropbox/godropbox/sync2/with.go +++ /dev/null @@ -1,11 +0,0 @@ -package sync2 - -import ( - "sync" -) - -func With(mu sync.Locker, f func()) { - mu.Lock() - defer mu.Unlock() - f() -}