From 961c04cae992eadb42d286d2f85f8a675bdc68c8 Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 29 Jan 2018 14:17:40 -0800 Subject: Upgrading server dependancies (#8154) --- .../appengine/memcache/memcache.go | 526 +++++++++++++++++++++ 1 file changed, 526 insertions(+) create mode 100644 vendor/google.golang.org/appengine/memcache/memcache.go (limited to 'vendor/google.golang.org/appengine/memcache/memcache.go') diff --git a/vendor/google.golang.org/appengine/memcache/memcache.go b/vendor/google.golang.org/appengine/memcache/memcache.go new file mode 100644 index 000000000..d8eed4be7 --- /dev/null +++ b/vendor/google.golang.org/appengine/memcache/memcache.go @@ -0,0 +1,526 @@ +// Copyright 2011 Google Inc. All rights reserved. +// Use of this source code is governed by the Apache 2.0 +// license that can be found in the LICENSE file. + +// Package memcache provides a client for App Engine's distributed in-memory +// key-value store for small chunks of arbitrary data. +// +// The fundamental operations get and set items, keyed by a string. +// +// item0, err := memcache.Get(c, "key") +// if err != nil && err != memcache.ErrCacheMiss { +// return err +// } +// if err == nil { +// fmt.Fprintf(w, "memcache hit: Key=%q Val=[% x]\n", item0.Key, item0.Value) +// } else { +// fmt.Fprintf(w, "memcache miss\n") +// } +// +// and +// +// item1 := &memcache.Item{ +// Key: "foo", +// Value: []byte("bar"), +// } +// if err := memcache.Set(c, item1); err != nil { +// return err +// } +package memcache // import "google.golang.org/appengine/memcache" + +import ( + "bytes" + "encoding/gob" + "encoding/json" + "errors" + "time" + + "github.com/golang/protobuf/proto" + "golang.org/x/net/context" + + "google.golang.org/appengine" + "google.golang.org/appengine/internal" + pb "google.golang.org/appengine/internal/memcache" +) + +var ( + // ErrCacheMiss means that an operation failed + // because the item wasn't present. + ErrCacheMiss = errors.New("memcache: cache miss") + // ErrCASConflict means that a CompareAndSwap call failed due to the + // cached value being modified between the Get and the CompareAndSwap. + // If the cached value was simply evicted rather than replaced, + // ErrNotStored will be returned instead. + ErrCASConflict = errors.New("memcache: compare-and-swap conflict") + // ErrNoStats means that no statistics were available. + ErrNoStats = errors.New("memcache: no statistics available") + // ErrNotStored means that a conditional write operation (i.e. Add or + // CompareAndSwap) failed because the condition was not satisfied. + ErrNotStored = errors.New("memcache: item not stored") + // ErrServerError means that a server error occurred. + ErrServerError = errors.New("memcache: server error") +) + +// Item is the unit of memcache gets and sets. +type Item struct { + // Key is the Item's key (250 bytes maximum). + Key string + // Value is the Item's value. + Value []byte + // Object is the Item's value for use with a Codec. + Object interface{} + // Flags are server-opaque flags whose semantics are entirely up to the + // App Engine app. + Flags uint32 + // Expiration is the maximum duration that the item will stay + // in the cache. + // The zero value means the Item has no expiration time. + // Subsecond precision is ignored. + // This is not set when getting items. + Expiration time.Duration + // casID is a client-opaque value used for compare-and-swap operations. + // Zero means that compare-and-swap is not used. + casID uint64 +} + +const ( + secondsIn30Years = 60 * 60 * 24 * 365 * 30 // from memcache server code + thirtyYears = time.Duration(secondsIn30Years) * time.Second +) + +// protoToItem converts a protocol buffer item to a Go struct. +func protoToItem(p *pb.MemcacheGetResponse_Item) *Item { + return &Item{ + Key: string(p.Key), + Value: p.Value, + Flags: p.GetFlags(), + casID: p.GetCasId(), + } +} + +// If err is an appengine.MultiError, return its first element. Otherwise, return err. +func singleError(err error) error { + if me, ok := err.(appengine.MultiError); ok { + return me[0] + } + return err +} + +// Get gets the item for the given key. ErrCacheMiss is returned for a memcache +// cache miss. The key must be at most 250 bytes in length. +func Get(c context.Context, key string) (*Item, error) { + m, err := GetMulti(c, []string{key}) + if err != nil { + return nil, err + } + if _, ok := m[key]; !ok { + return nil, ErrCacheMiss + } + return m[key], nil +} + +// GetMulti is a batch version of Get. The returned map from keys to items may +// have fewer elements than the input slice, due to memcache cache misses. +// Each key must be at most 250 bytes in length. +func GetMulti(c context.Context, key []string) (map[string]*Item, error) { + if len(key) == 0 { + return nil, nil + } + keyAsBytes := make([][]byte, len(key)) + for i, k := range key { + keyAsBytes[i] = []byte(k) + } + req := &pb.MemcacheGetRequest{ + Key: keyAsBytes, + ForCas: proto.Bool(true), + } + res := &pb.MemcacheGetResponse{} + if err := internal.Call(c, "memcache", "Get", req, res); err != nil { + return nil, err + } + m := make(map[string]*Item, len(res.Item)) + for _, p := range res.Item { + t := protoToItem(p) + m[t.Key] = t + } + return m, nil +} + +// Delete deletes the item for the given key. +// ErrCacheMiss is returned if the specified item can not be found. +// The key must be at most 250 bytes in length. +func Delete(c context.Context, key string) error { + return singleError(DeleteMulti(c, []string{key})) +} + +// DeleteMulti is a batch version of Delete. +// If any keys cannot be found, an appengine.MultiError is returned. +// Each key must be at most 250 bytes in length. +func DeleteMulti(c context.Context, key []string) error { + if len(key) == 0 { + return nil + } + req := &pb.MemcacheDeleteRequest{ + Item: make([]*pb.MemcacheDeleteRequest_Item, len(key)), + } + for i, k := range key { + req.Item[i] = &pb.MemcacheDeleteRequest_Item{Key: []byte(k)} + } + res := &pb.MemcacheDeleteResponse{} + if err := internal.Call(c, "memcache", "Delete", req, res); err != nil { + return err + } + if len(res.DeleteStatus) != len(key) { + return ErrServerError + } + me, any := make(appengine.MultiError, len(key)), false + for i, s := range res.DeleteStatus { + switch s { + case pb.MemcacheDeleteResponse_DELETED: + // OK + case pb.MemcacheDeleteResponse_NOT_FOUND: + me[i] = ErrCacheMiss + any = true + default: + me[i] = ErrServerError + any = true + } + } + if any { + return me + } + return nil +} + +// Increment atomically increments the decimal value in the given key +// by delta and returns the new value. The value must fit in a uint64. +// Overflow wraps around, and underflow is capped to zero. The +// provided delta may be negative. If the key doesn't exist in +// memcache, the provided initial value is used to atomically +// populate it before the delta is applied. +// The key must be at most 250 bytes in length. +func Increment(c context.Context, key string, delta int64, initialValue uint64) (newValue uint64, err error) { + return incr(c, key, delta, &initialValue) +} + +// IncrementExisting works like Increment but assumes that the key +// already exists in memcache and doesn't take an initial value. +// IncrementExisting can save work if calculating the initial value is +// expensive. +// An error is returned if the specified item can not be found. +func IncrementExisting(c context.Context, key string, delta int64) (newValue uint64, err error) { + return incr(c, key, delta, nil) +} + +func incr(c context.Context, key string, delta int64, initialValue *uint64) (newValue uint64, err error) { + req := &pb.MemcacheIncrementRequest{ + Key: []byte(key), + InitialValue: initialValue, + } + if delta >= 0 { + req.Delta = proto.Uint64(uint64(delta)) + } else { + req.Delta = proto.Uint64(uint64(-delta)) + req.Direction = pb.MemcacheIncrementRequest_DECREMENT.Enum() + } + res := &pb.MemcacheIncrementResponse{} + err = internal.Call(c, "memcache", "Increment", req, res) + if err != nil { + return + } + if res.NewValue == nil { + return 0, ErrCacheMiss + } + return *res.NewValue, nil +} + +// set sets the given items using the given conflict resolution policy. +// appengine.MultiError may be returned. +func set(c context.Context, item []*Item, value [][]byte, policy pb.MemcacheSetRequest_SetPolicy) error { + if len(item) == 0 { + return nil + } + req := &pb.MemcacheSetRequest{ + Item: make([]*pb.MemcacheSetRequest_Item, len(item)), + } + for i, t := range item { + p := &pb.MemcacheSetRequest_Item{ + Key: []byte(t.Key), + } + if value == nil { + p.Value = t.Value + } else { + p.Value = value[i] + } + if t.Flags != 0 { + p.Flags = proto.Uint32(t.Flags) + } + if t.Expiration != 0 { + // In the .proto file, MemcacheSetRequest_Item uses a fixed32 (i.e. unsigned) + // for expiration time, while MemcacheGetRequest_Item uses int32 (i.e. signed). + // Throughout this .go file, we use int32. + // Also, in the proto, the expiration value is either a duration (in seconds) + // or an absolute Unix timestamp (in seconds), depending on whether the + // value is less than or greater than or equal to 30 years, respectively. + if t.Expiration < time.Second { + // Because an Expiration of 0 means no expiration, we take + // care here to translate an item with an expiration + // Duration between 0-1 seconds as immediately expiring + // (saying it expired a few seconds ago), rather than + // rounding it down to 0 and making it live forever. + p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) - 5) + } else if t.Expiration >= thirtyYears { + p.ExpirationTime = proto.Uint32(uint32(time.Now().Unix()) + uint32(t.Expiration/time.Second)) + } else { + p.ExpirationTime = proto.Uint32(uint32(t.Expiration / time.Second)) + } + } + if t.casID != 0 { + p.CasId = proto.Uint64(t.casID) + p.ForCas = proto.Bool(true) + } + p.SetPolicy = policy.Enum() + req.Item[i] = p + } + res := &pb.MemcacheSetResponse{} + if err := internal.Call(c, "memcache", "Set", req, res); err != nil { + return err + } + if len(res.SetStatus) != len(item) { + return ErrServerError + } + me, any := make(appengine.MultiError, len(item)), false + for i, st := range res.SetStatus { + var err error + switch st { + case pb.MemcacheSetResponse_STORED: + // OK + case pb.MemcacheSetResponse_NOT_STORED: + err = ErrNotStored + case pb.MemcacheSetResponse_EXISTS: + err = ErrCASConflict + default: + err = ErrServerError + } + if err != nil { + me[i] = err + any = true + } + } + if any { + return me + } + return nil +} + +// Set writes the given item, unconditionally. +func Set(c context.Context, item *Item) error { + return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_SET)) +} + +// SetMulti is a batch version of Set. +// appengine.MultiError may be returned. +func SetMulti(c context.Context, item []*Item) error { + return set(c, item, nil, pb.MemcacheSetRequest_SET) +} + +// Add writes the given item, if no value already exists for its key. +// ErrNotStored is returned if that condition is not met. +func Add(c context.Context, item *Item) error { + return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_ADD)) +} + +// AddMulti is a batch version of Add. +// appengine.MultiError may be returned. +func AddMulti(c context.Context, item []*Item) error { + return set(c, item, nil, pb.MemcacheSetRequest_ADD) +} + +// CompareAndSwap writes the given item that was previously returned by Get, +// if the value was neither modified or evicted between the Get and the +// CompareAndSwap calls. The item's Key should not change between calls but +// all other item fields may differ. +// ErrCASConflict is returned if the value was modified in between the calls. +// ErrNotStored is returned if the value was evicted in between the calls. +func CompareAndSwap(c context.Context, item *Item) error { + return singleError(set(c, []*Item{item}, nil, pb.MemcacheSetRequest_CAS)) +} + +// CompareAndSwapMulti is a batch version of CompareAndSwap. +// appengine.MultiError may be returned. +func CompareAndSwapMulti(c context.Context, item []*Item) error { + return set(c, item, nil, pb.MemcacheSetRequest_CAS) +} + +// Codec represents a symmetric pair of functions that implement a codec. +// Items stored into or retrieved from memcache using a Codec have their +// values marshaled or unmarshaled. +// +// All the methods provided for Codec behave analogously to the package level +// function with same name. +type Codec struct { + Marshal func(interface{}) ([]byte, error) + Unmarshal func([]byte, interface{}) error +} + +// Get gets the item for the given key and decodes the obtained value into v. +// ErrCacheMiss is returned for a memcache cache miss. +// The key must be at most 250 bytes in length. +func (cd Codec) Get(c context.Context, key string, v interface{}) (*Item, error) { + i, err := Get(c, key) + if err != nil { + return nil, err + } + if err := cd.Unmarshal(i.Value, v); err != nil { + return nil, err + } + return i, nil +} + +func (cd Codec) set(c context.Context, items []*Item, policy pb.MemcacheSetRequest_SetPolicy) error { + var vs [][]byte + var me appengine.MultiError + for i, item := range items { + v, err := cd.Marshal(item.Object) + if err != nil { + if me == nil { + me = make(appengine.MultiError, len(items)) + } + me[i] = err + continue + } + if me == nil { + vs = append(vs, v) + } + } + if me != nil { + return me + } + + return set(c, items, vs, policy) +} + +// Set writes the given item, unconditionally. +func (cd Codec) Set(c context.Context, item *Item) error { + return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_SET)) +} + +// SetMulti is a batch version of Set. +// appengine.MultiError may be returned. +func (cd Codec) SetMulti(c context.Context, items []*Item) error { + return cd.set(c, items, pb.MemcacheSetRequest_SET) +} + +// Add writes the given item, if no value already exists for its key. +// ErrNotStored is returned if that condition is not met. +func (cd Codec) Add(c context.Context, item *Item) error { + return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_ADD)) +} + +// AddMulti is a batch version of Add. +// appengine.MultiError may be returned. +func (cd Codec) AddMulti(c context.Context, items []*Item) error { + return cd.set(c, items, pb.MemcacheSetRequest_ADD) +} + +// CompareAndSwap writes the given item that was previously returned by Get, +// if the value was neither modified or evicted between the Get and the +// CompareAndSwap calls. The item's Key should not change between calls but +// all other item fields may differ. +// ErrCASConflict is returned if the value was modified in between the calls. +// ErrNotStored is returned if the value was evicted in between the calls. +func (cd Codec) CompareAndSwap(c context.Context, item *Item) error { + return singleError(cd.set(c, []*Item{item}, pb.MemcacheSetRequest_CAS)) +} + +// CompareAndSwapMulti is a batch version of CompareAndSwap. +// appengine.MultiError may be returned. +func (cd Codec) CompareAndSwapMulti(c context.Context, items []*Item) error { + return cd.set(c, items, pb.MemcacheSetRequest_CAS) +} + +var ( + // Gob is a Codec that uses the gob package. + Gob = Codec{gobMarshal, gobUnmarshal} + // JSON is a Codec that uses the json package. + JSON = Codec{json.Marshal, json.Unmarshal} +) + +func gobMarshal(v interface{}) ([]byte, error) { + var buf bytes.Buffer + if err := gob.NewEncoder(&buf).Encode(v); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func gobUnmarshal(data []byte, v interface{}) error { + return gob.NewDecoder(bytes.NewBuffer(data)).Decode(v) +} + +// Statistics represents a set of statistics about the memcache cache. +// This may include items that have expired but have not yet been removed from the cache. +type Statistics struct { + Hits uint64 // Counter of cache hits + Misses uint64 // Counter of cache misses + ByteHits uint64 // Counter of bytes transferred for gets + + Items uint64 // Items currently in the cache + Bytes uint64 // Size of all items currently in the cache + + Oldest int64 // Age of access of the oldest item, in seconds +} + +// Stats retrieves the current memcache statistics. +func Stats(c context.Context) (*Statistics, error) { + req := &pb.MemcacheStatsRequest{} + res := &pb.MemcacheStatsResponse{} + if err := internal.Call(c, "memcache", "Stats", req, res); err != nil { + return nil, err + } + if res.Stats == nil { + return nil, ErrNoStats + } + return &Statistics{ + Hits: *res.Stats.Hits, + Misses: *res.Stats.Misses, + ByteHits: *res.Stats.ByteHits, + Items: *res.Stats.Items, + Bytes: *res.Stats.Bytes, + Oldest: int64(*res.Stats.OldestItemAge), + }, nil +} + +// Flush flushes all items from memcache. +func Flush(c context.Context) error { + req := &pb.MemcacheFlushRequest{} + res := &pb.MemcacheFlushResponse{} + return internal.Call(c, "memcache", "FlushAll", req, res) +} + +func namespaceMod(m proto.Message, namespace string) { + switch m := m.(type) { + case *pb.MemcacheDeleteRequest: + if m.NameSpace == nil { + m.NameSpace = &namespace + } + case *pb.MemcacheGetRequest: + if m.NameSpace == nil { + m.NameSpace = &namespace + } + case *pb.MemcacheIncrementRequest: + if m.NameSpace == nil { + m.NameSpace = &namespace + } + case *pb.MemcacheSetRequest: + if m.NameSpace == nil { + m.NameSpace = &namespace + } + // MemcacheFlushRequest, MemcacheStatsRequest do not apply namespace. + } +} + +func init() { + internal.RegisterErrorCodeMap("memcache", pb.MemcacheServiceError_ErrorCode_name) + internal.NamespaceMods["memcache"] = namespaceMod +} -- cgit v1.2.3-1-g7c22