Documentation ¶
Overview ¶
This is the internal version of the charmstore package. It exposes details to the various API packages that we do not wish to expose to the world at large.
Index ¶
- Constants
- Variables
- func EntityResolvedURL(e *mongodoc.Entity) *router.ResolvedURL
- func EntityStatsKey(url *charm.Reference, kind string) []string
- func NewServer(db *mgo.Database, si *SearchIndex, config ServerParams, ...) (http.Handler, error)
- func NewZipFile(f *zip.File) (mongodoc.ZipFile, error)
- func ReaderAtSeeker(r io.ReadSeeker) io.ReaderAt
- func ZipFileReader(zipr io.ReadSeeker, f mongodoc.ZipFile) (io.Reader, error)
- type AddParams
- type AggregatedCounts
- type Counter
- type CounterRequest
- type CounterRequestBy
- type NewAPIHandlerFunc
- type Pool
- type SearchDoc
- type SearchIndex
- type SearchParams
- type SearchResult
- type ServerParams
- type Store
- func (s *Store) AddBundle(b charm.Bundle, p AddParams) error
- func (s *Store) AddBundleWithArchive(url *router.ResolvedURL, b charm.Bundle) error
- func (s *Store) AddCharm(c charm.Charm, p AddParams) (err error)
- func (s *Store) AddCharmWithArchive(url *router.ResolvedURL, ch charm.Charm) error
- func (s *Store) AddLog(data *json.RawMessage, logLevel mongodoc.LogLevel, logType mongodoc.LogType, ...) error
- func (s *Store) ArchiveDownloadCounts(id *charm.Reference) (thisRevision, allRevisions AggregatedCounts, err error)
- func (s *Store) BlobNameAndHash(id *router.ResolvedURL) (name, hash string, err error)
- func (s *Store) Close()
- func (s *Store) Copy() *Store
- func (s *Store) Counters(req *CounterRequest) ([]Counter, error)
- func (s *Store) EntitiesQuery(url *charm.Reference) *mgo.Query
- func (s *Store) FindBaseEntity(url *charm.Reference, fields ...string) (*mongodoc.BaseEntity, error)
- func (s *Store) FindBestEntity(url *charm.Reference, fields ...string) (*mongodoc.Entity, error)
- func (s *Store) FindEntities(url *charm.Reference, fields ...string) ([]*mongodoc.Entity, error)
- func (s *Store) FindEntity(url *router.ResolvedURL, fields ...string) (*mongodoc.Entity, error)
- func (s *Store) Go(f func(*Store))
- func (s *Store) IncCounter(key []string) error
- func (s *Store) IncCounterAsync(key []string)
- func (s *Store) IncCounterAtTime(key []string, t time.Time) error
- func (s *Store) IncrementDownloadCounts(id *router.ResolvedURL) error
- func (s *Store) IncrementDownloadCountsAsync(id *router.ResolvedURL)
- func (s *Store) OpenBlob(id *router.ResolvedURL) (r blobstore.ReadSeekCloser, size int64, hash string, err error)
- func (s *Store) OpenCachedBlobFile(entity *mongodoc.Entity, fileId mongodoc.FileId, isFile func(f *zip.File) bool) (_ io.ReadCloser, err error)
- func (s *Store) Pool() *Pool
- func (store *Store) Search(sp SearchParams) (SearchResult, error)
- func (s *Store) SetPerms(id *charm.Reference, which string, acl ...string) error
- func (s *Store) SetPromulgated(url *router.ResolvedURL, promulgate bool) error
- func (s *Store) SetReconnectTimeout(d time.Duration)
- func (s *Store) SynchroniseElasticsearch() error
- func (s *Store) UpdateBaseEntity(url *router.ResolvedURL, update interface{}) error
- func (s *Store) UpdateEntity(url *router.ResolvedURL, update interface{}) error
- func (s *Store) UpdateEntitySHA256(id *router.ResolvedURL) (string, error)
- func (s *Store) UpdateSearch(r *router.ResolvedURL) error
- func (s *Store) UpdateSearchAsync(r *router.ResolvedURL)
- func (s *Store) UpdateSearchBaseURL(baseURL *charm.Reference) error
- func (s *Store) UpdateSearchFields(r *router.ResolvedURL, fields map[string]interface{}) error
- type StoreDatabase
- func (s StoreDatabase) BaseEntities() *mgo.Collection
- func (s StoreDatabase) Close()
- func (s StoreDatabase) Collections() []*mgo.Collection
- func (s StoreDatabase) Copy() StoreDatabase
- func (s StoreDatabase) Entities() *mgo.Collection
- func (s StoreDatabase) Logs() *mgo.Collection
- func (s StoreDatabase) Macaroons() *mgo.Collection
- func (s StoreDatabase) Migrations() *mgo.Collection
- func (s StoreDatabase) StatCounters() *mgo.Collection
- func (s StoreDatabase) StatTokens() *mgo.Collection
Constants ¶
const StatsGranularity = time.Minute
StatsGranularity holds the time granularity of statistics gathering. IncCounter(Async) calls within this duration may be aggregated.
Variables ¶
var LegacyDownloadCountsEnabled = true
LegacyDownloadCountsEnabled represents whether aggregated download counts must be retrieved from the legacy infrastructure. In essence, if the value is true (enabled), aggregated counts are not calculated based on the data stored in the charm store stats; they are instead retrieved from the entity extra-info. For this reason, enabling this we assume an external program updated the extra-info for the entity, specifically the "legacy-download-stats" key. TODO (frankban): this is a temporary hack, and can be removed once we have a more consistent way to import the download counts from the legacy charm store (charms) and from charmworld (bundles). To remove the legacy download counts logic in the future, grep the code for "LegacyDownloadCountsEnabled" and remove as required.
var UpdateEntitySHA256 = func(store *Store, id *router.ResolvedURL, sum256 string) { err := store.DB.Entities().UpdateId(&id.URL, bson.D{{"$set", bson.D{{"blobhash256", sum256}}}}) if err != nil && err != mgo.ErrNotFound { logger.Errorf("cannot update sha256 of archive: %v", err) } }
UpdateEntitySHA256 updates the BlobHash256 entry for the entity. It is defined as a variable so that it can be mocked in tests. This function will be removed soon.
Functions ¶
func EntityResolvedURL ¶
func EntityResolvedURL(e *mongodoc.Entity) *router.ResolvedURL
EntityResolvedURL returns the ResolvedURL for the entity. It requires the PromulgatedURL field to have been filled out in the entity.
func EntityStatsKey ¶
EntityStatsKey returns a stats key for the given charm or bundle reference and the given kind. Entity stats keys are generated using the following schema:
kind:series:name:user:revision
where user can be empty (for promulgated charms/bundles) and revision is optional (e.g. when uploading an entity the revision is not specified). For instance, entities' stats can then be retrieved like the following:
- kind:utopic:* -> all charms of a specific series;
- kind:trusty:django:* -> all revisions and user variations of a charm;
- kind:trusty:django::* -> all revisions of a promulgated charm;
- kind:trusty:django::42 -> a specific promulgated charm;
- kind:trusty:django:who:* -> all revisions of a user owned charm;
- kind:trusty:django:who:42 -> a specific user owned charm;
The above also applies to bundles (where the series is "bundle").
func NewServer ¶
func NewServer(db *mgo.Database, si *SearchIndex, config ServerParams, versions map[string]NewAPIHandlerFunc) (http.Handler, error)
NewServer returns a handler that serves the given charm store API versions using db to store that charm store data. An optional elasticsearch configuration can be specified in si. If elasticsearch is not being used then si can be set to nil. The key of the versions map is the version name. The handler configuration is provided to all version handlers.
func NewZipFile ¶
NewZipFile returns a new mongodoc zip file reference to the given zip file.
func ReaderAtSeeker ¶
func ReaderAtSeeker(r io.ReadSeeker) io.ReaderAt
ReaderAtSeeker adapts r so that it can be used as a ReaderAt. Note that, unlike some implementations of ReaderAt, it is not OK to use concurrently.
func ZipFileReader ¶
ZipFileReader returns a reader that will read content referred to by f within zipr, which should refer to the contents of a zip file,
Types ¶
type AddParams ¶
type AddParams struct { // URL holds the id to be associated with the stored entity. // If URL.PromulgatedRevision is not -1, the entity will // be promulgated. URL *router.ResolvedURL // BlobName holds the name of the entity's archive blob. BlobName string // BlobHash holds the hash of the entity's archive blob. BlobHash string // BlobHash256 holds the sha256 hash of the entity's archive blob. BlobHash256 string // BlobHash holds the size of the entity's archive blob. BlobSize int64 // Contents holds references to files inside the // entity's archive blob. Contents map[mongodoc.FileId]mongodoc.ZipFile }
AddParams holds parameters held in common between the Store.AddCharm and Store.AddBundle methods.
type AggregatedCounts ¶
type AggregatedCounts struct {
LastDay, LastWeek, LastMonth, Total int64
}
AggregatedCounts contains counts for a statistic aggregated over the lastDay, lastWeek, lastMonth and all time.
type CounterRequest ¶
type CounterRequest struct { // Key and Prefix determine the counter keys to match. // If Prefix is false, Key must match exactly. Otherwise, counters // must begin with Key and have at least one more key token. Key []string Prefix bool // If List is true, matching counters are aggregated under their // prefixes instead of being returned as a single overall sum. // // For example, given the following counts: // // {"a", "b"}: 1, // {"a", "c"}: 3 // {"a", "c", "d"}: 5 // {"a", "c", "e"}: 7 // // and assuming that Prefix is true, the following keys will // present the respective results if List is true: // // {"a"} => {{"a", "b"}, 1, false}, // {{"a", "c"}, 3, false}, // {{"a", "c"}, 12, true} // {"a", "c"} => {{"a", "c", "d"}, 3, false}, // {{"a", "c", "e"}, 5, false} // // If List is false, the same key prefixes will present: // // {"a"} => {{"a"}, 16, true} // {"a", "c"} => {{"a", "c"}, 12, false} // List bool // By defines the period covered by each aggregated data point. // If unspecified, it defaults to ByAll, which aggregates all // matching data points in a single entry. By CounterRequestBy // Start, if provided, changes the query so that only data points // ocurring at the given time or afterwards are considered. Start time.Time // Stop, if provided, changes the query so that only data points // ocurring at the given time or before are considered. Stop time.Time }
CounterRequest represents a request to aggregate counter values.
type CounterRequestBy ¶
type CounterRequestBy int
const ( ByAll CounterRequestBy = iota ByDay ByWeek )
type NewAPIHandlerFunc ¶
type NewAPIHandlerFunc func(*Pool, ServerParams) http.Handler
NewAPIHandlerFunc is a function that returns a new API handler that uses the given Store.
type Pool ¶
Pool holds a connection to the underlying charm and blob data stores. Calling its Store method returns a new Store from the pool that can be used to process short-lived requests to access and modify the store.
func NewPool ¶
func NewPool(db *mgo.Database, si *SearchIndex, bakeryParams *bakery.NewServiceParams) (*Pool, error)
NewPool returns a Pool that uses the given database and search index. If bakeryParams is not nil, the Bakery field in the resulting Store will be set to a new Service that stores macaroons in mongo.
type SearchDoc ¶
SearchDoc is a mongodoc.Entity with additional fields useful for searching. This is the document that is stored in the search index.
type SearchIndex ¶
type SearchIndex struct { *elasticsearch.Database Index string }
func (*SearchIndex) GetSearchDocument ¶
func (si *SearchIndex) GetSearchDocument(id *charm.Reference) (*SearchDoc, error)
GetSearchDocument retrieves the current search record for the charm reference id.
type SearchParams ¶
type SearchParams struct { // The text to use in the full text search query. Text string // If autocomplete is specified, the search will return only charms and // bundles with a name that has text as a prefix. AutoComplete bool // Limit the search to items with attributes that match the specified filter value. Filters map[string][]string // Limit the number of returned items to the specified count. Limit int // Include the following metadata items in the search results. Include []string // Start the the returned items at a specific offset. Skip int // ACL values to search in addition to everyone. ACL values may represent user names // or group names. Groups []string // Admin searches will not filter on the ACL and will show results for all matching // charms. Admin bool // contains filtered or unexported fields }
SearchParams represents the search parameters used to search the store.
func (*SearchParams) ParseSortFields ¶
func (sp *SearchParams) ParseSortFields(f ...string) error
type SearchResult ¶
type SearchResult struct { SearchTime time.Duration Total int Results []*router.ResolvedURL }
SearchResult represents the result of performing a search.
type ServerParams ¶
type ServerParams struct { // AuthUsername and AuthPassword hold the credentials // used for HTTP basic authentication. AuthUsername string AuthPassword string // IdentityLocation holds the location of the third party authorization // service to use when creating third party caveats, // for example: http://api.jujucharms.com/identity/v1/discharger // If it is empty, IdentityURL+"/v1/discharger" will be used. IdentityLocation string // PublicKeyLocator holds a public key store. // It may be nil. PublicKeyLocator bakery.PublicKeyLocator // IdentityAPIURL holds the URL of the identity manager, // for example http://api.jujucharms.com/identity IdentityAPIURL string // IdentityAPIUsername and IdentityAPIPassword hold the credentials // to be used when querying the identity manager API. IdentityAPIUsername string IdentityAPIPassword string }
ServerParams holds configuration for a new internal API server.
type Store ¶
type Store struct { DB StoreDatabase BlobStore *blobstore.Store ES *SearchIndex Bakery *bakery.Service // contains filtered or unexported fields }
Store holds a connection to the underlying charm and blob data stores that is appropriate for short term use.
func (*Store) AddBundle ¶
AddBundle adds a bundle to the entities collection with the given parameters.
func (*Store) AddBundleWithArchive ¶
func (s *Store) AddBundleWithArchive(url *router.ResolvedURL, b charm.Bundle) error
AddBundleWithArchive is like AddBundle but also adds the charm archive to the blob store. This method is provided principally so that tests can easily create content in the store.
If purl is not nil then the bundle will also be available at the promulgated url specified.
TODO This could take a *router.ResolvedURL as an argument instead of two *charm.References.
func (*Store) AddCharmWithArchive ¶
func (s *Store) AddCharmWithArchive(url *router.ResolvedURL, ch charm.Charm) error
AddCharmWithArchive is like AddCharm but also adds the charm archive to the blob store. This method is provided principally so that tests can easily create content in the store.
If purl is not nil then the charm will also be available at the promulgated url specified.
func (*Store) AddLog ¶
func (s *Store) AddLog(data *json.RawMessage, logLevel mongodoc.LogLevel, logType mongodoc.LogType, urls []*charm.Reference) error
AddLog adds a log message to the database.
func (*Store) ArchiveDownloadCounts ¶
func (s *Store) ArchiveDownloadCounts(id *charm.Reference) (thisRevision, allRevisions AggregatedCounts, err error)
ArchiveDownloadCounts calculates the aggregated download counts for a charm or bundle.
func (*Store) BlobNameAndHash ¶
func (s *Store) BlobNameAndHash(id *router.ResolvedURL) (name, hash string, err error)
BlobNameAndHash returns the name that is used to store the blob for the entity with the given id and its hash. It returns a params.ErrNotFound error if the entity does not exist.
func (*Store) Copy ¶
Copy returns a new store with a lifetime independent of s. Use this method if you need to use a store in an independent goroutine.
It must be closed (with the Close method) after use.
func (*Store) Counters ¶
func (s *Store) Counters(req *CounterRequest) ([]Counter, error)
Counters aggregates and returns counter values according to the provided request.
func (*Store) EntitiesQuery ¶
func (s *Store) EntitiesQuery(url *charm.Reference) *mgo.Query
EntitiesQuery creates a mgo.Query object that can be used to find entities matching the given URL. If the given URL has no user then the produced query will only match promulgated entities.
func (*Store) FindBaseEntity ¶
func (s *Store) FindBaseEntity(url *charm.Reference, fields ...string) (*mongodoc.BaseEntity, error)
FindBaseEntity finds the base entity in the store using the given URL, which can either represent a fully qualified entity or a base id. If any fields are specified, only those fields will be populated in the returned base entity.
func (*Store) FindBestEntity ¶
FindBestEntity finds the entity that provides the preferred match to the given URL. If any fields are specified, only those fields will be populated in the returned entities. If the given URL has no user then only promulgated entities will be queried.
func (*Store) FindEntities ¶
FindEntities finds all entities in the store matching the given URL. If any fields are specified, only those fields will be populated in the returned entities. If the given URL has no user then only promulgated entities will be queried.
func (*Store) FindEntity ¶
FindEntity finds the entity in the store with the given URL, which must be fully qualified. If any fields are specified, only those fields will be populated in the returned entities. If the given URL has no user then it is assumed to be a promulgated entity.
func (*Store) Go ¶
Go runs the given function in a new goroutine, passing it a copy of s, which will be closed after the function returns.
func (*Store) IncCounter ¶
IncCounter increases by one the counter associated with the composed key.
func (*Store) IncCounterAsync ¶
IncCounterAsync increases by one the counter associated with the composed key. The action is done in the background using a separate goroutine.
func (*Store) IncCounterAtTime ¶
IncCounterAtTime increases by one the counter associated with the composed key, associating it with the given time, which should be time.Now. This method is exposed for testing purposes only - production code should always call IncCounter or IncCounterAsync.
func (*Store) IncrementDownloadCounts ¶
func (s *Store) IncrementDownloadCounts(id *router.ResolvedURL) error
IncrementDownloadCounts updates the download statistics for entity id in both the statistics database and the search database.
func (*Store) IncrementDownloadCountsAsync ¶
func (s *Store) IncrementDownloadCountsAsync(id *router.ResolvedURL)
IncrementDownloadCountsAsync updates the download statistics for entity id in both the statistics database and the search database. The action is done in the background using a separate goroutine.
func (*Store) OpenBlob ¶
func (s *Store) OpenBlob(id *router.ResolvedURL) (r blobstore.ReadSeekCloser, size int64, hash string, err error)
OpenBlob opens a blob given its entity id; it returns the blob's data source, its size and its hash. It returns a params.ErrNotFound error if the entity does not exist.
func (*Store) OpenCachedBlobFile ¶
func (s *Store) OpenCachedBlobFile( entity *mongodoc.Entity, fileId mongodoc.FileId, isFile func(f *zip.File) bool, ) (_ io.ReadCloser, err error)
OpenCachedBlobFile opens a file from the given entity's archive blob. The file is identified by the provided fileId. If the file has not previously been opened on this entity, the isFile function will be used to determine which file in the zip file to use. The result will be cached for the next time.
When retrieving the entity, at least the BlobName and Contents fields must be populated.
func (*Store) Search ¶
func (store *Store) Search(sp SearchParams) (SearchResult, error)
Search searches the store for the given SearchParams. It returns a SearchResult containing the results of the search.
func (*Store) SetPerms ¶
SetPerms sets the permissions for the base entity with the given id for "which" operations ("read" or "write") to the given ACL. This is mostly provided for testing.
func (*Store) SetPromulgated ¶
func (s *Store) SetPromulgated(url *router.ResolvedURL, promulgate bool) error
SetPromulgated sets whether the base entity of url is promulgated, If promulgated is true it also unsets promulgated on any other base entity for entities with the same name. It also calculates the next promulgated URL for the entities owned by the new owner and sets those entities appropriately.
Note: This code is known to have some unfortunate (but not dangerous) race conditions. It is possible that if one or more promulgations happens concurrently for the same entity name then it could result in more than one base entity being promulgated. If this happens then uploads to either user will get promulgated names, these names will never clash. This situation is easily remedied by setting the promulgated user for this charm again, even to one of the ones that is already promulgated. It can also result in the latest promulgated revision of the charm not being one created by the promulgated user. This will be remedied when a new charm is uploaded by the promulgated user. As promulgation is a rare operation, it is considered that the chances this will happen are slim.
func (*Store) SetReconnectTimeout ¶
SetReconnectTimeout sets the length of time that mongo requests will block waiting to reconnect to a disconnected mongo server. If it is zero, requests may block forever.
func (*Store) SynchroniseElasticsearch ¶
SynchroniseElasticsearch creates new indexes in elasticsearch and populates them with the current data from the mongodb database.
func (*Store) UpdateBaseEntity ¶
func (s *Store) UpdateBaseEntity(url *router.ResolvedURL, update interface{}) error
UpdateBaseEntity applies the provided update to the base entity of url.
func (*Store) UpdateEntity ¶
func (s *Store) UpdateEntity(url *router.ResolvedURL, update interface{}) error
UpdateEntity applies the provided update to the entity described by url.
func (*Store) UpdateEntitySHA256 ¶
func (s *Store) UpdateEntitySHA256(id *router.ResolvedURL) (string, error)
UpdateEntitySHA256 calculates and return the SHA256 hash of the archive of the given entity id. The entity document is then asynchronously updated with the resulting hash. This method will be removed soon.
func (*Store) UpdateSearch ¶
func (s *Store) UpdateSearch(r *router.ResolvedURL) error
UpdateSearch updates the search record for the entity reference r. The search index only includes the latest revision of each entity so the latest revision of the charm specified by r will be indexed.
func (*Store) UpdateSearchAsync ¶
func (s *Store) UpdateSearchAsync(r *router.ResolvedURL)
UpdateSearchAsync will update the search record for the entity reference r in the backgroud.
func (*Store) UpdateSearchBaseURL ¶
UpdateSearchBaseURL updates the search record for all entities with the specified base URL. It must be called whenever the entry for the given URL in the BaseEntitites collection has changed.
func (*Store) UpdateSearchFields ¶
func (s *Store) UpdateSearchFields(r *router.ResolvedURL, fields map[string]interface{}) error
UpdateSearchFields updates the search record for the entity reference r with the updated values in fields.
type StoreDatabase ¶
type StoreDatabase struct {
*mgo.Database
}
StoreDatabase wraps an mgo.DB ands adds a few convenience methods.
func (StoreDatabase) BaseEntities ¶
func (s StoreDatabase) BaseEntities() *mgo.Collection
BaseEntities returns the mongo collection where base entities are stored.
func (StoreDatabase) Close ¶
func (s StoreDatabase) Close()
Close closes the store database's underlying session.
func (StoreDatabase) Collections ¶
func (s StoreDatabase) Collections() []*mgo.Collection
Collections returns a slice of all the collections used by the charm store.
func (StoreDatabase) Copy ¶
func (s StoreDatabase) Copy() StoreDatabase
Copy copies the StoreDatabase and its underlying mgo session.
func (StoreDatabase) Entities ¶
func (s StoreDatabase) Entities() *mgo.Collection
Entities returns the mongo collection where entities are stored.
func (StoreDatabase) Logs ¶
func (s StoreDatabase) Logs() *mgo.Collection
Logs returns the Mongo collection where charm store logs are stored.
func (StoreDatabase) Macaroons ¶
func (s StoreDatabase) Macaroons() *mgo.Collection
func (StoreDatabase) Migrations ¶
func (s StoreDatabase) Migrations() *mgo.Collection
Migrations returns the Mongo collection where the migration info is stored.
func (StoreDatabase) StatCounters ¶
func (s StoreDatabase) StatCounters() *mgo.Collection
func (StoreDatabase) StatTokens ¶
func (s StoreDatabase) StatTokens() *mgo.Collection