Repository navigation
Cache plugin configs #3656
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Cache plugin configs #3656
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -244,8 +244,8 @@ func pkg(s *scope, args []pyObject) pyObject { | |
| s.Assert(configVal != nil, "error calling package(): %s is not a known config value", k) | ||
|
|
||
| // Merge in the existing config for dictionaries | ||
| if overrides, ok := v.(pyDict); ok { | ||
| if pluginConfig, ok := configVal.(pyDict); ok { | ||
| if overrides, ok := asDict(v); ok { | ||
| if pluginConfig, ok := asDict(configVal); ok { | ||
| newPluginConfig := pluginConfig.Copy() | ||
| for pluginKey, override := range overrides { | ||
| pluginKey = strings.ToUpper(pluginKey) | ||
|
|
@@ -515,11 +515,11 @@ func isType(obj pyObject, name string) bool { | |
| return name == "str" | ||
| case *pyRange: | ||
| return name == "range" | ||
| case pyList: | ||
| case pyList, pyFrozenList: | ||
| return name == "list" | ||
| case pyDict: | ||
| case pyDict, pyFrozenDict: | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I ran a quick check against my repo and noticed differences in output, I suspect a change in behaviour; top-level variables in a sub included .build_defs file are frozen once it has loaded, so This changed output (ran a plz query print comparison with plz-before and plz-after), it's probably worth noting the slight change in behaviour (which probably makes more sense).
on would it make sense to check val != _UNSET before the isinstance call?
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes exactly; there are some fixes in #3657 which I'll land in the same release as this and I've landed a fix internally to deal with that Thanks for looking into it! |
||
| return name == "dict" | ||
| case *pyConfig: | ||
| case *pyConfig, *pyFrozenConfig: | ||
| return name == "config" | ||
| case *pyFunc: | ||
| return name == "callable" | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,6 +11,13 @@ import ( | |
| "github.com/thought-machine/please/src/core" | ||
| ) | ||
|
|
||
| // Used to key a cache to memoise building plugin configs. | ||
| type pluginConfigKey struct { | ||
| PluginState *core.BuildState // State for the plugin itself | ||
| State *core.BuildState // State for the receiving package (or its parent if it has no overrides) | ||
| InheritOnly bool // True if State is the parent of a state with no overrides for this plugin | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: Do we need InheritOnly since keys with it set should match their parent config with it set to false? We do have a comment on the State field. Either way the impact is pretty negligible , so either way is fine.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, we do. There's a test case covering this: say package P overrides both bar and baz for plugin foo;
Without InheritOnly, whichever was built first would be cached and returned for the other. |
||
| } | ||
|
|
||
| // valueToPyObject converts a field value to a pyObject | ||
| func valueToPyObject(value reflect.Value) pyObject { | ||
| switch value.Kind() { | ||
|
|
@@ -134,17 +141,16 @@ func getConfigKey(aspKey, configKey string) string { | |
| return strings.ToLower(configKey) | ||
| } | ||
|
|
||
| // pluginConfig loads the plugin's config into a pyDict. It will load con | ||
| func pluginConfig(pluginState *core.BuildState, pkgState *core.BuildState) pyDict { | ||
| pluginName := strings.ToLower(pluginState.RepoConfig.PluginDefinition.Name) | ||
| // pluginConfig loads the plugin's config into a pyDict. | ||
| func pluginConfig(pluginState *core.BuildState, pkgState *core.BuildState, pluginName string) pyDict { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Nit: Does this function need the whole state or just a couple of things in it? Can we just pass those things in instead? I think this will make the function signature a little clearer.
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. No not really, but it needs a few things from each, I think it will make it less clear if I break those out as a series of separate arguments |
||
| var extraVals map[string][]string | ||
| var ret pyDict | ||
| if pkgState.ParentState == nil { | ||
| extraVals = getExtraVals(pkgState.RepoConfig, pluginName) | ||
| ret = pyDict{} | ||
| } else { | ||
| extraVals = getExtraVals(pkgState.RepoConfig, pluginName) | ||
| ret = pluginConfig(pluginState, pkgState.ParentState) | ||
| ret = pluginConfig(pluginState, pkgState.ParentState, pluginName) | ||
| } | ||
|
|
||
| for key, definition := range pluginState.RepoConfig.PluginConfig { | ||
|
|
@@ -193,6 +199,35 @@ func pluginConfig(pluginState *core.BuildState, pkgState *core.BuildState) pyDic | |
| return ret | ||
| } | ||
|
|
||
| // loadCachedPluginConfig loads the config for a particular plugin, caching it to minimise rebuilds. | ||
| func (i *interpreter) loadCachedPluginConfig(pluginState *core.BuildState, pkgState *core.BuildState, name string) pyFrozenDict { | ||
| key := pluginConfigKey{ | ||
| PluginState: pluginState, | ||
| State: pkgState, | ||
| } | ||
| if len(getExtraVals(pkgState.RepoConfig, name)) == 0 { | ||
| // This package doesn't have any overrides, so we can reuse the parent state's config. | ||
| key.State = pkgState.ParentState | ||
| key.InheritOnly = true | ||
| } | ||
| i.pluginConfigLock.RLock() | ||
| if cfg, present := i.pluginConfigs[key]; present { | ||
| i.pluginConfigLock.RUnlock() | ||
| return cfg | ||
| } | ||
| i.pluginConfigLock.RUnlock() | ||
| i.pluginConfigLock.Lock() | ||
| defer i.pluginConfigLock.Unlock() | ||
| // Check again since we just released the lock, we may have raced. | ||
| if cfg, present := i.pluginConfigs[key]; present { | ||
| return cfg | ||
| } | ||
| log.Debugf("Loading configuration for plugin %s", name) | ||
| cfg := pluginConfig(pluginState, pkgState, name).Freeze().(pyFrozenDict) | ||
| i.pluginConfigs[key] = cfg | ||
| return cfg | ||
| } | ||
|
|
||
| func (i *interpreter) loadPluginConfig(s *scope, pluginState *core.BuildState) { | ||
| if pluginState.RepoConfig == nil { | ||
| return | ||
|
|
@@ -204,19 +239,14 @@ func (i *interpreter) loadPluginConfig(s *scope, pluginState *core.BuildState) { | |
| return | ||
| } | ||
|
|
||
| log.Debugf("Loading configuration for plugin %s", pluginName) | ||
|
|
||
| if s.config.overlay == nil { | ||
| s.config.overlay = pyDict{} | ||
| } | ||
|
|
||
| key := strings.ToUpper(pluginName) | ||
| if _, ok := s.config.overlay[key]; ok { | ||
| return | ||
| } | ||
|
|
||
| cfg := pluginConfig(pluginState, s.state) | ||
| s.config.overlay[key] = cfg | ||
| if s.config.overlay == nil { | ||
| s.config.overlay = pyDict{} | ||
| } | ||
| s.config.overlay[key] = i.loadCachedPluginConfig(pluginState, s.state, strings.ToLower(pluginName)) | ||
| } | ||
|
|
||
| func toPyObject(key, val, toType string, optional bool) pyObject { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -510,7 +510,7 @@ func (d pyDict) Operator(operator Operator, operand pyObject) pyObject { | |
| } | ||
| panic("unknown dict key: " + s.String()) | ||
| case Union: | ||
| d2, ok := operand.(pyDict) | ||
| d2, ok := asDict(operand) | ||
|
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. another drive-by about using a union operator of a dict and a frozen dict |
||
| if !ok { | ||
| panic("Operator to | must be another dict, not " + operand.Type()) | ||
| } | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1638,6 +1638,7 @@ func execute(command string) int { | |
| } | ||
| defer f.Close() | ||
| defer pprof.WriteHeapProfile(f) | ||
| defer runtime.GC() | ||
|
Collaborator
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. and here; we only get up-to-date stats for the heap profile once a GC cycle has completed, without this that might have been ages ago |
||
| } | ||
| if opts.MutexProfile != "" { | ||
| runtime.SetMutexProfileFraction(1) | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this is not strictly related but a bit of a drive-by while I was here;
isinstanceshould treat a frozen list as being a list