Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 120 additions & 31 deletions src/query/graph.go
Original file line number Diff line number Diff line change
@@ -1,32 +1,137 @@
package query

import (
"cmp"
"encoding/base64"
"encoding/json"
"encoding/json/jsontext"
"encoding/json/v2"
"io"
"iter"
"os"
"path/filepath"
"sync"
"runtime"
"slices"

"github.com/thought-machine/please/src/build"
"github.com/thought-machine/please/src/core"
)

// graphEncodeOptions are the options used to encode the graph.
var graphEncodeOptions = json.JoinOptions(
jsontext.WithIndent(" "),
json.Deterministic(true), // Sort map keys so the output is stable
// Paths and commands aren't guaranteed to be valid UTF-8; replace bad bytes with U+FFFD rather
// than failing (which would leave a truncated document since we're streaming).
jsontext.AllowInvalidUTF8(true),
jsontext.AllowDuplicateNames(true), // We construct all the keys ourselves; this saves tracking every name in each object
)

// Graph prints a representation of the build graph as JSON.
func Graph(state *core.BuildState, targets []core.BuildLabel) {
log.Notice("Generating graph...")
g := makeJSONGraph(state, targets)

encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
encoder.SetEscapeHTML(false)

log.Notice("Encoding...")
if err := encoder.Encode(g); err != nil {
if err := writeGraph(os.Stdout, state, targets); err != nil {
log.Fatalf("Failed to serialise JSON: %s\n", err)
}
log.Notice("Done")
}

// writeGraph writes the JSON representation of the build graph to the given writer.
// When writing the entire graph, it is streamed out one package at a time rather than
// being constructed in memory first.
func writeGraph(w io.Writer, state *core.BuildState, targets []core.BuildLabel) error {
enc := jsontext.NewEncoder(w, graphEncodeOptions)
if len(targets) != 0 {
return json.MarshalEncode(enc, makeJSONGraph(state, targets))
}
return writeAllPackages(enc, state)
}

// writeAllPackages streams the entire graph to the given encoder. The output is identical to
// encoding a JSONGraph containing every package.
func writeAllPackages(enc *jsontext.Encoder, state *core.BuildState) error {
// Sort the packages so the output is deterministic; this matches the order of the keys in
// JSONGraph (packages in the root repo, then each subrepo in turn, then by name within those).
pkgs := slices.SortedFunc(state.Graph.AllPackages(), func(a, b *core.Package) int {
return cmp.Or(cmp.Compare(a.SubrepoName, b.SubrepoName), cmp.Compare(a.Name, b.Name))
})
w := tokenWriter{enc: enc}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can this fail? I wonder if it's worth looking at w.err and returning it if not nil, instead of proceeding to range orderedParallel<..>

w.Write(jsontext.BeginObject, jsontext.String("packages"), jsontext.BeginObject)
subrepo := ""
// The window is fairly generous since per-package cost varies a lot and a slow one blocks
// everything queued behind it. Completed packages are small relative to the whole graph.
window := 100 * runtime.GOMAXPROCS(0)
for pkg := range orderedParallel(pkgs, window, func(pkg *core.Package) JSONPackage {
return makeJSONPackage(state, pkg)
}) {
if pkg.subrepo != subrepo {
if subrepo == "" {
// End of the root packages, start of the subrepos
w.Write(jsontext.EndObject, jsontext.String("subrepos"), jsontext.BeginObject)
} else {
// End of the previous subrepo's packages and the subrepo itself
w.Write(jsontext.EndObject, jsontext.EndObject)
}
w.Write(jsontext.String(pkg.subrepo), jsontext.BeginObject, jsontext.String("packages"), jsontext.BeginObject)
subrepo = pkg.subrepo
}
w.Write(jsontext.String(pkg.name))
if w.err == nil {
w.err = json.MarshalEncode(enc, pkg)
}
if w.err != nil {
return w.err
}
}
if subrepo != "" {
w.Write(jsontext.EndObject, jsontext.EndObject) // the last subrepo's packages and the subrepo itself
}
w.Write(jsontext.EndObject, jsontext.EndObject) // packages / subrepos, and the root object
return w.err
}

// A tokenWriter writes tokens to an encoder, retaining the first error that occurs.
type tokenWriter struct {
enc *jsontext.Encoder
err error
}

func (w *tokenWriter) Write(tokens ...jsontext.Token) {
for _, tok := range tokens {
if w.err != nil {
return
}
w.err = w.enc.WriteToken(tok)
}
}

// orderedParallel applies f to each element of in, running up to roughly n of them concurrently,
// and yields the results in the same order as the input.
// Limiting the window bounds how many results can be waiting in memory behind a slow one.
func orderedParallel[T, U any](in []T, n int, f func(T) U) iter.Seq[U] {
return func(yield func(U) bool) {
results := make(chan chan U, n)
done := make(chan struct{})
defer close(done)
go func() {
defer close(results)
for _, x := range in {
ch := make(chan U, 1)
select {
case results <- ch:
case <-done:
return
}
go func() { ch <- f(x) }()
}
}()
for ch := range results {
if !yield(<-ch) {
return
}
}
}
}

// JSONGraph is an alternate representation of our build graph; will contain different information
// to the structures in core (also those ones can't be printed as JSON directly).
type JSONGraph struct {
Expand All @@ -53,9 +158,9 @@ type JSONTarget struct {
Requires []string `json:"requires,omitempty" note:"corresponds to requires in rule declaration"`
Command string `json:"command,omitempty" note:"the currently active command of the target. not present on filegroup or remote_file actions"`
Hash string `json:"hash" note:"partial hash of target, does not include source hash"`
Test bool `json:"test,omitempty" note:"true if target is a test"`
Binary bool `json:"binary,omitempty" note:"true if target is a binary"`
TestOnly bool `json:"test_only,omitempty" note:"true if target should be restricted to test code"`
Test bool `json:"test,omitzero" note:"true if target is a test"`
Binary bool `json:"binary,omitzero" note:"true if target is a binary"`
TestOnly bool `json:"test_only,omitzero" note:"true if target should be restricted to test code"`
}

func makeJSONGraph(state *core.BuildState, targets []core.BuildLabel) *JSONGraph {
Expand All @@ -64,8 +169,8 @@ func makeJSONGraph(state *core.BuildState, targets []core.BuildLabel) *JSONGraph
Subrepos: map[string]*JSONGraph{},
}
if len(targets) == 0 {
for pkg := range makeAllPackages(state) {
ret.Subrepo(pkg.subrepo).Packages[pkg.name] = pkg
for pkg := range state.Graph.AllPackages() {
ret.Subrepo(pkg.SubrepoName).Packages[pkg.Name] = makeJSONPackage(state, pkg)
}
} else {
done := map[core.BuildLabel]struct{}{}
Expand All @@ -91,22 +196,6 @@ func (graph *JSONGraph) Subrepo(name string) *JSONGraph {
return subrepo
}

// makeAllPackages constructs all the JSONPackage objects for this graph in parallel.
func makeAllPackages(state *core.BuildState) <-chan JSONPackage {
ch := make(chan JSONPackage, 100)
go func() {
var wg sync.WaitGroup
for pkg := range state.Graph.AllPackages() {
wg.Go(func() {
ch <- makeJSONPackage(state, pkg)
})
}
wg.Wait()
close(ch)
}()
return ch
}

func addJSONTarget(state *core.BuildState, graph *JSONGraph, label core.BuildLabel, done map[core.BuildLabel]struct{}) {
if _, present := done[label]; present {
return
Expand Down
56 changes: 56 additions & 0 deletions src/query/graph_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package query

import (
"bytes"
"encoding/json/jsontext"
"encoding/json/v2"
"slices"
"testing"

"github.com/stretchr/testify/assert"
Expand Down Expand Up @@ -38,6 +42,58 @@ func TestQueryPackage(t *testing.T) {
assert.Equal(t, []string{"//package1:target1"}, pkg1.Targets["target2"].Deps)
}

func TestStreamedGraphMatchesInMemoryGraph(t *testing.T) {
state := makeGraph(t)
// Add a couple of subrepos (in non-sorted order) to check they get streamed correctly too.
for _, subrepo := range []string{"zzz", "aaa"} {
pkg := core.NewPackage("package3", core.WithPackageSubrepo(subrepo))
target := makeTarget("///" + subrepo + "//package3:target4")
pkg.AddTarget(target)
state.Graph.AddTarget(target)
state.Graph.AddPackage(pkg)
}

var streamed bytes.Buffer
assert.NoError(t, writeGraph(&streamed, state, nil))

var expected bytes.Buffer
assert.NoError(t, json.MarshalEncode(jsontext.NewEncoder(&expected, graphEncodeOptions), makeJSONGraph(state, nil)))
assert.Equal(t, expected.String(), streamed.String())

// Sanity check the structure
graph := &JSONGraph{}
assert.NoError(t, json.Unmarshal(streamed.Bytes(), graph))
assert.Equal(t, 2, len(graph.Packages))
assert.Equal(t, 2, len(graph.Subrepos))
assert.Contains(t, graph.Subrepos["aaa"].Packages["package3"].Targets, "target4")
assert.Contains(t, graph.Subrepos["zzz"].Packages["package3"].Targets, "target4")
}

func TestStreamedGraphNoSubrepos(t *testing.T) {
var streamed bytes.Buffer
assert.NoError(t, writeGraph(&streamed, makeGraph(t), nil))
assert.NotContains(t, streamed.String(), "subrepos")
assert.True(t, jsontext.Value(streamed.Bytes()).IsValid())
}

func TestOrderedParallel(t *testing.T) {
in := make([]int, 1000)
for i := range in {
in[i] = i
}
out := slices.Collect(orderedParallel(in, 8, func(i int) int { return i * 2 }))
assert.Equal(t, len(in), len(out))
for i, x := range out {
assert.Equal(t, i*2, x)
}
// Stopping early must not deadlock.
for x := range orderedParallel(in, 8, func(i int) int { return i }) {
if x == 10 {
break
}
}
}

func makeGraph(t *testing.T) *core.BuildState {
t.Helper()
state := core.NewDefaultBuildState()
Expand Down
Loading