Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 48 additions & 32 deletions src/remote/remote.go
Original file line number Diff line number Diff line change
Expand Up @@ -481,6 +481,10 @@ func (c *Client) Download(target *core.BuildTarget) error {
log.Debug("Not downloading outputs for %s, they're already up-to-date", target)
return nil
}
if target.IsTextFile {
// We know the contents locally so there's no need to download anything.
return c.writeTextFile(target, buildAction)
}
// Show the target as active while it downloads; otherwise nothing appears to be happening during a long
// download (which also looks like a stall to cycle detection). Failures aren't logged here since all
// callers report them already.
Expand Down Expand Up @@ -686,6 +690,11 @@ func (c *Client) maybeRetrieveResults(target *core.BuildTarget, command *pb.Comm
// execute submits an action to the remote executor and monitors its progress.
// The returned ActionResult may be nil on failure.
func (c *Client) execute(target *core.BuildTarget, command *pb.Command, digest *pb.Digest, isTest, needStdout bool, run int) (*core.BuildMetadata, *pb.ActionResult, error) {
// Text files are constructed entirely locally; there's no point checking the action cache since
// it's no more expensive to recreate the result than to look it up.
if target.IsTextFile {
return c.buildTextFile(target, command)
}
if !isTest || (!c.state.ForceRerun && c.state.NumTestRuns == 1) {
if metadata, ar := c.maybeRetrieveResults(target, command, digest, isTest, needStdout, run); metadata != nil {
return metadata, ar, nil
Expand All @@ -696,11 +705,9 @@ func (c *Client) execute(target *core.BuildTarget, command *pb.Command, digest *
if err != nil {
return nil, nil, fmt.Errorf("Failed to upload build action: %s", err)
}
// Remote files & text files get special treatment at this point.
// Remote files get special treatment at this point.
if target.IsRemoteFile {
return c.fetchRemoteFile(target, digest)
} else if target.IsTextFile {
return c.buildTextFile(c.state, target, command, digest)
}

// We should skip the cache lookup (and override any existing action result) if we --rebuild, or --rerun and this is
Expand Down Expand Up @@ -943,42 +950,51 @@ func (c *Client) fetchRemoteFile(target *core.BuildTarget, actionDigest *pb.Dige
return md, ar, err
}

// buildTextFile "builds" uploads a text file to the CAS
func (c *Client) buildTextFile(state *core.BuildState, target *core.BuildTarget, command *pb.Command, actionDigest *pb.Digest) (*core.BuildMetadata, *pb.ActionResult, error) {
ar := &pb.ActionResult{}
if err := c.uploadBlobs(func(ch chan<- *uploadinfo.Entry) error {
defer close(ch)
if len(command.OutputPaths) != 1 {
return fmt.Errorf("text_file %s should have a single output, has %d", target.Label, len(command.OutputPaths))
}
content, err := target.GetFileContent(state)
if err != nil {
return err
}
entry := uploadinfo.EntryFromBlob([]byte(content))
ch <- entry
ar.OutputFiles = append(ar.OutputFiles, &pb.OutputFile{
// buildTextFile "builds" a text file by uploading its contents to the CAS.
// The ActionResult is constructed locally and never written to the action cache.
func (c *Client) buildTextFile(target *core.BuildTarget, command *pb.Command) (*core.BuildMetadata, *pb.ActionResult, error) {
if len(command.OutputPaths) != 1 {
return nil, nil, fmt.Errorf("text_file %s should have a single output, has %d", target.Label, len(command.OutputPaths))
}
content, err := target.GetFileContent(c.state)
if err != nil {
return nil, nil, err
}
entry := uploadinfo.EntryFromBlob([]byte(content))
if err := c.uploadIfMissing(context.Background(), []*uploadinfo.Entry{entry}); err != nil {
return nil, nil, err
}
ar := &pb.ActionResult{
OutputFiles: []*pb.OutputFile{{
Path: command.OutputPaths[0],
Digest: entry.Digest.ToProto(),
IsExecutable: target.IsBinary,
})
return nil
}); err != nil {
return nil, nil, err
}
if _, err := c.client.UpdateActionResult(context.Background(), &pb.UpdateActionResultRequest{
InstanceName: c.instance,
ActionDigest: actionDigest,
ActionResult: ar,
}); err != nil {
return nil, nil, fmt.Errorf("Error updating action result: %s", err)
}},
}
md, err := c.buildMetadata(target, ar, false, false)
return md, ar, err
}

// writeTextFile writes the output of a text_file target directly, instead of downloading it from the CAS.
func (c *Client) writeTextFile(target *core.BuildTarget, digest *pb.Digest) error {
outs := target.FullOutputs(c.state.Graph)
if len(outs) != 1 {
return fmt.Errorf("text_file %s should have a single output, has %d", target.Label, len(outs))
}
content, err := target.GetFileContent(c.state)
if err != nil {
return nil, nil, err
return err
}
if err := c.removeOutputs(target); err != nil {
return err
} else if err := fs.EnsureDir(outs[0]); err != nil {
return err
} else if err := os.WriteFile(outs[0], []byte(content), target.OutMode()); err != nil {
return err
}
c.locallyCacheResults(target, actionDigest, md)
return md, ar, nil
c.recordAttrs(target, digest)
c.state.LogBuildResult(target, core.TargetBuilt, "Built")
return nil
}

// logActionResult logs the state of an action while it's building or testing
Expand Down
20 changes: 20 additions & 0 deletions src/remote/remote_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,26 @@ func TestExecuteFetch(t *testing.T) {
assert.NoError(t, err)
}

func TestBuildTextFile(t *testing.T) {
c := newClient()
server.calls.Reset()
target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "text_file"})
target.IsTextFile = true
target.FileContent = "this is a text file"
target.AddOutput("text_file.txt")
target.BuildTimeout = time.Minute
c.state.Graph.AddTarget(target)
_, err := c.Build(target)
require.NoError(t, err)
assert.Zero(t, server.calls.GetActionResult.Load(), "should not look up any action results")
assert.Zero(t, server.calls.UpdateActionResult.Load(), "should not write any action results")
assert.Zero(t, server.calls.Execute.Load(), "should not execute anything")
dg := digest.NewFromBlob([]byte("this is a text file"))
assert.Equal(t, []byte("this is a text file"), server.blobs[dg.Hash], "content should be uploaded to the CAS")
require.NoError(t, c.Download(target))
assertFileContent(t, filepath.Join(target.OutDir(), "text_file.txt"), []byte("this is a text file"))
}

func TestExecuteTest(t *testing.T) {
c := newClientInstance("test")
target := core.NewBuildTarget(core.BuildLabel{PackageName: "package", Name: "target3"})
Expand Down
Loading