diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1650e363..a3e2bb42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1058,7 +1058,7 @@ jobs: # in somebody else's file. run: | set -euo pipefail - watched='internal/format/registry.go internal/damage/damage.go cmd/tfg/main.go internal/gui/window/run.go internal/gui/run_cgo.go internal/gui/window/tidy.go internal/audit/parallel.go internal/engine/parallel.go internal/tool/tool.go internal/gui/window/tools.go go.mod .github/workflows/ci.yml .github/build-tags' + watched='internal/format/registry.go internal/damage/damage.go cmd/tfg/main.go internal/gui/window/run.go internal/gui/run_cgo.go internal/gui/window/tidy.go internal/audit/parallel.go internal/engine/parallel.go internal/tool/tool.go internal/gui/window/tools.go internal/format/video/ahead.go go.mod .github/workflows/ci.yml .github/build-tags' # On a pull request there is no "before" - the field belongs to a push # - so this asked for something empty and every pull request answered # "touched". That quietly undid the decision of 2026-08-20, because diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b64247..9c95574e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,12 @@ because it turns other people's test suites red. reads the time the picture starts, the way a player's position does - `00:00:07`, with milliseconds when the changes do not fall on whole seconds. Each change is a whole new picture in bytes and in coding time, - and the frames between changes cost a few bytes each, so an hour at thirty + and when a film has more than one picture they are coded on several cores + at once - a thirty-minute 1920x1080 film took 68 s on sixteen threads, and + the + progress bar counts the pictures as well as the bytes, so the time it says + is left holds while they are coded. The frames + between changes cost a few bytes each, so an hour at thirty frames a second with a change every second fits in about a megabyte (1 075 831 B at the smallest), and a `change_interval` as long as the film or longer keeps one picture throughout (977 434 B for that hour). Set diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 12679d60..5576fe38 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -58,7 +58,7 @@ func (p *progressBar) report(pr engine.Progress) { line := fmt.Sprintf(" %d/%d files %s of %s %d%%%s", pr.FilesDone, pr.FilesTotal, core.HumanBytes(pr.BytesDone), core.HumanBytes(pr.BytesTotal), - core.Percent(pr.BytesDone, pr.BytesTotal), + pr.Percent(), p.remaining(pr)) // Pad to cover whatever the last line left behind, so a shorter line does @@ -82,14 +82,12 @@ func (p *progressBar) clear() { } // remaining is an estimate and says so by staying quiet until it has enough to -// go on. A number that swings wildly for the first second is worse than none. +// go on (engine.Progress.Left, which the window asks too). func (p *progressBar) remaining(pr engine.Progress) string { - elapsed := time.Since(p.started) - if elapsed < time.Second || pr.BytesDone <= 0 || pr.BytesDone >= pr.BytesTotal { + left, ok := pr.Left(time.Since(p.started)) + if !ok { return "" } - left := time.Duration(float64(elapsed) * - float64(pr.BytesTotal-pr.BytesDone) / float64(pr.BytesDone)) return " " + core.Roughly(left) + " left" } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4482f151..b083f96c 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -451,25 +451,6 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF return pl.out, nil } -// TotalBytes is what a plan will occupy on disk. Known before the first byte -// is written, which is what the free space guard and --dry-run stand on. -func TotalBytes(files []PlannedFile) int64 { - var n int64 - for _, f := range files { - n += f.Plan.Bytes - } - return n -} - -// Progress is how far a run has got. Both counts are known from the plan, so -// the fractions are exact rather than estimated. -type Progress struct { - FilesDone int - FilesTotal int - BytesDone int64 - BytesTotal int64 -} - // Run writes a planned set of files. // // Each file is written under a temporary name and only then renamed, so the diff --git a/internal/engine/parallel.go b/internal/engine/parallel.go index 1eb30761..e1dea444 100644 --- a/internal/engine/parallel.go +++ b/internal/engine/parallel.go @@ -18,6 +18,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/damage" + "github.com/donislawdev/TestingFilesGenerator/internal/format" ) // This file is the only place in internal/engine that runs anything beside @@ -157,7 +158,7 @@ func drain(ctx context.Context, next *atomic.Int64, files []PlannedFile, outDir p := fileProgress{gate: gate} sum, err := writeOne(ctx, files[i], outDir, &p) out[i] = fileResult{sha: sum, ok: err == nil, err: err} - p.finished(files[i].Plan.Bytes, err == nil) + p.finished(files[i].Plan, err == nil) } } @@ -182,12 +183,17 @@ func drain(ctx context.Context, next *atomic.Int64, files []PlannedFile, outDir // // A nil gate is a run nobody is watching. Every method takes a nil receiver, // so a run without progress does no locking and allocates nothing for it. +// +// Work moves with the bytes, and on its own where a generator reports work its +// bytes do not show (format.Plan.Work) - the pictures of a film. type progressGate struct { mu sync.Mutex filesDone int bytesDone int64 + workDone int64 filesTotal int bytesTotal int64 + workTotal int64 report func(Progress) } @@ -198,6 +204,7 @@ func newProgressGate(files []PlannedFile, report func(Progress)) *progressGate { return &progressGate{ filesTotal: len(files), bytesTotal: TotalBytes(files), + workTotal: totalWork(files), report: report, } } @@ -209,25 +216,27 @@ func newProgressGate(files []PlannedFile, report func(Progress)) *progressGate { // file" describes no run at all. At one writer the numbers this produces are // the same ones the sequential loop produced, which is what lets the guards // that watch the bar stay as they were. -func (g *progressGate) advance(delta int64) { +func (g *progressGate) advance(bytes, work int64) { if g == nil { return } g.mu.Lock() defer g.mu.Unlock() - g.bytesDone += delta + g.bytesDone += bytes + g.workDone += work g.say() } // finished is the end of one file: the delta that squares this writer's // reporting with what the plan promised, and one more file done. -func (g *progressGate) finished(delta int64) { +func (g *progressGate) finished(bytes, work int64) { if g == nil { return } g.mu.Lock() defer g.mu.Unlock() - g.bytesDone += delta + g.bytesDone += bytes + g.workDone += work g.filesDone++ g.say() } @@ -237,6 +246,7 @@ func (g *progressGate) say() { g.report(Progress{ FilesDone: g.filesDone, FilesTotal: g.filesTotal, BytesDone: g.bytesDone, BytesTotal: g.bytesTotal, + WorkDone: g.workDone, WorkTotal: g.workTotal, }) } @@ -248,14 +258,38 @@ func (g *progressGate) say() { // delta, so the gate never has to know which file it came from. type fileProgress struct { gate *progressGate - reported int64 + reported int64 // bytes + worked int64 // work, the bytes among it } // advance is the counting writer's callback: n is the running total for this -// file, and the gate is told the difference. +// file, and the gate is told the difference - as bytes, and as work, because +// writing a byte is a byte of work. func (p *fileProgress) advance(n int64) { - p.gate.advance(n - p.reported) + d := n - p.reported + p.gate.advance(d, d) p.reported = n + p.worked += d +} + +// listen has this writer hear the file being written: its bytes through the +// counter, and the work its bytes do not show through the context the +// generator writes under (format.Worked). Nothing is set when nobody is +// listening, so a run without progress does no locking at all and allocates +// nothing for it. +func (p *fileProgress) listen(ctx context.Context, counter *countingWriter) context.Context { + if p.gate == nil { + return ctx + } + counter.report = p.advance + return format.WithWork(ctx, p.work) +} + +// work is the generator's report of work its bytes do not show +// (format.Worked), n more of what its plan counted in Work. +func (p *fileProgress) work(n int64) { + p.gate.advance(0, n) + p.worked += n } // finished squares this file up. A file that succeeded is topped up to exactly @@ -265,12 +299,14 @@ func (p *fileProgress) advance(n int64) { // find. That the total can go backwards on a failed file is how it already // behaves. What is new is only that another writer's bytes may sit in the same // total while it happens. -func (p *fileProgress) finished(planned int64, ok bool) { +// +// Work is squared up the same way, to the plan's bytes and Work together. +func (p *fileProgress) finished(plan format.Plan, ok bool) { if ok { - p.gate.finished(planned - p.reported) + p.gate.finished(plan.Bytes-p.reported, plan.Bytes+plan.Work-p.worked) return } - p.gate.finished(-p.reported) + p.gate.finished(-p.reported, -p.worked) } // writeOne writes one file under a temporary name and renames it only once it @@ -334,11 +370,7 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress h := sha256.New() buffered := bufio.NewWriterSize(fh, 64<<10) counter := &countingWriter{w: io.MultiWriter(buffered, h)} - // Left nil when nobody is listening, so a run without progress does no - // locking at all and allocates nothing for it. - if p.gate != nil { - counter.report = p.advance - } + ctx = p.listen(ctx, counter) // Damage sits between the generator and the counter, and the order is the // design rather than a convenience. diff --git a/internal/engine/progress.go b/internal/engine/progress.go new file mode 100644 index 00000000..005b0619 --- /dev/null +++ b/internal/engine/progress.go @@ -0,0 +1,70 @@ +// Part of package engine. See engine.go. +package engine + +import ( + "time" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" +) + +// How far a run has got, and what it is measured against. Moved out of +// engine.go on 2026-10-06, when progress began to count the work a file costs +// as well as its bytes and that file passed its ceiling on length - the ceiling +// is a ratchet, so the answer is a split by subject. The gate that hands the +// numbers out while files are written in parallel stays in parallel.go, +// because it holds a lock and that file is where everything concurrent in this +// package lives. + +// TotalBytes is what a plan will occupy on disk. Known before the first byte +// is written, which is what the free space guard and --dry-run stand on. +func TotalBytes(files []PlannedFile) int64 { + var n int64 + for _, f := range files { + n += f.Plan.Bytes + } + return n +} + +// totalWork is what the planned files cost to make, their bytes included. +func totalWork(files []PlannedFile) int64 { + var n int64 + for _, f := range files { + n += f.Plan.Bytes + f.Plan.Work + } + return n +} + +// Progress is how far a run has got. Both counts are known from the plan, so +// the fractions are exact rather than estimated. +// +// A fraction of the run and the time it has left come from the work, not the +// bytes. The work is the bytes plus what the plans counted beyond them +// (format.Plan.Work), so for a run of files whose cost is their bytes the two +// are the same numbers, and for a film they are the pictures as well as the +// padding - the bytes alone stood at five percent while a film's pictures were +// coded and promised hours for a minute's work. +type Progress struct { + FilesDone int + FilesTotal int + BytesDone int64 + BytesTotal int64 + WorkDone int64 + WorkTotal int64 +} + +// Percent is how far the run has got, out of a hundred, by the work. The +// window's bar and the command line's both draw this, so the two cannot +// disagree and neither can fall back to the bytes on its own. +func (p Progress) Percent() int { return core.Percent(p.WorkDone, p.WorkTotal) } + +// Left is how long the run has left at the pace it has kept so far, by the +// work, and false while there is not enough to go on: in the first second, +// before any work is done and once all of it is. A number that swings wildly +// for the first second is worse than no number. The two surfaces carried a +// copy of this each until 2026-10-06. +func (p Progress) Left(elapsed time.Duration) (time.Duration, bool) { + if elapsed < time.Second || p.WorkDone <= 0 || p.WorkDone >= p.WorkTotal { + return 0, false + } + return time.Duration(float64(elapsed) * float64(p.WorkTotal-p.WorkDone) / float64(p.WorkDone)), true +} diff --git a/internal/format/format.go b/internal/format/format.go index 68181f0c..662bd803 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -789,6 +789,10 @@ type Plan struct { // Memo is the generator's own scratch space, carried from planning to // writing. Nothing outside the generator reads it. Memo any + // Work is what writing the file costs beyond writing its bytes, counted + // as the bytes that would take as long to write - nought for every file + // whose cost is its bytes. See work.go. + Work int64 } // PropertyLabelEmbedded is the key a generator sets to say whether the label diff --git a/internal/format/video/ahead.go b/internal/format/video/ahead.go new file mode 100644 index 00000000..9f99f24b --- /dev/null +++ b/internal/format/video/ahead.go @@ -0,0 +1,269 @@ +package video + +import ( + "runtime" + "sync" + "sync/atomic" +) + +// The one place in this package where anything runs beside anything else, and +// it is listed in internal/guard/concurrency_test.go with that reason. +// +// Why it exists. A film is its pictures: every change is a whole picture +// through gav1d, which codes on one goroutine and has no setting that searches +// less than the one already used. A 1920x1080 picture is about 0.35 s, so a +// thirty minute film at a change a second was ten minutes on one core while +// fifteen others idled - measured 2026-10-06 at 611.7 s for the owner's own +// order, 2 GB, 1920x1080, thirty minutes (docs/WEBM-WYDAJNOSC-2026-10-06.md). +// The pictures do not depend on one another, so they can be coded at once and +// written in order, and the bytes are the ones one goroutine makes. Measured +// with tools/probes/videoparallel, 48 pictures at 1920x1080, three rounds: +// +// W1 1.00x W2 1.81x W4 3.24x W8 5.12x W12 6.95x W16 7.60x +// +// with every picture the same size at every width. +// +// The limit is the process's, not the film's. The engine already writes +// GOMAXPROCS files at once, so helpers taken per film would be sixteen films +// times sixteen pictures in memory. Here the goroutine writing a film always +// codes, which is what guarantees it moves on, and the helpers beside it come +// from one count for the whole process, GOMAXPROCS less one. A film that +// finds none free codes alone, as every film did before, and asks again at its +// next picture. + +// helping is how many helpers are coding beside the writers right now. +var helping atomic.Int64 + +// codedBeside counts the pictures a helper coded rather than the goroutine +// that asked for them, since the process started. +var codedBeside atomic.Int64 + +// Helpers is how many helpers are coding right now and how many pictures +// helpers have coded since the process started. Guards read it: a film held to +// the bytes of one goroutine says nothing unless several really coded it, and +// a film abandoned half way has to leave none of them running. +func Helpers() (running, coded int64) { return helping.Load(), codedBeside.Load() } + +// takeHelper claims a place for one more helper, if the process has one. +func takeHelper() bool { + limit := int64(runtime.GOMAXPROCS(0) - 1) + for { + n := helping.Load() + if n >= limit { + return false + } + if helping.CompareAndSwap(n, n+1) { + return true + } + } +} + +// job is one picture to code. Whoever takes it first - a helper, or the +// writer that needs it - codes it, and done is released when it is coded, so +// the writer waits only for a picture somebody else is coding. A WaitGroup +// inside the job rather than a channel beside it, because a channel is one +// more allocation for every picture of every film. +type job struct { + look look + change int64 + taken atomic.Bool + done sync.WaitGroup + coded Coded + err error + // panicked is what coding it panicked with. A panic on a helper would end + // the process, where the same panic on the writer becomes the run's error + // (internal/engine, writeWithoutCrashing), so a helper keeps it here and + // the writer panics with it when it reaches the picture. + panicked any +} + +func newJob(l look, change int64) *job { + j := &job{look: l, change: change} + j.done.Add(1) + return j +} + +// codedJob is a picture already coded, which nobody codes again. +func codedJob(c Coded) *job { + j := &job{coded: c} + j.taken.Store(true) + return j +} + +// crew codes the pictures of one film: the goroutine that owns it, and the +// helpers it has been able to take. Jobs are offered in the order the film +// shows them and waited for in the same order, so the first error a film +// meets is the first in its order, whoever coded what. +type crew struct { + film *film + qindex int + own *painter + queue chan *job + most int + helpers int + wg sync.WaitGroup + stopped atomic.Bool + closed bool + // encode stands in for painting a picture and coding it through gav1d, + // told whether a helper is the one asking. Nil in every film - only + // CodeBeside sets it. + encode func(change int64, beside bool) (Coded, error) +} + +// newCrew is the crew for a film of this many pictures. A film of one picture +// takes no helper, and neither does a process of one thread. +func newCrew(f *film, qindex int, pictures int64) *crew { + c := &crew{film: f, qindex: qindex, own: f.painter()} + if most := min(int64(runtime.GOMAXPROCS(0)-1), pictures-1); most > 0 { + c.most = int(most) + c.queue = make(chan *job, c.ahead()) + } + return c +} + +// ahead is how many pictures past the one being written may be offered: two +// for each goroutine that codes, so none of them waits for the next picture +// while the writer is still busy with the last. Nought without helpers - a +// film coded alone looks at no picture before it needs it. +func (c *crew) ahead() int { + if c.most == 0 { + return 0 + } + return 2 * (c.most + 1) +} + +// offer hands a picture to the helpers, taking one more first when the film +// may have more and the process has a place for it. With no helper the +// writer codes it when it gets there. Called by the owning goroutine only. +func (c *crew) offer(j *job) { + if c.queue == nil || c.closed { + return + } + if c.helpers < c.most && takeHelper() { + c.helpers++ + c.wg.Add(1) + go c.help() + } + if c.helpers == 0 { + return + } + select { + case c.queue <- j: + default: + // The queue is as long as the window, so this is a picture the + // writer will reach before any helper would have. + } +} + +func (c *crew) help() { + defer c.wg.Done() + defer helping.Add(-1) + p := c.film.painter() + for j := range c.queue { + if c.stopped.Load() { + continue + } + if j.taken.CompareAndSwap(false, true) { + c.code(p, j, true) + codedBeside.Add(1) + } + } +} + +func (c *crew) code(p *painter, j *job, beside bool) { + defer j.done.Done() + defer func() { + if v := recover(); v != nil { + j.panicked = v + } + }() + if c.encode != nil { + j.coded, j.err = c.encode(j.change, beside) + return + } + j.coded, j.err = Encode(p.draw(j.look), c.qindex) +} + +// wait is the picture of j, coded here when no helper has taken it. +func (c *crew) wait(j *job) (Coded, error) { + if j.taken.CompareAndSwap(false, true) { + c.code(c.own, j, false) + } + j.done.Wait() + if j.panicked != nil { + panic(j.panicked) + } + return j.coded, j.err +} + +// finish says no picture is coming after the ones offered, so the helpers end +// once the queue is empty and give their places back to the process. +func (c *crew) finish() { + if c.queue != nil && !c.closed { + c.closed = true + close(c.queue) + } +} + +// stop ends the crew and waits for its helpers. A helper in the middle of a +// picture finishes that picture - gav1d cannot be interrupted, and that was +// so before there were helpers - and codes nothing after it. Nothing a crew +// started is running once this returns. +func (c *crew) stop() { + c.stopped.Store(true) + c.finish() + c.wg.Wait() +} + +// all offers every job at once and gives back their pictures in order - a +// set known in advance, like the sample planning codes, rather than a film +// coded as it is written. +func (c *crew) all(jobs []*job) ([]Coded, error) { + for _, j := range jobs { + c.offer(j) + } + c.finish() + out := make([]Coded, len(jobs)) + for i, j := range jobs { + coded, err := c.wait(j) + if err != nil { + return nil, err + } + out[i] = coded + } + return out, nil +} + +// CodeBeside codes pictures 0 to n-1 the way a film's are coded - this +// goroutine and the helpers it can take, offered ahead and taken in order - +// with code standing in for painting and gav1d, told whether a helper is the +// one calling, and gives back the size code returned for each. +// +// It is for the guards, and only because what it lets them hold cannot be +// reached another way: a panic on a helper has to come out on the goroutine +// that asked for the picture, where the engine turns it into the run's error, +// rather than end the process - and nothing outside makes gav1d panic. +// Measured on 2026-10-06: a quantizer out of range is refused with an error, +// and only planes shorter than the picture panic, which no painter makes. +func CodeBeside(n int64, code func(change int64, beside bool) (int, error)) ([]int, error) { + t := Timeline{FPS: 1, Frames: n, DurationMs: n * 1000, KeyEvery: n, ChangeEvery: 1} + c := newCrew(newFilm(1, 1, 0, "", t), 0, n) + defer c.stop() + c.encode = func(change int64, beside bool) (Coded, error) { + size, err := code(change, beside) + return Coded{tile: make([]byte, size)}, err + } + jobs := make([]*job, n) + for i := range jobs { + jobs[i] = newJob(look{}, int64(i)) + } + coded, err := c.all(jobs) + if err != nil { + return nil, err + } + sizes := make([]int, n) + for i, p := range coded { + sizes[i] = p.Size() + } + return sizes, nil +} diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go index f702d2e6..5b37c90f 100644 --- a/internal/format/video/choose.go +++ b/internal/format/video/choose.go @@ -88,20 +88,30 @@ func (c Choice) ClockShown(t Timeline) bool { return ClockShown(c.Width, c.Heigh // ceilings keep over their own measurement. The gradient does not move between // pictures, by the owner's decision of the same day, because moving it changed // a picture's size by up to 1.9 times. +// +// The sample is coded the way a film is (ahead.go), all of it offered at once, +// because at 1920x1080 ten pictures are three seconds of planning on one +// goroutine, and planning is what a preview waits for. func sampleReserve(w, h int, seed uint64, label string, t Timeline, qindex int) (Coded, int, error) { - p := newPainter(w, h, seed, label, t) + f := newFilm(w, h, seed, label, t) sample := SampleChanges(t) + c := newCrew(f, qindex, int64(len(sample))) + defer c.stop() + jobs := make([]*job, len(sample)) + for i, ch := range sample { + jobs[i] = newJob(f.lookOf(ch), ch) + } + coded, err := c.all(jobs) + if err != nil { + return Coded{}, 0, err + } var first Coded largest := 0 - for _, c := range sample { - coded, err := Encode(p.paint(c), qindex) - if err != nil { - return Coded{}, 0, err - } - if c == 0 { - first = coded + for i, ch := range sample { + if ch == 0 { + first = coded[i] } - largest = max(largest, coded.Size()) + largest = max(largest, coded[i].Size()) } if int64(len(sample)) == t.Changes() { return first, largest, nil diff --git a/internal/format/video/picture.go b/internal/format/video/picture.go index df3fba2e..9f2c9026 100644 --- a/internal/format/video/picture.go +++ b/internal/format/video/picture.go @@ -5,6 +5,7 @@ import ( "image" "image/color" "image/draw" + "slices" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" ) @@ -33,11 +34,22 @@ var ink = color.RGBA{R: 240, G: 240, B: 240, A: 255} // Picture draws picture c of a film: the gradient every image format here // draws, moved by the seed, with the label burned into the top of it when // there is room, the clock under it reading when the picture starts, and the -// square at its step. A film draws its pictures through one painter, so this -// is for planning, which codes the first, and for the probes and guards that -// measure any one of them. +// square at its step. It is drawn the way a film draws it, and a film draws +// its pictures through painters of its own, so this is for the probes and +// guards that measure any one of them. func Picture(width, height int, seed uint64, label string, t Timeline, c int64) Planes { - return newPainter(width, height, seed, label, t).paint(c) + f := newFilm(width, height, seed, label, t) + return f.painter().draw(f.lookOf(c)) +} + +// WholePicture is picture c painted whole - the gradient copied, the clock and +// the square drawn on the copy, and every pixel of it converted - which is how +// every picture of a film was painted until 2026-10-06. A film now paints only +// the rows that can change (painter.draw), and a guard holds the two to the +// same planes, because the bytes of every film depend on them being the same. +func WholePicture(width, height int, seed uint64, label string, t Timeline, c int64) Planes { + f := newFilm(width, height, seed, label, t) + return f.whole(f.lookOf(c)) } // Labelled says whether a picture this wide carries a readable label - asked @@ -76,21 +88,28 @@ func Clock(c int64, t Timeline) string { return fmt.Sprintf("%s.%03d", hms, ms%1000) } -// painter draws the pictures of one film. The gradient and the label are the -// same in every one of them, so they are drawn once, and each change copies -// them and adds only what moves - the clock and the square. -type painter struct { - t Timeline - base *image.RGBA - work *image.RGBA - planes Planes - clockFrom int // the row the clock's band starts at - showClock bool - side int - square *image.Uniform +// film is what every picture of one film has in common, made once and only +// read after: the gradient with the label burned in, the same gradient in +// planes, and where the clock and the square go. The painters of one film +// share it, which is what lets several of them paint at once (ahead.go). +type film struct { + t Timeline + base *image.RGBA + basePlanes Planes + planesBuf []uint8 // what basePlanes lie over, copied whole by a painter + clockFrom int // the row the clock's band starts at + clockEnd int // the row under the band, where Draw stops painting it + showClock bool + side int + squareY int + square *image.Uniform + // strips are the only rows a picture can differ from the gradient in - + // the clock's band and the square's - each widened to whole pairs of + // rows, because one chroma sample covers two, and merged where they meet. + strips []image.Rectangle } -func newPainter(width, height int, seed uint64, label string, t Timeline) *painter { +func newFilm(width, height int, seed uint64, label string, t Timeline) *film { off := int(seed % 256) base := image.NewRGBA(image.Rect(0, 0, width, height)) for y := range height { @@ -102,15 +121,61 @@ func newPainter(width, height int, seed uint64, label string, t Timeline) *paint if Labelled(width, label) { imagelabel.Draw(base, label) } - cw, ch := (width+1)/2, (height+1)/2 - return &painter{ - t: t, base: base, work: image.NewRGBA(base.Rect), - planes: Planes{Width: width, Height: height, Y: make([]uint8, width*height), U: make([]uint8, cw*ch), V: make([]uint8, cw*ch)}, + f := &film{ + t: t, base: base, clockFrom: labelBand(width, label), showClock: ClockShown(width, height, label, t), side: min(max(width/squareDivisor, 1), maxSquareSide, height), square: image.NewUniform(ink), } + f.basePlanes, f.planesBuf = newPlanes(width, height) + toPlanes(base, f.basePlanes) + f.squareY = (height - f.side) * 3 / 4 + f.clockEnd = f.clockFrom + if f.showClock { + f.clockEnd = min(height, f.clockFrom+imagelabel.BandHeight(width, len(Clock(0, t)))) + } + f.strips = changingRows(width, height, [][2]int{{f.clockFrom, f.clockEnd}, {f.squareY, f.squareY + f.side}}) + return f +} + +// newPlanes is the three planes of a picture this size in one allocation, +// and that allocation, which a painter copies whole (film.painter). +func newPlanes(width, height int) (Planes, []uint8) { + cw, ch := (width+1)/2, (height+1)/2 + buf := make([]uint8, width*height+2*cw*ch) + return planesOver(buf, width, height), buf +} + +// planesOver lays the three planes out over buf, luma first. Each is capped +// at its own length, so no plane can grow into the next. +func planesOver(buf []uint8, width, height int) Planes { + n, c := width*height, ((width+1)/2)*((height+1)/2) + return Planes{Width: width, Height: height, Y: buf[:n:n], U: buf[n : n+c : n+c], V: buf[n+c : n+2*c : n+2*c]} +} + +// changingRows turns runs of rows into strips the width of the picture, +// each starting on an even row and ending on an even row or the last one, so +// that every chroma sample a strip converts covers only rows the strip holds, +// and merges the strips that overlap or touch. A run of no rows - the clock's, +// on a picture with no room for it - gives no strip. +func changingRows(width, height int, runs [][2]int) []image.Rectangle { + var spans [][2]int + for _, r := range runs { + if r[0] < r[1] { + spans = append(spans, [2]int{r[0] &^ 1, min(height, (r[1]+1)&^1)}) + } + } + slices.SortFunc(spans, func(a, b [2]int) int { return a[0] - b[0] }) + var out []image.Rectangle + for _, s := range spans { + if n := len(out); n > 0 && s[0] <= out[n-1].Max.Y { + out[n-1].Max.Y = max(out[n-1].Max.Y, s[1]) + continue + } + out = append(out, image.Rect(0, s[0], width, s[1])) + } + return out } // look is what tells two pictures of one film apart: the clock's text when it @@ -122,28 +187,84 @@ type look struct { x int } -func (p *painter) lookOf(c int64) look { - l := look{x: (p.base.Rect.Dx() - p.side) * int(c%squareSteps) / (squareSteps - 1)} - if p.showClock { - l.clock = Clock(c, p.t) +func (f *film) lookOf(c int64) look { + l := look{x: (f.base.Rect.Dx() - f.side) * int(c%squareSteps) / (squareSteps - 1)} + if f.showClock { + l.clock = Clock(c, f.t) } return l } -// paint draws picture c into the painter's own planes and returns them. They -// are overwritten by the next call. -func (p *painter) paint(c int64) Planes { return p.draw(p.lookOf(c)) } +// whole paints a picture the long way, into planes of its own: the whole +// gradient copied, the clock and the square drawn on the copy, every pixel +// converted. It is the reference painter.draw is held to (WholePicture). +func (f *film) whole(l look) Planes { + work := image.NewRGBA(f.base.Rect) + copy(work.Pix, f.base.Pix) + if f.showClock { + b := work.Rect + imagelabel.Draw(work.SubImage(image.Rect(0, f.clockFrom, b.Dx(), b.Dy())).(*image.RGBA), l.clock) + } + draw.Draw(work, image.Rect(l.x, f.squareY, l.x+f.side, f.squareY+f.side), f.square, image.Point{}, draw.Src) + planes, _ := newPlanes(f.base.Rect.Dx(), f.base.Rect.Dy()) + toPlanes(work, planes) + return planes +} + +// painter draws pictures of one film into planes of its own. They start as +// the gradient's, and every row outside the film's strips stays the +// gradient's in every picture, so a picture repaints and converts the strips +// and nothing else - at 1920x1080 the clock's band and the square's are about +// a sixth of the rows. One painter is used by one goroutine at a time. +type painter struct { + f *film + planes Planes + strips []image.RGBA +} + +// painter is a painter of this film, in four allocations whatever the size: +// itself, its planes, its strips, and the pixels of all of them. Every helper +// coding a film makes one, and the allocations a file may cost are counted +// (the AllocCeiling of the formats built on this). +func (f *film) painter() *painter { + w := f.base.Rect.Dx() + stride := 4 * w + rows := 0 + for _, r := range f.strips { + rows += r.Dy() + } + pix := make([]uint8, rows*stride) + p := &painter{f: f, planes: planesOver(slices.Clone(f.planesBuf), w, f.base.Rect.Dy()), strips: make([]image.RGBA, len(f.strips))} + for i, r := range f.strips { + n := r.Dy() * stride + p.strips[i] = image.RGBA{Pix: pix[:n:n], Stride: stride, Rect: r} + pix = pix[n:] + } + return p +} -// draw is paint for a picture whose look is already worked out. +// draw paints the picture with this look into the painter's planes and +// returns them. They are overwritten by the next call. +// +// Each strip is the gradient's rows copied, the clock drawn when its band +// starts in the strip, the square drawn where it crosses the strip, and the +// rows converted - the same steps, in the same order, as whole, on fewer +// rows. The clock's band always lies whole inside one strip, because the +// strips were cut around it, so Draw sizes it as it would on the whole +// picture. func (p *painter) draw(l look) Planes { - copy(p.work.Pix, p.base.Pix) - if p.showClock { - b := p.work.Rect - imagelabel.Draw(p.work.SubImage(image.Rect(0, p.clockFrom, b.Dx(), b.Dy())).(*image.RGBA), l.clock) - } - y := (p.base.Rect.Dy() - p.side) * 3 / 4 - draw.Draw(p.work, image.Rect(l.x, y, l.x+p.side, y+p.side), p.square, image.Point{}, draw.Src) - toPlanes(p.work, p.planes) + f := p.f + w := f.base.Rect.Dx() + for i := range p.strips { + s := &p.strips[i] + r := s.Rect + copy(s.Pix, f.base.Pix[r.Min.Y*f.base.Stride:r.Max.Y*f.base.Stride]) + if f.showClock && r.Min.Y <= f.clockFrom && f.clockFrom < r.Max.Y { + imagelabel.Draw(s.SubImage(image.Rect(0, f.clockFrom, w, r.Max.Y)).(*image.RGBA), l.clock) + } + draw.Draw(s, image.Rect(l.x, f.squareY, l.x+f.side, f.squareY+f.side), f.square, image.Point{}, draw.Src) + convertRows(s, p.planes, r.Min.Y, r.Max.Y) + } return p.planes } @@ -162,30 +283,38 @@ func (p *painter) draw(l look) Planes { // // Chroma is the rounded mean of the two by two block it covers, so an odd // width or height averages the pixels that are there. -func toPlanes(img *image.RGBA, p Planes) { - w, h := img.Rect.Dx(), img.Rect.Dy() - cw, ch := (w+1)/2, (h+1)/2 - for y := range h { - row := img.Pix[y*img.Stride:] +func toPlanes(img *image.RGBA, p Planes) { convertRows(img, p, 0, p.Height) } + +// convertRows converts rows r0 up to r1 of the picture p is, read from img - +// the whole picture, or a strip of it that starts at row r0. r0 is even and r1 +// is even or the picture's height, so every chroma sample written here covers +// rows img holds. +func convertRows(img *image.RGBA, p Planes, r0, r1 int) { + w, h := p.Width, p.Height + top := img.Rect.Min.Y + for y := r0; y < r1; y++ { + row := img.Pix[(y-top)*img.Stride:] for x := range w { r, g, b := int(row[4*x]), int(row[4*x+1]), int(row[4*x+2]) p.Y[y*w+x] = uint8(16 + (47*r+157*g+16*b+128)>>8) } } - for cy := range ch { + cw := (w + 1) / 2 + for cy := r0 / 2; cy < (r1+1)/2; cy++ { for cx := range cw { - p.U[cy*cw+cx], p.V[cy*cw+cx] = chroma(img, 2*cx, 2*cy) + p.U[cy*cw+cx], p.V[cy*cw+cx] = chroma(img, 2*cx, 2*cy, w, h) } } } // chroma is the Cb and Cr of the two by two block whose top left is x0, y0, -// over the pixels of it that are inside the picture. -func chroma(img *image.RGBA, x0, y0 int) (cb, cr uint8) { +// over the pixels of it that are inside a picture w by h. +func chroma(img *image.RGBA, x0, y0, w, h int) (cb, cr uint8) { sumB, sumR, n := 0, 0, 0 - for y := y0; y < min(y0+2, img.Rect.Dy()); y++ { - row := img.Pix[y*img.Stride:] - for x := x0; x < min(x0+2, img.Rect.Dx()); x++ { + top := img.Rect.Min.Y + for y := y0; y < min(y0+2, h); y++ { + row := img.Pix[(y-top)*img.Stride:] + for x := x0; x < min(x0+2, w); x++ { r, g, b := int(row[4*x]), int(row[4*x+1]), int(row[4*x+2]) sumB += -26*r - 86*g + 112*b sumR += 112*r - 102*g - 10*b diff --git a/internal/format/video/stream.go b/internal/format/video/stream.go index 48f6a70f..8923f8d4 100644 --- a/internal/format/video/stream.go +++ b/internal/format/video/stream.go @@ -88,32 +88,69 @@ func (s Stream) BoundBytes() (key, copied, shown int) { return len(s.seq) + k, h + len(s.show), len(s.show) } +// workPerPixel is how many bytes of writing coding one pixel of a picture is +// worth, for the bar a run draws (format.Plan.Work). Measured on 2026-10-06 +// with the owner's film, 1810 pictures of 1920x1080 and 2 GB of padding +// (docs/WEBM-WYDAJNOSC-2026-10-06.md): coding 16.9 ns a pixel on sixteen +// threads and 162 on one, writing 2.2 ns a byte. So the true figure is +// between 8 and 74 depending on the machine, and this is one of four to eight +// threads. A wrong figure misjudges only the padding against the pictures, +// and the padding takes seconds, so the estimate is off by seconds rather +// than by the film. +const workPerPixel = 16 + +// PictureWork is what coding one picture of the film is worth in bytes of +// writing, which a container reports as each change is coded. +func (s Stream) PictureWork() int64 { return int64(s.Width) * int64(s.Height) * workPerPixel } + +// Work is what coding every picture of the film is worth (format.Plan.Work). +func (s Stream) Work() int64 { return s.Changes() * s.PictureWork() } + // joined is a followed by b in one allocation of the right size. func joined(a, b []byte) []byte { return append(append(make([]byte, 0, len(a)+len(b)), a...), b...) } -// Pictures codes the pictures of one film in order, one change at a time, and -// keeps only the last - an hour at a change a second is 3600 pictures, and -// holding them all would hold the film. +// Pictures codes the pictures of one film in the order the film shows them, +// on the goroutine that asks for them and on the helpers it can take +// (ahead.go), and keeps only the last one it was asked for and the few coded +// ahead of it - an hour at a change a second is 3600 pictures, and holding +// them all would hold the film. +// +// Close has to be called when the film is written or abandoned, because the +// helpers are goroutines. type Pictures struct { - choice Choice - stream Stream - painter *painter + choice Choice + stream Stream + film *film + crew *crew + // pending are the pictures offered and not yet reached, in order, each + // with the change it opens. Changes that look the same as the one before + // open none, so they are not in it. + pending []openedBy + next int64 // the first change not looked at yet + last look // the look of change next-1 change int64 - look look key []byte copied []byte } +type openedBy struct { + change int64 + job *job +} + // Pictures is the coder of this choice's film. func (c Choice) Pictures(st Stream) *Pictures { - return &Pictures{choice: c, stream: st, change: -1, - painter: newPainter(c.Width, c.Height, c.Seed, c.Label, st.Timeline)} + f := newFilm(c.Width, c.Height, c.Seed, c.Label, st.Timeline) + cr := newCrew(f, c.QIndex, st.Changes()) + return &Pictures{choice: c, stream: st, film: f, crew: cr, change: -1, + pending: make([]openedBy, 0, cr.ahead()+1)} } // At makes picture c the current one, coding it unless it looks the same as -// the one before. +// the one before. Pictures are asked for in the order the film shows them - +// the same change again is fine, an earlier one is a defect of the caller. // // Every picture is held to the stream's reserve, because the level and the // file around it were planned on it. A picture over it is a ceiling that did @@ -124,30 +161,73 @@ func (p *Pictures) At(c int64) error { if c == p.change { return nil } - l := p.painter.lookOf(c) - if p.change >= 0 && l == p.look { + if c < p.change { + return core.Defect(fmt.Errorf("video: picture %d was asked for after picture %d, and a film's pictures are coded in the order it shows them", c, p.change)) + } + p.offerTo(c) + var opened *job + reached := 0 + for reached < len(p.pending) && p.pending[reached].change <= c { + opened = p.pending[reached].job + reached++ + } + // Moved down rather than sliced off, so the one array made for the + // window lasts the whole film. + p.pending = p.pending[:copy(p.pending, p.pending[reached:])] + if opened == nil { + // The same look as the picture before it: shown again, not coded. p.change = c return nil } - var coded Coded - if c == 0 && p.choice.First != nil { - coded = *p.choice.First - } else { - var err error - if coded, err = Encode(p.painter.draw(l), p.choice.QIndex); err != nil { - return err - } + coded, err := p.crew.wait(opened) + if err != nil { + return err } if coded.Size() > p.stream.Reserve { return core.Defect(fmt.Errorf("video: picture %d of a %dx%d film coded to a %d B tile and the film was planned on %d B a picture, so the file cannot be kept", c, p.choice.Width, p.choice.Height, coded.Size(), p.stream.Reserve)) } - p.change, p.look = c, l + p.change = c p.key = joined(p.stream.seq, keyFrame(coded)) p.copied = joined(hiddenCopy(coded), p.stream.show) return nil } +// offerTo looks at every change up to c and as far past it as the crew codes +// ahead, and offers each one that looks different from the one before it. +// Change 0 is the picture planning already coded, when it coded one. +func (p *Pictures) offerTo(c int64) { + n := p.stream.Changes() + for p.next < n && (p.next <= c || len(p.pending) < p.crew.ahead()) { + l := p.film.lookOf(p.next) + if p.next == 0 || l != p.last { + p.pending = append(p.pending, openedBy{change: p.next, job: p.jobFor(l)}) + } + p.last = l + p.next++ + } + if p.next == n { + p.crew.finish() + } +} + +// jobFor is the job of the picture with this look at change p.next, offered +// to the helpers - or, for change 0 when planning coded it, the picture +// planning already has. +func (p *Pictures) jobFor(l look) *job { + if p.next == 0 && p.choice.First != nil { + return codedJob(*p.choice.First) + } + j := newJob(l, p.next) + p.crew.offer(j) + return j +} + +// Close stops the helpers and waits for them, so nothing this film started is +// running once it returns. A helper in the middle of a picture finishes it +// first, which is at most one picture's time. +func (p *Pictures) Close() { p.crew.stop() } + // KeySample is the current picture as a key frame, with the sequence header // before it. func (p *Pictures) KeySample() []byte { return p.key } diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index 3d0456f7..ffcf941e 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -52,6 +52,11 @@ func init() { // object allocated per frame, the defect the ceiling exists for. The // number below sits between the two, and like every ceiling here it // goes down when work makes it lowerable, never up to turn a run green. + // Since the pictures are coded beside each other (internal/format/video, + // ahead.go) the default film is 474 to 477 objects at sixteen, four and + // one threads alike - each picture is a job, each helper one painter of + // four allocations - measured the same day, still under the ceiling, + // which was not moved. AllocCeiling: 512, Padding: format.PaddingChannel{ @@ -113,6 +118,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Determinism: format.DeterminismByte, Properties: video.Facts(c, s), Memo: memo{choice: c, settings: s, total: r.Bytes}, + Work: st.Work(), } if r.Label && !c.Labelled() { p.Notes = append(p.Notes, format.Note{ @@ -183,7 +189,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { out.write(seekHead(l.seekHeadLen, l.seekHeadLen+uint64(len(l.info)), cuesAt)) out.write(l.info) out.write(l.tracks) - keys, end, err := writeClusters(ctx, out, l, m.choice.Pictures(st)) + pics := m.choice.Pictures(st) + defer pics.Close() + keys, end, err := writeClusters(ctx, out, l, pics) if err != nil { return err } @@ -226,20 +234,27 @@ func (s *sticky) uint(id uint32, v uint64) { // writeClusters codes the pictures one change at a time and writes the // clusters as it goes, giving back where each key frame's cluster starts, for -// the Cues, and where the last one ends. +// the Cues, and where the last one ends. Every change opens a cluster, so each +// is reported as worked once, which adds up to the Work the plan counted. func writeClusters(ctx context.Context, out *sticky, l layout, pics *video.Pictures) ([]uint64, uint64, error) { s := l.stream keys := make([]uint64, 0, s.Keys()) pos := l.front() + worked := int64(-1) for first := int64(0); first < s.Frames; { if err := ctx.Err(); err != nil { return nil, 0, err } // A cluster thirty seconds into a picture asks for the picture it // already has, which At answers without coding anything. - if err := pics.At(s.ChangeOf(first)); err != nil { + change := s.ChangeOf(first) + if err := pics.At(change); err != nil { return nil, 0, err } + if change != worked { + format.Worked(ctx, s.PictureWork()) + worked = change + } if s.IsKey(first) { keys = append(keys, pos) } diff --git a/internal/format/work.go b/internal/format/work.go new file mode 100644 index 00000000..c844f2dd --- /dev/null +++ b/internal/format/work.go @@ -0,0 +1,32 @@ +package format + +import "context" + +// The work a file costs beyond its bytes (Plan.Work), and how a generator says +// how much of it is done. +// +// A film is the case this exists for: thirty minutes of 1920x1080 is a +// hundred megabytes of pictures that take a minute to code and two gigabytes +// of padding that take five seconds to write, so a bar counting bytes stood at +// five percent for the whole minute and promised three hours +// (docs/WEBM-WYDAJNOSC-2026-10-06.md). Kept out of format.go, which was at its +// ceiling on length, because it is a subject of its own. + +type workKey struct{} + +// WithWork is ctx carrying report, which Worked calls. The engine sets it on +// the context a generator writes under, when somebody is watching the run. +func WithWork(ctx context.Context, report func(int64)) context.Context { + return context.WithValue(ctx, workKey{}, report) +} + +// Worked says that n more of the plan's Work is done. It goes through the +// context rather than the writer, because the writer a generator is handed is +// not always the engine's - a damaged file is written through the damage - +// and from the goroutine Write runs on, which is the one the engine counts +// the bytes on. With nobody watching it does nothing. +func Worked(ctx context.Context, n int64) { + if report, ok := ctx.Value(workKey{}).(func(int64)); ok { + report(n) + } +} diff --git a/internal/guard/concurrency_test.go b/internal/guard/concurrency_test.go index 622dff6b..a279e3d0 100644 --- a/internal/guard/concurrency_test.go +++ b/internal/guard/concurrency_test.go @@ -101,6 +101,18 @@ var mayBeConcurrent = map[string]string{ // Numbers, the instrument, and the two mistakes made getting them: // docs/PERFORMANCE-REVIEW-2026-09-05.md section 14. "internal/engine/parallel.go": "the planned files are written beside each other, and nothing else in the package does", + // A film is its pictures, each a whole picture through gav1d on one + // goroutine: the owner's thirty minute 1920x1080 film was 611.7 s on one + // core of sixteen, and the window's estimate from the bytes said three + // hours. The pictures are independent, so several goroutines code them and + // the writer takes them in order - 7.60x at sixteen threads, 5.12x at + // eight, every picture the same size at every width (tools/probes/ + // videoparallel). Helpers come from one count for the whole process, + // because the engine above already writes a file per thread. + // + // Added 2026-10-06 and THE OWNER DECIDED IT: + // docs/WEBM-WYDAJNOSC-2026-10-06.md sections 4 and 5. + "internal/format/video/ahead.go": "the pictures of one film are coded beside each other, from one count of helpers for the process", } // Waiting on cancellation is not the same thing as running in parallel. Every diff --git a/internal/guard/filmahead_test.go b/internal/guard/filmahead_test.go new file mode 100644 index 00000000..9ef553ac --- /dev/null +++ b/internal/guard/filmahead_test.go @@ -0,0 +1,217 @@ +package guard + +import ( + "bytes" + "context" + "runtime" + "slices" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// The pictures of one film are coded by several goroutines at once and +// painted a strip at a time (internal/format/video, ahead.go and picture.go, +// docs/WEBM-WYDAJNOSC-2026-10-06.md). Neither may change a byte of any film, +// and neither may leave anything running once the film is done with. + +// A film paints only the rows that can change - the clock's band and the +// square's, widened to whole pairs of rows - onto planes made once from the +// gradient. It has to come out as the picture painted whole, the way every +// film was painted before, in every plane and every sample. +// +// The sizes are the ones where strips go wrong: one pixel, odd sides that +// leave a chroma sample half outside, a picture short enough that the square +// meets the clock's band, the smallest that still shows the clock, and the +// ladder's top rung. Each with the label and without, and each with the clock +// in whole seconds and with milliseconds, because those are bands of two +// lengths. +func TestAFilmPaintsOnlyWhatChangesAndGetsTheWholePicture(t *testing.T) { + sizes := [][2]int{{1, 1}, {2, 2}, {3, 3}, {16, 9}, {17, 11}, {48, 32}, {52, 20}, {64, 48}, {97, 41}, {160, 90}, {161, 91}, {640, 360}} + timelines := map[string]video.Timeline{} + for name, ms := range map[string][2]int64{"seconds": {3_600_000, 1_000}, "milliseconds": {60_000, 100}} { + tl, err := video.NewTimeline("webm", ms[0], 60_000, ms[1], 30) + if err != nil { + t.Fatal(err) + } + timelines[name] = tl + } + labels := map[string]string{"labelled": core.Label("webm", 1<<20, 7), "unlabelled": ""} + + compared, clocked, moved := 0, 0, 0 + for _, size := range sizes { + w, h := size[0], size[1] + for tName, tl := range timelines { + for lName, label := range labels { + changes := []int64{0, 1, 2, 9, 10, 11, tl.Changes() / 2, tl.Changes() - 1} + var first video.Planes + for i, c := range changes { + got := video.Picture(w, h, 7, label, tl, c) + want := video.WholePicture(w, h, 7, label, tl, c) + for plane, pair := range map[string][2][]uint8{"Y": {got.Y, want.Y}, "U": {got.U, want.U}, "V": {got.V, want.V}} { + if !bytes.Equal(pair[0], pair[1]) { + t.Errorf("%dx%d %s %s, picture %d: the %s plane painted a strip at a time differs from the picture painted whole", w, h, tName, lName, c, plane) + } + } + compared++ + if i == 0 { + first = got + first.Y = slices.Clone(got.Y) + } else if !bytes.Equal(first.Y, got.Y) { + moved++ + } + } + if video.ClockShown(w, h, label, tl) { + clocked++ + } + } + } + } + // Asserted, not assumed: a strip painter that drew nothing would agree with + // a whole painter that drew nothing, so the pictures have to have moved, + // and the clock has to have been in some of them. + if compared < 300 || moved < 100 || clocked < 10 { + t.Fatalf("%d pictures compared, %d of them different from their film's first, %d films with a clock - the guard did not see pictures that change", + compared, moved, clocked) + } +} + +// A film coded by several goroutines has the bytes of the same film coded by +// one, and the several really did code it. +// +// The second half is the one that keeps the first honest. With one thread the +// process has no helper to give, so the run under one thread is the reference +// and has to have used none, and the run under eight has to have used some - +// otherwise both runs are the same goroutine and the comparison is of a film +// with itself. One film is sized by hand, so planning codes its sample through +// the helpers too, and one is chosen to fit. +func TestAFilmCodedBySeveralGoroutinesHasTheBytesOfOne(t *testing.T) { + targets := []engine.Target{ + filmTarget(2<<20, map[string]string{"width": "64", "height": "48", "duration": "40s"}), + filmTarget(1<<20, map[string]string{"duration": "30s", "change_interval": "500ms"}), + } + for _, target := range targets { + alone, helped := filmUnder(t, 1, target), filmUnder(t, 8, target) + if alone.coded != 0 { + t.Errorf("%v: under one thread helpers coded %d pictures, so there is no film of one goroutine to compare with", target.Properties, alone.coded) + } + if helped.coded == 0 { + t.Errorf("%v: under eight threads no helper coded anything, so the film of several goroutines was never made", target.Properties) + } + if !bytes.Equal(alone.bytes, helped.bytes) { + t.Errorf("%v: the film coded with helpers differs from the film coded by one goroutine (%d B and %d B)", target.Properties, len(helped.bytes), len(alone.bytes)) + } + } +} + +type filmRun struct { + bytes []byte + coded int64 +} + +func filmUnder(t *testing.T, threads int, target engine.Target) filmRun { + t.Helper() + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(threads)) + _, before := video.Helpers() + b, _ := filmOne(t, target) + _, after := video.Helpers() + return filmRun{bytes: b, coded: after - before} +} + +// A film's progress moves with its pictures, not with its padding. +// +// The bytes of a film are mostly padding written in its last seconds, so a bar +// counting bytes stood at five percent while the pictures were coded and +// promised the owner three hours for a film that took ten minutes +// (docs/WEBM-WYDAJNOSC-2026-10-06.md). Here the pictures are about a +// twenty-fifth of the bytes and two thirds of the work: at the moment they are +// all written the work has to be far ahead of the bytes. And a run of files +// whose cost is their bytes has to report the same numbers as before - work +// equal to bytes at every report. +func TestAFilmsProgressMovesWithItsPicturesNotItsPadding(t *testing.T) { + reports, planned := progressOf(t, filmTarget(32<<20, map[string]string{"width": "160", "height": "90", "duration": "5m"})) + if changes, _ := planned.Properties["change_count"].(int64); changes < 100 || planned.Work <= 0 { + t.Fatalf("the film has %d pictures and declares %d of work, so there were no pictures for the bar to move with", changes, planned.Work) + } + ahead := 0.0 + for _, r := range reports { + ahead = max(ahead, float64(r.WorkDone)/float64(r.WorkTotal)-float64(r.BytesDone)/float64(r.BytesTotal)) + } + if ahead < 0.4 { + t.Errorf("the work was never more than %.0f%% ahead of the bytes, so the bar counts the padding as the film", 100*ahead) + } + if last := reports[len(reports)-1]; last.WorkDone != last.WorkTotal || last.BytesDone != last.BytesTotal { + t.Errorf("the run ended at %d of %d work and %d of %d bytes", last.WorkDone, last.WorkTotal, last.BytesDone, last.BytesTotal) + } + + plain, _ := progressOf(t, engine.Target{ID: "plain", Format: "txt", Sizes: engine.Uniform(1, 1<<20)}) + for _, r := range plain { + if r.WorkDone != r.BytesDone || r.WorkTotal != r.BytesTotal { + t.Fatalf("a text file reported %d of %d work against %d of %d bytes, and its cost is its bytes", r.WorkDone, r.WorkTotal, r.BytesDone, r.BytesTotal) + } + } +} + +// progressOf runs one target with every progress report kept, and its plan. +func progressOf(t *testing.T, target engine.Target) ([]engine.Progress, format.Plan) { + t.Helper() + opt := engine.Options{OutDir: t.TempDir(), Seed: goldenSeed, Command: "test"} + planned, err := engine.Plan([]engine.Target{target}, opt) + if err != nil { + t.Fatal(err) + } + var reports []engine.Progress + opt.OnProgress = func(p engine.Progress) { reports = append(reports, p) } + if _, err := engine.Run(context.Background(), planned, opt); err != nil { + t.Fatal(err) + } + if len(reports) < 2 { + t.Fatalf("%s: %d progress reports, too few to say how the bar moved", target.Format, len(reports)) + } + return reports, planned[0].Plan +} + +// A film stopped half way leaves no helper running when the run returns. +// +// Helpers are goroutines, and one left behind holds a picture's memory and a +// place the next film cannot take - and the window, which waits for the run +// before it closes (G7), would be waiting for something that no longer waits +// for anybody. The run is stopped once pictures are being written, which is +// asserted: stopped at the file's first bytes it would end before any helper +// was taken and pass without asking anything. +func TestAFilmStoppedHalfWayLeavesNoHelperRunning(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8)) + target := filmTarget(8<<20, map[string]string{"width": "160", "height": "90", "duration": "10m"}) + dir := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + opt := engine.Options{OutDir: dir, Seed: goldenSeed, Command: "test"} + planned, err := engine.Plan([]engine.Target{target}, opt) + if err != nil { + t.Fatal(err) + } + if changes, _ := planned[0].Plan.Properties["change_count"].(int64); changes < 100 { + t.Fatalf("the film has %d pictures, too few to be stopped in the middle of them", changes) + } + _, codedBefore := video.Helpers() + opt.OnProgress = func(p engine.Progress) { + // Some forty pictures in, of six hundred. + if p.BytesDone > 100_000 { + cancel() + } + } + res, runErr := engine.Run(ctx, planned, opt) + running, codedAfter := video.Helpers() + if runErr == nil && (res == nil || len(res.Manifest.Files) > 0) { + t.Fatalf("the run finished the film, so it was never stopped half way") + } + if codedAfter == codedBefore { + t.Fatalf("no helper coded a picture before the stop, so there was nothing that could have been left running") + } + if running != 0 { + t.Errorf("%d helpers are still running after the stopped run returned", running) + } +} diff --git a/internal/guard/progressbywork_test.go b/internal/guard/progressbywork_test.go new file mode 100644 index 00000000..379072d9 --- /dev/null +++ b/internal/guard/progressbywork_test.go @@ -0,0 +1,166 @@ +package guard + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// The bar's percentage and the time it says is left come from the work, on +// both surfaces (docs/WEBM-WYDAJNOSC-2026-10-06.md). +// +// Two halves, because the property lives in two places. engine.Progress +// works the two numbers out, and that is held here by what it answers for a +// report whose bytes and work disagree - five percent of the bytes, half of +// the work, the shape a film has while its pictures are coded. And the window +// and the command line must take both from there rather than work out their +// own: read from their source, every use they make of a report's bytes or work +// is an argument to HumanBytes, the size the line shows. A surface going back +// to a percentage of the bytes would use BytesDone somewhere else, and both of +// its own bars are out of a guard's reach - the window's moves on a goroutine +// the guards must not read beside, and the command line draws only on a +// terminal. +func TestTheBarAndTheTimeLeftComeFromTheWorkOnBothSurfaces(t *testing.T) { + p := engine.Progress{FilesTotal: 1, BytesDone: 5, BytesTotal: 100, WorkDone: 50, WorkTotal: 100} + if got := p.Percent(); got != 50 { + t.Errorf("half the work and a twentieth of the bytes is %d%%, and the work says 50", got) + } + if left, ok := p.Left(10 * time.Second); !ok || left != 10*time.Second { + t.Errorf("half the work in ten seconds leaves %v (%v), and the work says ten seconds", left, ok) + } + for name, quiet := range map[string]struct { + p engine.Progress + elapsed time.Duration + }{ + "the first second": {p, 500 * time.Millisecond}, + "no work done yet": {engine.Progress{BytesTotal: 100, WorkTotal: 100}, time.Minute}, + "all the work done": {engine.Progress{BytesDone: 100, BytesTotal: 100, WorkDone: 100, WorkTotal: 100}, time.Minute}, + } { + if left, ok := quiet.p.Left(quiet.elapsed); ok { + t.Errorf("%s: the estimate said %v where it has nothing to go on", name, left) + } + } + + shown, percents, lefts := 0, 0, 0 + for _, dir := range []string{"internal/cli", "internal/gui"} { + walkGo(t, filepath.Join(repoRoot(t), dir), func(path string, file *ast.File) { + counted := map[ast.Node]bool{} + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch calledName(call) { + case "HumanBytes": + for _, arg := range call.Args { + counted[arg] = true + } + case "Percent": + if len(call.Args) == 0 { + percents++ + } + case "Left": + lefts++ + } + return true + }) + ast.Inspect(file, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + switch sel.Sel.Name { + case "BytesDone", "BytesTotal", "WorkDone", "WorkTotal": + if counted[sel] { + shown++ + return true + } + rel, _ := filepath.Rel(repoRoot(t), path) + t.Errorf("%s reads a report's %s other than to show it as a size - a surface working out its own percentage or estimate rather than asking engine.Progress", filepath.ToSlash(rel), sel.Sel.Name) + } + return true + }) + }) + } + // Asserted, not assumed: both lines show their bytes, all three places + // that draw a percentage ask Progress for it, and both estimates come + // from Left. A scan that found none of it would pass about nothing. + if shown < 4 || percents < 3 || lefts < 2 { + t.Fatalf("the surfaces show %d sizes from a report, ask %d times for a percentage and %d times for the time left - the scan did not reach the bars it is about", shown, percents, lefts) + } +} + +// walkGo parses every Go file under dir that is not a test. +func walkGo(t *testing.T, dir string, visit func(path string, file *ast.File)) { + t.Helper() + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return err + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + return err + } + visit(path, file) + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// A panic while a helper codes a picture comes out on the goroutine that asked +// for the picture, and leaves no helper running. +// +// The engine turns a generator's panic into the run's error only on the +// goroutine that writes the file (writeWithoutCrashing), and a panic on any +// other goroutine ends the process - the window with it. So a helper keeps its +// panic and the writer raises it when it reaches that picture. gav1d cannot be +// made to panic from outside, so the pictures here are coded by a stand-in +// that panics the first time a helper calls it, through video.CodeBeside, +// which codes them exactly as a film's are. Under a helper that did not keep +// its panic this test does not fail - the test binary dies, which is red too. +func TestAPanicOnAHelperComesOutOnTheGoroutineThatAskedForIt(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8)) + var fromHelper atomic.Bool + helped := make(chan struct{}) + got := func() (v any) { + defer func() { v = recover() }() + _, _ = video.CodeBeside(40, func(change int64, beside bool) (int, error) { + if beside && fromHelper.CompareAndSwap(false, true) { + close(helped) + panic("a helper's picture") + } + if !beside { + // The writer's own picture waits for a helper to have + // started, or the writer, with nothing to code, takes all + // forty before any helper is scheduled. + select { + case <-helped: + case <-time.After(5 * time.Second): + } + } + return 1, nil + }) + return nil + }() + if !fromHelper.Load() { + t.Fatal("no helper coded a picture, so no helper could have panicked") + } + if got != "a helper's picture" { + t.Errorf("the goroutine that asked for the pictures got %v, and a helper panicked with %q", got, "a helper's picture") + } + if running, _ := video.Helpers(); running != 0 { + t.Errorf("%d helpers are still running after the panic came out", running) + } +} diff --git a/internal/gui/window/run.go b/internal/gui/window/run.go index 9eb785aa..c0266611 100644 --- a/internal/gui/window/run.go +++ b/internal/gui/window/run.go @@ -616,7 +616,7 @@ func (r *runner) startRun(targets []engine.Target, opt engine.Options) { } elapsed := time.Since(started) fyne.Do(func() { - r.busy.bar.SetValue(float64(core.Percent(p.BytesDone, p.BytesTotal))) + r.busy.bar.SetValue(float64(p.Percent())) r.status.SetText(progressText(p, elapsed)) }) } diff --git a/internal/gui/window/runreport.go b/internal/gui/window/runreport.go index a04c8456..bafa24ea 100644 --- a/internal/gui/window/runreport.go +++ b/internal/gui/window/runreport.go @@ -4,7 +4,6 @@ import ( "fmt" "time" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/gui/text" "github.com/donislawdev/TestingFilesGenerator/internal/manifest" @@ -51,18 +50,18 @@ func manifestReachNote(res *engine.Result) []string { // progressText is the line under the bar. Bytes rather than files, because one // large file is a run where the file count says nothing for minutes. +// +// The percentage and the time left come from the work rather than the bytes +// (engine.Progress), because a film's bytes are mostly padding written in its +// last seconds - by the bytes, a thirty minute film stood at five percent for +// the whole minute its pictures took and promised three hours. func progressText(p engine.Progress, elapsed time.Duration) string { line := text.Progress(p.FilesDone, p.FilesTotal, - text.HumanBytes(p.BytesDone), text.HumanBytes(p.BytesTotal), - core.Percent(p.BytesDone, p.BytesTotal)) - - // The estimate stays quiet until it has enough to go on. A number that - // swings wildly for the first second is worse than no number. - if elapsed < time.Second || p.BytesDone <= 0 || p.BytesDone >= p.BytesTotal { + text.HumanBytes(p.BytesDone), text.HumanBytes(p.BytesTotal), p.Percent()) + left, ok := p.Left(elapsed) + if !ok { return line } - left := time.Duration(float64(elapsed) * - float64(p.BytesTotal-p.BytesDone) / float64(p.BytesDone)) return line + text.TimeLeft(text.Roughly(left)) }