From 0cf26bffdc83f8ca8c295f29cea672c5722e53fd Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Tue, 6 Oct 2026 20:30:39 +0200 Subject: [PATCH 1/3] format: a film's pictures are coded beside each other - the same bytes, nine times sooner A film is its pictures, each a whole picture through gav1d on one goroutine. A thirty minute 1920x1080 film at a change a second was 1800 pictures and 611.7 s on one core of sixteen. The pictures are independent, so internal/format/video/ahead.go codes them on helpers and the writer takes them in order. The goroutine writing a film always codes, and the helpers come from one count for the whole process, GOMAXPROCS less one, because the engine above already writes a file per thread. A helper's panic is kept and raised by the writer, where the engine turns it into the run's error as before. Pictures.Close waits for the helpers. Each painter now repaints only the rows that change - the clock's band and the square's, widened to whole pairs of rows - onto planes made once from the gradient, so a helper costs a few megabytes rather than two full frames. WholePicture keeps the old way as the reference. Measured on the same order end to end: 611.7 s before, 67.9 s after, and the 2 GB file has the same SHA-256. The golden hashes did not move. Guards: the strip painter gives the whole picture at twelve sizes, a film coded with helpers has the bytes of one coded alone and helpers really coded it, and a film stopped half way leaves no helper running. Each reddened by hand. The race detector watches ahead.go. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 2 +- CHANGELOG.md | 4 +- internal/format/video/ahead.go | 207 ++++++++++++++++++++++++++ internal/format/video/choose.go | 20 ++- internal/format/video/picture.go | 227 ++++++++++++++++++++++------- internal/format/video/stream.go | 99 ++++++++++--- internal/format/webm/webm.go | 9 +- internal/guard/concurrency_test.go | 12 ++ internal/guard/filmahead_test.go | 163 +++++++++++++++++++++ 9 files changed, 666 insertions(+), 77 deletions(-) create mode 100644 internal/format/video/ahead.go create mode 100644 internal/guard/filmahead_test.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1650e363..a3e2bb42 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1058,7 +1058,7 @@ jobs: # in somebody else's file. run: | set -euo pipefail - watched='internal/format/registry.go internal/damage/damage.go cmd/tfg/main.go internal/gui/window/run.go internal/gui/run_cgo.go internal/gui/window/tidy.go internal/audit/parallel.go internal/engine/parallel.go internal/tool/tool.go internal/gui/window/tools.go go.mod .github/workflows/ci.yml .github/build-tags' + watched='internal/format/registry.go internal/damage/damage.go cmd/tfg/main.go internal/gui/window/run.go internal/gui/run_cgo.go internal/gui/window/tidy.go internal/audit/parallel.go internal/engine/parallel.go internal/tool/tool.go internal/gui/window/tools.go internal/format/video/ahead.go go.mod .github/workflows/ci.yml .github/build-tags' # On a pull request there is no "before" - the field belongs to a push # - so this asked for something empty and every pull request answered # "touched". That quietly undid the decision of 2026-08-20, because diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b64247..e86f3f19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,7 +25,9 @@ because it turns other people's test suites red. reads the time the picture starts, the way a player's position does - `00:00:07`, with milliseconds when the changes do not fall on whole seconds. Each change is a whole new picture in bytes and in coding time, - and the frames between changes cost a few bytes each, so an hour at thirty + and the pictures of a film are coded on every core of the machine at once - + a thirty minute 1920x1080 film took 68 s on sixteen threads. The frames + between changes cost a few bytes each, so an hour at thirty frames a second with a change every second fits in about a megabyte (1 075 831 B at the smallest), and a `change_interval` as long as the film or longer keeps one picture throughout (977 434 B for that hour). Set diff --git a/internal/format/video/ahead.go b/internal/format/video/ahead.go new file mode 100644 index 00000000..2984b62b --- /dev/null +++ b/internal/format/video/ahead.go @@ -0,0 +1,207 @@ +package video + +import ( + "runtime" + "sync" + "sync/atomic" +) + +// The one place in this package where anything runs beside anything else, and +// it is listed in internal/guard/concurrency_test.go with that reason. +// +// Why it exists. A film is its pictures: every change is a whole picture +// through gav1d, which codes on one goroutine and has no setting that searches +// less than the one already used. A 1920x1080 picture is about 0.35 s, so a +// thirty minute film at a change a second was ten minutes on one core while +// fifteen others idled - measured 2026-10-06 at 611.7 s for the owner's own +// order, 2 GB, 1920x1080, thirty minutes (docs/WEBM-WYDAJNOSC-2026-10-06.md). +// The pictures do not depend on one another, so they can be coded at once and +// written in order, and the bytes are the ones one goroutine makes. Measured +// with tools/probes/videoparallel, 48 pictures at 1920x1080, three rounds: +// +// W1 1.00x W2 1.81x W4 3.24x W8 5.12x W12 6.95x W16 7.60x +// +// with every picture the same size at every width. +// +// The limit is the process's, not the film's. The engine already writes +// GOMAXPROCS files at once, so helpers taken per film would be sixteen films +// times sixteen pictures in memory. Here the goroutine writing a film always +// codes, which is what guarantees it moves on, and the helpers beside it come +// from one count for the whole process, GOMAXPROCS less one. A film that +// finds none free codes alone, as every film did before, and asks again at its +// next picture. + +// helping is how many helpers are coding beside the writers right now. +var helping atomic.Int64 + +// codedBeside counts the pictures a helper coded rather than the goroutine +// that asked for them, since the process started. +var codedBeside atomic.Int64 + +// Helpers is how many helpers are coding right now and how many pictures +// helpers have coded since the process started. Guards read it: a film held to +// the bytes of one goroutine says nothing unless several really coded it, and +// a film abandoned half way has to leave none of them running. +func Helpers() (running, coded int64) { return helping.Load(), codedBeside.Load() } + +// takeHelper claims a place for one more helper, if the process has one. +func takeHelper() bool { + limit := int64(runtime.GOMAXPROCS(0) - 1) + for { + n := helping.Load() + if n >= limit { + return false + } + if helping.CompareAndSwap(n, n+1) { + return true + } + } +} + +// job is one picture to code. Whoever takes it first - a helper, or the +// writer that needs it - codes it, and done is released when it is coded, so +// the writer waits only for a picture somebody else is coding. A WaitGroup +// inside the job rather than a channel beside it, because a channel is one +// more allocation for every picture of every film. +type job struct { + look look + taken atomic.Bool + done sync.WaitGroup + coded Coded + err error + // panicked is what coding it panicked with. A panic on a helper would end + // the process, where the same panic on the writer becomes the run's error + // (internal/engine, writeWithoutCrashing), so a helper keeps it here and + // the writer panics with it when it reaches the picture. + panicked any +} + +func newJob(l look) *job { + j := &job{look: l} + j.done.Add(1) + return j +} + +// codedJob is a picture already coded, which nobody codes again. +func codedJob(c Coded) *job { + j := &job{coded: c} + j.taken.Store(true) + return j +} + +// crew codes the pictures of one film: the goroutine that owns it, and the +// helpers it has been able to take. Jobs are offered in the order the film +// shows them and waited for in the same order, so the first error a film +// meets is the first in its order, whoever coded what. +type crew struct { + film *film + qindex int + own *painter + queue chan *job + most int + helpers int + wg sync.WaitGroup + stopped atomic.Bool + closed bool +} + +// newCrew is the crew for a film of this many pictures. A film of one picture +// takes no helper, and neither does a process of one thread. +func newCrew(f *film, qindex int, pictures int64) *crew { + c := &crew{film: f, qindex: qindex, own: f.painter()} + if most := min(int64(runtime.GOMAXPROCS(0)-1), pictures-1); most > 0 { + c.most = int(most) + c.queue = make(chan *job, c.ahead()) + } + return c +} + +// ahead is how many pictures past the one being written may be offered: two +// for each goroutine that codes, so none of them waits for the next picture +// while the writer is still busy with the last. Nought without helpers - a +// film coded alone looks at no picture before it needs it. +func (c *crew) ahead() int { + if c.most == 0 { + return 0 + } + return 2 * (c.most + 1) +} + +// offer hands a picture to the helpers, taking one more first when the film +// may have more and the process has a place for it. With no helper the +// writer codes it when it gets there. Called by the owning goroutine only. +func (c *crew) offer(j *job) { + if c.queue == nil || c.closed { + return + } + if c.helpers < c.most && takeHelper() { + c.helpers++ + c.wg.Add(1) + go c.help() + } + if c.helpers == 0 { + return + } + select { + case c.queue <- j: + default: + // The queue is as long as the window, so this is a picture the + // writer will reach before any helper would have. + } +} + +func (c *crew) help() { + defer c.wg.Done() + defer helping.Add(-1) + p := c.film.painter() + for j := range c.queue { + if c.stopped.Load() { + continue + } + if j.taken.CompareAndSwap(false, true) { + c.code(p, j) + codedBeside.Add(1) + } + } +} + +func (c *crew) code(p *painter, j *job) { + defer j.done.Done() + defer func() { + if v := recover(); v != nil { + j.panicked = v + } + }() + j.coded, j.err = Encode(p.draw(j.look), c.qindex) +} + +// wait is the picture of j, coded here when no helper has taken it. +func (c *crew) wait(j *job) (Coded, error) { + if j.taken.CompareAndSwap(false, true) { + c.code(c.own, j) + } + j.done.Wait() + if j.panicked != nil { + panic(j.panicked) + } + return j.coded, j.err +} + +// finish says no picture is coming after the ones offered, so the helpers end +// once the queue is empty and give their places back to the process. +func (c *crew) finish() { + if c.queue != nil && !c.closed { + c.closed = true + close(c.queue) + } +} + +// stop ends the crew and waits for its helpers. A helper in the middle of a +// picture finishes that picture - gav1d cannot be interrupted, and that was +// so before there were helpers - and codes nothing after it. Nothing a crew +// started is running once this returns. +func (c *crew) stop() { + c.stopped.Store(true) + c.finish() + c.wg.Wait() +} diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go index f702d2e6..f3f86323 100644 --- a/internal/format/video/choose.go +++ b/internal/format/video/choose.go @@ -88,17 +88,29 @@ func (c Choice) ClockShown(t Timeline) bool { return ClockShown(c.Width, c.Heigh // ceilings keep over their own measurement. The gradient does not move between // pictures, by the owner's decision of the same day, because moving it changed // a picture's size by up to 1.9 times. +// +// The sample is coded the way a film is (ahead.go), all of it offered at once, +// because at 1920x1080 ten pictures are three seconds of planning on one +// goroutine, and planning is what a preview waits for. func sampleReserve(w, h int, seed uint64, label string, t Timeline, qindex int) (Coded, int, error) { - p := newPainter(w, h, seed, label, t) + f := newFilm(w, h, seed, label, t) sample := SampleChanges(t) + c := newCrew(f, qindex, int64(len(sample))) + defer c.stop() + jobs := make([]*job, len(sample)) + for i, ch := range sample { + jobs[i] = newJob(f.lookOf(ch)) + c.offer(jobs[i]) + } + c.finish() var first Coded largest := 0 - for _, c := range sample { - coded, err := Encode(p.paint(c), qindex) + for i, ch := range sample { + coded, err := c.wait(jobs[i]) if err != nil { return Coded{}, 0, err } - if c == 0 { + if ch == 0 { first = coded } largest = max(largest, coded.Size()) diff --git a/internal/format/video/picture.go b/internal/format/video/picture.go index df3fba2e..9f2c9026 100644 --- a/internal/format/video/picture.go +++ b/internal/format/video/picture.go @@ -5,6 +5,7 @@ import ( "image" "image/color" "image/draw" + "slices" "github.com/donislawdev/TestingFilesGenerator/internal/format/imagelabel" ) @@ -33,11 +34,22 @@ var ink = color.RGBA{R: 240, G: 240, B: 240, A: 255} // Picture draws picture c of a film: the gradient every image format here // draws, moved by the seed, with the label burned into the top of it when // there is room, the clock under it reading when the picture starts, and the -// square at its step. A film draws its pictures through one painter, so this -// is for planning, which codes the first, and for the probes and guards that -// measure any one of them. +// square at its step. It is drawn the way a film draws it, and a film draws +// its pictures through painters of its own, so this is for the probes and +// guards that measure any one of them. func Picture(width, height int, seed uint64, label string, t Timeline, c int64) Planes { - return newPainter(width, height, seed, label, t).paint(c) + f := newFilm(width, height, seed, label, t) + return f.painter().draw(f.lookOf(c)) +} + +// WholePicture is picture c painted whole - the gradient copied, the clock and +// the square drawn on the copy, and every pixel of it converted - which is how +// every picture of a film was painted until 2026-10-06. A film now paints only +// the rows that can change (painter.draw), and a guard holds the two to the +// same planes, because the bytes of every film depend on them being the same. +func WholePicture(width, height int, seed uint64, label string, t Timeline, c int64) Planes { + f := newFilm(width, height, seed, label, t) + return f.whole(f.lookOf(c)) } // Labelled says whether a picture this wide carries a readable label - asked @@ -76,21 +88,28 @@ func Clock(c int64, t Timeline) string { return fmt.Sprintf("%s.%03d", hms, ms%1000) } -// painter draws the pictures of one film. The gradient and the label are the -// same in every one of them, so they are drawn once, and each change copies -// them and adds only what moves - the clock and the square. -type painter struct { - t Timeline - base *image.RGBA - work *image.RGBA - planes Planes - clockFrom int // the row the clock's band starts at - showClock bool - side int - square *image.Uniform +// film is what every picture of one film has in common, made once and only +// read after: the gradient with the label burned in, the same gradient in +// planes, and where the clock and the square go. The painters of one film +// share it, which is what lets several of them paint at once (ahead.go). +type film struct { + t Timeline + base *image.RGBA + basePlanes Planes + planesBuf []uint8 // what basePlanes lie over, copied whole by a painter + clockFrom int // the row the clock's band starts at + clockEnd int // the row under the band, where Draw stops painting it + showClock bool + side int + squareY int + square *image.Uniform + // strips are the only rows a picture can differ from the gradient in - + // the clock's band and the square's - each widened to whole pairs of + // rows, because one chroma sample covers two, and merged where they meet. + strips []image.Rectangle } -func newPainter(width, height int, seed uint64, label string, t Timeline) *painter { +func newFilm(width, height int, seed uint64, label string, t Timeline) *film { off := int(seed % 256) base := image.NewRGBA(image.Rect(0, 0, width, height)) for y := range height { @@ -102,15 +121,61 @@ func newPainter(width, height int, seed uint64, label string, t Timeline) *paint if Labelled(width, label) { imagelabel.Draw(base, label) } - cw, ch := (width+1)/2, (height+1)/2 - return &painter{ - t: t, base: base, work: image.NewRGBA(base.Rect), - planes: Planes{Width: width, Height: height, Y: make([]uint8, width*height), U: make([]uint8, cw*ch), V: make([]uint8, cw*ch)}, + f := &film{ + t: t, base: base, clockFrom: labelBand(width, label), showClock: ClockShown(width, height, label, t), side: min(max(width/squareDivisor, 1), maxSquareSide, height), square: image.NewUniform(ink), } + f.basePlanes, f.planesBuf = newPlanes(width, height) + toPlanes(base, f.basePlanes) + f.squareY = (height - f.side) * 3 / 4 + f.clockEnd = f.clockFrom + if f.showClock { + f.clockEnd = min(height, f.clockFrom+imagelabel.BandHeight(width, len(Clock(0, t)))) + } + f.strips = changingRows(width, height, [][2]int{{f.clockFrom, f.clockEnd}, {f.squareY, f.squareY + f.side}}) + return f +} + +// newPlanes is the three planes of a picture this size in one allocation, +// and that allocation, which a painter copies whole (film.painter). +func newPlanes(width, height int) (Planes, []uint8) { + cw, ch := (width+1)/2, (height+1)/2 + buf := make([]uint8, width*height+2*cw*ch) + return planesOver(buf, width, height), buf +} + +// planesOver lays the three planes out over buf, luma first. Each is capped +// at its own length, so no plane can grow into the next. +func planesOver(buf []uint8, width, height int) Planes { + n, c := width*height, ((width+1)/2)*((height+1)/2) + return Planes{Width: width, Height: height, Y: buf[:n:n], U: buf[n : n+c : n+c], V: buf[n+c : n+2*c : n+2*c]} +} + +// changingRows turns runs of rows into strips the width of the picture, +// each starting on an even row and ending on an even row or the last one, so +// that every chroma sample a strip converts covers only rows the strip holds, +// and merges the strips that overlap or touch. A run of no rows - the clock's, +// on a picture with no room for it - gives no strip. +func changingRows(width, height int, runs [][2]int) []image.Rectangle { + var spans [][2]int + for _, r := range runs { + if r[0] < r[1] { + spans = append(spans, [2]int{r[0] &^ 1, min(height, (r[1]+1)&^1)}) + } + } + slices.SortFunc(spans, func(a, b [2]int) int { return a[0] - b[0] }) + var out []image.Rectangle + for _, s := range spans { + if n := len(out); n > 0 && s[0] <= out[n-1].Max.Y { + out[n-1].Max.Y = max(out[n-1].Max.Y, s[1]) + continue + } + out = append(out, image.Rect(0, s[0], width, s[1])) + } + return out } // look is what tells two pictures of one film apart: the clock's text when it @@ -122,28 +187,84 @@ type look struct { x int } -func (p *painter) lookOf(c int64) look { - l := look{x: (p.base.Rect.Dx() - p.side) * int(c%squareSteps) / (squareSteps - 1)} - if p.showClock { - l.clock = Clock(c, p.t) +func (f *film) lookOf(c int64) look { + l := look{x: (f.base.Rect.Dx() - f.side) * int(c%squareSteps) / (squareSteps - 1)} + if f.showClock { + l.clock = Clock(c, f.t) } return l } -// paint draws picture c into the painter's own planes and returns them. They -// are overwritten by the next call. -func (p *painter) paint(c int64) Planes { return p.draw(p.lookOf(c)) } +// whole paints a picture the long way, into planes of its own: the whole +// gradient copied, the clock and the square drawn on the copy, every pixel +// converted. It is the reference painter.draw is held to (WholePicture). +func (f *film) whole(l look) Planes { + work := image.NewRGBA(f.base.Rect) + copy(work.Pix, f.base.Pix) + if f.showClock { + b := work.Rect + imagelabel.Draw(work.SubImage(image.Rect(0, f.clockFrom, b.Dx(), b.Dy())).(*image.RGBA), l.clock) + } + draw.Draw(work, image.Rect(l.x, f.squareY, l.x+f.side, f.squareY+f.side), f.square, image.Point{}, draw.Src) + planes, _ := newPlanes(f.base.Rect.Dx(), f.base.Rect.Dy()) + toPlanes(work, planes) + return planes +} + +// painter draws pictures of one film into planes of its own. They start as +// the gradient's, and every row outside the film's strips stays the +// gradient's in every picture, so a picture repaints and converts the strips +// and nothing else - at 1920x1080 the clock's band and the square's are about +// a sixth of the rows. One painter is used by one goroutine at a time. +type painter struct { + f *film + planes Planes + strips []image.RGBA +} + +// painter is a painter of this film, in four allocations whatever the size: +// itself, its planes, its strips, and the pixels of all of them. Every helper +// coding a film makes one, and the allocations a file may cost are counted +// (the AllocCeiling of the formats built on this). +func (f *film) painter() *painter { + w := f.base.Rect.Dx() + stride := 4 * w + rows := 0 + for _, r := range f.strips { + rows += r.Dy() + } + pix := make([]uint8, rows*stride) + p := &painter{f: f, planes: planesOver(slices.Clone(f.planesBuf), w, f.base.Rect.Dy()), strips: make([]image.RGBA, len(f.strips))} + for i, r := range f.strips { + n := r.Dy() * stride + p.strips[i] = image.RGBA{Pix: pix[:n:n], Stride: stride, Rect: r} + pix = pix[n:] + } + return p +} -// draw is paint for a picture whose look is already worked out. +// draw paints the picture with this look into the painter's planes and +// returns them. They are overwritten by the next call. +// +// Each strip is the gradient's rows copied, the clock drawn when its band +// starts in the strip, the square drawn where it crosses the strip, and the +// rows converted - the same steps, in the same order, as whole, on fewer +// rows. The clock's band always lies whole inside one strip, because the +// strips were cut around it, so Draw sizes it as it would on the whole +// picture. func (p *painter) draw(l look) Planes { - copy(p.work.Pix, p.base.Pix) - if p.showClock { - b := p.work.Rect - imagelabel.Draw(p.work.SubImage(image.Rect(0, p.clockFrom, b.Dx(), b.Dy())).(*image.RGBA), l.clock) - } - y := (p.base.Rect.Dy() - p.side) * 3 / 4 - draw.Draw(p.work, image.Rect(l.x, y, l.x+p.side, y+p.side), p.square, image.Point{}, draw.Src) - toPlanes(p.work, p.planes) + f := p.f + w := f.base.Rect.Dx() + for i := range p.strips { + s := &p.strips[i] + r := s.Rect + copy(s.Pix, f.base.Pix[r.Min.Y*f.base.Stride:r.Max.Y*f.base.Stride]) + if f.showClock && r.Min.Y <= f.clockFrom && f.clockFrom < r.Max.Y { + imagelabel.Draw(s.SubImage(image.Rect(0, f.clockFrom, w, r.Max.Y)).(*image.RGBA), l.clock) + } + draw.Draw(s, image.Rect(l.x, f.squareY, l.x+f.side, f.squareY+f.side), f.square, image.Point{}, draw.Src) + convertRows(s, p.planes, r.Min.Y, r.Max.Y) + } return p.planes } @@ -162,30 +283,38 @@ func (p *painter) draw(l look) Planes { // // Chroma is the rounded mean of the two by two block it covers, so an odd // width or height averages the pixels that are there. -func toPlanes(img *image.RGBA, p Planes) { - w, h := img.Rect.Dx(), img.Rect.Dy() - cw, ch := (w+1)/2, (h+1)/2 - for y := range h { - row := img.Pix[y*img.Stride:] +func toPlanes(img *image.RGBA, p Planes) { convertRows(img, p, 0, p.Height) } + +// convertRows converts rows r0 up to r1 of the picture p is, read from img - +// the whole picture, or a strip of it that starts at row r0. r0 is even and r1 +// is even or the picture's height, so every chroma sample written here covers +// rows img holds. +func convertRows(img *image.RGBA, p Planes, r0, r1 int) { + w, h := p.Width, p.Height + top := img.Rect.Min.Y + for y := r0; y < r1; y++ { + row := img.Pix[(y-top)*img.Stride:] for x := range w { r, g, b := int(row[4*x]), int(row[4*x+1]), int(row[4*x+2]) p.Y[y*w+x] = uint8(16 + (47*r+157*g+16*b+128)>>8) } } - for cy := range ch { + cw := (w + 1) / 2 + for cy := r0 / 2; cy < (r1+1)/2; cy++ { for cx := range cw { - p.U[cy*cw+cx], p.V[cy*cw+cx] = chroma(img, 2*cx, 2*cy) + p.U[cy*cw+cx], p.V[cy*cw+cx] = chroma(img, 2*cx, 2*cy, w, h) } } } // chroma is the Cb and Cr of the two by two block whose top left is x0, y0, -// over the pixels of it that are inside the picture. -func chroma(img *image.RGBA, x0, y0 int) (cb, cr uint8) { +// over the pixels of it that are inside a picture w by h. +func chroma(img *image.RGBA, x0, y0, w, h int) (cb, cr uint8) { sumB, sumR, n := 0, 0, 0 - for y := y0; y < min(y0+2, img.Rect.Dy()); y++ { - row := img.Pix[y*img.Stride:] - for x := x0; x < min(x0+2, img.Rect.Dx()); x++ { + top := img.Rect.Min.Y + for y := y0; y < min(y0+2, h); y++ { + row := img.Pix[(y-top)*img.Stride:] + for x := x0; x < min(x0+2, w); x++ { r, g, b := int(row[4*x]), int(row[4*x+1]), int(row[4*x+2]) sumB += -26*r - 86*g + 112*b sumR += 112*r - 102*g - 10*b diff --git a/internal/format/video/stream.go b/internal/format/video/stream.go index 48f6a70f..5848bb05 100644 --- a/internal/format/video/stream.go +++ b/internal/format/video/stream.go @@ -93,27 +93,46 @@ func joined(a, b []byte) []byte { return append(append(make([]byte, 0, len(a)+len(b)), a...), b...) } -// Pictures codes the pictures of one film in order, one change at a time, and -// keeps only the last - an hour at a change a second is 3600 pictures, and -// holding them all would hold the film. +// Pictures codes the pictures of one film in the order the film shows them, +// on the goroutine that asks for them and on the helpers it can take +// (ahead.go), and keeps only the last one it was asked for and the few coded +// ahead of it - an hour at a change a second is 3600 pictures, and holding +// them all would hold the film. +// +// Close has to be called when the film is written or abandoned, because the +// helpers are goroutines. type Pictures struct { - choice Choice - stream Stream - painter *painter + choice Choice + stream Stream + film *film + crew *crew + // pending are the pictures offered and not yet reached, in order, each + // with the change it opens. Changes that look the same as the one before + // open none, so they are not in it. + pending []openedBy + next int64 // the first change not looked at yet + last look // the look of change next-1 change int64 - look look key []byte copied []byte } +type openedBy struct { + change int64 + job *job +} + // Pictures is the coder of this choice's film. func (c Choice) Pictures(st Stream) *Pictures { - return &Pictures{choice: c, stream: st, change: -1, - painter: newPainter(c.Width, c.Height, c.Seed, c.Label, st.Timeline)} + f := newFilm(c.Width, c.Height, c.Seed, c.Label, st.Timeline) + cr := newCrew(f, c.QIndex, st.Changes()) + return &Pictures{choice: c, stream: st, film: f, crew: cr, change: -1, + pending: make([]openedBy, 0, cr.ahead()+1)} } // At makes picture c the current one, coding it unless it looks the same as -// the one before. +// the one before. Pictures are asked for in the order the film shows them - +// the same change again is fine, an earlier one is a defect of the caller. // // Every picture is held to the stream's reserve, because the level and the // file around it were planned on it. A picture over it is a ceiling that did @@ -124,30 +143,68 @@ func (p *Pictures) At(c int64) error { if c == p.change { return nil } - l := p.painter.lookOf(c) - if p.change >= 0 && l == p.look { + if c < p.change { + return core.Defect(fmt.Errorf("video: picture %d was asked for after picture %d, and a film's pictures are coded in the order it shows them", c, p.change)) + } + p.offerTo(c) + var opened *job + reached := 0 + for reached < len(p.pending) && p.pending[reached].change <= c { + opened = p.pending[reached].job + reached++ + } + // Moved down rather than sliced off, so the one array made for the + // window lasts the whole film. + p.pending = p.pending[:copy(p.pending, p.pending[reached:])] + if opened == nil { + // The same look as the picture before it: shown again, not coded. p.change = c return nil } - var coded Coded - if c == 0 && p.choice.First != nil { - coded = *p.choice.First - } else { - var err error - if coded, err = Encode(p.painter.draw(l), p.choice.QIndex); err != nil { - return err - } + coded, err := p.crew.wait(opened) + if err != nil { + return err } if coded.Size() > p.stream.Reserve { return core.Defect(fmt.Errorf("video: picture %d of a %dx%d film coded to a %d B tile and the film was planned on %d B a picture, so the file cannot be kept", c, p.choice.Width, p.choice.Height, coded.Size(), p.stream.Reserve)) } - p.change, p.look = c, l + p.change = c p.key = joined(p.stream.seq, keyFrame(coded)) p.copied = joined(hiddenCopy(coded), p.stream.show) return nil } +// offerTo looks at every change up to c and as far past it as the crew codes +// ahead, and offers each one that looks different from the one before it. +// Change 0 is the picture planning already coded, when it coded one. +func (p *Pictures) offerTo(c int64) { + n := p.stream.Changes() + for p.next < n && (p.next <= c || len(p.pending) < p.crew.ahead()) { + l := p.film.lookOf(p.next) + if p.next == 0 || l != p.last { + j := (*job)(nil) + if p.next == 0 && p.choice.First != nil { + j = codedJob(*p.choice.First) + } else { + j = newJob(l) + p.crew.offer(j) + } + p.pending = append(p.pending, openedBy{change: p.next, job: j}) + } + p.last = l + p.next++ + } + if p.next == n { + p.crew.finish() + } +} + +// Close stops the helpers and waits for them, so nothing this film started is +// running once it returns. A helper in the middle of a picture finishes it +// first, which is at most one picture's time. +func (p *Pictures) Close() { p.crew.stop() } + // KeySample is the current picture as a key frame, with the sequence header // before it. func (p *Pictures) KeySample() []byte { return p.key } diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index 3d0456f7..cdfdb5d1 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -52,6 +52,11 @@ func init() { // object allocated per frame, the defect the ceiling exists for. The // number below sits between the two, and like every ceiling here it // goes down when work makes it lowerable, never up to turn a run green. + // Since the pictures are coded beside each other (internal/format/video, + // ahead.go) the default film is 474 to 477 objects at sixteen, four and + // one threads alike - each picture is a job, each helper one painter of + // four allocations - measured the same day, still under the ceiling, + // which was not moved. AllocCeiling: 512, Padding: format.PaddingChannel{ @@ -183,7 +188,9 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { out.write(seekHead(l.seekHeadLen, l.seekHeadLen+uint64(len(l.info)), cuesAt)) out.write(l.info) out.write(l.tracks) - keys, end, err := writeClusters(ctx, out, l, m.choice.Pictures(st)) + pics := m.choice.Pictures(st) + defer pics.Close() + keys, end, err := writeClusters(ctx, out, l, pics) if err != nil { return err } diff --git a/internal/guard/concurrency_test.go b/internal/guard/concurrency_test.go index 622dff6b..a279e3d0 100644 --- a/internal/guard/concurrency_test.go +++ b/internal/guard/concurrency_test.go @@ -101,6 +101,18 @@ var mayBeConcurrent = map[string]string{ // Numbers, the instrument, and the two mistakes made getting them: // docs/PERFORMANCE-REVIEW-2026-09-05.md section 14. "internal/engine/parallel.go": "the planned files are written beside each other, and nothing else in the package does", + // A film is its pictures, each a whole picture through gav1d on one + // goroutine: the owner's thirty minute 1920x1080 film was 611.7 s on one + // core of sixteen, and the window's estimate from the bytes said three + // hours. The pictures are independent, so several goroutines code them and + // the writer takes them in order - 7.60x at sixteen threads, 5.12x at + // eight, every picture the same size at every width (tools/probes/ + // videoparallel). Helpers come from one count for the whole process, + // because the engine above already writes a file per thread. + // + // Added 2026-10-06 and THE OWNER DECIDED IT: + // docs/WEBM-WYDAJNOSC-2026-10-06.md sections 4 and 5. + "internal/format/video/ahead.go": "the pictures of one film are coded beside each other, from one count of helpers for the process", } // Waiting on cancellation is not the same thing as running in parallel. Every diff --git a/internal/guard/filmahead_test.go b/internal/guard/filmahead_test.go new file mode 100644 index 00000000..6cdb611b --- /dev/null +++ b/internal/guard/filmahead_test.go @@ -0,0 +1,163 @@ +package guard + +import ( + "bytes" + "context" + "runtime" + "slices" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// The pictures of one film are coded by several goroutines at once and +// painted a strip at a time (internal/format/video, ahead.go and picture.go, +// docs/WEBM-WYDAJNOSC-2026-10-06.md). Neither may change a byte of any film, +// and neither may leave anything running once the film is done with. + +// A film paints only the rows that can change - the clock's band and the +// square's, widened to whole pairs of rows - onto planes made once from the +// gradient. It has to come out as the picture painted whole, the way every +// film was painted before, in every plane and every sample. +// +// The sizes are the ones where strips go wrong: one pixel, odd sides that +// leave a chroma sample half outside, a picture short enough that the square +// meets the clock's band, the smallest that still shows the clock, and the +// ladder's top rung. Each with the label and without, and each with the clock +// in whole seconds and with milliseconds, because those are bands of two +// lengths. +func TestAFilmPaintsOnlyWhatChangesAndGetsTheWholePicture(t *testing.T) { + sizes := [][2]int{{1, 1}, {2, 2}, {3, 3}, {16, 9}, {17, 11}, {48, 32}, {52, 20}, {64, 48}, {97, 41}, {160, 90}, {161, 91}, {640, 360}} + timelines := map[string]video.Timeline{} + for name, ms := range map[string][2]int64{"seconds": {3_600_000, 1_000}, "milliseconds": {60_000, 100}} { + tl, err := video.NewTimeline("webm", ms[0], 60_000, ms[1], 30) + if err != nil { + t.Fatal(err) + } + timelines[name] = tl + } + labels := map[string]string{"labelled": core.Label("webm", 1<<20, 7), "unlabelled": ""} + + compared, clocked, moved := 0, 0, 0 + for _, size := range sizes { + w, h := size[0], size[1] + for tName, tl := range timelines { + for lName, label := range labels { + changes := []int64{0, 1, 2, 9, 10, 11, tl.Changes() / 2, tl.Changes() - 1} + var first video.Planes + for i, c := range changes { + got := video.Picture(w, h, 7, label, tl, c) + want := video.WholePicture(w, h, 7, label, tl, c) + for plane, pair := range map[string][2][]uint8{"Y": {got.Y, want.Y}, "U": {got.U, want.U}, "V": {got.V, want.V}} { + if !bytes.Equal(pair[0], pair[1]) { + t.Errorf("%dx%d %s %s, picture %d: the %s plane painted a strip at a time differs from the picture painted whole", w, h, tName, lName, c, plane) + } + } + compared++ + if i == 0 { + first = got + first.Y = slices.Clone(got.Y) + } else if !bytes.Equal(first.Y, got.Y) { + moved++ + } + } + if video.ClockShown(w, h, label, tl) { + clocked++ + } + } + } + } + // Asserted, not assumed: a strip painter that drew nothing would agree with + // a whole painter that drew nothing, so the pictures have to have moved, + // and the clock has to have been in some of them. + if compared < 300 || moved < 100 || clocked < 10 { + t.Fatalf("%d pictures compared, %d of them different from their film's first, %d films with a clock - the guard did not see pictures that change", + compared, moved, clocked) + } +} + +// A film coded by several goroutines has the bytes of the same film coded by +// one, and the several really did code it. +// +// The second half is the one that keeps the first honest. With one thread the +// process has no helper to give, so the run under one thread is the reference +// and has to have used none, and the run under eight has to have used some - +// otherwise both runs are the same goroutine and the comparison is of a film +// with itself. One film is sized by hand, so planning codes its sample through +// the helpers too, and one is chosen to fit. +func TestAFilmCodedBySeveralGoroutinesHasTheBytesOfOne(t *testing.T) { + targets := []engine.Target{ + filmTarget(2<<20, map[string]string{"width": "64", "height": "48", "duration": "40s"}), + filmTarget(1<<20, map[string]string{"duration": "30s", "change_interval": "500ms"}), + } + for _, target := range targets { + alone, helped := filmUnder(t, 1, target), filmUnder(t, 8, target) + if alone.coded != 0 { + t.Errorf("%v: under one thread helpers coded %d pictures, so there is no film of one goroutine to compare with", target.Properties, alone.coded) + } + if helped.coded == 0 { + t.Errorf("%v: under eight threads no helper coded anything, so the film of several goroutines was never made", target.Properties) + } + if !bytes.Equal(alone.bytes, helped.bytes) { + t.Errorf("%v: the film coded with helpers differs from the film coded by one goroutine (%d B and %d B)", target.Properties, len(helped.bytes), len(alone.bytes)) + } + } +} + +type filmRun struct { + bytes []byte + coded int64 +} + +func filmUnder(t *testing.T, threads int, target engine.Target) filmRun { + t.Helper() + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(threads)) + _, before := video.Helpers() + b, _ := filmOne(t, target) + _, after := video.Helpers() + return filmRun{bytes: b, coded: after - before} +} + +// A film stopped half way leaves no helper running when the run returns. +// +// Helpers are goroutines, and one left behind holds a picture's memory and a +// place the next film cannot take - and the window, which waits for the run +// before it closes (G7), would be waiting for something that no longer waits +// for anybody. The run is stopped once pictures are being written, which is +// asserted: stopped at the file's first bytes it would end before any helper +// was taken and pass without asking anything. +func TestAFilmStoppedHalfWayLeavesNoHelperRunning(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8)) + target := filmTarget(8<<20, map[string]string{"width": "160", "height": "90", "duration": "10m"}) + dir := t.TempDir() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + opt := engine.Options{OutDir: dir, Seed: goldenSeed, Command: "test"} + planned, err := engine.Plan([]engine.Target{target}, opt) + if err != nil { + t.Fatal(err) + } + if changes, _ := planned[0].Plan.Properties["change_count"].(int64); changes < 100 { + t.Fatalf("the film has %d pictures, too few to be stopped in the middle of them", changes) + } + _, codedBefore := video.Helpers() + opt.OnProgress = func(p engine.Progress) { + // Some forty pictures in, of six hundred. + if p.BytesDone > 100_000 { + cancel() + } + } + res, runErr := engine.Run(ctx, planned, opt) + running, codedAfter := video.Helpers() + if runErr == nil && (res == nil || len(res.Manifest.Files) > 0) { + t.Fatalf("the run finished the film, so it was never stopped half way") + } + if codedAfter == codedBefore { + t.Fatalf("no helper coded a picture before the stop, so there was nothing that could have been left running") + } + if running != 0 { + t.Errorf("%d helpers are still running after the stopped run returned", running) + } +} From d2a4097255133f28c202f9502c1121276ca8a96d Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Tue, 6 Oct 2026 20:36:46 +0200 Subject: [PATCH 2/3] progress: the bar counts a film's pictures, not only its bytes A film's bytes are mostly padding written in its last seconds. Counting bytes, the bar stood at five percent while the pictures were coded and the estimate for a thirty minute 1920x1080 film said three hours for a run of ten minutes - 1315 s for one of 66 s on the parallel coder. format.Plan gains Work: what writing a file costs beyond its bytes, counted as the bytes that would take as long to write. A generator reports it through format.Worked on the context it writes under, not through the writer, because a damaged file is written through the damage. The engine keeps work beside bytes in every report and squares both up when a file ends. Every file whose cost is its bytes declares no Work, so its reports are what they were. webm declares its pictures' work from their pixels and reports each change as its cluster is written. The window and the command line take the percentage and the time left from the work. The line still shows the bytes. Measured on the owner's film with tools/probes/filmeta: the estimate from the work is within two seconds of the time really left from the tenth second on. Guard: a film's work runs well ahead of its bytes while the pictures are written, it ends on both totals, and a text file reports work equal to its bytes at every step. Reddened by hand by taking the report out. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++- internal/cli/progress.go | 8 +++-- internal/engine/engine.go | 18 +++++++++++ internal/engine/parallel.go | 46 ++++++++++++++++++++------- internal/format/format.go | 28 +++++++++++++++++ internal/format/video/stream.go | 18 +++++++++++ internal/format/webm/webm.go | 12 +++++-- internal/guard/filmahead_test.go | 54 ++++++++++++++++++++++++++++++++ internal/gui/window/run.go | 2 +- internal/gui/window/runreport.go | 11 +++++-- 10 files changed, 180 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e86f3f19..462f4c85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,7 +26,9 @@ because it turns other people's test suites red. `00:00:07`, with milliseconds when the changes do not fall on whole seconds. Each change is a whole new picture in bytes and in coding time, and the pictures of a film are coded on every core of the machine at once - - a thirty minute 1920x1080 film took 68 s on sixteen threads. The frames + a thirty minute 1920x1080 film took 68 s on sixteen threads, and the + progress bar counts the pictures as well as the bytes, so the time it says + is left holds while they are coded. The frames between changes cost a few bytes each, so an hour at thirty frames a second with a change every second fits in about a megabyte (1 075 831 B at the smallest), and a `change_interval` as long as the film diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 12679d60..140b08e3 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -58,7 +58,7 @@ func (p *progressBar) report(pr engine.Progress) { line := fmt.Sprintf(" %d/%d files %s of %s %d%%%s", pr.FilesDone, pr.FilesTotal, core.HumanBytes(pr.BytesDone), core.HumanBytes(pr.BytesTotal), - core.Percent(pr.BytesDone, pr.BytesTotal), + core.Percent(pr.WorkDone, pr.WorkTotal), p.remaining(pr)) // Pad to cover whatever the last line left behind, so a shorter line does @@ -83,13 +83,15 @@ func (p *progressBar) clear() { // remaining is an estimate and says so by staying quiet until it has enough to // go on. A number that swings wildly for the first second is worse than none. +// It comes from the work, like the percentage, and not from the bytes - see +// engine.Progress for the film that promised three hours for a minute. func (p *progressBar) remaining(pr engine.Progress) string { elapsed := time.Since(p.started) - if elapsed < time.Second || pr.BytesDone <= 0 || pr.BytesDone >= pr.BytesTotal { + if elapsed < time.Second || pr.WorkDone <= 0 || pr.WorkDone >= pr.WorkTotal { return "" } left := time.Duration(float64(elapsed) * - float64(pr.BytesTotal-pr.BytesDone) / float64(pr.BytesDone)) + float64(pr.WorkTotal-pr.WorkDone) / float64(pr.WorkDone)) return " " + core.Roughly(left) + " left" } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4482f151..4a084b50 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -461,13 +461,31 @@ func TotalBytes(files []PlannedFile) int64 { return n } +// totalWork is what the planned files cost to make, their bytes included. +func totalWork(files []PlannedFile) int64 { + var n int64 + for _, f := range files { + n += f.Plan.Bytes + f.Plan.Work + } + return n +} + // Progress is how far a run has got. Both counts are known from the plan, so // the fractions are exact rather than estimated. +// +// A fraction of the run and the time it has left come from the work, not the +// bytes. The work is the bytes plus what the plans counted beyond them +// (format.Plan.Work), so for a run of files whose cost is their bytes the two +// are the same numbers, and for a film they are the pictures as well as the +// padding - the bytes alone stood at five percent while a film's pictures were +// coded and promised hours for a minute's work. type Progress struct { FilesDone int FilesTotal int BytesDone int64 BytesTotal int64 + WorkDone int64 + WorkTotal int64 } // Run writes a planned set of files. diff --git a/internal/engine/parallel.go b/internal/engine/parallel.go index 1eb30761..2a1868ff 100644 --- a/internal/engine/parallel.go +++ b/internal/engine/parallel.go @@ -18,6 +18,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/damage" + "github.com/donislawdev/TestingFilesGenerator/internal/format" ) // This file is the only place in internal/engine that runs anything beside @@ -157,7 +158,7 @@ func drain(ctx context.Context, next *atomic.Int64, files []PlannedFile, outDir p := fileProgress{gate: gate} sum, err := writeOne(ctx, files[i], outDir, &p) out[i] = fileResult{sha: sum, ok: err == nil, err: err} - p.finished(files[i].Plan.Bytes, err == nil) + p.finished(files[i].Plan, err == nil) } } @@ -182,12 +183,17 @@ func drain(ctx context.Context, next *atomic.Int64, files []PlannedFile, outDir // // A nil gate is a run nobody is watching. Every method takes a nil receiver, // so a run without progress does no locking and allocates nothing for it. +// +// Work moves with the bytes, and on its own where a generator reports work its +// bytes do not show (format.Plan.Work) - the pictures of a film. type progressGate struct { mu sync.Mutex filesDone int bytesDone int64 + workDone int64 filesTotal int bytesTotal int64 + workTotal int64 report func(Progress) } @@ -198,6 +204,7 @@ func newProgressGate(files []PlannedFile, report func(Progress)) *progressGate { return &progressGate{ filesTotal: len(files), bytesTotal: TotalBytes(files), + workTotal: totalWork(files), report: report, } } @@ -209,25 +216,27 @@ func newProgressGate(files []PlannedFile, report func(Progress)) *progressGate { // file" describes no run at all. At one writer the numbers this produces are // the same ones the sequential loop produced, which is what lets the guards // that watch the bar stay as they were. -func (g *progressGate) advance(delta int64) { +func (g *progressGate) advance(bytes, work int64) { if g == nil { return } g.mu.Lock() defer g.mu.Unlock() - g.bytesDone += delta + g.bytesDone += bytes + g.workDone += work g.say() } // finished is the end of one file: the delta that squares this writer's // reporting with what the plan promised, and one more file done. -func (g *progressGate) finished(delta int64) { +func (g *progressGate) finished(bytes, work int64) { if g == nil { return } g.mu.Lock() defer g.mu.Unlock() - g.bytesDone += delta + g.bytesDone += bytes + g.workDone += work g.filesDone++ g.say() } @@ -237,6 +246,7 @@ func (g *progressGate) say() { g.report(Progress{ FilesDone: g.filesDone, FilesTotal: g.filesTotal, BytesDone: g.bytesDone, BytesTotal: g.bytesTotal, + WorkDone: g.workDone, WorkTotal: g.workTotal, }) } @@ -248,14 +258,25 @@ func (g *progressGate) say() { // delta, so the gate never has to know which file it came from. type fileProgress struct { gate *progressGate - reported int64 + reported int64 // bytes + worked int64 // work, the bytes among it } // advance is the counting writer's callback: n is the running total for this -// file, and the gate is told the difference. +// file, and the gate is told the difference - as bytes, and as work, because +// writing a byte is a byte of work. func (p *fileProgress) advance(n int64) { - p.gate.advance(n - p.reported) + d := n - p.reported + p.gate.advance(d, d) p.reported = n + p.worked += d +} + +// work is the generator's report of work its bytes do not show +// (format.Worked), n more of what its plan counted in Work. +func (p *fileProgress) work(n int64) { + p.gate.advance(0, n) + p.worked += n } // finished squares this file up. A file that succeeded is topped up to exactly @@ -265,12 +286,14 @@ func (p *fileProgress) advance(n int64) { // find. That the total can go backwards on a failed file is how it already // behaves. What is new is only that another writer's bytes may sit in the same // total while it happens. -func (p *fileProgress) finished(planned int64, ok bool) { +// +// Work is squared up the same way, to the plan's bytes and Work together. +func (p *fileProgress) finished(plan format.Plan, ok bool) { if ok { - p.gate.finished(planned - p.reported) + p.gate.finished(plan.Bytes-p.reported, plan.Bytes+plan.Work-p.worked) return } - p.gate.finished(-p.reported) + p.gate.finished(-p.reported, -p.worked) } // writeOne writes one file under a temporary name and renames it only once it @@ -338,6 +361,7 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress // locking at all and allocates nothing for it. if p.gate != nil { counter.report = p.advance + ctx = format.WithWork(ctx, p.work) } // Damage sits between the generator and the counter, and the order is the diff --git a/internal/format/format.go b/internal/format/format.go index 68181f0c..a754b20d 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -789,6 +789,34 @@ type Plan struct { // Memo is the generator's own scratch space, carried from planning to // writing. Nothing outside the generator reads it. Memo any + // Work is what writing the file costs beyond writing its bytes, counted + // as the bytes that would take as long to write - nought for every file + // whose cost is its bytes. A film is the case it exists for: thirty + // minutes of 1920x1080 is a hundred megabytes of pictures that take a + // minute to code and two gigabytes of padding that take five seconds to + // write, so a bar counting bytes stood at five percent for the whole + // minute and promised three hours (docs/WEBM-WYDAJNOSC-2026-10-06.md). + // The generator says how much of it is done through Worked as it goes. + Work int64 +} + +type workKey struct{} + +// WithWork is ctx carrying report, which Worked calls. The engine sets it on +// the context a generator writes under, when somebody is watching the run. +func WithWork(ctx context.Context, report func(int64)) context.Context { + return context.WithValue(ctx, workKey{}, report) +} + +// Worked says that n more of the plan's Work is done. It goes through the +// context rather than the writer, because the writer a generator is handed is +// not always the engine's - a damaged file is written through the damage - +// and from the goroutine Write runs on, which is the one the engine counts +// the bytes on. With nobody watching it does nothing. +func Worked(ctx context.Context, n int64) { + if report, ok := ctx.Value(workKey{}).(func(int64)); ok { + report(n) + } } // PropertyLabelEmbedded is the key a generator sets to say whether the label diff --git a/internal/format/video/stream.go b/internal/format/video/stream.go index 5848bb05..05a41551 100644 --- a/internal/format/video/stream.go +++ b/internal/format/video/stream.go @@ -88,6 +88,24 @@ func (s Stream) BoundBytes() (key, copied, shown int) { return len(s.seq) + k, h + len(s.show), len(s.show) } +// workPerPixel is how many bytes of writing coding one pixel of a picture is +// worth, for the bar a run draws (format.Plan.Work). Measured on 2026-10-06 +// with the owner's film, 1810 pictures of 1920x1080 and 2 GB of padding +// (docs/WEBM-WYDAJNOSC-2026-10-06.md): coding 16.9 ns a pixel on sixteen +// threads and 162 on one, writing 2.2 ns a byte. So the true figure is +// between 8 and 74 depending on the machine, and this is one of four to eight +// threads. A wrong figure misjudges only the padding against the pictures, +// and the padding takes seconds, so the estimate is off by seconds rather +// than by the film. +const workPerPixel = 16 + +// PictureWork is what coding one picture of the film is worth in bytes of +// writing, which a container reports as each change is coded. +func (s Stream) PictureWork() int64 { return int64(s.Width) * int64(s.Height) * workPerPixel } + +// Work is what coding every picture of the film is worth (format.Plan.Work). +func (s Stream) Work() int64 { return s.Changes() * s.PictureWork() } + // joined is a followed by b in one allocation of the right size. func joined(a, b []byte) []byte { return append(append(make([]byte, 0, len(a)+len(b)), a...), b...) diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index cdfdb5d1..ffcf941e 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -118,6 +118,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { Determinism: format.DeterminismByte, Properties: video.Facts(c, s), Memo: memo{choice: c, settings: s, total: r.Bytes}, + Work: st.Work(), } if r.Label && !c.Labelled() { p.Notes = append(p.Notes, format.Note{ @@ -233,20 +234,27 @@ func (s *sticky) uint(id uint32, v uint64) { // writeClusters codes the pictures one change at a time and writes the // clusters as it goes, giving back where each key frame's cluster starts, for -// the Cues, and where the last one ends. +// the Cues, and where the last one ends. Every change opens a cluster, so each +// is reported as worked once, which adds up to the Work the plan counted. func writeClusters(ctx context.Context, out *sticky, l layout, pics *video.Pictures) ([]uint64, uint64, error) { s := l.stream keys := make([]uint64, 0, s.Keys()) pos := l.front() + worked := int64(-1) for first := int64(0); first < s.Frames; { if err := ctx.Err(); err != nil { return nil, 0, err } // A cluster thirty seconds into a picture asks for the picture it // already has, which At answers without coding anything. - if err := pics.At(s.ChangeOf(first)); err != nil { + change := s.ChangeOf(first) + if err := pics.At(change); err != nil { return nil, 0, err } + if change != worked { + format.Worked(ctx, s.PictureWork()) + worked = change + } if s.IsKey(first) { keys = append(keys, pos) } diff --git a/internal/guard/filmahead_test.go b/internal/guard/filmahead_test.go index 6cdb611b..9ef553ac 100644 --- a/internal/guard/filmahead_test.go +++ b/internal/guard/filmahead_test.go @@ -9,6 +9,7 @@ import ( "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format" "github.com/donislawdev/TestingFilesGenerator/internal/format/video" ) @@ -120,6 +121,59 @@ func filmUnder(t *testing.T, threads int, target engine.Target) filmRun { return filmRun{bytes: b, coded: after - before} } +// A film's progress moves with its pictures, not with its padding. +// +// The bytes of a film are mostly padding written in its last seconds, so a bar +// counting bytes stood at five percent while the pictures were coded and +// promised the owner three hours for a film that took ten minutes +// (docs/WEBM-WYDAJNOSC-2026-10-06.md). Here the pictures are about a +// twenty-fifth of the bytes and two thirds of the work: at the moment they are +// all written the work has to be far ahead of the bytes. And a run of files +// whose cost is their bytes has to report the same numbers as before - work +// equal to bytes at every report. +func TestAFilmsProgressMovesWithItsPicturesNotItsPadding(t *testing.T) { + reports, planned := progressOf(t, filmTarget(32<<20, map[string]string{"width": "160", "height": "90", "duration": "5m"})) + if changes, _ := planned.Properties["change_count"].(int64); changes < 100 || planned.Work <= 0 { + t.Fatalf("the film has %d pictures and declares %d of work, so there were no pictures for the bar to move with", changes, planned.Work) + } + ahead := 0.0 + for _, r := range reports { + ahead = max(ahead, float64(r.WorkDone)/float64(r.WorkTotal)-float64(r.BytesDone)/float64(r.BytesTotal)) + } + if ahead < 0.4 { + t.Errorf("the work was never more than %.0f%% ahead of the bytes, so the bar counts the padding as the film", 100*ahead) + } + if last := reports[len(reports)-1]; last.WorkDone != last.WorkTotal || last.BytesDone != last.BytesTotal { + t.Errorf("the run ended at %d of %d work and %d of %d bytes", last.WorkDone, last.WorkTotal, last.BytesDone, last.BytesTotal) + } + + plain, _ := progressOf(t, engine.Target{ID: "plain", Format: "txt", Sizes: engine.Uniform(1, 1<<20)}) + for _, r := range plain { + if r.WorkDone != r.BytesDone || r.WorkTotal != r.BytesTotal { + t.Fatalf("a text file reported %d of %d work against %d of %d bytes, and its cost is its bytes", r.WorkDone, r.WorkTotal, r.BytesDone, r.BytesTotal) + } + } +} + +// progressOf runs one target with every progress report kept, and its plan. +func progressOf(t *testing.T, target engine.Target) ([]engine.Progress, format.Plan) { + t.Helper() + opt := engine.Options{OutDir: t.TempDir(), Seed: goldenSeed, Command: "test"} + planned, err := engine.Plan([]engine.Target{target}, opt) + if err != nil { + t.Fatal(err) + } + var reports []engine.Progress + opt.OnProgress = func(p engine.Progress) { reports = append(reports, p) } + if _, err := engine.Run(context.Background(), planned, opt); err != nil { + t.Fatal(err) + } + if len(reports) < 2 { + t.Fatalf("%s: %d progress reports, too few to say how the bar moved", target.Format, len(reports)) + } + return reports, planned[0].Plan +} + // A film stopped half way leaves no helper running when the run returns. // // Helpers are goroutines, and one left behind holds a picture's memory and a diff --git a/internal/gui/window/run.go b/internal/gui/window/run.go index 9eb785aa..b11a5c02 100644 --- a/internal/gui/window/run.go +++ b/internal/gui/window/run.go @@ -616,7 +616,7 @@ func (r *runner) startRun(targets []engine.Target, opt engine.Options) { } elapsed := time.Since(started) fyne.Do(func() { - r.busy.bar.SetValue(float64(core.Percent(p.BytesDone, p.BytesTotal))) + r.busy.bar.SetValue(float64(core.Percent(p.WorkDone, p.WorkTotal))) r.status.SetText(progressText(p, elapsed)) }) } diff --git a/internal/gui/window/runreport.go b/internal/gui/window/runreport.go index a04c8456..5f14105a 100644 --- a/internal/gui/window/runreport.go +++ b/internal/gui/window/runreport.go @@ -51,18 +51,23 @@ func manifestReachNote(res *engine.Result) []string { // progressText is the line under the bar. Bytes rather than files, because one // large file is a run where the file count says nothing for minutes. +// +// The percentage and the time left come from the work rather than the bytes +// (engine.Progress), because a film's bytes are mostly padding written in its +// last seconds - by the bytes, a thirty minute film stood at five percent for +// the whole minute its pictures took and promised three hours. func progressText(p engine.Progress, elapsed time.Duration) string { line := text.Progress(p.FilesDone, p.FilesTotal, text.HumanBytes(p.BytesDone), text.HumanBytes(p.BytesTotal), - core.Percent(p.BytesDone, p.BytesTotal)) + core.Percent(p.WorkDone, p.WorkTotal)) // The estimate stays quiet until it has enough to go on. A number that // swings wildly for the first second is worse than no number. - if elapsed < time.Second || p.BytesDone <= 0 || p.BytesDone >= p.BytesTotal { + if elapsed < time.Second || p.WorkDone <= 0 || p.WorkDone >= p.WorkTotal { return line } left := time.Duration(float64(elapsed) * - float64(p.BytesTotal-p.BytesDone) / float64(p.BytesDone)) + float64(p.WorkTotal-p.WorkDone) / float64(p.WorkDone)) return line + text.TimeLeft(text.Roughly(left)) } From 1e4d295154c6cf0de44aa8114eaca8a7e5e0b2e2 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Tue, 6 Oct 2026 20:59:45 +0200 Subject: [PATCH 3/3] review #167: one source for the bar's percentage and time left, a guard on a helper's panic, and files split under their ceilings engine.Progress.Percent and Left work out the bar's two numbers from the work. The window and the command line each carried a copy of the estimate and now both ask Progress. A guard holds the two methods to a report whose bytes and work disagree, and reads the surfaces' source: every use they make of a report's bytes or work is an argument to HumanBytes. A helper's panic coming out on the goroutine that asked for the picture now has a guard. gav1d cannot be made to panic from outside - a quantizer out of range is an error - so video.CodeBeside codes pictures the way a film does with a stand-in for the encoder, which panics the first time a helper calls. The changelog says the pictures are coded on several cores when a film has more than one. The shape gates were red: engine.go and format.go passed 399 lines, writeOne reached 60 and offerTo nested three deep. Progress moved to engine/progress.go, the work context to format/work.go, the job of a change to Pictures.jobFor, and wiring a file's progress to fileProgress.listen. No ceiling moved. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- internal/cli/progress.go | 12 +- internal/engine/engine.go | 37 ------ internal/engine/parallel.go | 20 +++- internal/engine/progress.go | 70 +++++++++++ internal/format/format.go | 26 +--- internal/format/video/ahead.go | 82 +++++++++++-- internal/format/video/choose.go | 16 ++- internal/format/video/stream.go | 21 ++-- internal/format/work.go | 32 +++++ internal/guard/progressbywork_test.go | 166 ++++++++++++++++++++++++++ internal/gui/window/run.go | 2 +- internal/gui/window/runreport.go | 12 +- 13 files changed, 386 insertions(+), 115 deletions(-) create mode 100644 internal/engine/progress.go create mode 100644 internal/format/work.go create mode 100644 internal/guard/progressbywork_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 462f4c85..9c95574e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,8 +25,9 @@ because it turns other people's test suites red. reads the time the picture starts, the way a player's position does - `00:00:07`, with milliseconds when the changes do not fall on whole seconds. Each change is a whole new picture in bytes and in coding time, - and the pictures of a film are coded on every core of the machine at once - - a thirty minute 1920x1080 film took 68 s on sixteen threads, and the + and when a film has more than one picture they are coded on several cores + at once - a thirty-minute 1920x1080 film took 68 s on sixteen threads, and + the progress bar counts the pictures as well as the bytes, so the time it says is left holds while they are coded. The frames between changes cost a few bytes each, so an hour at thirty diff --git a/internal/cli/progress.go b/internal/cli/progress.go index 140b08e3..5576fe38 100644 --- a/internal/cli/progress.go +++ b/internal/cli/progress.go @@ -58,7 +58,7 @@ func (p *progressBar) report(pr engine.Progress) { line := fmt.Sprintf(" %d/%d files %s of %s %d%%%s", pr.FilesDone, pr.FilesTotal, core.HumanBytes(pr.BytesDone), core.HumanBytes(pr.BytesTotal), - core.Percent(pr.WorkDone, pr.WorkTotal), + pr.Percent(), p.remaining(pr)) // Pad to cover whatever the last line left behind, so a shorter line does @@ -82,16 +82,12 @@ func (p *progressBar) clear() { } // remaining is an estimate and says so by staying quiet until it has enough to -// go on. A number that swings wildly for the first second is worse than none. -// It comes from the work, like the percentage, and not from the bytes - see -// engine.Progress for the film that promised three hours for a minute. +// go on (engine.Progress.Left, which the window asks too). func (p *progressBar) remaining(pr engine.Progress) string { - elapsed := time.Since(p.started) - if elapsed < time.Second || pr.WorkDone <= 0 || pr.WorkDone >= pr.WorkTotal { + left, ok := pr.Left(time.Since(p.started)) + if !ok { return "" } - left := time.Duration(float64(elapsed) * - float64(pr.WorkTotal-pr.WorkDone) / float64(pr.WorkDone)) return " " + core.Roughly(left) + " left" } diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 4a084b50..b083f96c 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -451,43 +451,6 @@ func PlanContext(ctx context.Context, targets []Target, opt Options) ([]PlannedF return pl.out, nil } -// TotalBytes is what a plan will occupy on disk. Known before the first byte -// is written, which is what the free space guard and --dry-run stand on. -func TotalBytes(files []PlannedFile) int64 { - var n int64 - for _, f := range files { - n += f.Plan.Bytes - } - return n -} - -// totalWork is what the planned files cost to make, their bytes included. -func totalWork(files []PlannedFile) int64 { - var n int64 - for _, f := range files { - n += f.Plan.Bytes + f.Plan.Work - } - return n -} - -// Progress is how far a run has got. Both counts are known from the plan, so -// the fractions are exact rather than estimated. -// -// A fraction of the run and the time it has left come from the work, not the -// bytes. The work is the bytes plus what the plans counted beyond them -// (format.Plan.Work), so for a run of files whose cost is their bytes the two -// are the same numbers, and for a film they are the pictures as well as the -// padding - the bytes alone stood at five percent while a film's pictures were -// coded and promised hours for a minute's work. -type Progress struct { - FilesDone int - FilesTotal int - BytesDone int64 - BytesTotal int64 - WorkDone int64 - WorkTotal int64 -} - // Run writes a planned set of files. // // Each file is written under a temporary name and only then renamed, so the diff --git a/internal/engine/parallel.go b/internal/engine/parallel.go index 2a1868ff..e1dea444 100644 --- a/internal/engine/parallel.go +++ b/internal/engine/parallel.go @@ -272,6 +272,19 @@ func (p *fileProgress) advance(n int64) { p.worked += d } +// listen has this writer hear the file being written: its bytes through the +// counter, and the work its bytes do not show through the context the +// generator writes under (format.Worked). Nothing is set when nobody is +// listening, so a run without progress does no locking at all and allocates +// nothing for it. +func (p *fileProgress) listen(ctx context.Context, counter *countingWriter) context.Context { + if p.gate == nil { + return ctx + } + counter.report = p.advance + return format.WithWork(ctx, p.work) +} + // work is the generator's report of work its bytes do not show // (format.Worked), n more of what its plan counted in Work. func (p *fileProgress) work(n int64) { @@ -357,12 +370,7 @@ func writeOne(ctx context.Context, f PlannedFile, outDir string, p *fileProgress h := sha256.New() buffered := bufio.NewWriterSize(fh, 64<<10) counter := &countingWriter{w: io.MultiWriter(buffered, h)} - // Left nil when nobody is listening, so a run without progress does no - // locking at all and allocates nothing for it. - if p.gate != nil { - counter.report = p.advance - ctx = format.WithWork(ctx, p.work) - } + ctx = p.listen(ctx, counter) // Damage sits between the generator and the counter, and the order is the // design rather than a convenience. diff --git a/internal/engine/progress.go b/internal/engine/progress.go new file mode 100644 index 00000000..005b0619 --- /dev/null +++ b/internal/engine/progress.go @@ -0,0 +1,70 @@ +// Part of package engine. See engine.go. +package engine + +import ( + "time" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" +) + +// How far a run has got, and what it is measured against. Moved out of +// engine.go on 2026-10-06, when progress began to count the work a file costs +// as well as its bytes and that file passed its ceiling on length - the ceiling +// is a ratchet, so the answer is a split by subject. The gate that hands the +// numbers out while files are written in parallel stays in parallel.go, +// because it holds a lock and that file is where everything concurrent in this +// package lives. + +// TotalBytes is what a plan will occupy on disk. Known before the first byte +// is written, which is what the free space guard and --dry-run stand on. +func TotalBytes(files []PlannedFile) int64 { + var n int64 + for _, f := range files { + n += f.Plan.Bytes + } + return n +} + +// totalWork is what the planned files cost to make, their bytes included. +func totalWork(files []PlannedFile) int64 { + var n int64 + for _, f := range files { + n += f.Plan.Bytes + f.Plan.Work + } + return n +} + +// Progress is how far a run has got. Both counts are known from the plan, so +// the fractions are exact rather than estimated. +// +// A fraction of the run and the time it has left come from the work, not the +// bytes. The work is the bytes plus what the plans counted beyond them +// (format.Plan.Work), so for a run of files whose cost is their bytes the two +// are the same numbers, and for a film they are the pictures as well as the +// padding - the bytes alone stood at five percent while a film's pictures were +// coded and promised hours for a minute's work. +type Progress struct { + FilesDone int + FilesTotal int + BytesDone int64 + BytesTotal int64 + WorkDone int64 + WorkTotal int64 +} + +// Percent is how far the run has got, out of a hundred, by the work. The +// window's bar and the command line's both draw this, so the two cannot +// disagree and neither can fall back to the bytes on its own. +func (p Progress) Percent() int { return core.Percent(p.WorkDone, p.WorkTotal) } + +// Left is how long the run has left at the pace it has kept so far, by the +// work, and false while there is not enough to go on: in the first second, +// before any work is done and once all of it is. A number that swings wildly +// for the first second is worse than no number. The two surfaces carried a +// copy of this each until 2026-10-06. +func (p Progress) Left(elapsed time.Duration) (time.Duration, bool) { + if elapsed < time.Second || p.WorkDone <= 0 || p.WorkDone >= p.WorkTotal { + return 0, false + } + return time.Duration(float64(elapsed) * float64(p.WorkTotal-p.WorkDone) / float64(p.WorkDone)), true +} diff --git a/internal/format/format.go b/internal/format/format.go index a754b20d..662bd803 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -791,34 +791,10 @@ type Plan struct { Memo any // Work is what writing the file costs beyond writing its bytes, counted // as the bytes that would take as long to write - nought for every file - // whose cost is its bytes. A film is the case it exists for: thirty - // minutes of 1920x1080 is a hundred megabytes of pictures that take a - // minute to code and two gigabytes of padding that take five seconds to - // write, so a bar counting bytes stood at five percent for the whole - // minute and promised three hours (docs/WEBM-WYDAJNOSC-2026-10-06.md). - // The generator says how much of it is done through Worked as it goes. + // whose cost is its bytes. See work.go. Work int64 } -type workKey struct{} - -// WithWork is ctx carrying report, which Worked calls. The engine sets it on -// the context a generator writes under, when somebody is watching the run. -func WithWork(ctx context.Context, report func(int64)) context.Context { - return context.WithValue(ctx, workKey{}, report) -} - -// Worked says that n more of the plan's Work is done. It goes through the -// context rather than the writer, because the writer a generator is handed is -// not always the engine's - a damaged file is written through the damage - -// and from the goroutine Write runs on, which is the one the engine counts -// the bytes on. With nobody watching it does nothing. -func Worked(ctx context.Context, n int64) { - if report, ok := ctx.Value(workKey{}).(func(int64)); ok { - report(n) - } -} - // PropertyLabelEmbedded is the key a generator sets to say whether the label // it was asked for actually reached the file. // diff --git a/internal/format/video/ahead.go b/internal/format/video/ahead.go index 2984b62b..9f99f24b 100644 --- a/internal/format/video/ahead.go +++ b/internal/format/video/ahead.go @@ -64,11 +64,12 @@ func takeHelper() bool { // inside the job rather than a channel beside it, because a channel is one // more allocation for every picture of every film. type job struct { - look look - taken atomic.Bool - done sync.WaitGroup - coded Coded - err error + look look + change int64 + taken atomic.Bool + done sync.WaitGroup + coded Coded + err error // panicked is what coding it panicked with. A panic on a helper would end // the process, where the same panic on the writer becomes the run's error // (internal/engine, writeWithoutCrashing), so a helper keeps it here and @@ -76,8 +77,8 @@ type job struct { panicked any } -func newJob(l look) *job { - j := &job{look: l} +func newJob(l look, change int64) *job { + j := &job{look: l, change: change} j.done.Add(1) return j } @@ -103,6 +104,10 @@ type crew struct { wg sync.WaitGroup stopped atomic.Bool closed bool + // encode stands in for painting a picture and coding it through gav1d, + // told whether a helper is the one asking. Nil in every film - only + // CodeBeside sets it. + encode func(change int64, beside bool) (Coded, error) } // newCrew is the crew for a film of this many pictures. A film of one picture @@ -159,26 +164,30 @@ func (c *crew) help() { continue } if j.taken.CompareAndSwap(false, true) { - c.code(p, j) + c.code(p, j, true) codedBeside.Add(1) } } } -func (c *crew) code(p *painter, j *job) { +func (c *crew) code(p *painter, j *job, beside bool) { defer j.done.Done() defer func() { if v := recover(); v != nil { j.panicked = v } }() + if c.encode != nil { + j.coded, j.err = c.encode(j.change, beside) + return + } j.coded, j.err = Encode(p.draw(j.look), c.qindex) } // wait is the picture of j, coded here when no helper has taken it. func (c *crew) wait(j *job) (Coded, error) { if j.taken.CompareAndSwap(false, true) { - c.code(c.own, j) + c.code(c.own, j, false) } j.done.Wait() if j.panicked != nil { @@ -205,3 +214,56 @@ func (c *crew) stop() { c.finish() c.wg.Wait() } + +// all offers every job at once and gives back their pictures in order - a +// set known in advance, like the sample planning codes, rather than a film +// coded as it is written. +func (c *crew) all(jobs []*job) ([]Coded, error) { + for _, j := range jobs { + c.offer(j) + } + c.finish() + out := make([]Coded, len(jobs)) + for i, j := range jobs { + coded, err := c.wait(j) + if err != nil { + return nil, err + } + out[i] = coded + } + return out, nil +} + +// CodeBeside codes pictures 0 to n-1 the way a film's are coded - this +// goroutine and the helpers it can take, offered ahead and taken in order - +// with code standing in for painting and gav1d, told whether a helper is the +// one calling, and gives back the size code returned for each. +// +// It is for the guards, and only because what it lets them hold cannot be +// reached another way: a panic on a helper has to come out on the goroutine +// that asked for the picture, where the engine turns it into the run's error, +// rather than end the process - and nothing outside makes gav1d panic. +// Measured on 2026-10-06: a quantizer out of range is refused with an error, +// and only planes shorter than the picture panic, which no painter makes. +func CodeBeside(n int64, code func(change int64, beside bool) (int, error)) ([]int, error) { + t := Timeline{FPS: 1, Frames: n, DurationMs: n * 1000, KeyEvery: n, ChangeEvery: 1} + c := newCrew(newFilm(1, 1, 0, "", t), 0, n) + defer c.stop() + c.encode = func(change int64, beside bool) (Coded, error) { + size, err := code(change, beside) + return Coded{tile: make([]byte, size)}, err + } + jobs := make([]*job, n) + for i := range jobs { + jobs[i] = newJob(look{}, int64(i)) + } + coded, err := c.all(jobs) + if err != nil { + return nil, err + } + sizes := make([]int, n) + for i, p := range coded { + sizes[i] = p.Size() + } + return sizes, nil +} diff --git a/internal/format/video/choose.go b/internal/format/video/choose.go index f3f86323..5b37c90f 100644 --- a/internal/format/video/choose.go +++ b/internal/format/video/choose.go @@ -99,21 +99,19 @@ func sampleReserve(w, h int, seed uint64, label string, t Timeline, qindex int) defer c.stop() jobs := make([]*job, len(sample)) for i, ch := range sample { - jobs[i] = newJob(f.lookOf(ch)) - c.offer(jobs[i]) + jobs[i] = newJob(f.lookOf(ch), ch) + } + coded, err := c.all(jobs) + if err != nil { + return Coded{}, 0, err } - c.finish() var first Coded largest := 0 for i, ch := range sample { - coded, err := c.wait(jobs[i]) - if err != nil { - return Coded{}, 0, err - } if ch == 0 { - first = coded + first = coded[i] } - largest = max(largest, coded.Size()) + largest = max(largest, coded[i].Size()) } if int64(len(sample)) == t.Changes() { return first, largest, nil diff --git a/internal/format/video/stream.go b/internal/format/video/stream.go index 05a41551..8923f8d4 100644 --- a/internal/format/video/stream.go +++ b/internal/format/video/stream.go @@ -201,14 +201,7 @@ func (p *Pictures) offerTo(c int64) { for p.next < n && (p.next <= c || len(p.pending) < p.crew.ahead()) { l := p.film.lookOf(p.next) if p.next == 0 || l != p.last { - j := (*job)(nil) - if p.next == 0 && p.choice.First != nil { - j = codedJob(*p.choice.First) - } else { - j = newJob(l) - p.crew.offer(j) - } - p.pending = append(p.pending, openedBy{change: p.next, job: j}) + p.pending = append(p.pending, openedBy{change: p.next, job: p.jobFor(l)}) } p.last = l p.next++ @@ -218,6 +211,18 @@ func (p *Pictures) offerTo(c int64) { } } +// jobFor is the job of the picture with this look at change p.next, offered +// to the helpers - or, for change 0 when planning coded it, the picture +// planning already has. +func (p *Pictures) jobFor(l look) *job { + if p.next == 0 && p.choice.First != nil { + return codedJob(*p.choice.First) + } + j := newJob(l, p.next) + p.crew.offer(j) + return j +} + // Close stops the helpers and waits for them, so nothing this film started is // running once it returns. A helper in the middle of a picture finishes it // first, which is at most one picture's time. diff --git a/internal/format/work.go b/internal/format/work.go new file mode 100644 index 00000000..c844f2dd --- /dev/null +++ b/internal/format/work.go @@ -0,0 +1,32 @@ +package format + +import "context" + +// The work a file costs beyond its bytes (Plan.Work), and how a generator says +// how much of it is done. +// +// A film is the case this exists for: thirty minutes of 1920x1080 is a +// hundred megabytes of pictures that take a minute to code and two gigabytes +// of padding that take five seconds to write, so a bar counting bytes stood at +// five percent for the whole minute and promised three hours +// (docs/WEBM-WYDAJNOSC-2026-10-06.md). Kept out of format.go, which was at its +// ceiling on length, because it is a subject of its own. + +type workKey struct{} + +// WithWork is ctx carrying report, which Worked calls. The engine sets it on +// the context a generator writes under, when somebody is watching the run. +func WithWork(ctx context.Context, report func(int64)) context.Context { + return context.WithValue(ctx, workKey{}, report) +} + +// Worked says that n more of the plan's Work is done. It goes through the +// context rather than the writer, because the writer a generator is handed is +// not always the engine's - a damaged file is written through the damage - +// and from the goroutine Write runs on, which is the one the engine counts +// the bytes on. With nobody watching it does nothing. +func Worked(ctx context.Context, n int64) { + if report, ok := ctx.Value(workKey{}).(func(int64)); ok { + report(n) + } +} diff --git a/internal/guard/progressbywork_test.go b/internal/guard/progressbywork_test.go new file mode 100644 index 00000000..379072d9 --- /dev/null +++ b/internal/guard/progressbywork_test.go @@ -0,0 +1,166 @@ +package guard + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/donislawdev/TestingFilesGenerator/internal/engine" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// The bar's percentage and the time it says is left come from the work, on +// both surfaces (docs/WEBM-WYDAJNOSC-2026-10-06.md). +// +// Two halves, because the property lives in two places. engine.Progress +// works the two numbers out, and that is held here by what it answers for a +// report whose bytes and work disagree - five percent of the bytes, half of +// the work, the shape a film has while its pictures are coded. And the window +// and the command line must take both from there rather than work out their +// own: read from their source, every use they make of a report's bytes or work +// is an argument to HumanBytes, the size the line shows. A surface going back +// to a percentage of the bytes would use BytesDone somewhere else, and both of +// its own bars are out of a guard's reach - the window's moves on a goroutine +// the guards must not read beside, and the command line draws only on a +// terminal. +func TestTheBarAndTheTimeLeftComeFromTheWorkOnBothSurfaces(t *testing.T) { + p := engine.Progress{FilesTotal: 1, BytesDone: 5, BytesTotal: 100, WorkDone: 50, WorkTotal: 100} + if got := p.Percent(); got != 50 { + t.Errorf("half the work and a twentieth of the bytes is %d%%, and the work says 50", got) + } + if left, ok := p.Left(10 * time.Second); !ok || left != 10*time.Second { + t.Errorf("half the work in ten seconds leaves %v (%v), and the work says ten seconds", left, ok) + } + for name, quiet := range map[string]struct { + p engine.Progress + elapsed time.Duration + }{ + "the first second": {p, 500 * time.Millisecond}, + "no work done yet": {engine.Progress{BytesTotal: 100, WorkTotal: 100}, time.Minute}, + "all the work done": {engine.Progress{BytesDone: 100, BytesTotal: 100, WorkDone: 100, WorkTotal: 100}, time.Minute}, + } { + if left, ok := quiet.p.Left(quiet.elapsed); ok { + t.Errorf("%s: the estimate said %v where it has nothing to go on", name, left) + } + } + + shown, percents, lefts := 0, 0, 0 + for _, dir := range []string{"internal/cli", "internal/gui"} { + walkGo(t, filepath.Join(repoRoot(t), dir), func(path string, file *ast.File) { + counted := map[ast.Node]bool{} + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + switch calledName(call) { + case "HumanBytes": + for _, arg := range call.Args { + counted[arg] = true + } + case "Percent": + if len(call.Args) == 0 { + percents++ + } + case "Left": + lefts++ + } + return true + }) + ast.Inspect(file, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + switch sel.Sel.Name { + case "BytesDone", "BytesTotal", "WorkDone", "WorkTotal": + if counted[sel] { + shown++ + return true + } + rel, _ := filepath.Rel(repoRoot(t), path) + t.Errorf("%s reads a report's %s other than to show it as a size - a surface working out its own percentage or estimate rather than asking engine.Progress", filepath.ToSlash(rel), sel.Sel.Name) + } + return true + }) + }) + } + // Asserted, not assumed: both lines show their bytes, all three places + // that draw a percentage ask Progress for it, and both estimates come + // from Left. A scan that found none of it would pass about nothing. + if shown < 4 || percents < 3 || lefts < 2 { + t.Fatalf("the surfaces show %d sizes from a report, ask %d times for a percentage and %d times for the time left - the scan did not reach the bars it is about", shown, percents, lefts) + } +} + +// walkGo parses every Go file under dir that is not a test. +func walkGo(t *testing.T, dir string, visit func(path string, file *ast.File)) { + t.Helper() + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return err + } + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + return err + } + visit(path, file) + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// A panic while a helper codes a picture comes out on the goroutine that asked +// for the picture, and leaves no helper running. +// +// The engine turns a generator's panic into the run's error only on the +// goroutine that writes the file (writeWithoutCrashing), and a panic on any +// other goroutine ends the process - the window with it. So a helper keeps its +// panic and the writer raises it when it reaches that picture. gav1d cannot be +// made to panic from outside, so the pictures here are coded by a stand-in +// that panics the first time a helper calls it, through video.CodeBeside, +// which codes them exactly as a film's are. Under a helper that did not keep +// its panic this test does not fail - the test binary dies, which is red too. +func TestAPanicOnAHelperComesOutOnTheGoroutineThatAskedForIt(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(8)) + var fromHelper atomic.Bool + helped := make(chan struct{}) + got := func() (v any) { + defer func() { v = recover() }() + _, _ = video.CodeBeside(40, func(change int64, beside bool) (int, error) { + if beside && fromHelper.CompareAndSwap(false, true) { + close(helped) + panic("a helper's picture") + } + if !beside { + // The writer's own picture waits for a helper to have + // started, or the writer, with nothing to code, takes all + // forty before any helper is scheduled. + select { + case <-helped: + case <-time.After(5 * time.Second): + } + } + return 1, nil + }) + return nil + }() + if !fromHelper.Load() { + t.Fatal("no helper coded a picture, so no helper could have panicked") + } + if got != "a helper's picture" { + t.Errorf("the goroutine that asked for the pictures got %v, and a helper panicked with %q", got, "a helper's picture") + } + if running, _ := video.Helpers(); running != 0 { + t.Errorf("%d helpers are still running after the panic came out", running) + } +} diff --git a/internal/gui/window/run.go b/internal/gui/window/run.go index b11a5c02..c0266611 100644 --- a/internal/gui/window/run.go +++ b/internal/gui/window/run.go @@ -616,7 +616,7 @@ func (r *runner) startRun(targets []engine.Target, opt engine.Options) { } elapsed := time.Since(started) fyne.Do(func() { - r.busy.bar.SetValue(float64(core.Percent(p.WorkDone, p.WorkTotal))) + r.busy.bar.SetValue(float64(p.Percent())) r.status.SetText(progressText(p, elapsed)) }) } diff --git a/internal/gui/window/runreport.go b/internal/gui/window/runreport.go index 5f14105a..bafa24ea 100644 --- a/internal/gui/window/runreport.go +++ b/internal/gui/window/runreport.go @@ -4,7 +4,6 @@ import ( "fmt" "time" - "github.com/donislawdev/TestingFilesGenerator/internal/core" "github.com/donislawdev/TestingFilesGenerator/internal/engine" "github.com/donislawdev/TestingFilesGenerator/internal/gui/text" "github.com/donislawdev/TestingFilesGenerator/internal/manifest" @@ -58,16 +57,11 @@ func manifestReachNote(res *engine.Result) []string { // the whole minute its pictures took and promised three hours. func progressText(p engine.Progress, elapsed time.Duration) string { line := text.Progress(p.FilesDone, p.FilesTotal, - text.HumanBytes(p.BytesDone), text.HumanBytes(p.BytesTotal), - core.Percent(p.WorkDone, p.WorkTotal)) - - // The estimate stays quiet until it has enough to go on. A number that - // swings wildly for the first second is worse than no number. - if elapsed < time.Second || p.WorkDone <= 0 || p.WorkDone >= p.WorkTotal { + text.HumanBytes(p.BytesDone), text.HumanBytes(p.BytesTotal), p.Percent()) + left, ok := p.Left(elapsed) + if !ok { return line } - left := time.Duration(float64(elapsed) * - float64(p.WorkTotal-p.WorkDone) / float64(p.WorkDone)) return line + text.TimeLeft(text.Roughly(left)) }