diff --git a/package.json b/package.json index 5e76e579a..3ecd3904e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "mapillary-js", - "version": "5.0.0-beta.5", + "version": "5.0.0-beta.5-fb15", "description": "A WebGL interactive street imagery library", "license": "MIT", "main": "dist/mapillary.js", diff --git a/src/component/direction/DirectionComponent.ts b/src/component/direction/DirectionComponent.ts index b490397fa..14b976f9f 100644 --- a/src/component/direction/DirectionComponent.ts +++ b/src/component/direction/DirectionComponent.ts @@ -1,6 +1,7 @@ import * as vd from "virtual-dom"; import { combineLatest as observableCombineLatest, + concat as observableConcat, of as observableOf, Observable, Subject, @@ -13,6 +14,7 @@ import { share, startWith, switchMap, + takeUntil, tap, withLatestFrom, } from "rxjs/operators"; @@ -151,18 +153,60 @@ export class DirectionComponent extends Component { withLatestFrom(this._configuration$), switchMap( ([image, configuration]: [Image, DirectionConfiguration]): Observable<[NavigationEdgeStatus, Sequence]> => { - return observableCombineLatest( - image.spatialEdges$, - configuration.distinguishSequence ? - this._navigator.graphService - .cacheSequence$(image.sequenceId).pipe( - catchError( - (error: Error): Observable => { - console.error(`Failed to cache sequence (${image.sequenceId})`, error); - - return observableOf(null); - })) : - observableOf(null)); + if (!configuration.distinguishSequence) { + return image.spatialEdges$.pipe( + map( + (edgeStatus: NavigationEdgeStatus): [NavigationEdgeStatus, Sequence] => { + return [edgeStatus, null]; + })); + } + + return this._navigator.graphService + .cacheSequence$(image.sequenceId).pipe( + catchError( + (error: Error): Observable => { + console.error(`Failed to cache sequence (${image.sequenceId})`, error); + + return observableOf(null); + }), + switchMap( + (sequence: Sequence): Observable<[NavigationEdgeStatus, Sequence]> => { + if (sequence == null) { + return image.spatialEdges$.pipe( + map( + (edgeStatus: NavigationEdgeStatus): + [NavigationEdgeStatus, Sequence] => { + return [edgeStatus, null]; + })); + } + + const spatialEdges$ = image.spatialEdges$.pipe( + filter( + (edgeStatus: NavigationEdgeStatus): boolean => { + return edgeStatus.cached; + })); + const sequenceEdges$ = this._navigator.graphService + .cacheSequenceSpatialEdges$(image.id).pipe( + takeUntil(spatialEdges$), + catchError( + (error: Error): Observable => { + console.error( + `Failed to cache sequence spatial edges (${image.id})`, + error); + + return observableOf({ + cached: false, + edges: [], + }); + })); + + return observableConcat(sequenceEdges$, spatialEdges$).pipe( + map( + (edgeStatus: NavigationEdgeStatus): + [NavigationEdgeStatus, Sequence] => { + return [edgeStatus, sequence]; + })); + })); })) .subscribe( ([edgeStatus, sequence]: [NavigationEdgeStatus, Sequence]): void => { diff --git a/src/component/direction/DirectionDOMRenderer.ts b/src/component/direction/DirectionDOMRenderer.ts index 8803b3f4a..731c731af 100644 --- a/src/component/direction/DirectionDOMRenderer.ts +++ b/src/component/direction/DirectionDOMRenderer.ts @@ -32,11 +32,13 @@ export class DirectionDOMRenderer { private _distinguishSequence: boolean; private _needsRender: boolean; + private _edgesCached: boolean; private _stepEdges: NavigationEdge[]; private _turnEdges: NavigationEdge[]; private _sphericalEdges: NavigationEdge[]; private _sequenceEdgeKeys: string[]; + private _sequenceAzimuthOffset: number; private _stepDirections: NavigationDirection[]; private _turnDirections: NavigationDirection[]; @@ -57,11 +59,13 @@ export class DirectionDOMRenderer { this._distinguishSequence = false; this._needsRender = false; + this._edgesCached = false; this._stepEdges = []; this._turnEdges = []; this._sphericalEdges = []; this._sequenceEdgeKeys = []; + this._sequenceAzimuthOffset = 0; this._stepDirections = [ NavigationDirection.StepForward, @@ -133,6 +137,11 @@ export class DirectionDOMRenderer { */ public setImage(image: Image): void { this._image = image; + this._sequenceAzimuthOffset = image != null && + isSpherical(image.cameraType) && + !Number.isFinite(image.computedCompassAngle) && + Number.isFinite(image.compassAngle) ? + this._spatial.degToRad(90 - image.compassAngle) : 0; this._clearEdges(); this._setNeedsRender(); @@ -200,6 +209,7 @@ export class DirectionDOMRenderer { } private _clearEdges(): void { + this._edgesCached = false; this._stepEdges = []; this._turnEdges = []; this._sphericalEdges = []; @@ -207,6 +217,7 @@ export class DirectionDOMRenderer { } private _setEdges(edgeStatus: NavigationEdgeStatus, sequence: Sequence): void { + this._edgesCached = edgeStatus.cached; this._stepEdges = []; this._turnEdges = []; this._sphericalEdges = []; @@ -237,13 +248,12 @@ export class DirectionDOMRenderer { for (let edge of edges) { let edgeKey: string = edge.target; - - for (let sequenceKey of sequence.imageIds) { - if (sequenceKey === edgeKey) { - this._sequenceEdgeKeys.push(edgeKey); - break; - } + const edgeIndex: number = sequence.imageIds.indexOf(edgeKey); + if (edgeIndex < 0) { + continue; } + + this._sequenceEdgeKeys.push(edgeKey); } } } @@ -256,10 +266,11 @@ export class DirectionDOMRenderer { this._createVNodeByKey( navigator, sphericalEdge.target, - sphericalEdge.data.worldMotionAzimuth, + this._getSphericalEdgeAzimuth(sphericalEdge), rotation, this._calculator.outerRadius, - "mapillary-direction-arrow-spherical")); + "mapillary-direction-arrow-spherical", + sphericalEdge.data.direction)); } for (let stepEdge of this._stepEdges) { @@ -267,7 +278,7 @@ export class DirectionDOMRenderer { this._createSphericalToPerspectiveArrow( navigator, stepEdge.target, - stepEdge.data.worldMotionAzimuth, + this._getSphericalEdgeAzimuth(stepEdge), rotation, stepEdge.data.direction)); } @@ -275,6 +286,14 @@ export class DirectionDOMRenderer { return arrows; } + private _getSphericalEdgeAzimuth(edge: NavigationEdge): number { + const sequenceEdge = this._sequenceEdgeKeys.indexOf(edge.target) > -1; + // Unmerged equirectangular captures are world-aligned at their east + // axis even when their compass field describes vehicle travel. + return edge.data.worldMotionAzimuth + + (sequenceEdge ? this._sequenceAzimuthOffset : 0); + } + private _createSphericalToPerspectiveArrow( navigator: Navigator, key: string, @@ -307,7 +326,8 @@ export class DirectionDOMRenderer { azimuth, rotation, this._calculator.outerRadius, - "mapillary-direction-arrow-step"); + "mapillary-direction-arrow-step", + direction); } return this._createVNodeInactive(key, azimuth, rotation); @@ -325,6 +345,7 @@ export class DirectionDOMRenderer { rotation, this._calculator.innerRadius, "mapillary-direction-arrow-spherical", + sphericalEdge.data.direction, true)); } @@ -372,11 +393,12 @@ export class DirectionDOMRenderer { rotation: EulerRotation, offset: number, className: string, + direction: NavigationDirection, shiftVertically?: boolean): vd.VNode { let onClick: (e: Event) => void = (e: Event): void => { - navigator.moveTo$(key) + navigator.moveTo$(key, direction) .subscribe( undefined, (error: Error): void => { @@ -406,7 +428,9 @@ export class DirectionDOMRenderer { let onClick: (e: Event) => void = (e: Event): void => { - navigator.moveDir$(direction) + (this._edgesCached ? + navigator.moveDir$(direction) : + navigator.moveTo$(key, direction)) .subscribe( undefined, (error: Error): void => { @@ -434,7 +458,9 @@ export class DirectionDOMRenderer { let onClick: (e: Event) => void = (e: Event): void => { - navigator.moveDir$(direction) + (this._edgesCached ? + navigator.moveDir$(direction) : + navigator.moveTo$(key, direction)) .subscribe( undefined, (error: Error): void => { diff --git a/src/component/image/ImageGLRenderer.ts b/src/component/image/ImageGLRenderer.ts index 4eb1eb8bf..acc8c7b88 100644 --- a/src/component/image/ImageGLRenderer.ts +++ b/src/component/image/ImageGLRenderer.ts @@ -4,6 +4,7 @@ import { Subscription } from "rxjs"; import { IAnimationState } from "../../state/interfaces/IAnimationState"; import { AnimationFrame } from "../../state/interfaces/AnimationFrame"; import { Image } from "../../graph/Image"; +import { Camera } from "../../geo/Camera"; import { Transform } from "../../geo/Transform"; import { TextureProvider } from "../../tile/TextureProvider"; import { MeshFactory } from "../util/MeshFactory"; @@ -27,6 +28,10 @@ export class ImageGLRenderer { private _frameId: number; private _needsRender: boolean; + private _motionless: boolean; + private _previousCamera: Camera; + private _previousPerspective: THREE.PerspectiveCamera; + constructor() { this._factory = new MeshFactory(); this._scene = new MeshScene(); @@ -41,6 +46,10 @@ export class ImageGLRenderer { this._frameId = 0; this._needsRender = false; + + this._motionless = false; + this._previousCamera = null; + this._previousPerspective = null; } public get frameId(): number { @@ -177,7 +186,9 @@ export class ImageGLRenderer { renderer.render(this._scene.scenePeriphery, perspectiveCamera); renderer.render(this._scene.scene, perspectiveCamera); - renderer.render(this._scene.sceneOld, perspectiveCamera); + renderer.render( + this._scene.sceneOld, + this._getPreviousPerspective(perspectiveCamera)); for (const plane of planes) { (plane.mesh.material).uniforms.opacity.value = this._alpha; @@ -197,9 +208,33 @@ export class ImageGLRenderer { disposeProvider(); } + this._motionless = false; + this._previousCamera = null; + this._previousPerspective = null; this._needsRender = true; } + private _getPreviousPerspective( + perspectiveCamera: THREE.PerspectiveCamera) + : THREE.PerspectiveCamera { + if (!this._motionless || this._previousCamera == null) { + return perspectiveCamera; + } + if (this._previousPerspective == null) { + this._previousPerspective = perspectiveCamera.clone(); + this._previousPerspective.position.copy( + this._previousCamera.position); + this._previousPerspective.up.copy(this._previousCamera.up); + this._previousPerspective.matrixAutoUpdate = true; + this._previousPerspective.lookAt(this._previousCamera.lookat); + this._previousPerspective.matrixAutoUpdate = false; + this._previousPerspective.updateMatrix(); + this._previousPerspective.updateMatrixWorld(false); + } + + return this._previousPerspective; + } + private _setShader(shader: GLShader, planes: MeshSceneItem[]): void { for (const plane of planes) { const material = plane.mesh.material; @@ -257,12 +292,16 @@ export class ImageGLRenderer { } if (previousKey != null) { - if (previousKey !== this._currentKey && previousKey !== this._previousKey) { + const freezePrevious = state.motionless && state.alpha < 1; + if (freezePrevious || + (previousKey !== this._currentKey && + previousKey !== this._previousKey)) { const previousMesh = this._factory.createMesh( state.previousImage, state.previousTransform, - shader); + shader, + !freezePrevious); const previousPlane: MeshSceneItem = { mesh: previousMesh, @@ -275,6 +314,10 @@ export class ImageGLRenderer { this._previousKey = previousKey; } + this._motionless = state.motionless && state.alpha < 1; + this._previousCamera = this._motionless ? + state.previousCamera.clone() : null; + this._previousPerspective = null; this._currentKey = currentKey; const currentMesh = this._factory.createMesh( diff --git a/src/component/interfaces/ReorientationConfiguration.ts b/src/component/interfaces/ReorientationConfiguration.ts index d6cdabfb1..d2cc018d8 100644 --- a/src/component/interfaces/ReorientationConfiguration.ts +++ b/src/component/interfaces/ReorientationConfiguration.ts @@ -18,6 +18,35 @@ import { ComponentConfiguration } from "./ComponentConfiguration"; * ``` */ export interface ReorientationConfiguration extends ComponentConfiguration { + /** + * Whether reconstructed images are automatically leveled to the horizon. + * + * @default true + */ + automaticHorizonLeveling?: boolean; + + /** + * Whether spherical images automatically face their direction of travel. + * + * @default true + */ + reorientToFront?: boolean; + + /** + * Whether a spatial navigation (the direction component's step, turn and + * spherical arrows, and their keyboard equivalents) reorients the landing + * image when it stays within the current sequence. + * + * Crossing into a new sequence with an arrow always keeps the carried view + * regardless of this setting: the state layer has already matched the + * angle, and reorienting on top of that fights the transition the arrow + * just made. Within a sequence the carried view instead drifts off-axis as + * the road bends, which is what reorientation corrects. + * + * @default true + */ + reorientOnSpatialNav?: boolean; + /** * Number of images ahead in the sequence to precompute the * reorientation for, so that a step lands on an already resolved @@ -41,7 +70,11 @@ export interface ReorientationConfiguration extends ComponentConfiguration { * with the travel bearing within {@link lowSpeedTurnMaxDeltaDeg} and * the step is longer than this distance in meters. * - * @default 2 + * Low enough to accept walking-pace capture, where frames are often only + * about a metre apart; the compass agreement above is what separates that + * from a stationary camera's GPS drift. + * + * @default 0.5 */ lowSpeedTurnDistanceM?: number; diff --git a/src/component/keyboard/KeySpatialNavigationHandler.ts b/src/component/keyboard/KeySpatialNavigationHandler.ts index 90184860f..8a18a5694 100644 --- a/src/component/keyboard/KeySpatialNavigationHandler.ts +++ b/src/component/keyboard/KeySpatialNavigationHandler.ts @@ -136,7 +136,7 @@ export class KeySpatialNavigationHandler extends HandlerBase { diff --git a/src/component/reorientation/ReorientationComponent.ts b/src/component/reorientation/ReorientationComponent.ts index 2ffb87fdd..f7c5a7888 100644 --- a/src/component/reorientation/ReorientationComponent.ts +++ b/src/component/reorientation/ReorientationComponent.ts @@ -1,4 +1,6 @@ -import { first } from "rxjs/operators"; +import { Observable, Subject } from "rxjs"; +import { filter, first } from "rxjs/operators"; +import * as THREE from "three"; import { Component } from "../Component"; import { ComponentName } from "../ComponentName"; @@ -6,30 +8,88 @@ import { ReorientationConfiguration } from "../interfaces/ReorientationConfiguration"; import { bearingToBasicX, + DEFAULT_REORIENTATION_CONFIGURATION, ReorientationEngine, ReorientationImage, ReorientationProvider, + ReorientationResult, wrapDelta, } from "./ReorientationEngine"; import { Image } from "../../graph/Image"; +import { NavigationDirection } from "../../graph/edge/NavigationDirection"; import { Sequence } from "../../graph/Sequence"; +import { isSpherical } from "../../geo/Geo"; +import { Transform } from "../../geo/Transform"; +import { ViewportCoords } from "../../geo/ViewportCoords"; +import { RenderCamera } from "../../render/RenderCamera"; +import { hasReconstructionMesh } from "../../util/Mesh"; import { Container } from "../../viewer/Container"; import { Navigator } from "../../viewer/Navigator"; // Skip a reorientation when it would move the current view less than this many // degrees (on either axis) — small moves are just jitter. const MIN_REORIENT_DEG = 15; +const MAX_HORIZON_CORRECTION_DEG = 75; +const MAX_REORIENTATION_ROLL_DEG = 45; +// Abrupt roll or pitch changes between neighboring captures are reconstruction +// errors, not plausible camera motion, so their horizon is not worth using. +const MAX_LEVEL_ROLL_DELTA_DEG = 15; +const MAX_LEVEL_PITCH_DELTA_DEG = 15; +const MAX_PERSPECTIVE_AUTO_ZOOM = 0.75; +const MAX_PERSPECTIVE_FOV = 125; +const PERSPECTIVE_EDGE_MARGIN = 1e-3; +const USER_ZOOM_EPSILON = 1e-2; + +/** Pitch correction, in degrees, that levelling to a horizon row implies. */ +export function horizonPitchDeg(horizonY: number): number { + return (0.5 - horizonY) * 180; +} + +/** + * Whether a reconstructed pose is plausible enough to level to. Judged on its + * roll, and — once the sequence has an accepted pose to compare against — on + * how far its pitch has moved since that pose. Roll is unsigned, as + * {@link ReorientationComponent._rollDeg} reports it. + */ +export function isLevelPlausible( + rollDeg: number, + pitchDeg: number, + baselineRollDeg: number, + baselinePitchDeg: number): boolean { + if (!Number.isFinite(rollDeg) || !Number.isFinite(pitchDeg)) { + return false; + } + return baselineRollDeg == null || baselinePitchDeg == null || + Math.abs(rollDeg - baselineRollDeg) <= MAX_LEVEL_ROLL_DELTA_DEG && + Math.abs(pitchDeg - baselinePitchDeg) <= MAX_LEVEL_PITCH_DELTA_DEG; +} + +/** + * The heading an image ended up being shown at, once reorientation has decided + * whether to turn it. Emitted on {@link ReorientationComponent.settled$}. + */ +export interface ReorientationSettledEvent { + id: string; + bearing: number; +} /** * @class ReorientationComponent * - * @classdesc Reorients each spherical image to face the direction of travel - * (the great-circle bearing toward the next image in the sequence) as the - * user navigates, instead of preserving the previous look direction. If the - * user drags to look around, that manual offset is preserved across the rest - * of the sequence rather than re-facing forward on every step. Active by - * default; disable with `component: { reorientation: false }`. + * @classdesc Reorients spherical images to face the direction of travel and + * levels their horizon. Perspective images keep their heading but are leveled + * and fitted so rotation does not expose the image boundary. If the user drags + * to look around, that manual offset is preserved across the rest of a + * spherical sequence rather than re-facing forward on every step. + * + * Crossing into a new spherical sequence with a direction arrow keeps the + * carried view rather than snapping to travel — the arrow's own transition + * already matched the angle — and adopts it as the new sequence's look-around + * offset. Every other way into a new sequence — Next/Prev, map click, or a + * fresh load without an explicitly adopted view — resets to travel + horizon. + * + * Active by default; disable with `component: { reorientation: false }`. * * @example * ```js @@ -53,33 +113,229 @@ export class ReorientationComponent // load or a deliberate jump to another capture) can be reoriented even when // the landing image's own GPS speed reads as stationary. private _lastSeq: string; + private _imageSequence: string; + private _sequenceChanged: boolean; + private _appliedPerspectiveZoom: number; + private _userZoomOverride: boolean; + private _automaticHorizonLeveling: boolean; + private _reorientToFront: boolean; + private _reorientOnSpatialNav: boolean; + private _adoptedView: number[]; + private _adoptedSequence: string; + private _settled$: Subject = + new Subject(); + + // Live viewer bearing, and its value snapshotted at the moment an image + // change arrives — see the currentImage$ subscription for why the snapshot + // is needed. + private _liveBearing: number; + private _incomingBearing: number; + private _dragging: boolean = false; + private _userViewChanged: boolean = false; + private _userViewRevision: number = 0; // Manual horizontal look-around offset, preserved within a sequence so the // engine doesn't yank the view back to the travel direction on every step. private _userOffsetX: number; - // Vertical look offset from the horizon. Seeded once per activation from the - // current view (so a shared link's y / carried pitch is kept) and updated on - // drag, then applied deterministically as 0.5 + offset. Re-reading the live + // Vertical look offset from the reconstructed horizon. Seeded once per + // activation from the current view and updated on drag. Re-reading the live // y every image round-trips through the spherical projection and drifts, so // it is held, not re-read. private _userOffsetY: number; private _ySeeded: boolean; + // Last accepted reconstructed pose. Rejected poses must not shift this + // baseline or their correct neighbors would be rejected instead. + private _levelRollDeg: number; + private _levelPitchDeg: number; + private _levelAccepted: boolean; + private _currentTransform: Transform; + private _viewportCoords: ViewportCoords = new ViewportCoords(); constructor(name: string, container: Container, navigator: Navigator) { super(name, container, navigator); } + /** + * The heading each image settles on, emitted once this component has + * decided whether to turn it. + * + * A host drawing its own indicator cannot infer this: the decision is + * asynchronous (it waits on the engine), and when the view is carried + * across — any direction arrow, or a turn below the reorientation + * threshold — no camera moves, so no bearing event is produced either. + */ + public get settled$(): Observable { + return this._settled$; + } + + /** + * The reoriented viewer bearing (degrees clockwise from north) the given + * image will be shown at once navigated to: its travel direction plus the + * manual look-around offset preserved across the sequence — the exact + * heading the reoriented view (and any cone tracking it) ends up at. + * + * Synchronous: reads only the engine's precomputed cache (populated for the + * images around the current one), so there is no network round-trip. + * Returns null when there is no valid reorientation for the id — not a + * reoriented pano, end of sequence, or not yet computed — so callers can + * fall back to the image's own compass angle. + */ + public getReorientedBearing(id: string): number | null { + const engine = this._engine; + if (!engine) { + return null; + } + const result = engine.get(id); + if (!result || !result.valid || typeof result.travel !== "number") { + return null; + } + if (!this._reorientToFront && + result.seq != null && result.seq === this._lastSeq) { + const active = this._engine.get(this._activeId); + if (active?.valid && + typeof active.viewCompassAngle === "number" && + typeof this._liveBearing === "number") { + return this._mapBearing(active, this._liveBearing); + } + } + // The look-around offset belongs to the active sequence and is reset on + // a cross-sequence jump, so apply it only to same-sequence ids — + // otherwise the hover cone predicts travel+offset while the actual + // landing (fresh sequence → offset 0) shows plain travel. + const offset = result.seq != null && result.seq === this._lastSeq ? + this._userOffsetX * 360 : 0; + return ((result.travel + offset) % 360 + 360) % 360; + } + + /** Resolve an uncached image before returning its predicted view bearing. */ + public getReorientedBearingAsync(id: string): Promise { + const engine = this._engine; + if (engine == null) { + return Promise.resolve(null); + } + + return engine.precompute(id, undefined, 0) + .then((): number | null => + this._engine === engine ? this.getReorientedBearing(id) : null) + .catch((): null => null); + } + + /** + * The GPS-derived direction of travel for an image, independent of its + * compass orientation and the viewer's look-around offset. + * + * Returns null until the reorientation engine has resolved the image or + * when the sequence cannot provide a valid neighboring segment. + */ + public getTravelBearing(id: string): number | null { + const result = this._engine == null ? null : this._engine.get(id); + if (!result || !result.valid || typeof result.travel !== "number") { + return null; + } + + return ((result.travel % 360) + 360) % 360; + } + + /** + * Treat the given basic coordinates as the user's look-around offset rather + * than reorienting away from them, and land the current image on them. + * + * For a view the host already knows about but this component never observed + * — a shared link carrying explicit basic coordinates, say — the offset + * would otherwise be discarded and the next navigation would snap to the + * travel direction. Coordinates are passed in rather than read from the + * viewer so the call does not race the host applying them. + */ + public adoptView(basic: number[]): void { + this._adoptedView = basic != null && basic.length === 2 ? + [basic[0], basic[1]] : null; + this._adoptedSequence = this._adoptedView != null ? + this._imageSequence : null; + } + + /** + * The heading the given image would be shown at if it were reached right + * now with a direction arrow, or null when that cannot be determined + * (reorientation absent, nothing cached yet, not a reorientable pano). + * + * Mirrors the decision _reorient makes on arrival, including the minimum + * turn threshold, so a host can draw a hover indicator that matches where + * the view will actually land. The travel direction alone is not that + * answer: an arrow carries the view across a sequence boundary or a + * sideways hop, and a turn smaller than the threshold is skipped. + */ + public predictBearingTo(id: string): number | null { + const engine = this._engine; + if (engine == null || + this._activeId == null || + typeof this._liveBearing !== "number") { + return null; + } + const from = engine.get(this._activeId); + if (from == null || !from.valid || + typeof from.cca !== "number" || + typeof from.viewCompassAngle !== "number") { + return null; + } + const liveBearing = this._mapBearing(from, this._liveBearing); + // Answered before looking the target up: anything that is not the + // in-sequence neighbour is a sideways hop or a sequence crossing, both + // of which carry the view. The engine only caches within the current + // sequence, so those targets are usually absent and requiring them here + // would return null for exactly the cases the host most needs. + const neighbor = from.nextId === id || from.prevId === id; + if (!neighbor || !this._reorientToFront || + !this._reorientOnSpatialNav) { + return liveBearing; + } + + const to = engine.get(id); + if (to == null || !to.valid || + typeof to.viewCompassAngle !== "number" || + typeof to.travel !== "number") { + return null; + } + if (to.seq == null || to.seq !== this._lastSeq) { + return liveBearing; + } + + const targetX = this._applyOffsetX(to.basicX); + const viewX = bearingToBasicX(liveBearing, to.viewCompassAngle); + const dxDeg = Math.abs(wrapDelta(targetX - viewX)) * 360; + + return dxDeg >= MIN_REORIENT_DEG ? + this._bearingForView(to, targetX) : + liveBearing; + } + protected _activate(): void { const subs = this._subscriptions; subs.push(this._configuration$.subscribe( (configuration: ReorientationConfiguration): void => { + this._automaticHorizonLeveling = + configuration.automaticHorizonLeveling ?? + DEFAULT_REORIENTATION_CONFIGURATION.automaticHorizonLeveling; + this._reorientToFront = configuration.reorientToFront ?? + DEFAULT_REORIENTATION_CONFIGURATION.reorientToFront; + this._reorientOnSpatialNav = + configuration.reorientOnSpatialNav ?? + DEFAULT_REORIENTATION_CONFIGURATION.reorientOnSpatialNav; this._engine = new ReorientationEngine( this._createProvider(), configuration); this._activeId = null; this._computedBasicX = 0.5; this._lastSeq = null; + this._imageSequence = null; + this._sequenceChanged = false; + this._appliedPerspectiveZoom = 0; + this._userZoomOverride = false; + this._userViewChanged = false; + // NOT _adoptedView: it is host intent that can be handed over + // before this fires (activation and configure() both re-run + // this), and clearing it here silently drops the view the host + // asked to keep. this._resetOffset(); })); @@ -90,14 +346,66 @@ export class ReorientationComponent if (!image) { return; } + this._currentTransform = new Transform( + image.exifOrientation, + image.width, + image.height, + image.scale, + image.rotation, + [0, 0, 0], + image.image, + image.camera); + this._levelAccepted = false; + const fromId = this._activeId; + if (this._adoptedView != null && + this._adoptedSequence == null) { + this._adoptedSequence = image.sequenceId; + } + this._sequenceChanged = fromId != null && + image.sequenceId !== this._imageSequence; + if (this._sequenceChanged) { + this._navigator.stateService.setZoom(0); + this._appliedPerspectiveZoom = 0; + this._userZoomOverride = false; + } this._activeId = image.id; - this._reorient(image); + this._imageSequence = image.sequenceId; + // Snapshot now, synchronously with the change. By the time the + // engine resolves, the transition has already begun moving the + // camera, so neither the live bearing nor getCenter() still + // describes the view the user carried in. + this._incomingBearing = this._liveBearing; + // Consume the direction here, synchronously with the landing, so + // it attributes to this image and not a later re-emit. + const direction = this._navigator.consumeMoveDirection(); + const commitSequenceView = this._userViewChanged && + (direction === NavigationDirection.Next || + direction === NavigationDirection.Prev); + this._userViewChanged = false; + this._reorient( + image, + direction, + fromId, + this._dragging, + commitSequenceView, + this._userViewRevision); })); + subs.push(this._container.renderService.bearing$.subscribe( + (bearing: number): void => { this._liveBearing = bearing; })); + + subs.push(this._container.mouseService.mouseDragStart$.subscribe( + (): void => { this._startUserViewChange(); })); + subs.push(this._container.touchService.singleTouchDragStart$.subscribe( + (): void => { this._startUserViewChange(); })); + // A finished drag is a genuine user look-around (our own steering goes - // through the state, not pointer events), so capture the offset. + // through the state, not pointer events), so capture the offset. During + // playback, stop residual momentum from spilling into later images. subs.push(this._container.mouseService.mouseDragEnd$.subscribe( - (): void => { this._captureOffset(); })); + (): void => { this._finishUserViewChange(); })); + subs.push(this._container.touchService.singleTouchDragEnd$.subscribe( + (): void => { this._finishUserViewChange(); })); } protected _deactivate(): void { @@ -105,6 +413,12 @@ export class ReorientationComponent this._engine = null; this._activeId = null; this._lastSeq = null; + this._imageSequence = null; + this._sequenceChanged = false; + this._appliedPerspectiveZoom = 0; + this._userZoomOverride = false; + this._dragging = false; + this._userViewChanged = false; this._resetOffset(); } @@ -112,7 +426,13 @@ export class ReorientationComponent return {}; } - private _reorient(image: Image): void { + private _reorient( + image: Image, + direction: NavigationDirection, + fromId: string, + draggingAtNavigation: boolean, + commitSequenceView: boolean, + userViewRevision: number): void { const id = image.id; const seed = this._seed(image); const engine = this._engine; @@ -123,55 +443,208 @@ export class ReorientationComponent return; } const result = engine.get(id); - // Read mesh now (after the precompute delay) so it's loaded: - // an image with SfM mesh eases to the travel direction, one - // without (disconnected) hard-cuts. The transition type is NOT - // used — an SfM image eases however you arrive (fresh URL, - // in-sequence step, or feed-click jump). - const meshV = image.mesh && image.mesh.vertices ? - image.mesh.vertices.length : -1; - const hardCut = meshV <= 0; + if (this._adoptedView != null && + this._adoptedSequence != null && + image.sequenceId !== this._adoptedSequence) { + this._clearAdoptedView(); + } + const sequenceId = result?.seq ?? image.sequenceId; + const freshSequence = + sequenceId != null && sequenceId !== this._lastSeq; + if (this._dragging || this._userViewChanged || + this._userViewRevision !== userViewRevision) { + if (freshSequence) { + this._navigator.stateService.clearReorientations(); + this._resetOffset(); + } + this._lastSeq = sequenceId; + if (result?.valid) { + this._computedBasicX = result.basicX; + } + this._clearAdoptedView(); + if (!this._dragging) { + this._captureOffset(); + } + return; + } + if (!isSpherical(image.cameraType)) { + this._levelPerspective(id); + return; + } + // A disconnected image still hard-cuts after navigation so its + // image change is not followed by a distracting pan. On initial + // load there is no preceding image cut, so use the same smooth + // orientation as a reconstructed image. + const hasReconstruction = + hasReconstructionMesh(image.mesh) && + result?.computedCompassOutlier !== true; + const hardCut = !hasReconstruction && fromId != null; + let levelingActive = + hasReconstruction && this._automaticHorizonLeveling; + const horizonY = (x: number): number => + levelingActive ? this._horizonY(x) : 0.5; + this._lastSeq = sequenceId; + + // An arrow move already lands at the heading the user was + // looking at, because the state layer carries the view across + // the image change. Crossing into a new sequence that way keeps + // that view and adopts it as this sequence's look-around offset, + // so nothing rotates here and the following in-sequence steps + // preserve the framing the user arrived with. A direct jump + // within the same sequence preserves that manual offset too; + // entering another sequence resets to travel. + const spatialNav = + this._isStep(direction) || this._isTurn(direction); + const leftAnother = fromId != null && fromId !== id; + const fromResult = leftAnother ? engine.get(fromId) : null; + const neighbor = fromResult != null && + (fromResult.nextId === id || fromResult.prevId === id); + // A user's drag can keep moving after mouse-up, but automatic + // camera movement must not become a persistent look offset. + if ((commitSequenceView || + (draggingAtNavigation && this._dragging)) && neighbor && + fromResult.valid && + typeof fromResult.cca === "number" && + typeof fromResult.basicX === "number" && + typeof this._incomingBearing === "number") { + const incomingX = bearingToBasicX( + this._incomingBearing, fromResult.cca); + this._userOffsetX = + wrapDelta(incomingX - fromResult.basicX); + } + // An arrow that lands somewhere other than the image next to + // the one we left is a sideways hop, not a step along the road: + // a parallel pass, or the return leg of a capture that doubles + // back, whose travel direction can be the reverse of ours. + // Facing its travel would swing the user around, so treat it + // like a sequence crossing and keep the carried view. + const lateralHop = + spatialNav && leftAnother && fromResult != null && !neighbor; + const carryView = spatialNav && (freshSequence || lateralHop); + // A directionless non-neighbor landing is a map/pKey jump. It + // still bypasses the low-motion guard, while resetView below + // distinguishes a same-capture jump from a new capture. + const jump = direction == null && leftAnother && !neighbor; + const resetView = freshSequence && !carryView; + + if (freshSequence) { + // Drop look-ahead hints from the prior sequence so nothing + // carries over; this sequence registers its own as it goes. + this._navigator.stateService.clearReorientations(); + } + // Comparing pitch against the previous image only means + // something between adjacent images. A jump lands anywhere in + // the capture, where a different pitch is the terrain rather + // than a bad pose, so it starts the comparison over. + if (freshSequence || !neighbor) { + this._levelRollDeg = null; + this._levelPitchDeg = null; + } + if (!result || !result.valid) { + this._levelAccepted = false; + // Switching out of Gravity can reset a center queued before + // the image loaded, so restore an explicit shared-link view + // after the fallback transition. Keep it pending because an + // endpoint has no travel result from which to derive the + // offset that its first valid same-sequence neighbor needs. + const adoptedView = this._adoptedView; + this._navigator.stateService.traverse(); + if (resetView) { + this._resetOffset(); + } + if (adoptedView != null) { + this._navigator.stateService.setCenter(adoptedView); + } else if (resetView) { + this._navigator.stateService.setCenter([0.5, 0.5]); + } + + return; + } + const horizonRow = levelingActive ? + this._horizonRow(result.basicX) : null; + const safetyCenter = [result.basicX, horizonRow ?? 0.5]; + const rollDeg = levelingActive ? + this._rollDeg(safetyCenter) : 0; + if (rollDeg == null || + rollDeg > MAX_REORIENTATION_ROLL_DEG) { + this._levelAccepted = false; + this._navigator.stateService.traverse(); + if (resetView) { + this._resetOffset(); + } + let fallbackCenter = + this._adoptedView ?? [result.basicX, 0.5]; + if (this._adoptedView == null && + this._isStep(direction) && + typeof this._incomingBearing === "number" && + typeof result.cca === "number") { + const carriedX = bearingToBasicX( + this._incomingBearing, result.cca); + fallbackCenter = [carriedX, 0.5]; + this._userOffsetX = + wrapDelta(carriedX - result.basicX); + } + this._navigator.stateService.setCenter(fallbackCenter); + this._clearAdoptedView(); return; } + if (levelingActive && !this._acceptLevel(rollDeg, horizonRow)) { + levelingActive = false; + } + this._levelAccepted = levelingActive; + if (levelingActive) { + this._navigator.stateService.gravityTraverse(); + } else { + this._navigator.stateService.traverse(); + } this._computedBasicX = result.basicX; + if (!levelingActive && !this._ySeeded) { + this._userOffsetY = 0; + this._ySeeded = true; + } + + if (draggingAtNavigation && this._dragging && neighbor) { + return; + } + + // Within a sequence an arrow step is where reorientation earns + // its keep: the carried view drifts off-axis as the road bends. + // Opt out to compare against plain carried-view navigation. + if (spatialNav && !freshSequence && + !this._reorientOnSpatialNav) { + this._clearAdoptedView(); + + return; + } - // A fresh load or a jump to another capture lands on a new - // sequence; reorient it to travel direction even if the landing - // image's GPS speed reads as stationary. Within a sequence, - // preserve the view when not moving rather than spinning on - // jitter. - const freshSequence = - result.seq != null && result.seq !== this._lastSeq; - this._lastSeq = result.seq; // Pitch is tracked deterministically via the offset (the live // view y drifts ~18° through the spherical projection, so it - // can't be measured per image). On a sequence change the held - // pitch resets to horizon; the reset amount is how far the - // carried pitch must move, used to force a reorientation even - // when the horizontal change is small. + // can't be measured per image). On a reset the held pitch resets + // to horizon; the reset amount is how far the carried pitch must + // move, used to force a reorientation even when the horizontal + // change is small. Carry/turn keep the pitch as part of the + // preserved look-around. let pitchResetDeg = 0; - if (freshSequence) { + if (resetView) { // Don't carry x/pitch/rotation offsets across sequences: // start fresh at travel direction + horizon. A cross-sequence // jump carries the prior image's view, which would otherwise // persist the previous sequence's pitch (e.g. "looking down"). - pitchResetDeg = Math.abs(this._userOffsetY) * 180; + pitchResetDeg = this._automaticHorizonLeveling ? + Math.abs(this._userOffsetY) * 180 : 0; this._userOffsetX = 0; - this._userOffsetY = 0; + if (this._automaticHorizonLeveling) { + this._userOffsetY = 0; + } this._ySeeded = true; - // Drop look-ahead hints from the prior sequence so nothing - // carries over; this sequence registers its own as it goes. - this._navigator.stateService.clearReorientations(); } - if (!result.moving && !freshSequence) { - // Low motion within the current sequence — preserve the view. + if (!result.moving && !freshSequence && !jump) { + // Low motion between adjacent images — preserve the view + // rather than spin on stationary GPS jitter. return; } - // Horizontal target: travel direction plus any manual offset. - const targetX = this._applyOffsetX(result.basicX); - // Read this image's current view and reorient only if doing so // would move it past the threshold on either axis — otherwise // the move is just jitter. Measured per image (current view → @@ -181,24 +654,63 @@ export class ReorientationComponent if (this._activeId !== id || this._engine !== engine) { return; } + // Adopt the host-supplied view as the offset before any + // of the reset/carry decisions above take effect, so a + // shared link's framing becomes the look-around offset + // this sequence preserves. Read the view as those coords + // too, not as whatever the viewer shows right now: the + // host may not have applied them yet, and steering there + // ourselves would animate a rotation the user did not + // ask for. Equal to targetX, so this image never moves. + let viewX = center[0]; + if (this._adoptedView != null) { + const adopted = this._adoptedView; + this._clearAdoptedView(); + viewX = adopted[0]; + this._userOffsetX = + wrapDelta(adopted[0] - result.basicX); + this._userOffsetY = + adopted[1] - horizonY(adopted[0]); + this._ySeeded = true; + } + else if (carryView && + typeof this._incomingBearing === "number" && + typeof result.cca === "number") { + // Not center[0]: on a sideways hop or a sequence + // crossing the two frames can be ~180 deg apart, and + // the state layer is still settling that when this + // resolves, so the sampled centre lags the carried + // view by however far it has got. + viewX = bearingToBasicX( + this._incomingBearing, result.cca); + } // Seed the held pitch offset once from the loaded view. if (!this._ySeeded) { - this._userOffsetY = center[1] - 0.5; + this._userOffsetY = center[1] - horizonY(center[0]); this._ySeeded = true; } - const targetY = - Math.max(0, Math.min(1, 0.5 + this._userOffsetY)); + // Arrow into a new sequence or a sideways hop: keep the + // view the user carried in. Seed the look-around offset + // from that view so following steps preserve it; + // targetX then equals viewX, so nothing rotates. Reads + // viewX rather than center so it agrees with an adopted + // view instead of overwriting the offset just set. + if (carryView) { + this._userOffsetX = + wrapDelta(viewX - result.basicX); + } + const targetX = this._reorientToFront ? + this._applyOffsetX(result.basicX) : viewX; + const targetY = this._automaticHorizonLeveling ? + Math.max(0, Math.min( + 1, horizonY(targetX) + this._userOffsetY)) : + center[1]; - // Horizontal only: the pitch is held deterministically, - // so the live y wobbles with spherical-projection round- - // trip noise (~0.1 ≈ 18°). Gating on it would just snap - // back projection drift — the very jitter we're avoiding. const dxDeg = - Math.abs(wrapDelta(targetX - center[0])) * 360; - // Reorient if the horizontal move clears the threshold, - // or (on a sequence change) the pitch must reset by more - // than the threshold to clear a carried look up/down. + Math.abs(wrapDelta(targetX - viewX)) * 360; + const dyDeg = Math.abs(targetY - center[1]) * 180; const move = dxDeg >= MIN_REORIENT_DEG || + dyDeg >= MIN_REORIENT_DEG || pitchResetDeg >= MIN_REORIENT_DEG; if (move) { if (hardCut) { @@ -210,48 +722,298 @@ export class ReorientationComponent } } - // Pre-orient the next image (hint) so a hard cut to it - // doesn't flash the carried view — but only if its - // reorientation would also clear the threshold. This is - // cross-image, so reason in absolute bearings: the view - // the next image carries in is where this one ends up. - if (result.nextId && typeof result.cca === "number") { - const endX = move ? targetX : center[0]; - const endBearing = - result.cca + (endX - 0.5) * 360; - const nid = String(result.nextId); - engine.precompute(nid) - .then((): void => { - if (this._engine !== engine) { - return; - } - const nr = engine.get(nid); - if (!nr || !nr.valid || - typeof nr.cca !== "number") { - return; - } - const nTargetX = this._applyOffsetX(nr.basicX); - const nCarriedX = - bearingToBasicX(endBearing, nr.cca); - const nDx = Math.abs( - wrapDelta(nTargetX - nCarriedX)) * 360; - if (nDx >= MIN_REORIENT_DEG) { - this._navigator.stateService - .setReorientation( - nid, [nTargetX, targetY]); - } - }) - .catch((): void => { /* skip */ }); + // Pre-orient BOTH neighbors (hints) so a hard cut + // doesn't flash the carried view and stepping either way + // lands on the travel direction deterministically. Prev + // needs this as much as next: without a hint, backward + // navigation only reorients on arrival past the 15° + // threshold, so on gentle stretches it holds the carried + // bearing and the prev cone (which predicts travel + + // offset) no longer matches. Cross-image, so reason in + // absolute bearings: the view a neighbor carries in is + // where this image ends up. + if (typeof result.viewCompassAngle === "number") { + const endX = move ? targetX : viewX; + const endBearing = this._bearingForView(result, endX); + // Carrying the view moves no camera, so a host + // watching bearing events would never learn where + // this image ended up. Tell it outright. + this._settled$.next({ + id, + bearing: endBearing, + }); + if (result.nextId) { + this._hintNeighbor( + engine, result.nextId, endBearing, targetY); + } + if (result.prevId) { + this._hintNeighbor( + engine, result.prevId, endBearing, targetY); + } } }); }) .catch((): void => { /* skip images we can't resolve */ }); } + private _startUserViewChange(): void { + this._dragging = true; + this._userViewRevision++; + this._navigator.stateService.clearReorientations(); + } + + private _finishUserViewChange(): void { + if (this._navigator.playService.playing) { + this._navigator.stateService.rotateBasicWithoutInertia([0, 0]); + } + this._dragging = false; + this._userViewChanged = true; + this._captureOffset(); + } + private _applyOffsetX(basicX: number): number { return ((basicX + this._userOffsetX) % 1 + 1) % 1; } + private _bearingForView( + result: ReorientationResult, + basicX: number): number { + const bearing = result.viewCompassAngle + (basicX - 0.5) * 360; + return ((bearing % 360) + 360) % 360; + } + + private _mapBearing( + result: ReorientationResult, + bearing: number): number { + return ((bearing + result.viewCompassAngle - result.cca) % 360 + 360) % 360; + } + + private _clearAdoptedView(): void { + this._adoptedView = null; + this._adoptedSequence = null; + } + + // Spherical is the pano-to-pano step, so it counts as one here. + private _isStep(direction: NavigationDirection): boolean { + return direction === NavigationDirection.StepLeft || + direction === NavigationDirection.StepRight || + direction === NavigationDirection.StepForward || + direction === NavigationDirection.StepBackward || + direction === NavigationDirection.Spherical; + } + + private _isTurn(direction: NavigationDirection): boolean { + return direction === NavigationDirection.TurnLeft || + direction === NavigationDirection.TurnRight || + direction === NavigationDirection.TurnU; + } + + // Pre-set a neighbor's reorientation to travel + offset, but only if it + // would clear the threshold against the view it carries in from this image + // (endBearing) — otherwise the step is small enough to leave alone. + private _hintNeighbor( + engine: ReorientationEngine, + id: string, + endBearing: number, + targetY: number): Promise { + if (!this._reorientToFront) { + return Promise.resolve(); + } + const userViewRevision = this._userViewRevision; + // Depth 0: we only need this neighbor's own result cached, not another + // forward prefetch cascade (the current image's _reorient already warms + // ahead). Avoids re-walking the already-scheduled chain per navigation. + return engine.precompute(id, undefined, 0) + .then((): void => { + if (this._engine !== engine || + this._userViewRevision !== userViewRevision) { + return; + } + const nr = engine.get(id); + if (!nr || !nr.valid || + typeof nr.viewCompassAngle !== "number") { + return; + } + const nTargetX = this._applyOffsetX(nr.basicX); + const nCarriedX = bearingToBasicX( + endBearing, nr.viewCompassAngle); + const nDx = Math.abs(wrapDelta(nTargetX - nCarriedX)) * 360; + const unsafeTransition = + nr.computedCompassOutlier === true || + nr.reconstructionDiscontinuity === true; + // Even when the final world bearings nearly match, a rejected + // or discontinuous pose can map that bearing to a very different + // basic x for its first frame. Cut before it can render. + if (unsafeTransition || nDx >= MIN_REORIENT_DEG) { + this._navigator.stateService.setReorientation( + id, + [nTargetX, targetY], + unsafeTransition); + } + }) + .catch((): void => { /* skip */ }); + } + + private _levelPerspective(id: string): void { + if (!this._automaticHorizonLeveling) { + this._clearAdoptedView(); + return; + } + if (this._adoptedView != null) { + this._navigator.stateService.gravityTraverse(); + this._clearAdoptedView(); + return; + } + + this._navigator.stateService.getCenter().pipe(first()).subscribe( + (): void => { + if (this._activeId !== id || this._adoptedView != null || + this._dragging || this._userViewChanged) { + this._clearAdoptedView(); + return; + } + const target = [0.5, this._horizonY(0.5)]; + this._container.renderService.renderCameraFrame$.pipe( + filter((render: RenderCamera): boolean => + render.currentImageId === id), + first(), + ).subscribe((render: RenderCamera): void => { + if (this._activeId !== id || this._adoptedView != null || + this._dragging || this._userViewChanged) { + this._clearAdoptedView(); + return; + } + const zoom = this._perspectiveAutoZoom( + target, render.unzoomedCurrentFov); + const stateService = this._navigator.stateService; + if (!this._sequenceChanged && + Math.abs(render.zoom - this._appliedPerspectiveZoom) > + USER_ZOOM_EPSILON) { + this._userZoomOverride = true; + } + if (zoom == null) { + stateService.traverse(); + stateService.zoomTo( + this._userZoomOverride ? render.zoom : 0); + this._appliedPerspectiveZoom = 0; + return; + } + stateService.gravityTraverse(); + stateService.rotateToBasicSmooth(target); + stateService.zoomTo( + this._userZoomOverride ? render.zoom : zoom); + this._appliedPerspectiveZoom = zoom; + }); + }); + } + + private _acceptLevel(rollDeg: number, horizonRow: number): boolean { + if (horizonRow == null) { + return false; + } + const pitchDeg = horizonPitchDeg(horizonRow); + if (!isLevelPlausible( + rollDeg, + pitchDeg, + this._levelRollDeg, + this._levelPitchDeg)) { + return false; + } + this._levelRollDeg = rollDeg; + this._levelPitchDeg = pitchDeg; + return true; + } + + private _rollDeg(center: number[]): number | null { + const transform = this._currentTransform; + if (transform == null) { + return null; + } + const origin = new THREE.Vector3().fromArray( + transform.unprojectSfM([0, 0], 0)); + const direction = new THREE.Vector3().fromArray( + transform.unprojectBasic(center, 10)) + .sub(origin) + .normalize(); + const imageUp = transform.upVector() + .addScaledVector(direction, -transform.upVector().dot(direction)) + .normalize(); + const gravityUp = new THREE.Vector3(0, 0, 1) + .addScaledVector(direction, -direction.z) + .normalize(); + const rollDeg = imageUp.angleTo(gravityUp) * 180 / Math.PI; + return Number.isFinite(rollDeg) ? rollDeg : null; + } + + private _perspectiveAutoZoom( + center: number[], baseFov: number): number | null { + const transform = this._currentTransform; + const element = this._container.container; + if (transform == null || element.offsetHeight === 0) { + return null; + } + + const origin = new THREE.Vector3().fromArray( + transform.unprojectSfM([0, 0], 0)); + const rollDeg = this._rollDeg(center); + if (rollDeg == null || + rollDeg > MAX_REORIENTATION_ROLL_DEG) { + return null; + } + + const camera = new THREE.PerspectiveCamera( + 60, + element.offsetWidth / element.offsetHeight, + 1e-1, + 1e4); + camera.position.copy(origin); + + const maxFov = (basic: number[], up: THREE.Vector3): number => { + camera.up.copy(up); + camera.lookAt(new THREE.Vector3().fromArray( + transform.unprojectBasic(basic, 10))); + camera.updateMatrixWorld(true); + + const fits = (fov: number): boolean => { + camera.fov = fov; + camera.updateProjectionMatrix(); + const corners = [ + this._viewportCoords.viewportToBasic( + -1, 1, transform, camera), + this._viewportCoords.viewportToBasic( + 1, 1, transform, camera), + this._viewportCoords.viewportToBasic( + 1, -1, transform, camera), + this._viewportCoords.viewportToBasic( + -1, -1, transform, camera), + ]; + return corners.every((point: number[]): boolean => + point != null && + point[0] >= PERSPECTIVE_EDGE_MARGIN && + point[0] <= 1 - PERSPECTIVE_EDGE_MARGIN && + point[1] >= PERSPECTIVE_EDGE_MARGIN && + point[1] <= 1 - PERSPECTIVE_EDGE_MARGIN); + }; + + let low = 0; + let high = MAX_PERSPECTIVE_FOV; + for (let i = 0; i < 16; i++) { + const middle = (low + high) / 2; + if (fits(middle)) { + low = middle; + } else { + high = middle; + } + } + return low; + }; + + const levelFov = maxFov(center, new THREE.Vector3(0, 0, 1)); + const zoom = Math.max(0, Math.log(baseFov / levelFov) / Math.log(2)); + return Number.isFinite(zoom) && zoom <= MAX_PERSPECTIVE_AUTO_ZOOM ? + zoom : null; + } + private _captureOffset(): void { const id = this._activeId; const engine = this._engine; @@ -267,24 +1029,93 @@ export class ReorientationComponent const basis = result && result.valid ? result.basicX : this._computedBasicX; this._userOffsetX = wrapDelta(center[0] - basis); - this._userOffsetY = center[1] - 0.5; + this._userOffsetY = + center[1] - this._levelRow(center[0]); this._ySeeded = true; }); } + private _levelRow(x: number): number { + return this._levelAccepted ? this._horizonY(x) : 0.5; + } + + private _horizonY(x: number): number { + return this._horizonRow(x) ?? 0.5; + } + + private _horizonRow(x: number): number | null { + const transform = this._currentTransform; + if (transform == null) { + return null; + } + + const cameraZ = transform.unprojectSfM([0, 0], 0)[2]; + let low = 0; + let high = 1; + let lowZ = transform.unprojectBasic([x, low], 10)[2] - cameraZ; + const highZ = transform.unprojectBasic([x, high], 10)[2] - cameraZ; + if (lowZ * highZ > 0) { + return null; + } + + for (let i = 0; i < 32; i++) { + const middle = (low + high) / 2; + const middleZ = + transform.unprojectBasic([x, middle], 10)[2] - cameraZ; + if (lowZ * middleZ > 0) { + low = middle; + lowZ = middleZ; + } else { + high = middle; + } + } + + const horizon = (low + high) / 2; + return Math.abs(horizon - 0.5) * 180 <= + MAX_HORIZON_CORRECTION_DEG ? horizon : null; + } + private _resetOffset(): void { this._userOffsetX = 0; this._userOffsetY = 0; this._ySeeded = false; + this._levelRollDeg = null; + this._levelPitchDeg = null; + this._levelAccepted = false; } private _seed(image: Image): ReorientationImage { const lngLat = image.lngLat; + const originalLngLat = image.originalLngLat; + const hasComputedCompass = + Number.isFinite(image.computedCompassAngle); + // Lookahead metadata has no loaded mesh. Its computed pose is + // provisional and is replaced by the real mesh classification when the + // image becomes current. + const hasReconstruction = image.hasInitializedCache() ? + hasReconstructionMesh(image.mesh) : hasComputedCompass; + // Placeholder geometry can carry a computed pose that is wildly + // inconsistent between neighboring frames. Its raw capture heading is + // the stable center axis of the underlying panorama. + const useOriginalCompass = !hasReconstruction && + hasComputedCompass && + Number.isFinite(image.originalCompassAngle); return { id: image.id, lat: lngLat ? lngLat.lat : null, lng: lngLat ? lngLat.lng : null, - cca: image.computedCompassAngle, + originalLat: originalLngLat ? originalLngLat.lat : null, + originalLng: originalLngLat ? originalLngLat.lng : null, + cca: useOriginalCompass ? + image.originalCompassAngle : image.compassAngle, + computedCca: hasComputedCompass ? + image.computedCompassAngle : undefined, + originalCca: Number.isFinite(image.originalCompassAngle) ? + image.originalCompassAngle : undefined, + // Unmerged equirectangular pixels use an east-facing axis; their + // raw compass describes travel rather than the panorama center. + viewCompassAngle: isSpherical(image.cameraType) && + !hasComputedCompass ? 90 : undefined, cam: image.cameraType, seq: image.sequenceId, ts: image.capturedAt, @@ -294,23 +1125,52 @@ export class ReorientationComponent private _createProvider(): ReorientationProvider { const graphService = this._navigator.graphService; const imageCache = new Map(); + const imagePending = new Map>(); + const fetchImages = (ids: string[]): Promise => { + const normalized = ids.map(String); + const missing = Array.from(new Set(normalized)).filter( + (id: string): boolean => + !imageCache.has(id) && !imagePending.has(id)); + if (missing.length > 0) { + const batch = new Promise((resolve, reject) => { + graphService.cacheImagesMetadata$(missing) + .pipe(first()) + .subscribe(resolve, reject); + }).then((images: Image[]): void => { + for (const image of images) { + imageCache.set(image.id, this._seed(image)); + } + }); + for (const id of missing) { + const pending = batch + .then((): ReorientationImage => { + const image = imageCache.get(id); + if (image == null) { + throw new Error(`Missing image metadata (${id})`); + } + imagePending.delete(id); + return image; + }) + .catch((error: Error): Promise => { + imagePending.delete(id); + return Promise.reject(error); + }); + imagePending.set(id, pending); + } + } + return Promise.all(normalized.map( + (id: string): Promise => + imageCache.has(id) ? + Promise.resolve(imageCache.get(id)) : + imagePending.get(id))); + }; return { fetchImage: (id: string): Promise => { - id = String(id); - const cached = imageCache.get(id); - if (cached) { - return Promise.resolve(cached); - } - return new Promise((resolve, reject) => { - graphService.cacheImage$(id).pipe(first()).subscribe( - (image: Image): void => { - const o = this._seed(image); - imageCache.set(id, o); - resolve(o); - }, - (e: Error): void => reject(e)); - }); + return fetchImages([id]).then( + (images: ReorientationImage[]): ReorientationImage => + images[0]); }, + fetchImages, cacheImage: (image: ReorientationImage): void => { if (image && image.id != null) { imageCache.set(String(image.id), image); diff --git a/src/component/reorientation/ReorientationEngine.ts b/src/component/reorientation/ReorientationEngine.ts index 28fc9b16a..26a255e81 100644 --- a/src/component/reorientation/ReorientationEngine.ts +++ b/src/component/reorientation/ReorientationEngine.ts @@ -9,7 +9,13 @@ export interface ReorientationImage { id: string; lat: number; lng: number; + originalLat?: number; + originalLng?: number; cca: number; + computedCca?: number; + originalCca?: number; + /** World bearing represented by basic x=0.5 when it differs from CCA. */ + viewCompassAngle?: number; cam: string; seq: string; ts: number; @@ -21,6 +27,7 @@ export interface ReorientationImage { */ export interface ReorientationProvider { fetchImage(id: string): Promise; + fetchImages?(ids: string[]): Promise; fetchSeqIds(seqId: string): Promise; cacheImage?(image: ReorientationImage): void; } @@ -32,10 +39,16 @@ export interface ReorientationResult { valid: boolean; reason?: string; nextId?: string; + prevId?: string; travel?: number; + computedTravel?: number; basicX?: number; dist?: number; cca?: number; + viewCompassAngle?: number; + computedCompassOutlier?: boolean; + reconstructionDiscontinuity?: boolean; + computedCompassOffset?: number; speed?: number; moving?: boolean; seq?: string; @@ -43,9 +56,12 @@ export interface ReorientationResult { export const DEFAULT_REORIENTATION_CONFIGURATION: Required = { + automaticHorizonLeveling: true, + reorientToFront: true, + reorientOnSpatialNav: true, prefetchAhead: 3, movingSpeedMps: 1, - lowSpeedTurnDistanceM: 2, + lowSpeedTurnDistanceM: 0.5, lowSpeedTurnMaxDeltaDeg: 30, outlierMaxDeltaDeg: 90, previousContextWindow: 5, @@ -56,6 +72,9 @@ export const DEFAULT_REORIENTATION_CONFIGURATION: const RAD = Math.PI / 180; const DEG = 180 / Math.PI; const EARTH_RADIUS_METERS = 6371000; +const MAX_REASONABLE_SPEED_MPS = 100; +const MAX_COMPUTED_COMPASS_DELTA_DEG = 45; +const MAX_COMPUTED_POSITION_HEADING_DELTA_DEG = 45; function isNum(v: number): boolean { return typeof v === "number" && Number.isFinite(v); @@ -110,6 +129,19 @@ interface PrevContext { moving: boolean; speed: number; travel: number; + computedTravel?: number; + compassOffset?: number; + computedCompassOutlier?: boolean; +} + +interface Segment { + dist: number; + speed: number; + speedExcessive: boolean; + travel: number; + computedSpeed: number; + computedTravel: number; + originalTravel?: number; } /** @@ -128,6 +160,7 @@ export class ReorientationEngine { private _config: Required; private _cache: Map; private _pending: Map>; + private _originalGeometrySequences: Set; constructor( provider: ReorientationProvider, @@ -139,6 +172,7 @@ export class ReorientationEngine { }; this._cache = new Map(); this._pending = new Map>(); + this._originalGeometrySequences = new Set(); } public get(id: string): ReorientationResult | null { @@ -154,21 +188,28 @@ export class ReorientationEngine { if (depth == null) { depth = cfg.prefetchAhead; } - if (this._cache.has(imgId)) { - this._prefetchNext(this._cache.get(imgId), depth); - return Promise.resolve(); + if (seed && isNum(seed.lat) && isNum(seed.lng) && + typeof this._provider.cacheImage === "function") { + this._provider.cacheImage(seed); + } + const cached = this._cache.get(imgId); + if (cached != null) { + const seedChanged = seed != null && + (cached.cca !== seed.cca || + cached.viewCompassAngle !== seed.viewCompassAngle); + if (!seedChanged) { + this._prefetchNext(cached, depth); + return Promise.resolve(); + } + this._cache.delete(imgId); } if (this._pending.has(imgId)) { - return this._pending.get(imgId).then(() => { - this._prefetchNext(this._cache.get(imgId), depth); - }); + return this._pending.get(imgId).then(() => + this.precompute(imgId, seed, depth)); } let p: Promise; if (seed && isNum(seed.lat) && isNum(seed.lng)) { - if (typeof this._provider.cacheImage === "function") { - this._provider.cacheImage(seed); - } p = Promise.resolve(seed); } else { p = this._provider.fetchImage(imgId); @@ -204,13 +245,29 @@ export class ReorientationEngine { return; } const nextId = ids[idx + 1]; - return this._provider.fetchImage(nextId).then((nxt) => { - if (!hasPosition(nxt)) { - this._invalid(imgId, "Next image missing geometry"); - return; + let warm: Promise = Promise.resolve([]); + if (this._provider.fetchImages != null) { + const requiredIds = [ids[idx - 1], nextId] + .filter((id: string): boolean => id != null); + warm = this._provider.fetchImages(requiredIds) + .catch((): ReorientationImage[] => []); + const aheadIds = ids.slice( + idx + 2, Math.min(ids.length, idx + depth + 2)); + if (aheadIds.length > 0) { + this._provider.fetchImages(aheadIds) + .catch((): ReorientationImage[] => []); } - return this._decide(imgId, cur, nxt, ids, idx, nextId, depth); - }); + } + return warm + .then(() => this._provider.fetchImage(nextId)) + .then((nxt) => { + if (!hasPosition(nxt)) { + this._invalid(imgId, "Next image missing geometry"); + return; + } + return this._decide( + imgId, cur, nxt, ids, idx, nextId, depth); + }); }); } @@ -223,10 +280,7 @@ export class ReorientationEngine { nextId: string, depth: number): Promise { const cfg = this._config; - const dist = haversineDist(cur.lat, cur.lng, nxt.lat, nxt.lng); - let tb = bearing(cur.lat, cur.lng, nxt.lat, nxt.lng); - const dt = (nxt.ts && cur.ts) ? (nxt.ts - cur.ts) / 1000 : 0; - const speed = dt > 0 ? dist / dt : 0; + let segment = this._segment(cur, nxt); let moving = false; let prevCtx: PrevContext = null; @@ -239,6 +293,9 @@ export class ReorientationEngine { moving: pd.moving, speed: pd.speed, travel: pd.travel, + computedTravel: pd.computedTravel, + compassOffset: pd.computedCompassOffset, + computedCompassOutlier: pd.computedCompassOutlier, }; break; } @@ -251,33 +308,50 @@ export class ReorientationEngine { if (!hasPosition(prev)) { return null; } - const pDist = - haversineDist(prev.lat, prev.lng, cur.lat, cur.lng); - const pTravel = - bearing(prev.lat, prev.lng, cur.lat, cur.lng); - const pDt = (cur.ts && prev.ts) ? - (cur.ts - prev.ts) / 1000 : 0; - const pSpeed = pDt > 0 ? pDist / pDt : 0; - let pMoving = pSpeed >= cfg.movingSpeedMps; + const previousSegment = this._segment(prev, cur); + let pMoving = previousSegment.speedExcessive ? + previousSegment.dist > cfg.lowSpeedTurnDistanceM : + previousSegment.speed >= cfg.movingSpeedMps; if (!pMoving && isNum(prev.cca)) { - const pDelta = angleDelta(pTravel, prev.cca); + const pDelta = + angleDelta(previousSegment.travel, prev.cca); if (pDelta < cfg.lowSpeedTurnMaxDeltaDeg && - pDist > cfg.lowSpeedTurnDistanceM) { + previousSegment.dist > + cfg.lowSpeedTurnDistanceM) { pMoving = true; } } + const compassOffset = + isNum(prev.computedCca) && isNum(prev.originalCca) ? + (prev.computedCca - prev.originalCca + 360) % 360 : + undefined; return { valid: true, moving: pMoving, - speed: pSpeed, - travel: pTravel, + speed: previousSegment.speed, + travel: previousSegment.travel, + computedTravel: previousSegment.computedTravel, + compassOffset, }; }) .catch(() => null); } return prevPromise.then((prev) => { - if (speed >= cfg.movingSpeedMps) { + if (this._shouldUseOriginalGeometry(cur, segment, prev)) { + this._originalGeometrySequences.add(cur.seq); + segment = this._segment(cur, nxt); + } + const dist = segment.dist; + let tb = segment.travel; + const speed = segment.speed; + + if (segment.speedExcessive) { + // Video frame timestamps can imply impossible speeds. There is + // still real displacement, but it must pass the bearing-outlier + // checks below rather than receiving the sustained-speed bypass. + moving = dist > cfg.lowSpeedTurnDistanceM; + } else if (speed >= cfg.movingSpeedMps) { // On direct jumps there is no prior segment, so only the // first image in a sequence may assume sustained motion. moving = prev ? @@ -293,9 +367,23 @@ export class ReorientationEngine { } } + // The outlier test exists to stop a single noisy fix from steering + // the view, and noise only reaches it through the low-speed branch + // above — two consecutive segments both at sustained speed are real + // displacement, so a large bearing change between them is a corner, + // not jitter. Rejecting those turned the sharpest corners (and, via + // the history window below, the whole stretch after them) into + // "not moving", which is exactly where reorientation is wanted. + const sustained = !segment.speedExcessive && + speed >= cfg.movingSpeedMps && + prev != null && + prev.speed >= cfg.movingSpeedMps && + prev.speed <= MAX_REASONABLE_SPEED_MPS; + if (moving) { if (prev && prev.moving) { - if (angleDelta(tb, prev.travel) > cfg.outlierMaxDeltaDeg) { + if (!sustained && + angleDelta(tb, prev.travel) > cfg.outlierMaxDeltaDeg) { moving = false; } } else { @@ -335,22 +423,133 @@ export class ReorientationEngine { } } + const hasCurrentCompassCalibration = + isNum(cur.computedCca) && isNum(cur.originalCca); + const hasNextCompassCalibration = + isNum(nxt.computedCca) && isNum(nxt.originalCca); + const currentCompassOffset = hasCurrentCompassCalibration ? + (cur.computedCca - cur.originalCca + 360) % 360 : 0; + const nextCompassOffset = hasNextCompassCalibration ? + (nxt.computedCca - nxt.originalCca + 360) % 360 : 0; + const reconstructionDiscontinuity = + hasCurrentCompassCalibration && + prev?.compassOffset != null && + angleDelta(currentCompassOffset, prev.compassOffset) > + MAX_COMPUTED_COMPASS_DELTA_DEG; + const continuesRejectedCalibration = + hasCurrentCompassCalibration && + prev?.computedCompassOutlier === true && + prev.compassOffset != null && + angleDelta(currentCompassOffset, prev.compassOffset) <= + MAX_COMPUTED_COMPASS_DELTA_DEG; + const computedCompassOutlier = + hasCurrentCompassCalibration && + angleDelta(cur.computedCca, cur.originalCca) > + MAX_COMPUTED_COMPASS_DELTA_DEG && + angleDelta(tb, cur.originalCca) < + cfg.lowSpeedTurnMaxDeltaDeg && + ((hasNextCompassCalibration && + angleDelta(currentCompassOffset, nextCompassOffset) > + MAX_COMPUTED_COMPASS_DELTA_DEG) || + reconstructionDiscontinuity || + continuesRejectedCalibration); + const viewCompassAngle = computedCompassOutlier ? + cur.originalCca : + (isNum(cur.viewCompassAngle) ? cur.viewCompassAngle : cur.cca); const result: ReorientationResult = { valid: true, nextId, + prevId: idx > 0 ? ids[idx - 1] : undefined, travel: tb, - basicX: bearingToBasicX(tb, cur.cca), + computedTravel: segment.computedTravel, + basicX: bearingToBasicX(tb, viewCompassAngle), dist, cca: cur.cca, + viewCompassAngle, + computedCompassOutlier, + reconstructionDiscontinuity, + computedCompassOffset: hasCurrentCompassCalibration ? + currentCompassOffset : undefined, speed, moving, seq: cur.seq, }; this._cache.set(imgId, result); this._prefetchNext(result, depth); + // Warm the immediate previous image so a "previous" hover can read + // its reoriented bearing instead of falling back to the raw compass + // angle. Only from the root request (depth === prefetchAhead) and at + // depth 0, so it resolves that one image without cascading backward. + // Load-bearing for the paths where the component early-returns + // before hinting neighbors (no motion, or no compass angle). + if (depth === cfg.prefetchAhead && result.prevId != null) { + this.precompute(result.prevId, undefined, 0) + .catch(() => { /* ignore prefetch errors */ }); + } }); } + private _segment(cur: ReorientationImage, nxt: ReorientationImage): Segment { + const dt = (nxt.ts && cur.ts) ? (nxt.ts - cur.ts) / 1000 : 0; + const computedDist = haversineDist(cur.lat, cur.lng, nxt.lat, nxt.lng); + const computedTravel = bearing(cur.lat, cur.lng, nxt.lat, nxt.lng); + const computedSpeed = dt > 0 ? computedDist / dt : 0; + const hasOriginal = + isNum(cur.originalLat) && isNum(cur.originalLng) && + isNum(nxt.originalLat) && isNum(nxt.originalLng); + const originalDist = hasOriginal ? haversineDist( + cur.originalLat, cur.originalLng, + nxt.originalLat, nxt.originalLng) : undefined; + const originalTravel = hasOriginal ? bearing( + cur.originalLat, cur.originalLng, + nxt.originalLat, nxt.originalLng) : undefined; + const useOriginal = hasOriginal && + this._originalGeometrySequences.has(cur.seq); + const dist = useOriginal ? originalDist : computedDist; + const travel = useOriginal ? originalTravel : computedTravel; + const speed = dt > 0 ? dist / dt : 0; + + return { + dist, + speed, + speedExcessive: speed > MAX_REASONABLE_SPEED_MPS, + travel, + computedSpeed, + computedTravel, + originalTravel, + }; + } + + private _shouldUseOriginalGeometry( + cur: ReorientationImage, + segment: Segment, + prev: PrevContext): boolean { + if (!isNum(segment.originalTravel)) { + return false; + } + if (segment.computedSpeed > MAX_REASONABLE_SPEED_MPS) { + return true; + } + + const originalHeading = isNum(cur.originalCca) ? + cur.originalCca : cur.cca; + const headingFavorsOriginal = + angleDelta(segment.computedTravel, cur.cca) > + MAX_COMPUTED_POSITION_HEADING_DELTA_DEG && + angleDelta(segment.originalTravel, originalHeading) < + this._config.lowSpeedTurnMaxDeltaDeg; + if (!headingFavorsOriginal) { + return false; + } + + // Raw GPS heading often agrees with raw compass by construction. Only + // prefer it when the computed track also breaks continuity; otherwise + // a smooth reconstructed track can be replaced by a noisy raw one. + return prev?.computedTravel == null || + angleDelta(segment.computedTravel, prev.computedTravel) > + MAX_COMPUTED_POSITION_HEADING_DELTA_DEG; + } + private _prefetchNext(d: ReorientationResult, depth: number): void { if (depth > 0 && d && d.valid && d.nextId) { this.precompute(d.nextId, null, depth - 1) diff --git a/src/component/sequence/SequenceDOMRenderer.ts b/src/component/sequence/SequenceDOMRenderer.ts index 26508fddf..9edb52c2a 100644 --- a/src/component/sequence/SequenceDOMRenderer.ts +++ b/src/component/sequence/SequenceDOMRenderer.ts @@ -15,6 +15,7 @@ import { ViewportSize } from "../../render/interfaces/ViewportSize"; import { Container } from "../../viewer/Container"; import { Navigator } from "../../viewer/Navigator"; import { SequenceConfiguration } from "../interfaces/SequenceConfiguration"; +import { createTooltipProperties } from "../util/Tooltip"; import { SequenceMode } from "./SequenceMode"; import { SequenceComponent } from "./SequenceComponent"; @@ -305,19 +306,27 @@ export class SequenceDOMRenderer { NavigationDirection.Prev : NavigationDirection.Next; const playing: boolean = configuration.playing; - const switchButtonProperties: vd.createProperties = { - onclick: (): void => { - if (!playing) { - component.configure({ direction }); - } - }, - }; + const switchButtonProperties: vd.createProperties = createTooltipProperties( + "Reverse direction", + { + onclick: (): void => { + if (!playing) { + component.configure({ direction }); + } + }, + }); const switchButtonClassName: string = configuration.playing ? ".mapillary-sequence-switch-button-inactive" : ".mapillary-sequence-switch-button"; const switchButton: vd.VNode = vd.h("div" + switchButtonClassName, switchButtonProperties, [switchIcon]); const slowIcon: vd.VNode = vd.h("div.mapillary-sequence-slow-icon.mapillary-sequence-icon-visible", []); - const slowContainer: vd.VNode = vd.h("div.mapillary-sequence-slow-container", [slowIcon]); + const slowContainer: vd.VNode = vd.h( + "div.mapillary-sequence-slow-container", + createTooltipProperties("Slower"), + [slowIcon]); const fastIcon: vd.VNode = vd.h("div.mapillary-sequence-fast-icon.mapillary-sequence-icon-visible", []); - const fastContainer: vd.VNode = vd.h("div.mapillary-sequence-fast-container", [fastIcon]); + const fastContainer: vd.VNode = vd.h( + "div.mapillary-sequence-fast-container", + createTooltipProperties("Faster"), + [fastIcon]); const closeIcon: vd.VNode = vd.h("div.mapillary-sequence-close-icon.mapillary-sequence-icon-visible", []); const closeButtonProperties: vd.createProperties = { onclick: (): void => { @@ -352,7 +361,9 @@ export class SequenceDOMRenderer { (): void => { component.stop(); } : canPlay ? (): void => { component.play(); } : null; - let buttonProperties: vd.createProperties = { onclick: onclick }; + let buttonProperties: vd.createProperties = createTooltipProperties( + configuration.playing ? "Stop" : "Play", + { onclick }); let iconProperties: vd.createProperties = {}; if (configuration.direction === NavigationDirection.Prev) { @@ -374,45 +385,51 @@ export class SequenceDOMRenderer { private _createSequenceControls(containerWidth: number): vd.VNode { const borderRadius: number = Math.round(8 / this._stepperDefaultWidth * containerWidth); - const expanderProperties: vd.createProperties = { - onclick: (): void => { - this._expandControls = !this._expandControls; - this._mode = SequenceMode.Default; - this._notifyChanged$.next(this); - }, - style: { - "border-bottom-right-radius": `${borderRadius}px`, - "border-top-right-radius": `${borderRadius}px`, - }, - }; + const expanderProperties: vd.createProperties = createTooltipProperties( + "Options", + { + onclick: (): void => { + this._expandControls = !this._expandControls; + this._mode = SequenceMode.Default; + this._notifyChanged$.next(this); + }, + style: { + "border-bottom-right-radius": `${borderRadius}px`, + "border-top-right-radius": `${borderRadius}px`, + }, + }); const expanderBar: vd.VNode = vd.h("div.mapillary-sequence-expander-bar", []); const expander: vd.VNode = vd.h("div.mapillary-sequence-expander-button", expanderProperties, [expanderBar]); const fastIconClassName: string = this._mode === SequenceMode.Playback ? ".mapillary-sequence-fast-icon-gray.mapillary-sequence-icon-visible" : ".mapillary-sequence-fast-icon"; const fastIcon: vd.VNode = vd.h("div" + fastIconClassName, []); - const playbackProperties: vd.createProperties = { - onclick: (): void => { - this._mode = this._mode === SequenceMode.Playback ? - SequenceMode.Default : - SequenceMode.Playback; - this._notifyChanged$.next(this); - }, - }; + const playbackProperties: vd.createProperties = createTooltipProperties( + "Playback speed", + { + onclick: (): void => { + this._mode = this._mode === SequenceMode.Playback ? + SequenceMode.Default : + SequenceMode.Playback; + this._notifyChanged$.next(this); + }, + }); const playback: vd.VNode = vd.h("div.mapillary-sequence-playback-button", playbackProperties, [fastIcon]); const timelineIconClassName: string = this._mode === SequenceMode.Timeline ? ".mapillary-sequence-timeline-icon-gray.mapillary-sequence-icon-visible" : ".mapillary-sequence-timeline-icon"; const timelineIcon: vd.VNode = vd.h("div" + timelineIconClassName, []); - const timelineProperties: vd.createProperties = { - onclick: (): void => { - this._mode = this._mode === SequenceMode.Timeline ? - SequenceMode.Default : - SequenceMode.Timeline; - this._notifyChanged$.next(this); - }, - }; + const timelineProperties: vd.createProperties = createTooltipProperties( + "Navigate capture", + { + onclick: (): void => { + this._mode = this._mode === SequenceMode.Timeline ? + SequenceMode.Default : + SequenceMode.Timeline; + this._notifyChanged$.next(this); + }, + }); const timeline: vd.VNode = vd.h("div.mapillary-sequence-timeline-button", timelineProperties, [timelineIcon]); @@ -437,44 +454,48 @@ export class SequenceDOMRenderer { configuration: SequenceConfiguration, navigator: Navigator): vd.VNode[] { - let nextProperties: vd.createProperties = { - onclick: nextId != null ? - (): void => { - navigator.moveDir$(NavigationDirection.Next) - .subscribe( - undefined, - (error: Error): void => { - if (!(error instanceof CancelMapillaryError)) { - console.error(error); - } - }); - } : - null, - onpointerenter: (): void => { this._mouseEnterDirection$.next(NavigationDirection.Next); }, - onpointerleave: (): void => { this._mouseLeaveDirection$.next(NavigationDirection.Next); }, - }; + let nextProperties: vd.createProperties = createTooltipProperties( + "Next", + { + onclick: nextId != null ? + (): void => { + navigator.moveDir$(NavigationDirection.Next) + .subscribe( + undefined, + (error: Error): void => { + if (!(error instanceof CancelMapillaryError)) { + console.error(error); + } + }); + } : + null, + onpointerenter: (): void => { this._mouseEnterDirection$.next(NavigationDirection.Next); }, + onpointerleave: (): void => { this._mouseLeaveDirection$.next(NavigationDirection.Next); }, + }); const borderRadius: number = Math.round(8 / this._stepperDefaultWidth * containerWidth); - let prevProperties: vd.createProperties = { - onclick: prevId != null ? - (): void => { - navigator.moveDir$(NavigationDirection.Prev) - .subscribe( - undefined, - (error: Error): void => { - if (!(error instanceof CancelMapillaryError)) { - console.error(error); - } - }); - } : - null, - onpointerenter: (): void => { this._mouseEnterDirection$.next(NavigationDirection.Prev); }, - onpointerleave: (): void => { this._mouseLeaveDirection$.next(NavigationDirection.Prev); }, - style: { - "border-bottom-left-radius": `${borderRadius}px`, - "border-top-left-radius": `${borderRadius}px`, - }, - }; + let prevProperties: vd.createProperties = createTooltipProperties( + "Previous", + { + onclick: prevId != null ? + (): void => { + navigator.moveDir$(NavigationDirection.Prev) + .subscribe( + undefined, + (error: Error): void => { + if (!(error instanceof CancelMapillaryError)) { + console.error(error); + } + }); + } : + null, + onpointerenter: (): void => { this._mouseEnterDirection$.next(NavigationDirection.Prev); }, + onpointerleave: (): void => { this._mouseLeaveDirection$.next(NavigationDirection.Prev); }, + style: { + "border-bottom-left-radius": `${borderRadius}px`, + "border-top-left-radius": `${borderRadius}px`, + }, + }); let nextClass: string = this._getStepClassName(NavigationDirection.Next, nextId, configuration.highlightId); let prevClass: string = this._getStepClassName(NavigationDirection.Prev, prevId, configuration.highlightId); diff --git a/src/component/slider/SliderMeshFactory.ts b/src/component/slider/SliderMeshFactory.ts index ad0c9af06..5bd9376fd 100644 --- a/src/component/slider/SliderMeshFactory.ts +++ b/src/component/slider/SliderMeshFactory.ts @@ -4,6 +4,7 @@ import { SliderShaders } from "./SliderShaders"; import { Image } from "../../graph/Image"; import { isFisheye, isSpherical } from "../../geo/Geo"; +import { hasReconstructionMesh } from "../../util/Mesh"; import { SliderTransform } from "./SliderTransform"; export class SliderMeshFactory { @@ -332,7 +333,7 @@ export class SliderMeshFactory { } private _useMesh(transform: SliderTransform, image: Image): boolean { - return image.mesh.vertices.length && transform.hasValidScale; + return hasReconstructionMesh(image.mesh) && transform.hasValidScale; } private _getImageSphereGeo(transform: SliderTransform, image: Image): THREE.BufferGeometry { diff --git a/src/component/util/MeshFactory.ts b/src/component/util/MeshFactory.ts index 297b29df7..a02294036 100644 --- a/src/component/util/MeshFactory.ts +++ b/src/component/util/MeshFactory.ts @@ -8,6 +8,7 @@ import { Camera } from "../../geometry/Camera"; import { resolveShader } from "../../shader/Resolver"; import { GLShader } from "../../shader/Shader"; +import { hasReconstructionMesh } from "../../util/Mesh"; function makeCameraUniforms(camera: Camera): { [key: string]: IUniform; } { const cameraUniforms: { [key: string]: IUniform; } = {}; @@ -69,7 +70,11 @@ export class MeshFactory { this._imageSphereRadius = imageSphereRadius != null ? imageSphereRadius : 200; } - public createMesh(image: Image, transform: Transform, shader: GLShader): THREE.Mesh { + public createMesh( + image: Image, + transform: Transform, + shader: GLShader, + useMesh: boolean = true): THREE.Mesh { const texture = this._createTexture(image.image); const materialParameters = this._createMaterialParameters( @@ -78,6 +83,14 @@ export class MeshFactory { shader); const material = new THREE.ShaderMaterial(materialParameters); + if (!useMesh) { + const geometry = isSpherical(transform.cameraType) ? + this._getFlatImageSphereGeo(transform) : + isFisheye(transform.cameraType) ? + this._getRegularFlatImagePlaneGeoFisheye(transform) : + this._getRegularFlatImagePlaneGeo(transform); + return new THREE.Mesh(geometry, material); + } if (isSpherical(transform.cameraType)) { return this._createImageSphere(image, transform, material); } else if (isFisheye(transform.cameraType)) { @@ -145,7 +158,7 @@ export class MeshFactory { } private _useMesh(transform: Transform, image: Image): boolean { - return image.mesh.vertices.length && transform.hasValidScale; + return hasReconstructionMesh(image.mesh) && transform.hasValidScale; } private _getImageSphereGeo(transform: Transform, image: Image): THREE.BufferGeometry { diff --git a/src/component/util/Tooltip.ts b/src/component/util/Tooltip.ts new file mode 100644 index 000000000..0ce33b827 --- /dev/null +++ b/src/component/util/Tooltip.ts @@ -0,0 +1,35 @@ +import * as vd from "virtual-dom"; + +export function createTooltipProperties( + text: string, + properties: vd.createProperties = {}, + position?: "left"): vd.createProperties { + + const onPointerDown: (event: PointerEvent) => void = properties.onpointerdown; + const onPointerLeave: (event: PointerEvent) => void = properties.onpointerleave; + const attributes: { [key: string]: string } = { + ...properties.attributes, + "data-mapillary-tooltip": text, + }; + + if (position) { + attributes["data-mapillary-tooltip-position"] = position; + } + + return { + ...properties, + attributes, + onpointerdown: (event: PointerEvent): void => { + (event.currentTarget as HTMLElement).setAttribute("data-mapillary-tooltip-dismissed", ""); + if (onPointerDown) { + onPointerDown(event); + } + }, + onpointerleave: (event: PointerEvent): void => { + (event.currentTarget as HTMLElement).removeAttribute("data-mapillary-tooltip-dismissed"); + if (onPointerLeave) { + onPointerLeave(event); + } + }, + }; +} diff --git a/src/component/zoom/ZoomComponent.ts b/src/component/zoom/ZoomComponent.ts index e875c775d..5c9fe80ca 100644 --- a/src/component/zoom/ZoomComponent.ts +++ b/src/component/zoom/ZoomComponent.ts @@ -13,6 +13,7 @@ import { import { Component } from "../Component"; import { ZoomConfiguration } from "../interfaces/ZoomConfiguration"; +import { createTooltipProperties } from "../util/Tooltip"; import { Transform } from "../../geo/Transform"; import { ViewportCoords } from "../../geo/ViewportCoords"; @@ -67,14 +68,22 @@ export class ZoomComponent extends Component { const zoom: number = frame.state.zoom; const zoomInIcon: vd.VNode = vd.h("div.mapillary-zoom-in-icon", []); + const zoomInProperties: vd.createProperties = createTooltipProperties("Zoom in", {}, "left"); const zoomInButton: vd.VNode = zoom >= 3 || state === State.Waiting ? - vd.h("div.mapillary-zoom-in-button-inactive", [zoomInIcon]) : - vd.h("div.mapillary-zoom-in-button", { onclick: (): void => { this._zoomDelta$.next(1); } }, [zoomInIcon]); + vd.h("div.mapillary-zoom-in-button-inactive", zoomInProperties, [zoomInIcon]) : + vd.h( + "div.mapillary-zoom-in-button", + { ...zoomInProperties, onclick: (): void => { this._zoomDelta$.next(1); } }, + [zoomInIcon]); const zoomOutIcon: vd.VNode = vd.h("div.mapillary-zoom-out-icon", []); + const zoomOutProperties: vd.createProperties = createTooltipProperties("Zoom out", {}, "left"); const zoomOutButton: vd.VNode = zoom <= 0 || state === State.Waiting ? - vd.h("div.mapillary-zoom-out-button-inactive", [zoomOutIcon]) : - vd.h("div.mapillary-zoom-out-button", { onclick: (): void => { this._zoomDelta$.next(-1); } }, [zoomOutIcon]); + vd.h("div.mapillary-zoom-out-button-inactive", zoomOutProperties, [zoomOutIcon]) : + vd.h( + "div.mapillary-zoom-out-button", + { ...zoomOutProperties, onclick: (): void => { this._zoomDelta$.next(-1); } }, + [zoomOutIcon]); const compact: string = configuration.size === ComponentSize.Small || configuration.size === ComponentSize.Automatic && size.width < 640 ? diff --git a/src/graph/Graph.ts b/src/graph/Graph.ts index 7dfb4a114..170d7284b 100644 --- a/src/graph/Graph.ts +++ b/src/graph/Graph.ts @@ -44,6 +44,9 @@ import { ImagesContract } from "../api/contracts/ImagesContract"; import { SequenceContract } from "../api/contracts/SequenceContract"; import { CoreImagesContract } from "../api/contracts/CoreImagesContract"; import { CancelMapillaryError } from "../error/CancelMapillaryError"; +import { geodeticToEnu } from "../geo/GeoCoords"; + +const MAX_GRAPH_IMAGE_BATCH_SIZE = 120; type NodeTiles = { cache: string[]; @@ -461,9 +464,9 @@ export class Graph { } const coreNodeBatches: string[][] = []; - const batchSize: number = 200; while (coreNodes.length > 0) { - coreNodeBatches.push(coreNodes.splice(0, batchSize)); + coreNodeBatches.push( + coreNodes.splice(0, MAX_GRAPH_IMAGE_BATCH_SIZE)); } const fullNodes$ = observableOf(fullNodes); @@ -558,6 +561,56 @@ export class Graph { return this._cachingFill$[key]; } + /** + * Retrieve and cache full node properties for multiple images in one request. + * + * @param {string[]} keys - Keys of nodes to fill. + * @returns {Observable} Observable emitting the graph + * when all nodes have been updated. + */ + public cacheFullImages$(keys: string[]): Observable { + const streams = new Set>(); + const uncached: string[] = []; + for (const key of Array.from(new Set(keys))) { + if (key in this._cachingFull$) { + streams.add(this._cachingFull$[key]); + } else if (key in this._cachingFill$) { + streams.add(this._cachingFill$[key]); + } else if (!this.hasNode(key)) { + uncached.push(key); + } else if (!this.getNode(key).complete) { + streams.add(this.cacheFill$(key)); + } + } + + while (uncached.length > 0) { + const batchKeys = uncached.splice(0, MAX_GRAPH_IMAGE_BATCH_SIZE); + const batch$ = this._api.getImages$(batchKeys).pipe( + tap((items: ImagesContract): void => { + this._storeFullImages(items); + }), + map((): Graph => this), + finalize((): void => { + for (const key of batchKeys) { + if (this._cachingFull$[key] === batch$) { + delete this._cachingFull$[key]; + } + } + this._changed$.next(this); + }), + publishReplay(1), + refCount()); + for (const key of batchKeys) { + this._cachingFull$[key] = batch$; + } + streams.add(batch$); + } + + return streams.size > 0 ? + observableMerge(...Array.from(streams)).pipe(last()) : + observableOf(this); + } + /** * Retrieve and cache full node properties. * @@ -577,44 +630,9 @@ export class Graph { } this._cachingFull$[key] = this._api.getImages$([key]).pipe( - tap( - (items: ImagesContract): void => { - for (const item of items) { - if (!item.node) { - throw new GraphMapillaryError( - `Image does not exist (${key}, ${item.node}).`); - } - - const id = item.node_id; - if (this.hasNode(id)) { - const node = this.getNode(key); - if (!node.complete) { - this._makeFull(node, item.node); - } - } else { - if (item.node.sequence.id == null) { - throw new GraphMapillaryError( - `Image has no sequence key (${key}).`); - } - - let node: Image = null; - if (this._preDeletedNodes.has(id)) { - node = this._unDeleteNode(id); - } else { - node = new Image(item.node); - } - this._makeFull(node, item.node); - - const lngLat = this._getNodeLngLat(node); - const cellId = this._api.data.geometry - .lngLatToCellId(lngLat); - this._preStore(cellId, node); - this._setNode(node); - - delete this._cachingFull$[id]; - } - } - }), + tap((items: ImagesContract): void => { + this._storeFullImages(items, key); + }), map((): Graph => this), finalize( (): void => { @@ -728,9 +746,8 @@ export class Graph { batches.push(keys.splice(startIndex, referenceBatchSize)); } - const batchSize: number = 200; while (keys.length > 0) { - batches.push(keys.splice(0, batchSize)); + batches.push(keys.splice(0, MAX_GRAPH_IMAGE_BATCH_SIZE)); } let batchesToCache: number = batches.length; @@ -908,6 +925,53 @@ export class Graph { return spatialArea$; } + /** + * Get sequence targets near the preferred spatial-navigation distance. + */ + public getSequenceSpatialTargetIds(key: string): string[] { + const node: Image = this.getNode(key); + if (!(node.sequenceId in this._sequences)) { + throw new GraphMapillaryError(`Sequence is not cached (${key}), (${node.sequenceId})`); + } + + const sequence = this._sequences[node.sequenceId].sequence; + return [ + this._getSequenceSpatialTarget(node, sequence, -1), + this._getSequenceSpatialTarget(node, sequence, 1), + ]; + } + + /** + * Compute spatial edges to nearby images in a node's sequence. + * + * @param {string} key - Key of node. + * @returns {Array} Spatial edges to cached sequence images. + * @throws {GraphMapillaryError} When the node or its sequence is not cached. + */ + public getSequenceSpatialEdges(key: string): NavigationEdge[] { + const node: Image = this.getNode(key); + const targetKeys = this.getSequenceSpatialTargetIds(key); + const prevKey = targetKeys[0]; + const nextKey = targetKeys[1]; + const potentialNodes: Image[] = []; + + for (const candidateKey of targetKeys) { + if (candidateKey == null || !this.hasNode(candidateKey)) { + continue; + } + + const candidate: Image = this.getNode(candidateKey); + if (candidate.complete && this._filter(candidate)) { + potentialNodes.push(candidate); + } + } + + const fallbackKeys = targetKeys + .filter((fallbackKey: string): boolean => fallbackKey != null); + + return this._computeSpatialEdges(node, potentialNodes, prevKey, nextKey, fallbackKeys); + } + /** * Cache spatial edges for a node. * @@ -920,49 +984,42 @@ export class Graph { throw new GraphMapillaryError(`Spatial edges already cached (${key}).`); } - let node: Image = this.getNode(key); - let sequence: Sequence = this._sequences[node.sequenceId].sequence; - - let fallbackKeys: string[] = []; - let prevKey: string = sequence.findPrev(node.id); - if (prevKey != null) { - fallbackKeys.push(prevKey); - } - - let nextKey: string = sequence.findNext(node.id); - if (nextKey != null) { - fallbackKeys.push(nextKey); - } - - let allSpatialNodes: { [key: string]: Image; } = this._requiredSpatialArea[key].all; - let potentialNodes: Image[] = []; - let filter: FilterFunction = this._filter; - for (let spatialNodeKey in allSpatialNodes) { + const node: Image = this.getNode(key); + const targetKeys = this.getSequenceSpatialTargetIds(key); + const prevKey = targetKeys[0]; + const nextKey = targetKeys[1]; + const fallbackKeys = targetKeys + .filter((fallbackKey: string): boolean => fallbackKey != null); + const allSpatialNodes: { [key: string]: Image; } = this._requiredSpatialArea[key].all; + const potentialNodes: Image[] = []; + const potentialNodeIds: { [key: string]: boolean; } = {}; + const filter: FilterFunction = this._filter; + + for (const spatialNodeKey in allSpatialNodes) { if (!allSpatialNodes.hasOwnProperty(spatialNodeKey)) { continue; } - let spatialNode: Image = allSpatialNodes[spatialNodeKey]; - + const spatialNode: Image = allSpatialNodes[spatialNodeKey]; if (spatialNode.complete && filter(spatialNode)) { potentialNodes.push(spatialNode); + potentialNodeIds[spatialNode.id] = true; } } - let potentialEdges: PotentialEdge[] = - this._edgeCalculator.getPotentialEdges(node, potentialNodes, fallbackKeys); + for (const fallbackKey of fallbackKeys) { + if (potentialNodeIds[fallbackKey] || !this.hasNode(fallbackKey)) { + continue; + } - let edges: NavigationEdge[] = - this._edgeCalculator.computeStepEdges( - node, - potentialEdges, - prevKey, - nextKey); + const fallbackNode: Image = this.getNode(fallbackKey); + if (fallbackNode.complete && filter(fallbackNode)) { + potentialNodes.push(fallbackNode); + } + } - edges = edges.concat(this._edgeCalculator.computeTurnEdges(node, potentialEdges)); - edges = edges.concat(this._edgeCalculator.computeSphericalEdges(node, potentialEdges)); - edges = edges.concat(this._edgeCalculator.computePerspectiveToSphericalEdges(node, potentialEdges)); - edges = edges.concat(this._edgeCalculator.computeSimilarEdges(node, potentialEdges)); + const edges: NavigationEdge[] = + this._computeSpatialEdges(node, potentialNodes, prevKey, nextKey, fallbackKeys); node.cacheSpatialEdges(edges); @@ -1266,12 +1323,29 @@ export class Graph { cacheNodes: {}, }; - for (let spatialItem of spatialItems) { - spatialNodes.all[spatialItem.node.id] = spatialItem.node; + for (const spatialItem of spatialItems) { + const spatialNode: Image = spatialItem.node; + const spatialLngLat: LngLat = spatialNode.lngLat; + const enu: number[] = geodeticToEnu( + spatialLngLat.lng, + spatialLngLat.lat, + 0, + node.lngLat.lng, + node.lngLat.lat, + 0); + const horizontalDistanceSquared: number = enu[0] * enu[0] + enu[1] * enu[1]; + + // Adjacent sequence images remain eligible as distance fallbacks. + if (horizontalDistanceSquared > this._tileThreshold * this._tileThreshold && + spatialNode.sequenceId !== node.sequenceId) { + continue; + } + + spatialNodes.all[spatialNode.id] = spatialNode; - if (!spatialItem.node.complete) { - spatialNodes.cacheKeys.push(spatialItem.node.id); - spatialNodes.cacheNodes[spatialItem.node.id] = spatialItem.node; + if (!spatialNode.complete) { + spatialNodes.cacheKeys.push(spatialNode.id); + spatialNodes.cacheNodes[spatialNode.id] = spatialNode; } } @@ -1776,6 +1850,65 @@ export class Graph { this._filterSubscription.unsubscribe(); } + private _getSequenceSpatialTarget( + node: Image, + sequence: Sequence, + direction: number): string { + + const index = sequence.imageIds.indexOf(node.id); + const adjacentIndex = index + direction; + if (index < 0 || adjacentIndex < 0 || adjacentIndex >= sequence.imageIds.length) { + return null; + } + + const adjacentId = sequence.imageIds[adjacentIndex]; + if (!this.hasNode(adjacentId) || !this.getNode(adjacentId).complete) { + return adjacentId; + } + + const adjacent = this.getNode(adjacentId); + const enu = geodeticToEnu( + adjacent.lngLat.lng, + adjacent.lngLat.lat, + adjacent.computedAltitude, + node.lngLat.lng, + node.lngLat.lat, + node.computedAltitude); + const distance = Math.sqrt(enu[0] * enu[0] + enu[1] * enu[1]); + if (!Number.isFinite(distance) || distance < 0.1) { + return adjacentId; + } + + const preferredDistance = this._edgeCalculator.getPreferredSpatialDistance(node); + const offset = Math.max(1, Math.min(10, Math.round(preferredDistance / distance))); + const targetIndex = Math.max( + 0, + Math.min(sequence.imageIds.length - 1, index + direction * offset)); + + return sequence.imageIds[targetIndex]; + } + + private _computeSpatialEdges( + node: Image, + potentialNodes: Image[], + prevKey: string, + nextKey: string, + fallbackKeys: string[]): NavigationEdge[] { + + const potentialEdges: PotentialEdge[] = + this._edgeCalculator.getPotentialEdges(node, potentialNodes, fallbackKeys); + + let edges: NavigationEdge[] = + this._edgeCalculator.computeStepEdges(node, potentialEdges, prevKey, nextKey); + + edges = edges.concat(this._edgeCalculator.computeTurnEdges(node, potentialEdges)); + edges = edges.concat(this._edgeCalculator.computeSphericalEdges(node, potentialEdges, fallbackKeys)); + edges = edges.concat(this._edgeCalculator.computePerspectiveToSphericalEdges(node, potentialEdges)); + edges = edges.concat(this._edgeCalculator.computeSimilarEdges(node, potentialEdges)); + + return edges; + } + private _addNewKeys(keys: { [key: string]: boolean; }, dict: { [key: string]: T; }): void { for (let key in dict) { if (!dict.hasOwnProperty(key) || !this.hasNode(key)) { @@ -1951,12 +2084,45 @@ export class Graph { } } + private _storeFullImages(items: ImagesContract, errorKey?: string): void { + for (const item of items) { + const key = errorKey ?? item.node_id; + if (!item.node) { + throw new GraphMapillaryError( + `Image does not exist (${key}, ${item.node}).`); + } + + const id = item.node_id; + if (this.hasNode(id)) { + const existingNode = this.getNode(id); + if (!existingNode.complete) { + this._makeFull(existingNode, item.node); + } + continue; + } + if (item.node.sequence.id == null) { + throw new GraphMapillaryError( + `Image has no sequence key (${key}).`); + } + + const node = this._preDeletedNodes.has(id) ? + this._unDeleteNode(id) : new Image(item.node); + this._makeFull(node, item.node); + const lngLat = this._getNodeLngLat(node); + const cellId = this._api.data.geometry.lngLatToCellId(lngLat); + this._preStore(cellId, node); + this._setNode(node); + delete this._cachingFull$[id]; + } + } + private _makeFull(node: Image, fillNode: SpatialImageEnt): void { if (fillNode.computed_altitude == null) { fillNode.computed_altitude = this._defaultAlt; } - if (fillNode.computed_rotation == null) { + if (fillNode.computed_rotation == null || + fillNode.computed_rotation.length !== 3) { fillNode.computed_rotation = this._graphCalculator.rotationFromCompass(fillNode.compass_angle, fillNode.exif_orientation); } diff --git a/src/graph/GraphService.ts b/src/graph/GraphService.ts index a13c8fe10..a67c56e94 100644 --- a/src/graph/GraphService.ts +++ b/src/graph/GraphService.ts @@ -28,6 +28,7 @@ import { FilterExpression } from "./FilterExpression"; import { Graph } from "./Graph"; import { GraphMode } from "./GraphMode"; import { Image } from "./Image"; +import { NavigationEdgeStatus } from "./interfaces/NavigationEdgeStatus"; import { Sequence } from "./Sequence"; import { LngLat } from "../api/interfaces/LngLat"; @@ -200,6 +201,26 @@ export class GraphService { })); } + /** Retrieve full image metadata without caching render assets or edges. */ + public cacheImagesMetadata$(ids: string[]): Observable { + return this._graph$.pipe( + first(), + mergeMap( + (graph: Graph): Observable => { + return graph.cacheFullImages$(ids); + }), + map( + (graph: Graph): Image[] => { + return ids.map((id: string): Image => { + if (!graph.hasNode(id)) { + throw new GraphMapillaryError( + `Failed to cache image metadata (${id})`); + } + return graph.getNode(id); + }); + })); + } + /** * Cache a image in the graph and retrieve it. * @@ -469,6 +490,50 @@ export class GraphService { })); } + /** + * Cache the adjacent sequence images and compute provisional spatial edges. + * + * @param {string} id - Id of the source image. + * @returns {Observable} Observable emitting edges to + * adjacent sequence images without waiting for the full spatial area. + */ + public cacheSequenceSpatialEdges$(id: string): Observable { + return this._graph$.pipe( + first(), + mergeMap( + (graph: Graph): Observable => { + if (graph.isCachingNodeSequence(id) || !graph.hasNodeSequence(id)) { + return graph.cacheNodeSequence$(id); + } + + return observableOf(graph); + }), + mergeMap( + (graph: Graph): Observable => { + const node: Image = graph.getNode(id); + const sequence: Sequence = graph.getSequence(node.sequenceId); + const adjacentIds = [ + sequence.findPrev(id), + sequence.findNext(id), + ].filter((adjacentId: string): boolean => adjacentId != null); + + return this._cacheFullImages$(graph, adjacentIds); + }), + mergeMap( + (graph: Graph): Observable => { + const targetIds = graph.getSequenceSpatialTargetIds(id) + .filter((targetId: string): boolean => targetId != null); + return this._cacheFullImages$(graph, targetIds); + }), + map( + (graph: Graph): NavigationEdgeStatus => { + return { + cached: false, + edges: graph.getSequenceSpatialEdges(id), + }; + })); + } + /** * Cache a sequence and its images in the graph and retrieve the sequence. * @@ -649,6 +714,27 @@ export class GraphService { })); } + private _cacheFullImages$(graph: Graph, ids: string[]): Observable { + if (ids.length === 0) { + return observableOf(graph); + } + + return observableFrom(ids).pipe( + mergeMap( + (id: string): Observable => { + if (graph.isCachingFull(id) || !graph.hasNode(id)) { + return graph.cacheFull$(id); + } + + if (graph.isCachingFill(id) || !graph.getNode(id).complete) { + return graph.cacheFill$(id); + } + + return observableOf(graph); + }), + takeLast(1)); + } + private _abortSubjects(subjects: Subject[]): void { for (const subject of subjects.slice()) { this._removeFromArray(subject, subjects); diff --git a/src/graph/edge/EdgeCalculator.ts b/src/graph/edge/EdgeCalculator.ts index 72c328669..f5646405b 100644 --- a/src/graph/edge/EdgeCalculator.ts +++ b/src/graph/edge/EdgeCalculator.ts @@ -49,6 +49,15 @@ export class EdgeCalculator { this._coefficients = coefficients != null ? coefficients : new EdgeCalculatorCoefficients(); } + /** + * Returns the preferred navigation distance for an image's camera type. + */ + public getPreferredSpatialDistance(node: Image): number { + return isSpherical(node.cameraType) ? + this._settings.sphericalPreferredDistance : + this._settings.stepPreferredDistance; + } + /** * Returns the potential edges to destination nodes for a set * of nodes with respect to a source node. @@ -65,7 +74,8 @@ export class EdgeCalculator { throw new ArgumentMapillaryError("Image has to be full."); } - if (!node.merged) { + if (!node.merged && + (fallbackIds.length === 0 || node.rotation.length !== 3)) { return []; } @@ -77,8 +87,9 @@ export class EdgeCalculator { let potentialEdges: PotentialEdge[] = []; for (let potential of potentialImages) { - if (!potential.merged || - potential.id === node.id) { + const fallback = fallbackIds.indexOf(potential.id) > -1; + if (((!node.merged || !potential.merged) && !fallback) || + potential.id === node.id || potential.rotation.length !== 3) { continue; } @@ -130,7 +141,7 @@ export class EdgeCalculator { potential.sequenceId === node.sequenceId; let sameMergeCC: boolean = - potential.mergeId === node.mergeId; + node.mergeId != null && potential.mergeId === node.mergeId; let sameUser: boolean = potential.creatorId === node.creatorId; @@ -573,9 +584,13 @@ export class EdgeCalculator { * * @param {Image} node - Source node. * @param {Array} potentialEdges - Potential edges. + * @param {Array} fallbackIds - Ids allowed beyond the maximum distance. * @throws {ArgumentMapillaryError} If node is not full. */ - public computeSphericalEdges(node: Image, potentialEdges: PotentialEdge[]): NavigationEdge[] { + public computeSphericalEdges( + node: Image, + potentialEdges: PotentialEdge[], + fallbackIds: string[] = []): NavigationEdge[] { if (!node.complete) { throw new ArgumentMapillaryError("Image has to be full."); } @@ -589,7 +604,8 @@ export class EdgeCalculator { let potentialSteps: [NavigationDirection, PotentialEdge][] = []; for (let potential of potentialEdges) { - if (potential.distance > this._settings.sphericalMaxDistance) { + if (potential.distance > this._settings.sphericalMaxDistance && + fallbackIds.indexOf(potential.id) < 0) { continue; } diff --git a/src/render/RenderCamera.ts b/src/render/RenderCamera.ts index 801c6c522..cbe2ae0b1 100644 --- a/src/render/RenderCamera.ts +++ b/src/render/RenderCamera.ts @@ -136,6 +136,10 @@ export class RenderCamera { return this._frameId === this._changedForFrame; } + public get currentImageId(): string { + return this._currentImageId; + } + public get frameId(): number { return this._frameId; } @@ -156,6 +160,10 @@ export class RenderCamera { return this._zoom; } + public get unzoomedCurrentFov(): number { + return this._computeCurrentFov(0); + } + public get size(): ViewportSize { return this._size; } diff --git a/src/state/StateContext.ts b/src/state/StateContext.ts index f8e18aa06..0cbcb599e 100644 --- a/src/state/StateContext.ts +++ b/src/state/StateContext.ts @@ -2,6 +2,7 @@ import { State } from "./State"; import { TransitionMode } from "./TransitionMode"; import { EulerRotation } from "./interfaces/EulerRotation"; import { IStateContext } from "./interfaces/IStateContext"; +import { ReorientationHint } from "./interfaces/IStateBase"; import { StateBase } from "./state/StateBase"; import { Camera } from "../geo/Camera"; @@ -28,7 +29,7 @@ export class StateContext implements IStateContext { currentIndex: -1, geometry, reference: { alt: 0, lat: 0, lng: 0 }, - reorientations: new Map(), + reorientations: new Map(), trajectory: [], transitionMode: transitionMode == null ? TransitionMode.Default : transitionMode, zoom: 0, @@ -139,6 +140,10 @@ export class StateContext implements IStateContext { this._state.setZoom(zoom); } + public zoomTo(zoom: number): void { + this._state.zoomTo(zoom); + } + public update(delta: number): void { this._state.update(delta); } @@ -207,8 +212,12 @@ export class StateContext implements IStateContext { this._state.rotateToBasicSmooth(basic); } - public setReorientation(imageId: string, basic: number[]): void { - this._state.setReorientation(imageId, basic); + public setReorientation( + imageId: string, + basic: number[], + forceOnReconstruction: boolean = false): void { + this._state.setReorientation( + imageId, basic, forceOnReconstruction); } public clearReorientations(): void { diff --git a/src/state/StateService.ts b/src/state/StateService.ts index d39b1ffd4..9b43e68f0 100644 --- a/src/state/StateService.ts +++ b/src/state/StateService.ts @@ -470,8 +470,15 @@ export class StateService { this._invokeContextOperation((context: IStateContext) => { context.rotateToBasicSmooth(basic); }); } - public setReorientation(imageId: string, basic: number[]): void { - this._invokeContextOperation((context: IStateContext) => { context.setReorientation(imageId, basic); }); + public setReorientation( + imageId: string, + basic: number[], + forceOnReconstruction: boolean = false): void { + this._invokeContextOperation( + (context: IStateContext) => { + context.setReorientation( + imageId, basic, forceOnReconstruction); + }); } public clearReorientations(): void { @@ -550,6 +557,11 @@ export class StateService { this._invokeContextOperation((context: IStateContext) => { context.setZoom(zoom); }); } + public zoomTo(zoom: number): void { + this._inMotionOperation$.next(true); + this._invokeContextOperation((context: IStateContext) => { context.zoomTo(zoom); }); + } + public start(): void { this._clock.start(); if (this._frameId == null) { diff --git a/src/state/interfaces/IStateBase.ts b/src/state/interfaces/IStateBase.ts index 7429790e2..8d9745d06 100644 --- a/src/state/interfaces/IStateBase.ts +++ b/src/state/interfaces/IStateBase.ts @@ -4,6 +4,11 @@ import { TransitionMode } from "../TransitionMode"; import { Image } from "../../graph/Image"; import { IGeometryProvider } from "../../mapillary"; +export interface ReorientationHint { + basic: number[]; + forceOnReconstruction: boolean; +} + export interface IStateBase { alpha: number; camera: Camera; @@ -14,7 +19,7 @@ export interface IStateBase { // current on a motionless (instant) transition so it renders already // oriented instead of flashing the carried view for one frame. Shared by // reference across state transitions. - reorientations?: Map; + reorientations?: Map; trajectory: Image[]; transitionMode: TransitionMode; zoom: number; diff --git a/src/state/interfaces/IStateContext.ts b/src/state/interfaces/IStateContext.ts index 702b3b0c3..d4323b6d6 100644 --- a/src/state/interfaces/IStateContext.ts +++ b/src/state/interfaces/IStateContext.ts @@ -30,7 +30,10 @@ export interface IStateContext extends IAnimationState { rotateBasicWithoutInertia(basicRotation: number[]): void; rotateToBasic(basic: number[]): void; rotateToBasicSmooth(basic: number[]): void; - setReorientation(imageId: string, basic: number[]): void; + setReorientation( + imageId: string, + basic: number[], + forceOnReconstruction?: boolean): void; clearReorientations(): void; move(delta: number): void; moveTo(position: number): void; @@ -39,6 +42,7 @@ export interface IStateContext extends IAnimationState { getCenter(): number[]; setCenter(center: number[]): void; setZoom(zoom: number): void; + zoomTo(zoom: number): void; setSpeed(speed: number): void; setTransitionMode(mode: TransitionMode): void; diff --git a/src/state/state/InteractiveStateBase.ts b/src/state/state/InteractiveStateBase.ts index ccf9ae34a..052e4eaa5 100644 --- a/src/state/state/InteractiveStateBase.ts +++ b/src/state/state/InteractiveStateBase.ts @@ -195,6 +195,9 @@ export abstract class InteractiveStateBase extends StateBase { this._desiredLookat = null; this._requestedRotationDelta = null; this._requestedBasicRotation = null; + this._requestedBasicRotationUnbounded = null; + this._rotationDelta.reset(); + this._basicRotation = [0, 0]; const threshold: number = 0.05 / Math.pow(2, this._zoom); @@ -322,6 +325,11 @@ export abstract class InteractiveStateBase extends StateBase { this._desiredZoom = this._zoom; } + public zoomTo(zoom: number): void { + this._desiredZoom = this._spatial.clamp( + zoom, this._minZoom, this._maxZoom); + } + protected _applyRotation(delta: EulerRotation, camera: Camera): void { if (camera == null) { return; @@ -549,9 +557,6 @@ export abstract class InteractiveStateBase extends StateBase { } protected _setDesiredZoom(): void { - this._desiredZoom = - isSpherical(this._currentImage.cameraType) || - this._previousImage == null ? - this._zoom : 0; + this._desiredZoom = this._zoom; } } diff --git a/src/state/state/StateBase.ts b/src/state/state/StateBase.ts index dd4b4c14b..62c4bad64 100644 --- a/src/state/state/StateBase.ts +++ b/src/state/state/StateBase.ts @@ -2,7 +2,7 @@ import * as Geo from "../../geo/Geo"; import { TransitionMode } from "../TransitionMode"; import { EulerRotation } from "../interfaces/EulerRotation"; -import { IStateBase } from "../interfaces/IStateBase"; +import { IStateBase, ReorientationHint } from "../interfaces/IStateBase"; import { ArgumentMapillaryError } from "../../error/ArgumentMapillaryError"; import { Camera } from "../../geo/Camera"; import { Spatial } from "../../geo/Spatial"; @@ -10,8 +10,12 @@ import { Transform } from "../../geo/Transform"; import { LngLatAlt } from "../../api/interfaces/LngLatAlt"; import { Image } from "../../graph/Image"; import { IGeometryProvider } from "../../mapillary"; +import { hasReconstructionMesh } from "../../util/Mesh"; import { connectedComponent } from "../../api/CellMath"; +const MAX_CAMERA_TRANSITION_DISTANCE = 20; +const MAX_CAMERA_UP_DELTA = Math.PI / 6; + export abstract class StateBase implements IStateBase { protected _spatial: Spatial; protected _geometry: IGeometryProvider; @@ -37,7 +41,7 @@ export abstract class StateBase implements IStateBase { protected _motionless: boolean; - protected _reorientations: Map; + protected _reorientations: Map; private _referenceThreshold: number; private _referenceCellIds: Set; @@ -54,7 +58,8 @@ export abstract class StateBase implements IStateBase { // Shared by reference across transitions so a reorientation registered // before navigating is available when the target image becomes current. - this._reorientations = state.reorientations || new Map(); + this._reorientations = state.reorientations || + new Map(); this._reference = state.reference; this._referenceCellIds = new Set( @@ -171,12 +176,16 @@ export abstract class StateBase implements IStateBase { return this._transitionMode; } - public get reorientations(): Map { + public get reorientations(): Map { return this._reorientations; } - public setReorientation(imageId: string, basic: number[]): void { - this._reorientations.set(imageId, basic); + public setReorientation( + imageId: string, + basic: number[], + forceOnReconstruction: boolean = false): void { + this._reorientations.set( + imageId, { basic, forceOnReconstruction }); } public clearReorientations(): void { @@ -213,6 +222,8 @@ export abstract class StateBase implements IStateBase { public setZoom(zoom: number): void { /*noop*/ } + public zoomTo(zoom: number): void { /*noop*/ } + public dolly(delta: number): void { /*noop*/ } public orbit(rotation: EulerRotation): void { /*noop*/ } @@ -347,7 +358,8 @@ export abstract class StateBase implements IStateBase { this._currentImage.merged && this._previousImage.merged && this._hasStructure() && - this._withinDistance() + this._withinDistance() && + this._withinCameraUpDelta() ); } @@ -483,8 +495,15 @@ export abstract class StateBase implements IStateBase { const current = this._currentImage; const previous = this._previousImage; - return current.mesh.vertices.length > 0 && - previous.mesh.vertices.length > 0; + return hasReconstructionMesh(current.mesh) && + hasReconstructionMesh(previous.mesh); + } + + private _withinCameraUpDelta(): boolean { + // Blending camera frames with sharply different up vectors visibly + // rolls the scene through an orientation neither image has. + return this._previousCamera.up.angleTo(this._currentCamera.up) <= + MAX_CAMERA_UP_DELTA; } private _withinDistance(): boolean { @@ -495,13 +514,19 @@ export abstract class StateBase implements IStateBase { return true; } + if (current.clusterId != null && + current.clusterId === previous.clusterId) { + return true; + } + const distance = this._spatial.distanceFromLngLat( current.lngLat.lng, current.lngLat.lat, previous.lngLat.lng, previous.lngLat.lat); - // 50 km/h moves 28m in 2s - return distance < 30; + // Across reconstruction frames, long-range mesh interpolation magnifies + // alignment errors into severe zooms and warped intermediate views. + return distance <= MAX_CAMERA_TRANSITION_DISTANCE; } } diff --git a/src/state/state/TraversingState.ts b/src/state/state/TraversingState.ts index 6ef5a19a4..c3721e7fe 100644 --- a/src/state/state/TraversingState.ts +++ b/src/state/state/TraversingState.ts @@ -5,6 +5,10 @@ import { IStateBase } from "../interfaces/IStateBase"; import { Image } from "../../graph/Image"; import { isSpherical } from "../../geo/Geo"; import { isNullImageId } from "../../util/Common"; +import { hasReconstructionMesh } from "../../util/Mesh"; +import { TransitionMode } from "../TransitionMode"; + +const FALLBACK_TRANSITION_SPEED = 2.5; export class TraversingState extends InteractiveStateBase { @@ -66,6 +70,7 @@ export class TraversingState extends InteractiveStateBase { this._setDesiredCenter(); this._setDesiredZoom(); + this._applyReorientation(); if (this._trajectory.length < 3) { this._smoothing = true; @@ -87,20 +92,22 @@ export class TraversingState extends InteractiveStateBase { this._resetTransition(); this._clearRotation(); - this._desiredZoom = - isSpherical(this._currentImage.cameraType) ? - this._zoom : 0; + this._desiredZoom = this._zoom; this._desiredLookat = null; // Orient the new image before it is rendered this frame so a - // motionless (instant) transition lands already facing the + // transition without camera motion lands already facing the // registered direction instead of flashing the carried view. this._applyReorientation(); } let animationSpeed: number = this._animationSpeed * delta / 1e-1 * 6; - this._baseAlpha = Math.min(1, this._baseAlpha + this._speedCoefficient * animationSpeed); + const transitionSpeed = this._motionless && this.transitionMode !== TransitionMode.Instantaneous ? + FALLBACK_TRANSITION_SPEED : 1; + this._baseAlpha = Math.min( + 1, + this._baseAlpha + this._speedCoefficient * animationSpeed * transitionSpeed); if (this._smoothing) { this._alpha = MathUtils.smootherstep(this._baseAlpha, 0, 1); } else { @@ -121,11 +128,15 @@ export class TraversingState extends InteractiveStateBase { this._updateZoom(animationSpeed); this._updateLookat(animationSpeed); - this._camera.lerpCameras(this._previousCamera, this._currentCamera, this.alpha); + // Fallback transitions cannot safely interpolate camera geometry, but + // their image alpha can still advance smoothly to produce a dissolve. + const cameraAlpha = this._motionless ? Math.ceil(this._alpha) : this._alpha; + this._camera.lerpCameras(this._previousCamera, this._currentCamera, cameraAlpha); } protected _getAlpha(): number { - return this._motionless ? Math.ceil(this._alpha) : this._alpha; + return this._motionless && this.transitionMode === TransitionMode.Instantaneous ? + Math.ceil(this._alpha) : this._alpha; } protected _setCurrentCamera(): void { @@ -158,27 +169,29 @@ export class TraversingState extends InteractiveStateBase { } private _applyReorientation(): void { - // Only for instant cuts: a smooth transition should ease into the new - // direction, not start already there. - if (!this._motionless || this._currentImage == null) { + if (this._currentImage == null) { return; } - // Only pre-orient (snap) mesh-less images. An image with SfM mesh eases - // to the travel direction, and easing must start from the carried view — - // pre-snapping it here would lose the ease. - if (this._currentImage.mesh != null && - this._currentImage.mesh.vertices.length > 0) { + const reorientation = this._reorientations.get(this._currentImage.id); + if (reorientation == null || + !isSpherical(this._currentImage.cameraType)) { return; } - const basic = this._reorientations.get(this._currentImage.id); - if (basic == null || !isSpherical(this._currentImage.cameraType)) { + // A forced hint means the reconstruction was rejected. Convert that + // transition to a fallback cut so the known-bad pose never renders. + if (reorientation.forceOnReconstruction) { + this._motionless = true; + } else if (!this._motionless || + hasReconstructionMesh(this._currentImage.mesh)) { return; } this._currentCamera.lookat.fromArray( - this.currentTransform.unprojectBasic(basic, this._lookatDepth)); + this.currentTransform.unprojectBasic( + reorientation.basic, this._lookatDepth)); const previousTransform = this.previousTransform != null ? this.previousTransform : this.currentTransform; this._previousCamera.lookat.fromArray( - previousTransform.unprojectBasic(basic, this._lookatDepth)); + previousTransform.unprojectBasic( + reorientation.basic, this._lookatDepth)); } } diff --git a/src/util/Mesh.ts b/src/util/Mesh.ts new file mode 100644 index 000000000..fa7b75db7 --- /dev/null +++ b/src/util/Mesh.ts @@ -0,0 +1,18 @@ +import { MeshContract } from "../api/contracts/MeshContract"; + +export function hasReconstructionMesh(mesh: MeshContract): boolean { + return mesh != null && + mesh.vertices.length > 0 && + !isDefaultCubeMesh(mesh); +} + +export function isDefaultCubeMesh(mesh: MeshContract): boolean { + if (mesh.vertices.length !== 24 || mesh.faces.length !== 36) { + return false; + } + + const radius = Math.abs(mesh.vertices[0]); + return radius > 0 && mesh.vertices.every( + (coordinate: number): boolean => + Math.abs(Math.abs(coordinate) - radius) < 1e-8); +} diff --git a/src/viewer/Navigator.ts b/src/viewer/Navigator.ts index 9f6b7029f..fed8dc0fc 100644 --- a/src/viewer/Navigator.ts +++ b/src/viewer/Navigator.ts @@ -58,6 +58,8 @@ export class Navigator { private _idRequested$: BehaviorSubject; private _movedToId$: BehaviorSubject; + private _lastMoveDirection: NavigationDirection; + private _request$: ReplaySubject; private _requestSubscription: Subscription; private _imageRequestSubscription: Subscription; @@ -117,6 +119,8 @@ export class Navigator { this._idRequested$ = new BehaviorSubject(null); this._movedToId$ = new BehaviorSubject(null); + this._lastMoveDirection = null; + this._request$ = null; this._requestSubscription = null; this._imageRequestSubscription = null; @@ -168,8 +172,29 @@ export class Navigator { this._stateService.dispose(); } + /** + * The direction of the most recent {@link moveDir$} navigation, consumed + * (reset to null) on read so it attributes to exactly one landing image. + * Null for direct id moves (map click, shared link, fresh load) and once + * read. Lets the reorientation component tell a spatial step/turn apart + * from a non-directional jump, which the direction-blind image stream can't. + */ + public consumeMoveDirection(): NavigationDirection { + const direction = this._lastMoveDirection; + this._lastMoveDirection = null; + return direction; + } + public moveDir$(direction: NavigationDirection): Observable { this._abortRequest(`in dir ${NavigationDirection[direction]}`); + this._lastMoveDirection = direction; + + // Sequence navigation commits the current framing; drag momentum must + // not keep rotating the newly loaded image. + if (direction === NavigationDirection.Next || + direction === NavigationDirection.Prev) { + this.stateService.rotateBasicWithoutInertia([0, 0]); + } this._loadingService.startLoading(this._loadingName); @@ -206,8 +231,14 @@ export class Navigator { return this._makeRequest$(image$); } - public moveTo$(id: string): Observable { + public moveTo$( + id: string, + direction: NavigationDirection = null): Observable { this._abortRequest(`to id ${id}`); + // Spatial handlers (keyboard, pano/step direction circles) resolve the + // edge themselves and move by id, but still know the intended direction; + // record it so reorientation can tell a step/turn from a bare jump. + this._lastMoveDirection = direction; this._loadingService.startLoading(this._loadingName); diff --git a/src/viewer/PlayService.ts b/src/viewer/PlayService.ts index 6cbbcd769..3df463260 100644 --- a/src/viewer/PlayService.ts +++ b/src/viewer/PlayService.ts @@ -128,12 +128,18 @@ export class PlayService { const stateSpeed: number = this._setSpeed(this._speed); this._stateService.setSpeed(stateSpeed); - this._graphModeSubscription = this._speed$.pipe( - map( - (speed: number): GraphMode => { - return speed > PlayService.sequenceSpeed ? GraphMode.Sequence : GraphMode.Spatial; - }), - distinctUntilChanged()) + this._graphModeSubscription = observableCombineLatest( + this._speed$, + this._direction$).pipe( + map( + ([speed, direction]: [number, NavigationDirection]): GraphMode => { + const sequenceDirection = + direction === NavigationDirection.Next || + direction === NavigationDirection.Prev; + return sequenceDirection || speed > PlayService.sequenceSpeed ? + GraphMode.Sequence : GraphMode.Spatial; + }), + distinctUntilChanged()) .subscribe( (mode: GraphMode): void => { this._graphService.setGraphMode(mode); @@ -150,19 +156,17 @@ export class PlayService { ([sequenceId]: [string, string]): string => { return sequenceId; })), - this._graphService.graphMode$, this._direction$).pipe( switchMap( - ([[sequenceId, imageId], mode, direction]: [[string, string], GraphMode, NavigationDirection]): + ([[sequenceId], direction]: [[string, string], NavigationDirection]): Observable<[Sequence, NavigationDirection]> => { if (direction !== NavigationDirection.Next && direction !== NavigationDirection.Prev) { return observableOf<[Sequence, NavigationDirection]>([undefined, direction]); } - const sequence$: Observable = (mode === GraphMode.Sequence ? - this._graphService.cacheSequenceImages$(sequenceId, imageId) : - this._graphService.cacheSequence$(sequenceId)).pipe( + const sequence$: Observable = this._graphService + .cacheSequence$(sequenceId).pipe( retry(3), catchError( (error: Error): Observable => { @@ -186,6 +190,21 @@ export class PlayService { imageIds.reverse(); } + // Positions are resolved on every animation frame, + // linear lookups would scale with the sequence length. + const imageIndices: Map = + new Map(); + for (let i: number = 0; i < imageIds.length; i++) { + imageIndices.set(imageIds[i], i); + } + + const indexOf: (id: string) => number = + (id: string): number => { + const index: number = imageIndices.get(id); + + return index === undefined ? -1 : index; + }; + return this._stateService.currentState$.pipe( map( (frame: AnimationFrame): [string, number] => { @@ -197,8 +216,12 @@ export class PlayService { [lastTrajectoryKey, imagesAhead]: [string, number]): [string, string[]] => { - if (lastRequestKey === undefined) { - lastRequestKey = lastTrajectoryKey; + // The trajectory reaches beyond the + // sequence when traversing into another + // one, there is nothing to request here. + const current: number = indexOf(lastTrajectoryKey); + if (current === -1) { + return [lastRequestKey, []]; } const lastIndex: number = imageIds.length - 1; @@ -206,8 +229,12 @@ export class PlayService { return [lastRequestKey, []]; } - const current: number = imageIds.indexOf(lastTrajectoryKey); - const start: number = imageIds.indexOf(lastRequestKey) + 1; + const lastRequestIndex: number = + lastRequestKey === undefined ? + -1 : indexOf(lastRequestKey); + const start: number = + (lastRequestIndex === -1 ? + current : lastRequestIndex) + 1; const end: number = Math.min(lastIndex, current + this._imagesAhead - imagesAhead) + 1; if (end <= start) { diff --git a/styles/mapillary.css b/styles/mapillary.css index 979f4d9e3..0b77fc165 100644 --- a/styles/mapillary.css +++ b/styles/mapillary.css @@ -5,6 +5,10 @@ -webkit-tap-highlight-color: rgba(0,0,0,0); } +.mapillary-canvas:focus { + outline: none; +} + .mapillary-interactive { cursor: grab; } diff --git a/styles/sequence.css b/styles/sequence.css index c6028bbe9..cbb1dd112 100644 --- a/styles/sequence.css +++ b/styles/sequence.css @@ -16,6 +16,11 @@ .mapillary-sequence-stepper { transform: translate(-50%, 0); + z-index: 1; +} + +.mapillary-sequence-controls { + z-index: 1; } .mapillary-sequence-expander-button, diff --git a/styles/tooltip.css b/styles/tooltip.css new file mode 100644 index 000000000..131eb3ca9 --- /dev/null +++ b/styles/tooltip.css @@ -0,0 +1,65 @@ +.mapillary-viewer [data-mapillary-tooltip] { + position: relative; +} + +.mapillary-viewer [data-mapillary-tooltip]::before, +.mapillary-viewer [data-mapillary-tooltip]::after { + opacity: 0; + pointer-events: none; + position: absolute; + transition: opacity 0.2s ease-in-out; + visibility: hidden; +} + +.mapillary-viewer [data-mapillary-tooltip]::before { + background-color: #212b36; + border-radius: 4px; + box-shadow: 0 4px 8px 3px rgba(0, 0, 0, 0.15), 0 1px 3px 0 rgba(0, 0, 0, 0.3); + color: white; + content: attr(data-mapillary-tooltip); + font-family: "Source Sans Pro", sans-serif; + font-size: 12px; + font-weight: 400; + left: 50%; + line-height: 18px; + max-width: 240px; + padding: 4px 8px; + text-align: center; + top: calc(100% + 8px); + transform: translateX(-50%); + white-space: nowrap; + z-index: 1001; +} + +.mapillary-viewer [data-mapillary-tooltip]::after { + background-color: #212b36; + border-radius: 2px; + content: ""; + height: 8px; + left: calc(50% - 4px); + top: calc(100% + 4px); + transform: rotate(45deg); + width: 8px; + z-index: 1000; +} + +.mapillary-viewer [data-mapillary-tooltip-position="left"]::before { + left: auto; + right: calc(100% + 8px); + top: 50%; + transform: translateY(-50%); +} + +.mapillary-viewer [data-mapillary-tooltip-position="left"]::after { + left: auto; + right: calc(100% + 4px); + top: calc(50% - 4px); +} + +@media (hover: hover) { + .mapillary-viewer [data-mapillary-tooltip]:hover:not([data-mapillary-tooltip-dismissed])::before, + .mapillary-viewer [data-mapillary-tooltip]:hover:not([data-mapillary-tooltip-dismissed])::after { + opacity: 1; + visibility: visible; + } +} diff --git a/test/component/direction/DirectionComponent.test.ts b/test/component/direction/DirectionComponent.test.ts index c35254c15..0d45afbff 100644 --- a/test/component/direction/DirectionComponent.test.ts +++ b/test/component/direction/DirectionComponent.test.ts @@ -2,6 +2,7 @@ import { throwError as observableThrowError, of as observableOf, Subject } from import { DirectionComponent } from "../../../src/component/direction/DirectionComponent"; import { DirectionDOMRenderer } from "../../../src/component/direction/DirectionDOMRenderer"; import { ImageCache } from "../../../src/graph/ImageCache"; +import { NavigationEdgeStatus } from "../../../src/graph/interfaces/NavigationEdgeStatus"; import { Sequence } from "../../../src/graph/Sequence"; import { Container } from "../../../src/viewer/Container"; import { Navigator } from "../../../src/viewer/Navigator"; @@ -79,6 +80,8 @@ describe("DirectionComponent.activate", () => { const sequence: Sequence = new Sequence({ id: "skey", image_ids: [] }); (navigatorMock.graphService.cacheSequence$).and.returnValue(observableOf(sequence)); + (navigatorMock.graphService.cacheSequenceSpatialEdges$).and.returnValue( + observableOf({ cached: false, edges: [] })); const image: Image = new ImageHelper().createImage(); image.initializeCache(new ImageCache(undefined)); @@ -108,6 +111,8 @@ describe("DirectionComponent.activate", () => { const sequence: Sequence = new Sequence({ id: "skey", image_ids: [] }); const cacheSequence$: Subject = new Subject(); (navigatorMock.graphService.cacheSequence$).and.returnValue(cacheSequence$); + (navigatorMock.graphService.cacheSequenceSpatialEdges$).and.returnValue( + observableOf({ cached: false, edges: [] })); const image: Image = new ImageHelper().createImage(); image.initializeCache(new ImageCache(undefined)); @@ -121,6 +126,45 @@ describe("DirectionComponent.activate", () => { expect(setEdgesSpy.calls.argsFor(0)[1]).toBe(sequence); }); + it("should set provisional sequence edges before spatial edges", () => { + const containerMock: Container = new ContainerMockCreator().create(); + const navigatorMock: Navigator = new NavigatorMockCreator().create(); + const renderer: DirectionDOMRenderer = new DirectionDOMRenderer({ distinguishSequence: true }, { height: 1, width: 1 }); + const setEdgesSpy: jasmine.Spy = spyOn(renderer, "setEdges").and.stub(); + + const directionComponent: DirectionComponent = + new DirectionComponent( + DirectionComponent.componentName, + containerMock, + navigatorMock, + renderer); + + directionComponent.configure({ distinguishSequence: true }); + directionComponent.activate(); + + const sequence: Sequence = new Sequence({ id: "skey", image_ids: [] }); + (navigatorMock.graphService.cacheSequence$).and.returnValue(observableOf(sequence)); + + const sequenceEdges$ = new Subject(); + (navigatorMock.graphService.cacheSequenceSpatialEdges$).and.returnValue(sequenceEdges$); + + const image: Image = new ImageHelper().createImage(); + image.initializeCache(new ImageCache(undefined)); + (>navigatorMock.stateService.currentImage$).next(image); + + const provisional: NavigationEdgeStatus = { cached: false, edges: [] }; + sequenceEdges$.next(provisional); + + expect(setEdgesSpy.calls.count()).toBe(1); + expect(setEdgesSpy.calls.argsFor(0)[0]).toBe(provisional); + + const spatial: NavigationEdgeStatus = { cached: true, edges: [] }; + image.cacheSpatialEdges(spatial.edges); + + expect(setEdgesSpy.calls.count()).toBe(2); + expect(setEdgesSpy.calls.argsFor(1)[0]).toEqual(spatial); + }); + it("should set edges when distinguishing sequence and cache sequence throws", () => { spyOn(console, "error").and.stub(); diff --git a/test/component/direction/DirectionDOMRenderer.test.ts b/test/component/direction/DirectionDOMRenderer.test.ts index 1b90a9ad0..a8347c219 100644 --- a/test/component/direction/DirectionDOMRenderer.test.ts +++ b/test/component/direction/DirectionDOMRenderer.test.ts @@ -5,6 +5,9 @@ import { ImageHelper } from "../../helper/ImageHelper"; import { Navigator } from "../../../src/viewer/Navigator"; import { Image } from "../../../src/graph/Image"; +import { Sequence } from "../../../src/graph/Sequence"; +import { NavigationDirection } from "../../../src/graph/edge/NavigationDirection"; +import { NavigationEdge } from "../../../src/graph/edge/interfaces/NavigationEdge"; import { DirectionDOMRenderer } from "../../../src/component/direction/DirectionDOMRenderer"; import { DirectionConfiguration } from "../../../src/component/interfaces/DirectionConfiguration"; import { RenderCamera } from "../../../src/render/RenderCamera"; @@ -226,3 +229,83 @@ describe("DirectionDOMRenderer.needsRender", () => { expect(renderer.needsRender).toBe(true); }); }); + +describe("DirectionDOMRenderer.render", () => { + it("should align unmerged spherical sequence arrows with the panorama", () => { + let configuration: DirectionConfiguration = { + distinguishSequence: true, + maxWidth: 200, + minWidth: 100, + }; + let renderer: DirectionDOMRenderer = new DirectionDOMRenderer(configuration, { height: 1, width: 1 }); + const imageEnt = new ImageHelper().createImageEnt(); + imageEnt.camera_type = "spherical"; + imageEnt.compass_angle = 180; + imageEnt.computed_compass_angle = null; + let image: Image = new Image(imageEnt); + image.makeComplete(imageEnt); + let sequence: Sequence = new Sequence({ + id: image.sequenceId, + image_ids: ["previous", image.id, "next"], + }); + let edges: NavigationEdge[] = [ + { + data: { + direction: NavigationDirection.Spherical, + worldMotionAzimuth: -Math.PI / 2, + }, + source: image.id, + target: "next", + }, + { + data: { + direction: NavigationDirection.Spherical, + worldMotionAzimuth: Math.PI / 2, + }, + source: image.id, + target: "previous", + }, + { + data: { + direction: NavigationDirection.Spherical, + worldMotionAzimuth: Math.PI / 2, + }, + source: image.id, + target: "spatial", + }, + ]; + + renderer.setConfiguration(configuration); + renderer.setImage(image); + renderer.setEdges({ cached: true, edges }, sequence); + + let renderCamera: RenderCamera = new RenderCamera(1, 1, RenderMode.Fill); + (renderCamera)._rotation = { phi: -Math.PI, theta: 0 }; + renderer.setRenderCamera(renderCamera); + + let navigator: Navigator = new Navigator({ container: "containerid" }); + let arrows: any[] = renderer.render(navigator).children; + + expect(arrows[0].properties.attributes["data-id"]).toBe("next"); + expect(arrows[0].properties.style.transform).toContain("rotate(0deg)"); + expect(arrows[1].properties.attributes["data-id"]).toBe("previous"); + expect(arrows[1].properties.style.transform).toContain("rotate(-180deg)"); + expect(arrows[2].properties.attributes["data-id"]).toBe("spatial"); + expect(arrows[2].properties.style.transform).toContain("rotate(-270deg)"); + + (renderCamera)._rotation = { phi: -Math.PI / 2, theta: 0 }; + renderer.setRenderCamera(renderCamera); + arrows = renderer.render(navigator).children; + + expect(arrows[0].properties.style.transform).toContain("rotate(90deg)"); + expect(arrows[1].properties.style.transform).toContain("rotate(-90deg)"); + expect(arrows[2].properties.style.transform).toContain("rotate(-180deg)"); + + renderer.setImage(image); + renderer.setEdges({ cached: true, edges }, sequence); + arrows = renderer.render(navigator).children; + + expect(arrows[0].properties.style.transform).toContain("rotate(90deg)"); + expect(arrows[1].properties.style.transform).toContain("rotate(-90deg)"); + }); +}); diff --git a/test/component/reorientation/ReorientationEngine.test.ts b/test/component/reorientation/ReorientationEngine.test.ts index 0b0fd4901..0def50579 100644 --- a/test/component/reorientation/ReorientationEngine.test.ts +++ b/test/component/reorientation/ReorientationEngine.test.ts @@ -57,6 +57,9 @@ function provider(images: Fixture, seqIds: string[]): ReorientationProvider { }, fetchSeqIds: (_seqId: string): Promise => Promise.resolve(seqIds.slice()), + cacheImage: (image: ReorientationImage): void => { + images[image.id] = image; + }, }; } @@ -65,11 +68,63 @@ function spherical( lat: number, lng: number, cca: number, - ts: number): ReorientationImage { - return { id, lat, lng, cca, cam: "spherical", seq: "s", ts }; + ts: number, + originalLat?: number, + originalLng?: number, + viewCompassAngle?: number): ReorientationImage { + return { + id, + lat, + lng, + originalLat, + originalLng, + cca, + viewCompassAngle, + cam: "spherical", + seq: "s", + ts, + }; } describe("ReorientationEngine.precompute", () => { + it("warms the configured sequence window in one batch", async () => { + const images: Fixture = { + a: spherical("a", 0, -0.0001, 0, 1000), + b: spherical("b", 0, 0, 0, 2000), + c: spherical("c", 0, 0.0001, 0, 3000), + d: spherical("d", 0, 0.0002, 0, 4000), + e: spherical("e", 0, 0.0003, 0, 5000), + }; + const dataProvider = provider(images, ["a", "b", "c", "d", "e"]); + dataProvider.fetchImages = jasmine.createSpy("fetchImages") + .and.callFake((ids: string[]): Promise => + Promise.resolve(ids.map(id => images[id]))); + const engine = new ReorientationEngine(dataProvider); + + await engine.precompute("b", images.b, 2); + + expect(dataProvider.fetchImages).toHaveBeenCalledWith(["a", "c"]); + expect(dataProvider.fetchImages).toHaveBeenCalledWith(["d", "e"]); + expect(engine.get("b").valid).toBe(true); + }); + + it("recomputes provisional metadata when the loaded seed changes", async () => { + const images: Fixture = { + a: spherical("a", 0, -0.0001, 0, 1000), + b: spherical("b", 0, 0, 0, 2000), + c: spherical("c", 0, 0.0001, 0, 3000), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + await engine.precompute("b", images.b, 0); + expect(engine.get("b").basicX).toBeCloseTo(0.75, 2); + + const loaded = { ...images.b, cca: 90, viewCompassAngle: 90 }; + await engine.precompute("b", loaded, 0); + + expect(engine.get("b").cca).toBe(90); + expect(engine.get("b").basicX).toBeCloseTo(0.5, 2); + }); + it("frames the travel direction for a moving image", async () => { // Straight eastward run, ~11 m apart, 1 s apart -> ~11 m/s, facing // north (cca 0) so travel east maps to basic-x 0.75. (capturedAt is an @@ -90,6 +145,222 @@ describe("ReorientationEngine.precompute", () => { expect(result.basicX).toBeCloseTo(0.75, 2); }); + it("frames travel against a distinct panorama-view axis", async () => { + const images: Fixture = { + a: spherical("a", 0.0002, 0, 180, 1000, undefined, undefined, 90), + b: spherical("b", 0.0001, 0, 180, 2000, undefined, undefined, 90), + c: spherical("c", 0, 0, 180, 3000, undefined, undefined, 90), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.travel).toBeCloseTo(180, 1); + expect(result.cca).toBeCloseTo(180, 1); + expect(result.viewCompassAngle).toBeCloseTo(90, 1); + expect(result.basicX).toBeCloseTo(0.75, 2); + }); + + it("uses raw compass when reconstructed heading conflicts with travel", async () => { + const current = spherical("b", 0, 0, 0, 2000); + current.computedCca = 0; + current.originalCca = 90; + const next = spherical("c", 0, 0.0001, 90, 3000); + next.computedCca = 90; + next.originalCca = 90; + const images: Fixture = { + a: spherical("a", 0, -0.0001, 90, 1000), + b: current, + c: next, + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.computedCompassOutlier).toBe(true); + expect(result.cca).toBeCloseTo(0, 1); + expect(result.viewCompassAngle).toBeCloseTo(90, 1); + expect(result.basicX).toBeCloseTo(0.5, 2); + }); + + it("keeps reconstructed heading when raw compass conflicts with travel", async () => { + const current = spherical("b", 0, 0, 0, 2000); + current.computedCca = 0; + current.originalCca = 180; + const next = spherical("c", 0, 0.0001, 90, 3000); + next.computedCca = 90; + next.originalCca = 90; + const images: Fixture = { + a: spherical("a", 0, -0.0001, 90, 1000), + b: current, + c: next, + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.computedCompassOutlier).toBe(false); + expect(result.viewCompassAngle).toBeCloseTo(0, 1); + expect(result.basicX).toBeCloseTo(0.75, 2); + }); + + it("keeps a stable reconstructed-to-raw compass calibration", async () => { + const current = spherical("b", 0, 0, 270, 2000); + current.computedCca = 270; + current.originalCca = 90; + const next = spherical("c", 0, 0.0001, 270, 3000); + next.computedCca = 270; + next.originalCca = 90; + const images: Fixture = { + a: spherical("a", 0, -0.0001, 270, 1000), + b: current, + c: next, + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.computedCompassOutlier).toBe(false); + expect(result.reconstructionDiscontinuity).toBe(false); + expect(result.viewCompassAngle).toBeCloseTo(270, 1); + expect(result.basicX).toBeCloseTo(0, 2); + }); + + it("rejects an abrupt reconstructed calibration change", async () => { + const previous = spherical("a", 0, -0.0001, 90, 1000); + previous.computedCca = 90; + previous.originalCca = 90; + const current = spherical("b", 0, 0, 270, 2000); + current.computedCca = 270; + current.originalCca = 90; + const next = spherical("c", 0, 0.0001, 270, 3000); + next.computedCca = 270; + next.originalCca = 90; + const following = spherical("d", 0, 0.0002, 270, 4000); + following.computedCca = 270; + following.originalCca = 90; + const images: Fixture = { + a: previous, + b: current, + c: next, + d: following, + }; + const engine = new ReorientationEngine( + provider(images, ["a", "b", "c", "d"])); + + await engine.precompute("b"); + await engine.precompute("c"); + const result = engine.get("b"); + const continued = engine.get("c"); + + expect(result.computedCompassOutlier).toBe(true); + expect(result.reconstructionDiscontinuity).toBe(true); + expect(result.viewCompassAngle).toBeCloseTo(90, 1); + expect(result.basicX).toBeCloseTo(0.5, 2); + expect(continued.computedCompassOutlier).toBe(true); + expect(continued.reconstructionDiscontinuity).toBe(false); + }); + + it("uses the original track when computed speed is impossible", async () => { + const images: Fixture = { + a: spherical("a", 0, 0, 90, 1000, 0, 0), + b: spherical("b", 0.001, 0, 90, 1040, 0, 0.00006), + c: spherical("c", 0.00101, 0, 90, 1080, 0, 0.00012), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("a"); + await engine.precompute("b"); + + expect(engine.get("a").travel).toBeCloseTo(90, 1); + expect(engine.get("b").travel).toBeCloseTo(90, 1); + }); + + it("uses the original track when computed geometry reverses", async () => { + const images: Fixture = { + a: spherical("a", 0, 0, 90, 1000, 0, 0), + b: spherical("b", 0, 0.00018, 90, 3000, 0, 0.00018), + c: spherical("c", 0, 0.00004, 90, 5000, 0, 0.00036), + d: spherical("d", 0, 0.00054, 90, 7000, 0, 0.00054), + }; + const engine = new ReorientationEngine( + provider(images, ["a", "b", "c", "d"])); + + await engine.precompute("a"); + await engine.precompute("b"); + await engine.precompute("c"); + + expect(engine.get("b").travel).toBeCloseTo(90, 1); + expect(engine.get("b").basicX).toBeCloseTo(0.5, 2); + expect(engine.get("c").travel).toBeCloseTo(90, 1); + }); + + it("uses the original track when computed geometry points sideways", async () => { + const images: Fixture = { + a: spherical("a", 0, 0, 90, 1000, 0, 0), + b: spherical("b", -0.0000819, 0.0000574, 90, 3000, 0, 0.0001), + }; + const engine = new ReorientationEngine(provider(images, ["a", "b"])); + + await engine.precompute("a"); + + expect(engine.get("a").travel).toBeCloseTo(90, 1); + expect(engine.get("a").basicX).toBeCloseTo(0.5, 2); + }); + + it("keeps a coherent computed track when raw positions zigzag", async () => { + const previous = spherical( + "a", 47.359316158503, 8.522159587028, 148.26818781405, + 1768975532063, 47.359348856991, 8.5221314103916); + previous.computedCca = 148.26818781405; + previous.originalCca = 210.92003060148; + const current = spherical( + "b", 47.359301627069, 8.5221372232107, 148.10483449986, + 1768975533663, 47.359324640976, 8.5221127109815); + current.computedCca = 148.10483449986; + current.originalCca = 208.06971887641; + const next = spherical( + "c", 47.359291658286, 8.5221209467221, 147.99487211037, + 1768975534743, 47.359297753971, 8.522101362013); + next.computedCca = 147.99487211037; + next.originalCca = 163.10847738317; + const engine = new ReorientationEngine(provider( + { a: previous, b: current, c: next }, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.travel).toBeCloseTo(227.9, 1); + expect(result.basicX).toBeCloseTo(0.72, 2); + }); + + it("uses raw track and compass when both computed values jump", async () => { + const current = spherical("b", 0, 0.00018, 0, 3000, 0, 0.00018); + current.computedCca = 0; + current.originalCca = 90; + const next = spherical("c", 0, 0.00004, 90, 5000, 0, 0.00036); + next.computedCca = 90; + next.originalCca = 90; + const images: Fixture = { + a: spherical("a", 0, 0, 90, 1000, 0, 0), + b: current, + c: next, + }; + const engine = new ReorientationEngine(provider(images, ["a", "b", "c"])); + + await engine.precompute("b"); + const result = engine.get("b"); + + expect(result.travel).toBeCloseTo(90, 1); + expect(result.computedCompassOutlier).toBe(true); + expect(result.viewCompassAngle).toBeCloseTo(90, 1); + expect(result.basicX).toBeCloseTo(0.5, 2); + }); + it("accepts a low-speed step as a turn when compass agrees", async () => { // ~11 m apart but 100 s apart -> ~0.11 m/s (below movingSpeedMps), // yet compass (90) agrees with eastward travel, so it counts. diff --git a/test/component/reorientation/ReorientationLeveling.test.ts b/test/component/reorientation/ReorientationLeveling.test.ts new file mode 100644 index 000000000..514f751e4 --- /dev/null +++ b/test/component/reorientation/ReorientationLeveling.test.ts @@ -0,0 +1,107 @@ +import { + horizonPitchDeg, + isLevelPlausible, +} from "../../../src/component/reorientation/ReorientationComponent"; + +/** + * Sequence 1lt7kgfezr0ysp9vmlirl4, measured in the viewer. Images without a + * reconstruction mesh never reach the plausibility check and are omitted. + */ +const MEASURED: { id: string, rollDeg: number, horizonY: number }[] = [ + { id: "915416489297910", rollDeg: 1.08, horizonY: 0.3795 }, + { id: "520184629350307", rollDeg: 11, horizonY: 0.3817 }, + { id: "535844770743193", rollDeg: 5.73, horizonY: 0.2645 }, + { id: "818284102398965", rollDeg: 4.85, horizonY: 0.374 }, + { id: "764978360865844", rollDeg: 0.29, horizonY: 0.3128 }, + { id: "4134207483305086", rollDeg: 34, horizonY: 0.4995 }, + { id: "784577412247663", rollDeg: 7.68, horizonY: 0.7393 }, +]; + +const BAD = [ + "535844770743193", + "4134207483305086", + "784577412247663", +]; + +function replay( + images: { id: string, rollDeg: number, horizonY: number }[]): string[] { + let baselineRoll: number = null; + let baselinePitch: number = null; + const rejected: string[] = []; + for (const image of images) { + const pitchDeg = horizonPitchDeg(image.horizonY); + if (isLevelPlausible( + image.rollDeg, + pitchDeg, + baselineRoll, + baselinePitch)) { + baselineRoll = image.rollDeg; + baselinePitch = pitchDeg; + } else { + rejected.push(image.id); + } + } + return rejected; +} + +describe("isLevelPlausible", () => { + it("accepts a level pose when the sequence has no baseline", () => { + expect(isLevelPlausible(0, 0, null, null)).toBe(true); + }); + + it("accepts a pose that stays close to the baseline", () => { + expect(isLevelPlausible(2, 21.3, 1, 21.7)).toBe(true); + }); + + it("rejects a pose whose roll jumps away from the baseline", () => { + expect(isLevelPlausible(34, 0, 1, 0)).toBe(false); + }); + + it("accepts a consistently tilted camera", () => { + expect(isLevelPlausible(34, 2, 30, 0)).toBe(true); + }); + + it("rejects a pose whose pitch jumps away from the baseline", () => { + expect(isLevelPlausible(5.73, 42.4, 11, 21.3)).toBe(false); + }); + + it("rejects a pose in either pitch direction", () => { + expect(isLevelPlausible(7.68, -43.1, 0.29, 33.7)).toBe(false); + }); + + it("rejects non-finite measurements", () => { + expect(isLevelPlausible(NaN, 0, null, null)).toBe(false); + expect(isLevelPlausible(0, NaN, null, null)).toBe(false); + }); + + it("accepts the first pose without a consistency baseline", () => { + expect(isLevelPlausible(34, 60, null, null)).toBe(true); + }); +}); + +describe("horizonPitchDeg", () => { + it("reports no correction for a centred horizon", () => { + expect(horizonPitchDeg(0.5)).toBeCloseTo(0, 6); + }); + + it("reports a positive correction for a horizon above centre", () => { + expect(horizonPitchDeg(0.25)).toBeCloseTo(45, 6); + }); +}); + +describe("isLevelPlausible.sequence", () => { + it("rejects exactly the badly reoriented images of the repro", () => { + expect(replay(MEASURED)).toEqual(BAD); + }); + + it("keeps levelling the images after a rejected one", () => { + const rejected = replay(MEASURED); + expect(rejected).not.toContain("818284102398965"); + expect(rejected).not.toContain("764978360865844"); + }); + + it("recovers a baseline after a sequence reset", () => { + const afterReset = replay([MEASURED[6]]); + expect(afterReset).toEqual([]); + }); +}); diff --git a/test/graph/Graph.test.ts b/test/graph/Graph.test.ts index 30a170448..653cb35ba 100644 --- a/test/graph/Graph.test.ts +++ b/test/graph/Graph.test.ts @@ -261,6 +261,64 @@ describe("Graph.cacheBoundingBox$", () => { }); }); +describe("Graph.cacheFullImages$", () => { + it("should fetch multiple images in one request", (done: Function) => { + const api = new APIWrapper(new DataProvider()); + const calculator = new GraphCalculator(); + const getImages = new Subject(); + const getImagesSpy = spyOn(api, "getImages$") + .and.returnValue(getImages); + spyOn(api.data.geometry, "lngLatToCellId").and.returnValue("cell-id"); + const graph = new Graph(api, undefined, undefined, calculator); + const firstNode = new ImageHelper().createImageEnt(); + const secondNode = new ImageHelper().createImageEnt(); + secondNode.id = `${firstNode.id}-2`; + + graph.cacheFullImages$([firstNode.id, secondNode.id]).subscribe( + (result: Graph): void => { + expect(result.getNode(firstNode.id).complete).toBe(true); + expect(result.getNode(secondNode.id).complete).toBe(true); + expect(getImagesSpy).toHaveBeenCalledWith( + [firstNode.id, secondNode.id]); + expect(getImagesSpy.calls.count()).toBe(1); + done(); + }); + + getImages.next([ + { node: firstNode, node_id: firstNode.id }, + { node: secondNode, node_id: secondNode.id }, + ]); + getImages.complete(); + }); + + it("should limit requests to 120 images", (done: Function) => { + const api = new APIWrapper(new DataProvider()); + const calculator = new GraphCalculator(); + const nodes = Array(121).fill(undefined).map((_, index: number) => { + const node = new ImageHelper().createImageEnt(); + node.id = `image-${index}`; + return node; + }); + const getImagesSpy = spyOn(api, "getImages$") + .and.callFake((ids: string[]): Observable => + observableOf(ids.map((id: string) => { + const node = nodes.find( + (candidate: ImageEnt): boolean => candidate.id === id); + return { node, node_id: id }; + }))); + spyOn(api.data.geometry, "lngLatToCellId").and.returnValue("cell-id"); + const graph = new Graph(api, undefined, undefined, calculator); + + graph.cacheFullImages$(nodes.map((node: ImageEnt) => node.id)).subscribe( + (): void => { + expect(getImagesSpy.calls.count()).toBe(2); + expect(getImagesSpy.calls.argsFor(0)[0].length).toBe(120); + expect(getImagesSpy.calls.argsFor(1)[0].length).toBe(1); + done(); + }); + }); +}); + describe("Graph.cacheFull$", () => { let helper: ImageHelper; @@ -320,6 +378,30 @@ describe("Graph.cacheFull$", () => { expect(graph.getNode(fullNode.id).id).toBe(fullNode.id); }); + it("should derive rotation when the API returns an empty vector", (done: Function) => { + const api = new APIWrapper(new DataProvider()); + const calculator = new GraphCalculator(); + const rotation = [1, 2, 3]; + const rotationSpy = spyOn(calculator, "rotationFromCompass") + .and.returnValue(rotation); + const getImages = new Subject(); + spyOn(api, "getImages$").and.returnValue(getImages); + spyOn(api.data.geometry, "lngLatToCellId").and.returnValue("cell-id"); + const graph = new Graph(api, undefined, undefined, calculator); + const fullNode = helper.createImageEnt(); + fullNode.computed_rotation = []; + + graph.cacheFull$(fullNode.id).subscribe( + (g: Graph): void => { + expect(rotationSpy).toHaveBeenCalledWith( + fullNode.compass_angle, fullNode.exif_orientation); + expect(g.getNode(fullNode.id).rotation).toEqual(rotation); + done(); + }); + getImages.next([{ node: fullNode, node_id: fullNode.id }]); + getImages.complete(); + }); + it("should not make additional calls when fetching same node twice", () => { const api = new APIWrapper(new DataProvider()); const calculator = new GraphCalculator(); @@ -1475,7 +1557,7 @@ describe("Graph.cacheSequenceNodes$", () => { expect(graph.hasNode(nodeKey)).toBe(false); }); - it("should start caching in with single batch when lass than or equal to 200 nodes", () => { + it("should start caching with a single batch for at most 120 nodes", () => { const api = new APIWrapper(new DataProvider()); const graphCalculator = new GraphCalculator(); const edgeCalculator = new EdgeCalculator(); @@ -1495,7 +1577,7 @@ describe("Graph.cacheSequenceNodes$", () => { const result: SequenceContract = { id: sequenceId, - image_ids: Array(200) + image_ids: Array(120) .fill(undefined) .map((_, i) => i.toString()) }; @@ -1507,15 +1589,15 @@ describe("Graph.cacheSequenceNodes$", () => { expect(graph.isCachingSequenceNodes(sequenceId)).toBe(true); expect(imageByKeySpy.calls.count()).toBe(1); - expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(200); + expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(120); expect( imageByKeySpy.calls.allArgs() .map((args: string[][]): number => { return args[0].length; }) .reduce((acc: number, cur: number): number => { return acc + cur; }, 0)) - .toBe(200); + .toBe(120); }); - it("should start caching in batches when more than 200 nodes", () => { + it("should start caching in batches when more than 120 nodes", () => { const api = new APIWrapper(new DataProvider()); const graphCalculator = new GraphCalculator(); const edgeCalculator = new EdgeCalculator(); @@ -1535,7 +1617,7 @@ describe("Graph.cacheSequenceNodes$", () => { const result: SequenceContract = { id: sequenceId, - image_ids: Array(201) + image_ids: Array(121) .fill(undefined) .map((_, i) => i.toString()), }; @@ -1547,13 +1629,13 @@ describe("Graph.cacheSequenceNodes$", () => { expect(graph.isCachingSequenceNodes(sequenceId)).toBe(true); expect(imageByKeySpy.calls.count()).toBe(2); - expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(200); + expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(120); expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(1); expect( imageByKeySpy.calls.allArgs() .map((args: string[][]): number => { return args[0].length; }) .reduce((acc: number, cur: number): number => { return acc + cur; }, 0)) - .toBe(201); + .toBe(121); }); it("should start caching prioritized batch when reference node key is specified at start", () => { @@ -1591,11 +1673,12 @@ describe("Graph.cacheSequenceNodes$", () => { expect(graph.isCachingSequenceNodes(sequenceId)).toBe(true); - expect(imageByKeySpy.calls.count()).toBe(3); + expect(imageByKeySpy.calls.count()).toBe(4); expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(50); expect(imageByKeySpy.calls.argsFor(0)[0][0]).toBe(referenceNodeKey); - expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(200); - expect(imageByKeySpy.calls.argsFor(2)[0].length).toBe(400 - 200 - 50); + expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(120); + expect(imageByKeySpy.calls.argsFor(2)[0].length).toBe(120); + expect(imageByKeySpy.calls.argsFor(3)[0].length).toBe(400 - 2 * 120 - 50); expect( imageByKeySpy.calls.allArgs() .map((args: string[][]): number => { return args[0].length; }) @@ -1638,12 +1721,13 @@ describe("Graph.cacheSequenceNodes$", () => { expect(graph.isCachingSequenceNodes(sequenceId)).toBe(true); - expect(imageByKeySpy.calls.count()).toBe(3); + expect(imageByKeySpy.calls.count()).toBe(4); expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(50); expect(imageByKeySpy.calls.argsFor(0)[0][0]).toBe((400 - 50).toString()); expect(imageByKeySpy.calls.argsFor(0)[0][49]).toBe(referenceNodeKey); - expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(200); - expect(imageByKeySpy.calls.argsFor(2)[0].length).toBe(400 - 200 - 50); + expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(120); + expect(imageByKeySpy.calls.argsFor(2)[0].length).toBe(120); + expect(imageByKeySpy.calls.argsFor(3)[0].length).toBe(400 - 2 * 120 - 50); expect( imageByKeySpy.calls.allArgs() .map((args: string[][]): number => { return args[0].length; }) @@ -1686,13 +1770,14 @@ describe("Graph.cacheSequenceNodes$", () => { expect(graph.isCachingSequenceNodes(sequenceId)).toBe(true); - expect(imageByKeySpy.calls.count()).toBe(3); + expect(imageByKeySpy.calls.count()).toBe(4); expect(imageByKeySpy.calls.argsFor(0)[0].length).toBe(50); expect(imageByKeySpy.calls.argsFor(0)[0][0]).toBe((200 - 25).toString()); expect(imageByKeySpy.calls.argsFor(0)[0][25]).toBe(referenceNodeKey); expect(imageByKeySpy.calls.argsFor(0)[0][49]).toBe((200 + 24).toString()); - expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(200); - expect(imageByKeySpy.calls.argsFor(2)[0].length).toBe(400 - 200 - 50); + expect(imageByKeySpy.calls.argsFor(1)[0].length).toBe(120); + expect(imageByKeySpy.calls.argsFor(2)[0].length).toBe(120); + expect(imageByKeySpy.calls.argsFor(3)[0].length).toBe(400 - 2 * 120 - 50); expect( imageByKeySpy.calls.allArgs() .map((args: string[][]): number => { return args[0].length; }) @@ -1929,6 +2014,17 @@ describe("Graph.cacheSpatialEdges", () => { getSequence.complete(); const node = graph.getNode(fullNode.id); + const previousEnt = helper.createImageEnt(); + previousEnt.id = "prev"; + previousEnt.sequence.id = fullNode.sequence.id; + const previous = new Image(previousEnt); + previous.makeComplete(previousEnt); + const hasNode = graph.hasNode.bind(graph); + const getNode = graph.getNode.bind(graph); + spyOn(graph, "hasNode").and.callFake( + (key: string): boolean => key === previous.id || hasNode(key)); + spyOn(graph, "getNode").and.callFake( + (key: string): Image => key === previous.id ? previous : getNode(key)); spyOn(graphCalculator, "boundingBoxCorners") .and.returnValue([ @@ -1951,12 +2047,59 @@ describe("Graph.cacheSpatialEdges", () => { graph.cacheSpatialEdges(fullNode.id); expect(getPotentialSpy.calls.first().args.length).toBe(3); + expect(getPotentialSpy.calls.first().args[1]).toEqual([previous]); expect(getPotentialSpy.calls.first().args[2].length).toBe(2); expect(getPotentialSpy.calls.first().args[2].indexOf("prev")).not.toBe(-1); expect(getPotentialSpy.calls.first().args[2].indexOf("next")).not.toBe(-1); expect(getPotentialSpy.calls.first().args[2].indexOf(fullNode.id)).toBe(-1); }); + it("should target the sequence image nearest the preferred distance", () => { + const api = new APIWrapper(new DataProvider()); + const graph = new Graph(api); + const fullNode = helper.createImageEnt(); + fullNode.camera_type = "spherical"; + + spyOn(api.data.geometry, "lngLatToCellId").and.returnValue("cell-id"); + const getImages = new Subject(); + spyOn(api, "getImages$").and.returnValue(getImages); + const getSequence = new Subject(); + spyOn(api, "getSequence$").and.returnValue(getSequence); + + graph.cacheFull$(fullNode.id).subscribe(() => { /*noop*/ }); + getImages.next([{ node: fullNode, node_id: fullNode.id }]); + getImages.complete(); + + graph.cacheNodeSequence$(fullNode.id).subscribe(() => { /*noop*/ }); + getSequence.next({ + id: fullNode.sequence.id, + image_ids: ["prev-target", "prev", fullNode.id, "next", "next-target"], + }); + getSequence.complete(); + + const adjacentNodes: { [id: string]: Image; } = {}; + for (const [id, lng] of [["prev", -0.000027], ["next", 0.000027]] as [string, number][]) { + const imageEnt = helper.createImageEnt(); + imageEnt.id = id; + imageEnt.sequence.id = fullNode.sequence.id; + imageEnt.computed_geometry = { lat: 0, lng }; + imageEnt.geometry = imageEnt.computed_geometry; + const image = new Image(imageEnt); + image.makeComplete(imageEnt); + adjacentNodes[id] = image; + } + + const hasNode = graph.hasNode.bind(graph); + const getNode = graph.getNode.bind(graph); + spyOn(graph, "hasNode").and.callFake( + (key: string): boolean => key in adjacentNodes || hasNode(key)); + spyOn(graph, "getNode").and.callFake( + (key: string): Image => key in adjacentNodes ? adjacentNodes[key] : getNode(key)); + + expect(graph.getSequenceSpatialTargetIds(fullNode.id)) + .toEqual(["prev-target", "next-target"]); + }); + test("should apply filter", () => { const cellId = "cell-id"; const dataProvider = new DataProvider(); diff --git a/test/graph/GraphService.test.ts b/test/graph/GraphService.test.ts index 3ed873565..8f082164b 100644 --- a/test/graph/GraphService.test.ts +++ b/test/graph/GraphService.test.ts @@ -665,6 +665,37 @@ class TestNode extends Image { } } +describe("GraphService.cacheImagesMetadata$", () => { + it("should cache image metadata as one graph batch", (done: Function) => { + const api: APIWrapper = new APIWrapper(new DataProvider()); + const graph: Graph = new Graph(api); + const cacheFullImages$: Subject = new Subject(); + const cacheSpy = spyOn(graph, "cacheFullImages$") + .and.returnValue(cacheFullImages$); + const firstImage: TestNode = new TestNode( + new ImageHelper().createCoreImageEnt()); + const secondEnt = new ImageHelper().createCoreImageEnt(); + secondEnt.id = `${firstImage.id}-2`; + const secondImage: TestNode = new TestNode(secondEnt); + spyOn(graph, "hasNode").and.returnValue(true); + spyOn(graph, "getNode").and.callFake((id: string): Image => + id === firstImage.id ? firstImage : secondImage); + const graphService: GraphService = new GraphService(graph); + + graphService.cacheImagesMetadata$( + [firstImage.id, secondImage.id]).subscribe( + (images: Image[]): void => { + expect(images).toEqual([firstImage, secondImage]); + expect(cacheSpy).toHaveBeenCalledWith( + [firstImage.id, secondImage.id]); + done(); + }); + + cacheFullImages$.next(graph); + cacheFullImages$.complete(); + }); +}); + describe("GraphService.cacheNode$", () => { let helper: ImageHelper; diff --git a/test/graph/edge/EdgeCalculator.Pano.test.ts b/test/graph/edge/EdgeCalculator.Pano.test.ts index a1a8ed47e..bcb92a2b4 100644 --- a/test/graph/edge/EdgeCalculator.Pano.test.ts +++ b/test/graph/edge/EdgeCalculator.Pano.test.ts @@ -77,6 +77,18 @@ describe("EdgeCalculator.computeSphericalEdges", () => { expect(sphericalEdges.length).toBe(0); }); + it("should have a distant spherical edge when it is a fallback", () => { + potentialEdge1.distance = settings.sphericalMaxDistance + 1; + + const sphericalEdges = edgeCalculator.computeSphericalEdges( + image, + [potentialEdge1], + [potentialEdge1.id]); + + expect(sphericalEdges.length).toBe(1); + expect(sphericalEdges[0].target).toBe(potentialEdge1.id); + }); + it("should not have a spherical edge with to short distance", () => { potentialEdge1.distance = settings.sphericalMinDistance / 2; diff --git a/test/graph/edge/EdgeCalculator.test.ts b/test/graph/edge/EdgeCalculator.test.ts index 8476b9e77..2db6e3907 100644 --- a/test/graph/edge/EdgeCalculator.test.ts +++ b/test/graph/edge/EdgeCalculator.test.ts @@ -80,6 +80,43 @@ describe("EdgeCalculator.getPotentialEdges", () => { expect(potentialEdges.length).toBe(0); }); + it("should return an unmerged fallback edge", () => { + let key = "key"; + let edgeKey = "edgeKey"; + let sequenceKey = "skey"; + + let lla: LngLatAlt = { alt: 0, lat: 0, lng: 0 }; + let image = helper.createCompleteImage( + key, + lla, + sequenceKey, + [0, -Math.PI / 2, 0], + null, + "spherical"); + let geodetic = GeoCoords.enuToGeodetic( + 10, + 0, + 0, + lla.lng, + lla.lat, + lla.alt); + let edgeImage = helper.createCompleteImage( + edgeKey, + { alt: geodetic[2], lat: geodetic[1], lng: geodetic[0] }, + sequenceKey, + [0, -Math.PI / 2, 0], + null, + "spherical"); + + let potentialEdges = + edgeCalculator.getPotentialEdges(image, [edgeImage], [edgeKey]); + + expect(potentialEdges.length).toBe(1); + expect(potentialEdges[0].id).toBe(edgeKey); + expect(potentialEdges[0].sameMergeCC).toBe(false); + expect(potentialEdges[0].sameSequence).toBe(true); + }); + it("should return a potential edge", () => { let key = "key"; let edgeKey = "edgeKey"; diff --git a/test/render/RenderCamera.test.ts b/test/render/RenderCamera.test.ts index 4f5c8d493..3ca90c4b7 100644 --- a/test/render/RenderCamera.test.ts +++ b/test/render/RenderCamera.test.ts @@ -198,6 +198,19 @@ describe("RenderCamera.perspective.fov", () => { expect(renderCamera.perspective.fov).toBeLessThan(fov); }); + + it("should expose the current image and unzoomed field of view", () => { + const renderCamera = new RenderCamera(1, 1, RenderMode.Fill); + const frame: AnimationFrame = new FrameHelper().createFrame(); + renderCamera.setFrame(frame); + const unzoomedFov = renderCamera.unzoomedCurrentFov; + + frame.state.zoom = 1; + renderCamera.setFrame(frame); + + expect(renderCamera.currentImageId).toBe(frame.state.currentImage.id); + expect(renderCamera.unzoomedCurrentFov).toBe(unzoomedFov); + }); }); describe("RenderCamera.setProjectionMatrix", () => { diff --git a/test/state/state/StateBase.test.ts b/test/state/state/StateBase.test.ts index 985d50972..82a19a83b 100644 --- a/test/state/state/StateBase.test.ts +++ b/test/state/state/StateBase.test.ts @@ -118,6 +118,137 @@ describe("StateBase.motionlessTransition", () => { expect(stateBase.motionlessTransition()).toBe(false); }); + it("should be true for placeholder cube meshes", () => { + const state: IStateBase = createState(); + const stateBase: TestStateBase = new TestStateBase(state); + const helper: ImageHelper = new ImageHelper(); + const vertices = [ + -1, -1, -1, -1, -1, 1, -1, 1, -1, -1, 1, 1, + 1, -1, -1, 1, -1, 1, 1, 1, -1, 1, 1, 1, + ]; + const faces = new Array(36).fill(0); + + const imageEnt1: ImageEnt = helper.createImageEnt(); + imageEnt1.merge_id = "1"; + const image1 = new TestImage(imageEnt1); + image1.makeComplete(imageEnt1); + image1.initializeCache(new ImageCache(new DataProvider())); + image1.cacheCamera(new ProjectionService()); + image1.mesh = { vertices, faces }; + + const imageEnt2: ImageEnt = helper.createImageEnt(); + imageEnt2.merge_id = "1"; + const image2 = new TestImage(imageEnt2); + image2.makeComplete(imageEnt2); + image2.initializeCache(new ImageCache(new DataProvider())); + image2.cacheCamera(new ProjectionService()); + image2.mesh = { vertices, faces }; + + stateBase.set([image1]); + stateBase.set([image2]); + + expect(stateBase.motionlessTransition()).toBe(true); + }); + + it("should be true beyond the spatial navigation range", () => { + const stateBase = new TestStateBase(createState()); + const helper = new ImageHelper(); + + const previousEnt = helper.createImageEnt(); + previousEnt.id = "previous"; + previousEnt.merge_id = "1"; + previousEnt.cluster = { id: "previous", url: null }; + const previous = new TestImage(previousEnt); + previous.makeComplete(previousEnt); + previous.initializeCache(new ImageCache(new DataProvider())); + previous.cacheCamera(new ProjectionService()); + previous.mesh = { + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + faces: [0, 1, 2], + }; + + const currentEnt = helper.createImageEnt(); + currentEnt.id = "current"; + currentEnt.merge_id = "1"; + currentEnt.cluster = { id: "current", url: null }; + currentEnt.computed_geometry = { lat: 0, lng: 0.0002 }; + currentEnt.geometry = currentEnt.computed_geometry; + const current = new TestImage(currentEnt); + current.makeComplete(currentEnt); + current.initializeCache(new ImageCache(new DataProvider())); + current.cacheCamera(new ProjectionService()); + current.mesh = { + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + faces: [0, 1, 2], + }; + + stateBase.set([previous]); + stateBase.set([current]); + + expect(stateBase.motionlessTransition()).toBe(true); + }); + + it("should allow distant transitions within one reconstruction", () => { + const stateBase = new TestStateBase(createState()); + const helper = new ImageHelper(); + + const previousEnt = helper.createImageEnt(); + previousEnt.id = "previous"; + previousEnt.merge_id = "1"; + previousEnt.cluster = { id: "same", url: null }; + const previous = new TestImage(previousEnt); + previous.makeComplete(previousEnt); + previous.initializeCache(new ImageCache(new DataProvider())); + previous.cacheCamera(new ProjectionService()); + previous.mesh = { + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + faces: [0, 1, 2], + }; + + const currentEnt = helper.createImageEnt(); + currentEnt.id = "current"; + currentEnt.merge_id = "1"; + currentEnt.cluster = { id: "same", url: null }; + currentEnt.computed_geometry = { lat: 0, lng: 0.0002 }; + currentEnt.geometry = currentEnt.computed_geometry; + const current = new TestImage(currentEnt); + current.makeComplete(currentEnt); + current.initializeCache(new ImageCache(new DataProvider())); + current.cacheCamera(new ProjectionService()); + current.mesh = { + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + faces: [0, 1, 2], + }; + + stateBase.set([previous]); + stateBase.set([current]); + + expect(stateBase.motionlessTransition()).toBe(false); + }); + + it("should be true if camera up vectors diverge", () => { + const state: IStateBase = createState(); + const stateBase: TestStateBase = new TestStateBase(state); + + const image1 = createCompleteImage() as TestImage; + image1.mesh = { + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + faces: [0, 1, 2], + }; + const image2 = createCompleteImage() as TestImage; + image2.mesh = { + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + faces: [0, 1, 2], + }; + + stateBase.set([image1]); + stateBase.set([image2]); + stateBase.previousCamera.up.set(0, 0, 1); + stateBase.currentCamera.up.set(0, 1, 0); + + expect(stateBase.motionlessTransition()).toBe(true); + }); + it("should be true if only previous image has structure", () => { const state: IStateBase = createState(); const stateBase: TestStateBase = new TestStateBase(state); diff --git a/test/state/state/TraversingState.test.ts b/test/state/state/TraversingState.test.ts index 419d34730..8ffae5d6d 100644 --- a/test/state/state/TraversingState.test.ts +++ b/test/state/state/TraversingState.test.ts @@ -40,22 +40,47 @@ class TestTraversingState extends TraversingState { return this._currentCamera; } + public get desiredZoom(): number { + return this._desiredZoom; + } + public get previousCamera(): Camera { return this._previousCamera; } } -function createTestImage(): TestImage { +function createTestImage(id: string = "key", lng: number = 0): TestImage { const image = new TestImage({ - computed_geometry: { lat: 0, lng: 0 }, - id: "key", - geometry: { lat: 0, lng: 0 }, + computed_geometry: { lat: 0, lng }, + id, + geometry: { lat: 0, lng }, sequence: { id: "skey" }, }); image.mesh = { vertices: [], faces: [] }; return image; } +describe("TraversingState.zoomTo", () => { + it("should update desired zoom without changing current zoom", () => { + const state: IStateBase = { + alpha: 1, + camera: new Camera(), + currentIndex: -1, + geometry: new S2GeometryProvider(), + reference: { alt: 0, lat: 0, lng: 0 }, + trajectory: [], + transitionMode: TransitionMode.Default, + zoom: 1, + }; + const traversingState = new TestTraversingState(state); + + traversingState.zoomTo(0.25); + + expect(traversingState.zoom).toBe(1); + expect(traversingState.desiredZoom).toBe(0.25); + }); +}); + describe("TraversingState.currentCamera.lookat", () => { let precision: number = 1e-8; @@ -280,3 +305,80 @@ describe("TraversingState.previousCamera.lookat", () => { expect(traversingState.previousCamera.lookat.z).toBeCloseTo(lookat.z, precision); }); }); + +describe("TraversingState mesh-less transition", () => { + function createCachedImage( + id: string, + lng: number, + spherical: boolean = false): TestImage { + const helper = new ImageHelper(); + const image = createTestImage(id, lng); + const spatialImage = helper.createSpatialImageEnt(); + spatialImage.camera_type = spherical ? "spherical" : "perspective"; + image.makeComplete(spatialImage); + image.initializeCache(new ImageCache(new DataProvider())); + image.cacheCamera(new ProjectionService()); + return image; + } + + function createTraversingState(mode: TransitionMode): TestTraversingState { + return new TestTraversingState({ + alpha: 1, + camera: new Camera(), + currentIndex: -1, + geometry: new S2GeometryProvider(), + reference: { alt: 0, lat: 0, lng: 0 }, + trajectory: [], + transitionMode: mode, + zoom: 0, + }); + } + + it("should cross-fade while snapping the camera", () => { + const traversingState = createTraversingState(TransitionMode.Default); + traversingState.set([createCachedImage("previous", 0)]); + traversingState.set([createCachedImage("current", 0.00001)]); + + traversingState.update(0.1); + + expect(traversingState.motionless).toBe(true); + expect(traversingState.alpha).toBeGreaterThan(0.2); + expect(traversingState.alpha).toBeLessThan(1); + expect(traversingState.camera.position.distanceTo( + traversingState.currentCamera.position)).toBeCloseTo(0); + }); + + it("should preserve instantaneous transition mode", () => { + const traversingState = createTraversingState(TransitionMode.Instantaneous); + traversingState.set([createCachedImage("previous", 0)]); + traversingState.set([createCachedImage("current", 0.00001)]); + + traversingState.update(0.1); + + expect(traversingState.alpha).toBe(1); + }); + + it("should force a reorientation hint onto rejected reconstruction", () => { + const current = createCachedImage("current", 0, true); + current.mesh = { + vertices: [0, 0, 1, 1, 0, 1, 0, 1, 1], + faces: [0, 1, 2], + }; + const unforced = createTraversingState(TransitionMode.Default); + unforced.setReorientation("current", [0.25, 0.5]); + unforced.set([current]); + const originalLookat = unforced.currentCamera.lookat.clone(); + + const forced = createTraversingState(TransitionMode.Default); + forced.setReorientation("current", [0.25, 0.5], true); + forced.set([current]); + + expect(forced.motionless).toBe(true); + expect(forced.currentCamera.lookat.equals(originalLookat)).toBe(false); + const expected = new THREE.Vector3().fromArray( + forced.currentTransform.unprojectBasic( + [0.25, 0.5], (forced as any)._lookatDepth)); + expect(forced.currentCamera.lookat.distanceTo(expected)) + .toBeCloseTo(0, 8); + }); +}); diff --git a/test/util/Mesh.test.ts b/test/util/Mesh.test.ts new file mode 100644 index 000000000..2cfc0b3f8 --- /dev/null +++ b/test/util/Mesh.test.ts @@ -0,0 +1,53 @@ +import { + hasReconstructionMesh, + isDefaultCubeMesh, +} from "../../src/util/Mesh"; + +const cubeVertices: number[] = [ + -2.8867513459481287, -2.8867513459481287, -2.8867513459481287, + -2.8867513459481287, -2.8867513459481287, 2.8867513459481287, + -2.8867513459481287, 2.8867513459481287, -2.8867513459481287, + -2.8867513459481287, 2.8867513459481287, 2.8867513459481287, + 2.8867513459481287, -2.8867513459481287, -2.8867513459481287, + 2.8867513459481287, -2.8867513459481287, 2.8867513459481287, + 2.8867513459481287, 2.8867513459481287, -2.8867513459481287, + 2.8867513459481287, 2.8867513459481287, 2.8867513459481287, +]; + +const cubeFaces: number[] = [ + 6, 2, 0, 6, 4, 0, 5, 4, 0, 5, 1, 0, + 5, 6, 4, 5, 6, 7, 3, 2, 0, 3, 1, 0, + 3, 6, 2, 3, 6, 7, 3, 5, 1, 3, 5, 7, +]; + +describe("hasReconstructionMesh", () => { + it("requires non-placeholder geometry", () => { + expect(hasReconstructionMesh({ faces: [], vertices: [] })).toBe(false); + expect(hasReconstructionMesh({ + faces: cubeFaces, + vertices: cubeVertices, + })).toBe(false); + expect(hasReconstructionMesh({ + faces: [0, 1, 2], + vertices: [0, 0, 0, 1, 1, 1, 2, 2, 2], + })).toBe(true); + }); +}); + +describe("isDefaultCubeMesh", () => { + it("identifies the default cube", () => { + expect(isDefaultCubeMesh({ faces: cubeFaces, vertices: cubeVertices })) + .toBe(true); + }); + + it("keeps reconstructed meshes", () => { + const vertices = cubeVertices.slice(); + vertices[0] = -3.5; + + expect(isDefaultCubeMesh({ faces: cubeFaces, vertices })).toBe(false); + }); + + it("keeps empty meshes", () => { + expect(isDefaultCubeMesh({ faces: [], vertices: [] })).toBe(false); + }); +}); diff --git a/test/viewer/PlayService.test.ts b/test/viewer/PlayService.test.ts index 14f381e4f..941f579b3 100644 --- a/test/viewer/PlayService.test.ts +++ b/test/viewer/PlayService.test.ts @@ -242,6 +242,7 @@ describe("PlayService.play", () => { const setGraphModeSpy: jasmine.Spy = spyOn(graphService, "setGraphMode").and.stub(); + playService.setDirection(NavigationDirection.TurnLeft); playService.setSpeed(0); playService.play(); @@ -255,6 +256,21 @@ describe("PlayService.play", () => { expect(setGraphModeSpy.calls.argsFor(2)[0]).toBe(GraphMode.Spatial); }); + it("should keep sequence graph mode for sequence playback", () => { + const playService: PlayService = new PlayService(graphService, stateService); + + const setGraphModeSpy: jasmine.Spy = spyOn(graphService, "setGraphMode").and.stub(); + + playService.setDirection(NavigationDirection.Next); + playService.setSpeed(0); + playService.play(); + playService.setSpeed(1); + playService.setSpeed(0); + + expect(setGraphModeSpy.calls.count()).toBe(1); + expect(setGraphModeSpy.calls.argsFor(0)[0]).toBe(GraphMode.Sequence); + }); + it("should stop immediately if image does not have an edge in current direction and no bridge", () => { const playService: PlayService = new PlayService(graphService, stateService); @@ -520,10 +536,9 @@ describe("PlayService.play", () => { expect(stopSpy.calls.count()).toBe(1); }); - it("should cache sequence when in spatial graph mode", () => { + it("should cache sequence metadata at low speed", () => { const playService: PlayService = new PlayService(graphService, stateService); playService.setDirection(NavigationDirection.Next); - // Set speed to zero so that graph mode is set to spatial when calling play playService.setSpeed(0); const cacheSequenceSpy: jasmine.Spy = spyOn(graphService, "cacheSequence$"); @@ -547,7 +562,7 @@ describe("PlayService.play", () => { playService.stop(); }); - it("should cache sequence images when in sequence graph mode", () => { + it("should not eagerly cache all sequence images", () => { const playService: PlayService = new PlayService(graphService, stateService); playService.setDirection(NavigationDirection.Next); // Set speed to one so that graph mode is set to sequence when calling play @@ -566,10 +581,10 @@ describe("PlayService.play", () => { const currentImageSubject: Subject = >stateService.currentImage$; currentImageSubject.next(currentImage); - expect(cacheSequenceSpy.calls.count()).toBe(0); + expect(cacheSequenceSpy.calls.count()).toBe(1); + expect(cacheSequenceSpy.calls.argsFor(0)[0]).toBe(currentImage.sequenceId); - expect(cacheSequenceImagesSpy.calls.count()).toBe(1); - expect(cacheSequenceImagesSpy.calls.argsFor(0)[0]).toBe(currentImage.sequenceId); + expect(cacheSequenceImagesSpy.calls.count()).toBe(0); playService.stop(); });