diff --git a/CHANGELOG.md b/CHANGELOG.md index 6840189..71273ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- In a dfs file opened for editing, or being created with `DfsBuilder`, + assigning an item's `Name` or `Quantity`, or changing an attribute of its + `SpatialAxis` (such as `X0`), writes the change to the file. For a static + item, `WriteStaticItemData` writes it. On a dfs1, dfs2 or dfs3 file, + changing an attribute of the file's `SpatialAxis` updates every dynamic + item on `Flush` or `Close`, and the static items with the same type of + axis on `Close`. As in the C# library, a new name is truncated or padded + to the length of the old one, and changing `Quantity.Item` or + `Quantity.Unit` in place is not written. `SpatialAxis` of a dfs1, dfs2 or + dfs3 file is now a copy of the first item's axis, as in the C# library, so + changing it in a file opened for reading no longer changes that item's + axis, and it can not be replaced, only changed. When a dynamic item has + another type of axis than the file, `Flush` and `Close` raise `ValueError` + without writing the file axis to any item. A name containing a NUL + character raises `ValueError` + ([#38](https://github.com/DHI/mikecore-python/issues/38)). +- A spatial axis value that is finite but too large for the 32 bit float + the file stores, such as `Dx = 1e300`, raises `ValueError` when the axis is + written, by `DfsBuilder` or by editing a file, instead of being written as + `inf`. +- `DfsuFile.Flush()` flushes the file instead of raising `TypeError`, and + `DfsFileFactory.DfsuFileOpenAppend` no longer requires `parameters`, like the + other `Open` methods. - Items read from a dfs file have the reference coordinates and orientation stored in the file. They had the `-1e-35` defaults, whatever the file held ([#31](https://github.com/DHI/mikecore-python/issues/31)). diff --git a/mikecore/Dfs123File.py b/mikecore/Dfs123File.py index dde5676..bbb70c0 100644 --- a/mikecore/Dfs123File.py +++ b/mikecore/Dfs123File.py @@ -1,18 +1,114 @@ -from mikecore.DfsFile import DfsFile, DfsFileMode, DfsItemData +import copy + +from mikecore.DfsFile import ( + DfsDLLUtil, + DfsFile, + DfsFileMode, + DfsItemData, + _WeakCallback, +) class Dfs123File(DfsFile): def __init__(self): super().__init__() - self.SpatialAxis = None + self._SpatialAxis = None self.reshape = False + self._spatialAxisUpdated = False def Open(self, filename, mode=DfsFileMode.Read, parameters=None, reshape=False): super().Open(filename, mode, parameters) self.reshape = reshape + self._spatialAxisUpdated = False # TODO: Need to check that this is the same for all items if len(self.ItemInfo) > 0: - self.SpatialAxis = self.ItemInfo[0].SpatialAxis + self._SpatialAxis = copy.copy(self.ItemInfo[0].SpatialAxis) + if self._SpatialAxis is not None and mode in ( + DfsFileMode.Edit, + DfsFileMode.Append, + ): + self._SpatialAxis._OnUpdate = _WeakCallback( + Dfs123File.__SpatialAxisUpdated, self + ) + + @property + def SpatialAxis(self): + """ + The axis of the file, a copy of the first item's axis. As in the C# + library it can not be replaced: change its attributes instead. + """ + return self._SpatialAxis + + def __SpatialAxisUpdated(self): + self._spatialAxisUpdated = True + + def __WriteSpatialAxis(self, staticItems): + """ + Write a changed SpatialAxis to all dynamic items, and with + staticItems to the static items with the same type of axis, as the + C# library does. Writing static items moves the file pointer. + """ + fileAxis = self.SpatialAxis + if not self._spatialAxisUpdated or fileAxis is None: + return + # Write to the item's axis in the file, also when the item was given + # another SpatialAxis object, which writes nothing itself. Check them + # all first, so that a file with another type of item axis is not + # left with only some items written. + itemAxes = [ + DfsDLLUtil.GetItemSpatialAxis(itemInfo.ItemPointer) + for itemInfo in self.ItemInfo + ] + for itemInfo, itemAxis in zip(self.ItemInfo, itemAxes, strict=True): + if itemAxis.AxisType != fileAxis.AxisType: + raise ValueError( + f"The axis of item {itemInfo.ItemNumber} is " + f"{itemAxis.AxisType.name}, not {fileAxis.AxisType.name} " + "like the axis of the file" + ) + for itemInfo, itemAxis in zip(self.ItemInfo, itemAxes, strict=True): + itemAxis._Update(fileAxis) + DfsDLLUtil.dfsSetItemSpatialAxis(itemInfo.ItemPointer, itemAxis) + axis = itemInfo.SpatialAxis + if axis is not None and axis.AxisType == fileAxis.AxisType: + axis._Update(fileAxis) + if not staticItems: + return + self.Reset() + staticItemNo = 1 + while (staticItem := self.ReadStaticItem(staticItemNo)) is not None: + staticItemNo += 1 + axis = staticItem.SpatialAxis + data = staticItem.Data + if axis is None or data is None or axis.AxisType != fileAxis.AxisType: + continue + axis._Update(fileAxis) + self.WriteStaticItemData(staticItem, data) + + def Flush(self): + """ + Flush cached data to the file. A changed SpatialAxis is written to the + dynamic items here, and to the static items only on Close: writing a + static item moves the file pointer, so a time step appended after + Flush would overwrite another. The C# library writes both on Flush. + A value the file axis cannot hold, or a dynamic item with another type + of axis, is only rejected here or on Close, before any item is written. + The whole file axis is written to each item, replacing changes made + to an item's own axis, as in the C# library. + """ + if self.filePointer.value is not None: + self.__WriteSpatialAxis(staticItems=False) + super().Flush() + + def Close(self): + try: + if self.filePointer.value is not None: + self.__WriteSpatialAxis(staticItems=True) + finally: + self._spatialAxisUpdated = False + if self._SpatialAxis is not None: + self._SpatialAxis._OnUpdate = None + super().Close() def Reshape(self, reshape): # Set True to reshape item data to multi-dimensional arrays. diff --git a/mikecore/DfsDLL.py b/mikecore/DfsDLL.py index a6f88fe..9a5d2d1 100644 --- a/mikecore/DfsDLL.py +++ b/mikecore/DfsDLL.py @@ -210,6 +210,14 @@ def Init(libfilepath: str | None = None): ctypes.c_void_p, ctypes.c_void_p, ] + lib.dfsUpdateItemInfo.argtypes = [ + ctypes.c_void_p, + ctypes.c_void_p, + ctypes.c_int32, + ctypes.c_char_p, + ctypes.c_int32, + ] + lib.dfsUpdateItemInfo.restype = ctypes.c_int32 lib.dfsAddCustomBlock.argtypes = [ ctypes.c_void_p, diff --git a/mikecore/DfsFile.py b/mikecore/DfsFile.py index 1e987e3..2d1f272 100644 --- a/mikecore/DfsFile.py +++ b/mikecore/DfsFile.py @@ -3,6 +3,8 @@ import datetime import ctypes import locale +import math +import weakref import numpy as np from mikecore.eum import * from mikecore.DfsDLL import DfsDLL @@ -385,14 +387,77 @@ def IncrementNumberOfTimeSteps(self, time): self.TimeSpan = time - self.StartTimeOffset +# Marks an attribute that was not set +_Missing = object() + + +def _WeakCallback(function, *args): + """ + A callback that calls function(*args) while all args are alive. It keeps + no reference to them, so an item or axis holding it in an attribute does + not form a reference cycle, and is freed as soon as it is unused. + """ + refs = [weakref.ref(arg) for arg in args] + + def callback(): + values = [ref() for ref in refs] + if all(value is not None for value in values): + function(*values) + + return callback + + class DfsSpatialAxis: + # Attributes that may change on an axis read from a file. Setting one of + # them invokes _OnUpdate, which writes the axis to an editable file. + _UpdatableAttributes = frozenset({"AxisUnit", "X0", "Dx", "Y0", "Dy", "Z0", "Dz"}) + def __init__(self, axisType, shape, sizeOfData, axisUnit): + self._OnUpdate = None self.AxisType = axisType self.Shape = shape self.Dimension = len(shape) self.SizeOfData = sizeOfData self.AxisUnit = axisUnit + def __getstate__(self): + # A copy is not in the file: it must not write to it + state = self.__dict__.copy() + state["_OnUpdate"] = None + return state + + def __setattr__(self, name, value): + onUpdate = self.__dict__.get("_OnUpdate") + if name not in self._UpdatableAttributes or onUpdate is None: + super().__setattr__(name, value) + return + oldValue = self.__dict__.get(name, _Missing) + super().__setattr__(name, value) + try: + onUpdate() + except BaseException: + if oldValue is _Missing: + super().__delattr__(name) + else: + super().__setattr__(name, oldValue) + raise + + def _Update(self, source): + """Copy the updatable attributes of source, an axis of the same type.""" + if self.AxisType != source.AxisType: + raise ValueError("Destination and source axis have different types") + # Invoke _OnUpdate once, not for every attribute + onUpdate = self._OnUpdate + self._OnUpdate = None + try: + for name in self._UpdatableAttributes: + if hasattr(source, name): + setattr(self, name, getattr(source, name)) + finally: + self._OnUpdate = onUpdate + if onUpdate is not None: + onUpdate() + class DfsAxisEqD0(DfsSpatialAxis): def __init__(self, axisUnit=eumUnit.eumUmeter): @@ -508,12 +573,17 @@ def __init__(self, axisUnit, xCount, yCount, zCount, xCoords, yCoords, zCoords): class DfsDynamicItemInfo: def __init__(self, itemPointer=None, itemNumber=0): + # Set on items of an editable file, to write Name and Quantity + self._OnUpdate = None + # The name read from the file and its bytes, which are written back + # while Name is unchanged, as cp1252 may not encode the decoded name + self._NameInFile = None self.ItemPointer = itemPointer self.ItemNumber = itemNumber self.DataType = DfsSimpleType.UShort - self.Name = "" + self._Name = "" self.ElementCount = -1 - self.Quantity = None + self._Quantity = None self.ReferenceCoordinateX = np.float32(-1e-35) self.ReferenceCoordinateY = np.float32(-1e-35) self.ReferenceCoordinateZ = np.float32(-1e-35) @@ -541,6 +611,50 @@ def __repr__(self): + ")" ) + @property + def Name(self): + return self._Name + + @Name.setter + def Name(self, name): + if self._OnUpdate is not None: + # As in the C# library, the name in the file keeps its length: the + # new name is truncated or padded with spaces to the old length. + name = name[: len(self._Name)].ljust(len(self._Name)) + oldName = self._Name + self._Name = name + try: + self._InvokeOnUpdate() + except BaseException: + self._Name = oldName + raise + + @property + def Quantity(self): + return self._Quantity + + @Quantity.setter + def Quantity(self, quantity): + # Only assigning a quantity updates the file, as in the C# library. + # Changing Item or Unit of the current quantity does not. + oldQuantity = self._Quantity + self._Quantity = quantity + try: + self._InvokeOnUpdate() + except BaseException: + self._Quantity = oldQuantity + raise + + def __getstate__(self): + # A copy is not in the file: it must not write to it + state = self.__dict__.copy() + state["_OnUpdate"] = None + return state + + def _InvokeOnUpdate(self): + if self._OnUpdate is not None: + self._OnUpdate() + def init(self, itemName, eumQuantity): self.Name = itemName self.Quantity = eumQuantity @@ -779,6 +893,10 @@ def __init__(self): self.fpTimeStepIndex = 0 self.headPointer = ctypes.c_void_p(0) self.filePointer = ctypes.c_void_p(0) + # Items and axes that write their changes to the file, until it is + # closed. Weak, so that the file does not keep the static items it + # read alive. + self._updatableItems = weakref.WeakSet() def __del__(self): self.Close() @@ -901,6 +1019,10 @@ def Close(self): ) if self.headPointer.value is not None: DfsDLL.Wrapper.dfsHeaderDestroy(ctypes.byref(self.headPointer)) + # The item pointers point into the destroyed header: stop writing to it + for itemOrAxis in list(self._updatableItems): + itemOrAxis._OnUpdate = None + self._updatableItems.clear() def GetNextItemNumber(self): return self.fpItemNumber @@ -1470,8 +1592,44 @@ def __DynamicItemInfoReadAndCreate(self, itemNumber, noOfItems): ) item = DfsDynamicItemInfo(itemPointer, itemNumber) self.__GetItemInfo(item) + self.__AttachItemUpdate(item) return item + def __AttachItemUpdate(self, item): + # In an editable file, changing the name, quantity or axis of an item + # updates its item info: in the header for a dynamic item, and in the + # static vector, written by WriteStaticItemData, for a static item + if self.FileMode not in (DfsFileMode.Edit, DfsFileMode.Append): + return + self._updatableItems.add(item) + item._OnUpdate = _WeakCallback(DfsFile.__UpdateItemInfo, self, item) + axis = item.SpatialAxis + if axis is not None: + self._updatableItems.add(axis) + axis._OnUpdate = _WeakCallback(DfsFile.__UpdateItemAxis, self, item, axis) + + def __UpdateItemAxis(self, item, axis): + # As in the C# library, an axis writes only while it is the item's + # axis: one replaced by assigning SpatialAxis writes nothing + if item.SpatialAxis is axis: + DfsDLLUtil.dfsSetItemSpatialAxis(item.ItemPointer, axis) + + def __UpdateItemInfo(self, item): + nameInFile, nameBytes = item._NameInFile + if item.Name != nameInFile: + # The native library reads the name up to the first NUL + if "\0" in item.Name: + raise ValueError(f"Item name contains a NUL character: {item.Name!r}") + nameBytes = item.Name.encode("cp1252") + rok = DfsDLL.Wrapper.dfsUpdateItemInfo( + self.headPointer, + item.ItemPointer, + ctypes.c_int32(item.Quantity.Item.value), + nameBytes, + ctypes.c_int32(item.Quantity.Unit.value), + ) + DfsDLL.CheckReturnCode(rok) + def __StaticItemReadAndCreate(self, number, ubgConversion): # Check if we can read it. staticVectorPointer contains both item info and data @@ -1503,6 +1661,7 @@ def __StaticItemReadAndCreate(self, number, ubgConversion): self.__GetItemInfo(staticItem) self.__GetStaticData(staticItem) + self.__AttachItemUpdate(staticItem) return staticItem @@ -1534,6 +1693,7 @@ def __GetItemInfo(self, item): quantity.UnitDescription = eumUnitDesc item.init(itemName, quantity) + item._NameInFile = (itemName, itemNameP.value) item.DataType = itemDataType item.ValueType = DfsDLLUtil.dfsGetItemValueType(item.ItemPointer) item.ElementCount = DfsDLL.Wrapper.dfsGetItemElements(item.ItemPointer) @@ -1908,6 +2068,18 @@ def GetItemSpatialAxis(itemPointer): return None + @staticmethod + def AxisFloat(axis, name): + """ + The axis attribute name as a C float. ctypes would turn a finite value + too large for a float into inf. + """ + value = getattr(axis, name) + result = ctypes.c_float(value) + if math.isinf(result.value) and math.isfinite(value): + raise ValueError(f"Axis {name} is out of the range of a float: {value}") + return result + @staticmethod def dfsSetItemSpatialAxis(itemPointer, spatialAxis): axis = spatialAxis @@ -1924,8 +2096,8 @@ def dfsSetItemSpatialAxis(itemPointer, spatialAxis): itemPointer, ctypes.c_int32(axis.AxisUnit.value), ctypes.c_int32(axis.XCount), - ctypes.c_float(axis.X0), - ctypes.c_float(axis.Dx), + DfsDLLUtil.AxisFloat(axis, "X0"), + DfsDLLUtil.AxisFloat(axis, "Dx"), ) DfsDLL.CheckReturnCode(rok) return @@ -1938,10 +2110,10 @@ def dfsSetItemSpatialAxis(itemPointer, spatialAxis): ctypes.c_int32(axis.AxisUnit.value), ctypes.c_int32(axis.XCount), ctypes.c_int32(axis.YCount), - ctypes.c_float(axis.X0), - ctypes.c_float(axis.Y0), - ctypes.c_float(axis.Dx), - ctypes.c_float(axis.Dy), + DfsDLLUtil.AxisFloat(axis, "X0"), + DfsDLLUtil.AxisFloat(axis, "Y0"), + DfsDLLUtil.AxisFloat(axis, "Dx"), + DfsDLLUtil.AxisFloat(axis, "Dy"), ) DfsDLL.CheckReturnCode(rok) return @@ -1957,12 +2129,12 @@ def dfsSetItemSpatialAxis(itemPointer, spatialAxis): ctypes.c_int32(axis.XCount), ctypes.c_int32(axis.YCount), axis.ZCount, - ctypes.c_float(axis.X0), - ctypes.c_float(axis.Y0), - ctypes.c_float(axis.Z0), - ctypes.c_float(axis.Dx), - ctypes.c_float(axis.Dy), - ctypes.c_float(axis.Dz), + DfsDLLUtil.AxisFloat(axis, "X0"), + DfsDLLUtil.AxisFloat(axis, "Y0"), + DfsDLLUtil.AxisFloat(axis, "Z0"), + DfsDLLUtil.AxisFloat(axis, "Dx"), + DfsDLLUtil.AxisFloat(axis, "Dy"), + DfsDLLUtil.AxisFloat(axis, "Dz"), ) DfsDLL.CheckReturnCode(rok) # elif spatialAxis.AxisType is SpaceAxisType.NeqD3: diff --git a/mikecore/DfsFileFactory.py b/mikecore/DfsFileFactory.py index b7a132d..302b136 100644 --- a/mikecore/DfsFileFactory.py +++ b/mikecore/DfsFileFactory.py @@ -71,7 +71,7 @@ def DfsuFileOpenEdit(filename, parameters=None): return DfsuFile(dfsFile) @staticmethod - def DfsuFileOpenAppend(filename, parameters): + def DfsuFileOpenAppend(filename, parameters=None): dfsFile = DfsFileFactory.DfsGenericOpenAppend(filename, parameters) return DfsuFile(dfsFile) diff --git a/mikecore/DfsuFile.py b/mikecore/DfsuFile.py index 9e8883d..9811f13 100644 --- a/mikecore/DfsuFile.py +++ b/mikecore/DfsuFile.py @@ -309,7 +309,7 @@ def Reset(self): self.dfsFile.Reset() def Flush(self): - self.dfsFile.Flush(self) + self.dfsFile.Flush() def FlushTimeStep(self): self.dfsFile.FlushTimeStep() diff --git a/tests/test_dfs2.py b/tests/test_dfs2.py index 7fa1706..d6a8529 100644 --- a/tests/test_dfs2.py +++ b/tests/test_dfs2.py @@ -213,69 +213,86 @@ def test_CreateOresundBathy900Test(self): FileOresundBathy900Dfs2.DynamicItemTester(dfsFile.ItemInfo) FileOresundBathy900Dfs2.ReadTester(dfsFile) - # def ModifyAxisTest(): - # originalFilename = "testdata/OresundHD.dfs2"; - # filename = "testdata/testtmp/test_modifyAxis_OresundHD.dfs2"; - # - # testUtil.copy_file(originalFilename, filename) - # - # ExamplesDfs2.ModifyDfs2ItemAxis(filename); - # - # # Check all dynamic items - # dfs2File = DfsFileFactory.DfsGenericOpen(filename); - # for itemInfo in dfs2File.ItemInfo: - # axisEqD2 = itemInfo.SpatialAxis; - # assert_equal( 55, axisEqD2.X0); - # assert_equal(905, axisEqD2.Dx); - # assert_equal(-55, axisEqD2.Y0); - # assert_equal(915, axisEqD2.Dy); - # - # # Check all static items - # while (True): - # staticItem = dfs2File.ReadStaticItemNext() - # if (staticItem is None): - # break; - # axisEqD2 = staticItem.SpatialAxis; - # assert_equal( 55, axisEqD2.X0); - # assert_equal(905, axisEqD2.Dx); - # assert_equal(-55, axisEqD2.Y0); - # assert_equal(915, axisEqD2.Dy); - # - # dfs2File.Close(); + def test_ModifyAxisTest(self): + originalFilename = "testdata/OresundHD.dfs2" + filename = "testdata/testtmp/test_modifyAxis_OresundHD.dfs2" - # def ModifyLanduseItemInfoTest(): - # - # originalFilename = "testdata/Landuse.dfs2"; - # filename = "testdata/testtmp/test_create_Landuse.dfs2"; - # - # File.Copy(originalFilename, filename, true); - # File.SetAttributes(filename, FileAttributes.Normal); # remove read-only flag - # - # # Check the new file - # - # file = DfsFileFactory.Dfs2FileOpen(filename); - # - # # Check existing data - # itemInfo = file.ItemInfo[0]; - # assert_equal("Landuse", itemInfo.Name); - # assert_equal(eumItem.eumIIntegerCode, itemInfo.Quantity.Item); - # assert_equal(eumUnit.eumUintCode, itemInfo.Quantity.Unit); - # - # file.Close(); - # - # # Method that modifies the Landuse.dfs2 file - # ExamplesDfs2.ModifyDfs2ItemInfo(filename); - # - # # Check the new file - # file = DfsFileFactory.Dfs2FileOpen(filename); - # - # # Check existing data - # itemInfo = file.ItemInfo[0]; - # assert_equal("GroundU", itemInfo.Name); - # assert_equal(eumItem.eumIAreaFraction, itemInfo.Quantity.Item); - # assert_equal(eumUnit.eumUPerCent, itemInfo.Quantity.Unit); - # - # file.Close(); + testUtil.copy_file(originalFilename, filename) + + # Check existing data + source = DfsFileFactory.DfsGenericOpen(filename) + for itemInfo in source.ItemInfo: + axisEqD2 = itemInfo.SpatialAxis + assert_equal(0, axisEqD2.X0) + assert_equal(900, axisEqD2.Dx) + assert_equal(0, axisEqD2.Y0) + assert_equal(900, axisEqD2.Dy) + sourceStaticItem = source.ReadStaticItemNext() + assert sourceStaticItem is not None + axisEqD2 = sourceStaticItem.SpatialAxis + assert axisEqD2 is not None + assert_equal(0, axisEqD2.X0) + assert_equal(900, axisEqD2.Dx) + assert_equal(0, axisEqD2.Y0) + assert_equal(900, axisEqD2.Dy) + sourceStaticData = sourceStaticItem.Data + assert source.ReadStaticItemNext() is None + source.Close() + + ExamplesDfs2.ModifyDfs2ItemAxis(filename) + + # Check all dynamic items + dfs2File = DfsFileFactory.DfsGenericOpen(filename) + for itemInfo in dfs2File.ItemInfo: + axisEqD2 = itemInfo.SpatialAxis + assert_equal(55, axisEqD2.X0) + assert_equal(905, axisEqD2.Dx) + assert_equal(-55, axisEqD2.Y0) + assert_equal(915, axisEqD2.Dy) + + # Check all static items + staticItemCount = 0 + while True: + staticItem = dfs2File.ReadStaticItemNext() + if staticItem is None: + break + staticItemCount += 1 + axisEqD2 = staticItem.SpatialAxis + assert axisEqD2 is not None + assert_equal(55, axisEqD2.X0) + assert_equal(905, axisEqD2.Dx) + assert_equal(-55, axisEqD2.Y0) + assert_equal(915, axisEqD2.Dy) + # Writing the axis keeps the static item data + assert_equal(sourceStaticData, staticItem.Data) + assert_equal(1, staticItemCount) + + dfs2File.Close() + + def test_ModifyLanduseItemInfoTest(self): + originalFilename = "testdata/Landuse.dfs2" + filename = "testdata/testtmp/test_modifyItemInfo_Landuse.dfs2" + + testUtil.copy_file(originalFilename, filename) + + # Check existing data + file = DfsFileFactory.Dfs2FileOpen(filename) + itemInfo = file.ItemInfo[0] + assert_equal("Landuse", itemInfo.Name) + assert_equal(eumItem.eumIIntegerCode, itemInfo.Quantity.Item) + assert_equal(eumUnit.eumUintCode, itemInfo.Quantity.Unit) + file.Close() + + # Method that modifies the Landuse.dfs2 file + ExamplesDfs2.ModifyDfs2ItemInfo(filename) + + # Check the new file + file = DfsFileFactory.Dfs2FileOpen(filename) + itemInfo = file.ItemInfo[0] + assert_equal("GroundU", itemInfo.Name) + assert_equal(eumItem.eumIAreaFraction, itemInfo.Quantity.Item) + assert_equal(eumUnit.eumUPerCent, itemInfo.Quantity.Unit) + file.Close() def test_ReadReshapedPastEndReturnsNone(self): dfsFile = DfsFileFactory.Dfs2FileOpen("testdata/OresundHD.dfs2") diff --git a/tests/test_dfs_edit.py b/tests/test_dfs_edit.py new file mode 100644 index 0000000..6858669 --- /dev/null +++ b/tests/test_dfs_edit.py @@ -0,0 +1,953 @@ +""" +Editing the item info and spatial axes of an existing dfs file, on every dfs +file type and every way of opening it for editing. +""" + +import copy +import gc +import os +import shutil +import weakref +from dataclasses import dataclass +from collections.abc import Callable +from typing import Any + +import numpy as np +import pytest + +from mikecore.DfsBuilder import DataValueType, DfsBuilder, DfsSimpleType +from mikecore.DfsFactory import DfsFactory +from mikecore.DfsFile import DfsAxisEqD0, SpaceAxisType +from mikecore.DfsFileFactory import DfsFileFactory +from mikecore.eum import eumItem, eumQuantity, eumUnit + + +@dataclass +class DfsType: + source: str + open: Callable[[str], Any] + openEdit: Callable[[str], Any] + openAppend: Callable[[str], Any] + # Encoded files store their data compressed, so the dynamic values are not + # the raw bytes at the end of the file + encoded: bool = False + + +def createDfs1(filename, beforeData=None): + # There is no dfs1 file in testdata, so the tests create one. beforeData + # is called with the file being built, before any data is written. + factory = DfsFactory() + builder = DfsBuilder.Create("title", "application", 1) + builder.SetDataType(0) + builder.SetGeographicalProjection(factory.CreateProjectionUndefined()) + builder.SetTemporalAxis(factory.CreateTemporalEqTimeAxis(eumUnit.eumUsec, 0, 60)) + builder.SetSpatialAxis(factory.CreateAxisEqD1(eumUnit.eumUmeter, 5, 0, 10)) + for name in ("Level", "Depth"): + item = builder.CreateDynamicItemBuilder() + item.Set( + name, + eumQuantity(eumItem.eumIWaterLevel, eumUnit.eumUmeter), + DfsSimpleType.Float, + ) + item.SetValueType(DataValueType.Instantaneous) + item.SetAxis(factory.CreateAxisEqD1(eumUnit.eumUmeter, 5, 0, 10)) + builder.AddDynamicItem(item.GetDynamicItemInfo()) + builder.CreateFile(filename) + builder.AddCreateStaticItem( + "Bathymetry", + eumQuantity(eumItem.eumIBathymetry, eumUnit.eumUmeter), + np.arange(5, dtype=np.float32), + factory.CreateAxisEqD1(eumUnit.eumUmeter, 5, 0, 10), + ) + file = builder.GetFile() + if beforeData is not None: + beforeData(file) + for timeStep in range(3): + for itemNumber in (1, 2): + data = np.arange(5, dtype=np.float32) + 10 * timeStep + itemNumber + file.WriteItemTimeStepNext(60.0 * timeStep, data) + file.Close() + + +def createDfs2(filename, dynamicAxes, staticAxes): + """ + A dfs2 file with a dynamic item for each of dynamicAxes and a static item + for each of staticAxes, each item with that axis, and 2 time steps. + """ + factory = DfsFactory() + builder = DfsBuilder.Create("title", "application", 1) + builder.SetDataType(0) + builder.SetGeographicalProjection(factory.CreateProjectionUndefined()) + builder.SetTemporalAxis(factory.CreateTemporalEqTimeAxis(eumUnit.eumUsec, 0, 60)) + builder.SetSpatialAxis(dynamicAxes[0]) + for itemNo, axis in enumerate(dynamicAxes): + item = builder.CreateDynamicItemBuilder() + item.Set( + f"Dynamic {itemNo + 1}", + eumQuantity(eumItem.eumIWaterLevel, eumUnit.eumUmeter), + DfsSimpleType.Float, + ) + item.SetValueType(DataValueType.Instantaneous) + item.SetAxis(axis) + builder.AddDynamicItem(item.GetDynamicItemInfo()) + builder.CreateFile(filename) + for itemNo, axis in enumerate(staticAxes): + builder.AddCreateStaticItem( + f"Static {itemNo + 1}", + eumQuantity(eumItem.eumIBathymetry, eumUnit.eumUmeter), + np.arange(np.prod(axis.Shape), dtype=np.float32) + 100 * itemNo, + axis, + ) + file = builder.GetFile() + for timeStep in range(2): + for axis in dynamicAxes: + # The Shape of an EqD0 axis is (0,), but it holds one value + size = max(1, int(np.prod(axis.Shape))) + data = np.arange(size, dtype=np.float32) + timeStep + file.WriteItemTimeStepNext(60.0 * timeStep, data) + file.Close() + + +F = DfsFileFactory +DFS_TYPES = { + "dfs0": DfsType( + "testdata/TemporalEqTime.dfs0", + F.DfsGenericOpen, + F.DfsGenericOpenEdit, + F.DfsGenericOpenAppend, + ), + "dfs1": DfsType("", F.Dfs1FileOpen, F.Dfs1FileOpenEdit, F.Dfs1FileOpenAppend), + "dfs2": DfsType( + "testdata/OresundHD.dfs2", + F.Dfs2FileOpen, + F.Dfs2FileOpenEdit, + F.Dfs2FileOpenAppend, + ), + "dfs3": DfsType( + "testdata/OresundHD.dfs3", + F.Dfs3FileOpen, + F.Dfs3FileOpenEdit, + F.Dfs3FileOpenAppend, + encoded=True, + ), + "dfsu": DfsType( + "testdata/OresundHD.dfsu", + F.DfsuFileOpen, + F.DfsuFileOpenEdit, + F.DfsuFileOpenAppend, + ), +} +FILE_AXIS_TYPES = ["dfs1", "dfs2", "dfs3"] + + +@dataclass +class Case: + filename: str + dfsType: DfsType + + +@pytest.fixture(params=list(DFS_TYPES)) +def case(request, tmp_path): + return makeCase(request.param, tmp_path) + + +@pytest.fixture(params=FILE_AXIS_TYPES) +def fileAxisCase(request, tmp_path): + return makeCase(request.param, tmp_path) + + +def makeCase(typeName, tmp_path): + dfsType = DFS_TYPES[typeName] + filename = str(tmp_path / ("test." + typeName)) + if dfsType.source: + shutil.copyfile(dfsType.source, filename) + os.chmod(filename, 0o644) + else: + createDfs1(filename) + return Case(filename, dfsType) + + +# Every way to open a file for editing: Edit and Append mode through the +# factory method of the file type, and Edit mode as a generic dfs file +OPENERS = { + "edit": lambda case: case.dfsType.openEdit(case.filename), + "append": lambda case: case.dfsType.openAppend(case.filename), + "generic": lambda case: DfsFileFactory.DfsGenericOpenEdit(case.filename), +} + +AXIS_ATTRIBUTES = ("AxisUnit", "X0", "Dx", "Y0", "Dy", "Z0", "Dz") + + +def axisValues(axis): + return { + name: getattr(axis, name) for name in AXIS_ATTRIBUTES if hasattr(axis, name) + } + + +def itemValues(itemInfo): + return ( + itemInfo.Name, + itemInfo.Quantity.Item, + itemInfo.Quantity.Unit, + axisValues(itemInfo.SpatialAxis), + ) + + +def readItems(filename): + file = DfsFileFactory.DfsGenericOpen(filename) + items = [itemValues(itemInfo) for itemInfo in file.ItemInfo] + file.Close() + return items + + +def readStaticItems(filename): + """The item values, axis type and data of each static item.""" + file = DfsFileFactory.DfsGenericOpen(filename) + staticItems = [] + while (staticItem := file.ReadStaticItemNext()) is not None: + axis = staticItem.SpatialAxis + assert axis is not None + staticItems.append((itemValues(staticItem), axis.AxisType, staticItem.Data)) + file.Close() + return staticItems + + +def readDynamicData(filename): + file = DfsFileFactory.DfsGenericOpen(filename) + data = [] + while (itemData := file.ReadItemTimeStepNext()) is not None: + data.append((itemData.Time, itemData.Data.copy())) + file.Close() + return data + + +def dynamicDataBytes(filename): + """Size of the raw values of all item time steps, from the header.""" + file = DfsFileFactory.DfsGenericOpen(filename) + timeAxis = file.FileInfo.TimeAxis + assert timeAxis is not None + timeSteps = timeAxis.NumberOfTimeSteps + valueBytes = {DfsSimpleType.Float: 4, DfsSimpleType.Double: 8} + itemBytes = [ + itemInfo.ElementCount * valueBytes[itemInfo.DataType] + for itemInfo in file.ItemInfo + ] + file.Close() + return timeSteps * sum(itemBytes) + + +def readBytes(filename): + with open(filename, "rb") as f: + return f.read() + + +def assertOnlyHeaderChanged(case, before): + """ + The edit changed only bytes in front of the data: the file keeps its size, + and every dynamic value, read raw from the end of the file and through the + reader, is unchanged. + """ + after = readBytes(case.filename) + assert len(before) == len(after) + if not case.dfsType.encoded: + dataBytes = dynamicDataBytes(case.filename) + assert 0 < dataBytes < len(before) + assert before[-dataBytes:] == after[-dataBytes:] + + +# Values that differ from those of every item in the test files +NEW_QUANTITY = (eumItem.eumIAreaFraction, eumUnit.eumUPerCent) +NEW_AXIS_UNIT = eumUnit.eumUfeet +NEW_X0 = 55.0 + + +def editName(itemInfo): + itemInfo.Name = "Q" * len(itemInfo.Name) + + +def editQuantity(itemInfo): + itemInfo.Quantity = eumQuantity(*NEW_QUANTITY) + + +def editAxisUnit(itemInfo): + itemInfo.SpatialAxis.AxisUnit = NEW_AXIS_UNIT + + +def editX0(itemInfo): + if not hasattr(itemInfo.SpatialAxis, "X0"): + pytest.skip("the axis of a dfs0 item has no origin") + itemInfo.SpatialAxis.X0 = NEW_X0 + + +def expectName(values): + name, item, unit, axis = values + return ("Q" * len(name), item, unit, axis) + + +def expectQuantity(values): + name, _, _, axis = values + return (name, *NEW_QUANTITY, axis) + + +def expectAxisUnit(values): + name, item, unit, axis = values + return (name, item, unit, {**axis, "AxisUnit": NEW_AXIS_UNIT}) + + +def expectX0(values): + name, item, unit, axis = values + return (name, item, unit, {**axis, "X0": NEW_X0}) + + +EDITS = { + "name": (editName, expectName), + "quantity": (editQuantity, expectQuantity), + "axisunit": (editAxisUnit, expectAxisUnit), + "x0": (editX0, expectX0), +} + + +@pytest.mark.parametrize("opener", list(OPENERS)) +@pytest.mark.parametrize("edit", list(EDITS)) +def test_edit_each_item(case, opener, edit): + """ + Editing one item writes that value to that item only, and leaves the + other items and all data unchanged. Checked for each item of the file. + """ + editItem, expectItem = EDITS[edit] + staticItems = readStaticItems(case.filename) + dynamicData = readDynamicData(case.filename) + expected = readItems(case.filename) + assert len(expected) > 1 + + for itemNo in range(len(expected)): + before = readBytes(case.filename) + file = OPENERS[opener](case) + editItem(file.ItemInfo[itemNo]) + file.Close() + + expected[itemNo] = expectItem(expected[itemNo]) + assert expected == readItems(case.filename) + assertOnlyHeaderChanged(case, before) + + for staticItem, newStaticItem in zip( + staticItems, readStaticItems(case.filename), strict=True + ): + assert staticItem[:2] == newStaticItem[:2] + np.testing.assert_array_equal(staticItem[2], newStaticItem[2]) + for (time, data), (newTime, newData) in zip( + dynamicData, readDynamicData(case.filename), strict=True + ): + assert time == newTime + np.testing.assert_array_equal(data, newData) + + +@pytest.mark.parametrize( + "length", + [ + pytest.param(lambda old: 0, id="empty"), + pytest.param(lambda old: 1, id="one"), + pytest.param(lambda old: old - 1, id="shorter"), + pytest.param(lambda old: old, id="same"), + pytest.param(lambda old: old + 5, id="longer"), + ], +) +def test_name_keeps_its_length_in_the_file(case, length): + """ + As in the C# library, a new name takes the place of the old one in the + file: it is padded with spaces or truncated to the old length. + """ + original = readBytes(case.filename) + oldName = readItems(case.filename)[0][0] + # The old name, null terminated, is stored once in the file + oldBytes = oldName.encode("cp1252") + b"\0" + assert original.count(oldBytes) == 1 + offset = original.index(oldBytes) + newName = ("Ab" * len(oldName))[: length(len(oldName))] + + file = case.dfsType.openEdit(case.filename) + file.ItemInfo[0].Name = newName + nameInMemory = file.ItemInfo[0].Name + file.Close() + + after = readBytes(case.filename) + written = after[offset : offset + len(oldBytes)] + padded = newName.encode("cp1252")[: len(oldName)].ljust(len(oldName), b" ") + assert padded + b"\0" == written + # The item shows the name the file holds + assert padded.decode("cp1252") == nameInMemory + assert padded.decode("cp1252") == readItems(case.filename)[0][0] + assertOnlyHeaderChanged(case, original) + + +def setName(itemInfo): + # Not in the cp1252 encoding of dfs names + itemInfo.Name = "Ω" + + +def setMismatchedQuantity(itemInfo): + # A unit of another dimension than the item + itemInfo.Quantity = eumQuantity(eumItem.eumIWaterLevel, eumUnit.eumUPerCent) + + +def setNoQuantity(itemInfo): + itemInfo.Quantity = None + + +def setNameWithNul(itemInfo): + # The file would get only the part in front of the NUL + itemInfo.Name = "A\0B" + + +def setAxisUnitNotUnit(itemInfo): + itemInfo.SpatialAxis.AxisUnit = "abc" + + +@pytest.mark.parametrize( + "setter, error", + [ + (setName, UnicodeEncodeError), + (setNameWithNul, ValueError), + (setMismatchedQuantity, Exception), + (setNoQuantity, AttributeError), + (setAxisUnitNotUnit, AttributeError), + ], + ids=["name", "nul", "quantity", "noquantity", "axisunit"], +) +def test_rejected_value_keeps_old_value(case, setter, error): + """A value the file cannot hold raises, and the item and file keep the old one.""" + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + itemInfo = file.ItemInfo[0] + values = itemValues(itemInfo) + with pytest.raises(error): + setter(itemInfo) + assert values == itemValues(itemInfo) + file.Close() + + assert original == readBytes(case.filename) + + +def test_axis_value_out_of_float_range_is_rejected(staticCase): + """ + The file holds axis values as 32 bit floats. A finite value too large for + one raises, instead of being written as inf, and the item and file keep + the old value. The files of staticCase have item axes with a Dx. + """ + case = staticCase + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + itemInfo = file.ItemInfo[0] + values = itemValues(itemInfo) + with pytest.raises(ValueError, match="Dx"): + itemInfo.SpatialAxis.Dx = 1e300 + assert values == itemValues(itemInfo) + file.Close() + + assert original == readBytes(case.filename) + + +def editItem(itemInfo): + itemInfo.Name = "abc" + itemInfo.Quantity = eumQuantity(*NEW_QUANTITY) + itemInfo.SpatialAxis.AxisUnit = NEW_AXIS_UNIT + + +def editEverything(file): + for itemInfo in file.ItemInfo: + editItem(itemInfo) + spatialAxis = getattr(file, "SpatialAxis", None) + if spatialAxis is not None: + spatialAxis.X0 = NEW_X0 + + +def test_read_mode_writes_nothing(case): + original = readBytes(case.filename) + + file = case.dfsType.open(case.filename) + editEverything(file) + # In Read mode the name is not fitted to the length in the file + assert "abc" == file.ItemInfo[0].Name + file.Close() + + assert original == readBytes(case.filename) + + +def test_edit_after_close_writes_nothing(case): + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + dfsFile = getattr(file, "dfsFile", file) + staticItem = dfsFile.ReadStaticItemNext() + file.Close() + editEverything(file) + assert "abc" == file.ItemInfo[0].Name + if staticItem is not None: + editItem(staticItem) + assert "abc" == staticItem.Name + with pytest.raises(OSError): + file.Flush() + + assert original == readBytes(case.filename) + + +def test_quantity_changed_in_place_is_not_written(case): + """As in the C# library, only assigning a new Quantity writes it.""" + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + for itemInfo in file.ItemInfo: + itemInfo.Quantity.Item = NEW_QUANTITY[0] + itemInfo.Quantity.Unit = NEW_QUANTITY[1] + file.Close() + + assert original == readBytes(case.filename) + + +def test_axis_attribute_the_axis_type_does_not_have(case): + """It is set on the Python object, as on an axis not in an editable file.""" + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + axis = file.ItemInfo[0].SpatialAxis + missing = [name for name in AXIS_ATTRIBUTES if not hasattr(axis, name)] + if not missing: + pytest.skip("a dfs3 axis has every attribute") + setattr(axis, missing[0], 1.0) + assert 1.0 == getattr(axis, missing[0]) + file.Close() + + assert original == readBytes(case.filename) + + +def test_flush_then_append(case): + """ + Changes written on Flush do not move the file pointer: a time step + appended after Flush is added at the end, and the edits are kept. A + changed file axis also reaches the static items on Close. + """ + dynamicData = readDynamicData(case.filename) + items = readItems(case.filename) + staticItems = readStaticItems(case.filename) + + file = case.dfsType.openAppend(case.filename) + file.ItemInfo[0].Name = "Q" * len(items[0][0]) + spatialAxis = getattr(file, "SpatialAxis", None) + if spatialAxis is not None: + spatialAxis.X0 = NEW_X0 + file.Flush() + # A copy of the last time step, which differs from the first, and which + # an encoded file stores as it is + appended = [data for _, data in dynamicData[-len(items) :]] + assert not np.array_equal(appended[0], dynamicData[0][1]) + for data in appended: + file.WriteItemTimeStepNext(0.0, data) + file.Close() + + newData = readDynamicData(case.filename) + assert len(dynamicData) + len(items) == len(newData) + for (_, data), (_, newDataValues) in zip(dynamicData, newData, strict=False): + np.testing.assert_array_equal(data, newDataValues) + for data, (_, newDataValues) in zip( + appended, newData[len(dynamicData) :], strict=True + ): + np.testing.assert_array_equal(data, newDataValues) + newItems = readItems(case.filename) + assert "Q" * len(items[0][0]) == newItems[0][0] + if spatialAxis is not None: + assert all(axis["X0"] == NEW_X0 for _, _, _, axis in newItems) + for (values, axisType, data), (newValues, _, newData) in zip( + staticItems, readStaticItems(case.filename), strict=True + ): + name, item, unit, axis = values + if spatialAxis is not None and axisType == spatialAxis.AxisType: + axis = {**axis, "X0": NEW_X0} + assert (name, item, unit, axis) == newValues + np.testing.assert_array_equal(data, newData) + + +def test_file_axis_is_written_to_every_item(fileAxisCase): + """ + Changing the axis of a dfs1, dfs2 or dfs3 file writes it to every dynamic + item and to every static item with that type of axis, and keeps the data. + """ + case = fileAxisCase + original = readBytes(case.filename) + staticItems = readStaticItems(case.filename) + dynamicData = readDynamicData(case.filename) + + file = case.dfsType.openEdit(case.filename) + axisType = file.SpatialAxis.AxisType + newValues = axisValues(file.SpatialAxis) + for name, value in axisValues(file.SpatialAxis).items(): + if name != "AxisUnit": + newValues[name] = value + 5 + setattr(file.SpatialAxis, name, value + 5) + file.Close() + + for _, _, _, axis in readItems(case.filename): + assert newValues == axis + assert axisType in [staticAxisType for _, staticAxisType, _ in staticItems] + for (values, staticAxisType, data), (newValues_, _, newData) in zip( + staticItems, readStaticItems(case.filename), strict=True + ): + name, item, unit, axis = values + if staticAxisType == axisType: + axis = newValues + assert (name, item, unit, axis) == newValues_ + np.testing.assert_array_equal(data, newData) + for (_, data), (_, newData) in zip( + dynamicData, readDynamicData(case.filename), strict=True + ): + np.testing.assert_array_equal(data, newData) + assertOnlyHeaderChanged(case, original) + + +STATIC_ITEM_TYPES = ["dfs1", "dfs2", "dfs3", "dfsu"] + + +@pytest.fixture(params=STATIC_ITEM_TYPES) +def staticCase(request, tmp_path): + return makeCase(request.param, tmp_path) + + +@pytest.mark.parametrize("edit", list(EDITS)) +def test_edit_each_static_item(staticCase, edit): + """ + As in the C# library, a changed static item is written by + WriteStaticItemData. Only that static item changes, and the data is kept. + Checked for each static item of the file. + """ + case = staticCase + editItem, expectItem = EDITS[edit] + items = readItems(case.filename) + dynamicData = readDynamicData(case.filename) + staticItems = readStaticItems(case.filename) + expected = [values for values, _, _ in staticItems] + assert len(expected) > 0 + + for staticItemNo in range(len(expected)): + before = readBytes(case.filename) + file = DfsFileFactory.DfsGenericOpenEdit(case.filename) + staticItem = file.ReadStaticItem(staticItemNo + 1) + assert staticItem is not None and staticItem.Data is not None + editItem(staticItem) + file.WriteStaticItemData(staticItem, staticItem.Data) + file.Close() + + expected[staticItemNo] = expectItem(expected[staticItemNo]) + newStaticItems = readStaticItems(case.filename) + assert expected == [values for values, _, _ in newStaticItems] + assertOnlyHeaderChanged(case, before) + + for (_, _, data), (_, _, newData) in zip( + staticItems, readStaticItems(case.filename), strict=True + ): + np.testing.assert_array_equal(data, newData) + assert items == readItems(case.filename) + for (_, data), (_, newData) in zip( + dynamicData, readDynamicData(case.filename), strict=True + ): + np.testing.assert_array_equal(data, newData) + + +def test_static_item_edit_is_written_only_by_write_static_item_data(staticCase): + original = readBytes(staticCase.filename) + + file = DfsFileFactory.DfsGenericOpenEdit(staticCase.filename) + staticItem = file.ReadStaticItemNext() + assert staticItem is not None + editItem(staticItem) + file.Close() + + assert original == readBytes(staticCase.filename) + + +def test_edit_while_building(tmp_path): + """ + A file made with DfsBuilder is open in Append mode, so item edits made + before its data is written go to the file, as in the C# library. + """ + filename = str(tmp_path / "test.dfs1") + namesInMemory = [] + + def editFirstItem(file): + itemInfo = file.ItemInfo[0] + itemInfo.Name = "LongerName" + itemInfo.Quantity = eumQuantity(*NEW_QUANTITY) + itemInfo.SpatialAxis.X0 = NEW_X0 + namesInMemory.append(itemInfo.Name) + + createDfs1(filename, editFirstItem) + + # The name keeps the length of "Level", the name the item was built with + assert ["Longe"] == namesInMemory + name, item, unit, axis = readItems(filename)[0] + assert ("Longe", NEW_QUANTITY, NEW_X0) == (name, (item, unit), axis["X0"]) + assert "Depth" == readItems(filename)[1][0] + assert 6 == len(readDynamicData(filename)) + + +def test_quantity_edit_keeps_a_name_cp1252_cannot_decode(case): + """ + A name with a byte cp1252 leaves undefined, such as one written as UTF-8 + by another tool, is read with U+FFFD in it. Assigning only a new + Quantity must still work, and must write the name bytes back unchanged. + """ + original = readBytes(case.filename) + oldName = readItems(case.filename)[0][0] + oldBytes = oldName.encode("cp1252") + b"\0" + assert original.count(oldBytes) == 1 + offset = original.index(oldBytes) + # 0x81 is undefined in cp1252 + nameBytes = b"\x81" + oldBytes[1:] + patched = original[:offset] + nameBytes + original[offset + len(oldBytes) :] + with open(case.filename, "wb") as f: + f.write(patched) + assert "\ufffd" == readItems(case.filename)[0][0][0] + + file = case.dfsType.openEdit(case.filename) + file.ItemInfo[0].Quantity = eumQuantity(*NEW_QUANTITY) + file.Close() + + after = readBytes(case.filename) + assert nameBytes == after[offset : offset + len(nameBytes)] + _, item, unit, _ = readItems(case.filename)[0] + assert NEW_QUANTITY == (item, unit) + assertOnlyHeaderChanged(case, patched) + + +@pytest.mark.parametrize("mode", ["read", "edit"]) +def test_file_axis_is_a_copy_of_the_item_axis(fileAxisCase, mode): + """ + As in the C# library, the axis of a dfs1, dfs2 or dfs3 file is a copy of + the first item's axis: changing it changes the items only when an + editable file is flushed or closed, and never in a file opened to read. + """ + case = fileAxisCase + opener = case.dfsType.open if mode == "read" else case.dfsType.openEdit + file = opener(case.filename) + itemAxis = file.ItemInfo[0].SpatialAxis + assert file.SpatialAxis is not itemAxis + oldX0 = itemAxis.X0 + file.SpatialAxis.X0 = oldX0 + 5 + assert oldX0 == itemAxis.X0 + if mode == "edit": + file.Flush() + assert oldX0 + 5 == itemAxis.X0 + file.Close() + + +@pytest.fixture +def gcDisabled(): + """Objects are freed only when nothing refers to them, not in a cycle.""" + gc.disable() + yield + gc.enable() + + +def test_static_item_read_from_an_editable_file_is_freed(staticCase, gcDisabled): + """ + The file does not keep the static items it read alive until Close, and a + static item and its axis do not refer to themselves. + """ + file = DfsFileFactory.DfsGenericOpenEdit(staticCase.filename) + staticItem = file.ReadStaticItemNext() + assert staticItem is not None + itemRef = weakref.ref(staticItem) + axisRef = weakref.ref(staticItem.SpatialAxis) + del staticItem + assert itemRef() is None + assert axisRef() is None + file.Close() + + +def copyItem(itemInfo): + itemCopy = copy.copy(itemInfo) + + def edit(): + # Not the axis: a shallow copy shares it with the item + itemCopy.Name = "abc" + itemCopy.Quantity = eumQuantity(*NEW_QUANTITY) + assert "abc" == itemCopy.Name + + return edit + + +def copyAxis(itemInfo, copyFunction=copy.copy): + axisCopy = copyFunction(itemInfo.SpatialAxis) + + def edit(): + axisCopy.AxisUnit = NEW_AXIS_UNIT + assert NEW_AXIS_UNIT == axisCopy.AxisUnit + + return edit + + +def deepcopyAxis(itemInfo): + # An item can't be deep-copied: its native pointer can't be pickled + return copyAxis(itemInfo, copy.deepcopy) + + +@pytest.mark.parametrize("copyForEdit", [copyItem, copyAxis, deepcopyAxis]) +@pytest.mark.parametrize("closeFirst", [False, True], ids=["open", "closed"]) +def test_copy_of_an_item_is_not_written(case, copyForEdit, closeFirst): + """ + A copy of an item or of its axis, taken from an open editable file, is a + plain object: changing it writes nothing, while the file is open or after + it is closed. + """ + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + edit = copyForEdit(file.ItemInfo[0]) + if closeFirst: + file.Close() + edit() + else: + edit() + file.Close() + + assert original == readBytes(case.filename) + + +def test_replaced_axis_writes_nothing(case): + """ + As in the C# library, an axis writes only while it is the item's axis. + Assigning another axis to an item writes nothing, and neither the old + nor the new axis writes when changed, before or after Close. + """ + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + itemInfo = file.ItemInfo[0] + oldAxis = itemInfo.SpatialAxis + newAxis = copy.copy(oldAxis) + itemInfo.SpatialAxis = newAxis + newAxis.AxisUnit = NEW_AXIS_UNIT + oldAxis.AxisUnit = eumUnit.eumUkilometer + file.Close() + oldAxis.AxisUnit = eumUnit.eumUmillimeter + newAxis.AxisUnit = eumUnit.eumUmillimeter + + assert original == readBytes(case.filename) + + +@pytest.mark.parametrize("replacement", ["copy", "othertype"]) +def test_file_axis_reaches_an_item_whose_axis_was_replaced(fileAxisCase, replacement): + """ + Changing the axis of a dfs1, dfs2 or dfs3 file writes it to every + dynamic item in the file, also one whose SpatialAxis object was replaced. + """ + case = fileAxisCase + file = case.dfsType.openEdit(case.filename) + itemInfo = file.ItemInfo[1] + if replacement == "copy": + itemInfo.SpatialAxis = copy.copy(itemInfo.SpatialAxis) + else: + itemInfo.SpatialAxis = DfsAxisEqD0() + newX0 = file.SpatialAxis.X0 + 5 + file.SpatialAxis.X0 = newX0 + file.Flush() + file.Close() + + assert all(axis["X0"] == newX0 for _, _, _, axis in readItems(case.filename)) + + +@pytest.mark.parametrize( + "name, value, error", + [("AxisUnit", "abc", AttributeError), ("Dx", 1e300, ValueError)], + ids=["axisunit", "dx"], +) +def test_bad_file_axis_value_is_rejected_on_flush_and_close( + fileAxisCase, name, value, error +): + """ + A value the file axis cannot hold is accepted when set and rejected on + Flush and Close. The file is still closed, unchanged. + """ + case = fileAxisCase + original = readBytes(case.filename) + + file = case.dfsType.openEdit(case.filename) + setattr(file.SpatialAxis, name, value) + with pytest.raises(error): + file.Flush() + with pytest.raises(error): + file.Close() + + with pytest.raises(OSError): + file.Flush() + assert original == readBytes(case.filename) + + +def test_file_axis_can_not_be_replaced(fileAxisCase): + """ + As in the C# library, the axis of a dfs1, dfs2 or dfs3 file can only be + changed, not replaced: a replaced axis would be written by nothing. + """ + file = fileAxisCase.dfsType.openEdit(fileAxisCase.filename) + with pytest.raises(AttributeError): + file.SpatialAxis = copy.copy(file.SpatialAxis) + file.Close() + + +def test_file_axis_is_not_written_when_an_item_has_another_axis_type(tmp_path): + """ + When a dynamic item has another type of axis than the file, Flush and + Close raise before writing the file axis to any item. + """ + factory = DfsFactory() + filename = str(tmp_path / "test.dfs2") + createDfs2( + filename, + [ + factory.CreateAxisEqD2(eumUnit.eumUmeter, 3, 0, 1, 2, 0, 1), + factory.CreateAxisEqD0(), + ], + [], + ) + original = readBytes(filename) + + file = DFS_TYPES["dfs2"].openEdit(filename) + file.SpatialAxis.X0 = NEW_X0 + with pytest.raises(ValueError, match="item 2 is EqD0, not EqD2"): + file.Flush() + with pytest.raises(ValueError, match="item 2 is EqD0, not EqD2"): + file.Close() + + assert original == readBytes(filename) + + +def test_file_axis_is_written_only_to_static_items_of_its_axis_type(tmp_path): + """ + On Close, the file axis is written to the static items with the same type + of axis, and the other static items are kept. The data of all is kept. + """ + factory = DfsFactory() + filename = str(tmp_path / "test.dfs2") + createDfs2( + filename, + [factory.CreateAxisEqD2(eumUnit.eumUmeter, 3, 0, 1, 2, 0, 1)], + [ + factory.CreateAxisEqD1(eumUnit.eumUmeter, 4, 0, 1), + factory.CreateAxisEqD2(eumUnit.eumUmeter, 3, 0, 1, 2, 0, 1), + ], + ) + staticItems = readStaticItems(filename) + assert [SpaceAxisType.EqD1, SpaceAxisType.EqD2] == [ + axisType for _, axisType, _ in staticItems + ] + + file = DFS_TYPES["dfs2"].openEdit(filename) + file.SpatialAxis.X0 = NEW_X0 + file.Close() + + newStaticItems = readStaticItems(filename) + eqD1Axis = staticItems[0][0][3] + assert eqD1Axis == newStaticItems[0][0][3] + assert {**staticItems[1][0][3], "X0": NEW_X0} == newStaticItems[1][0][3] + for (_, _, data), (_, _, newData) in zip(staticItems, newStaticItems, strict=True): + np.testing.assert_array_equal(data, newData) diff --git a/tests/test_dfsbuilder.py b/tests/test_dfsbuilder.py index 681b77f..a7f7843 100644 --- a/tests/test_dfsbuilder.py +++ b/tests/test_dfsbuilder.py @@ -428,3 +428,24 @@ def test_reference_coordinates_and_orientation_are_read_back(tmp_path): staticItemInfo.OrientationPhi, staticItemInfo.OrientationTheta, ) + + +def test_axis_value_out_of_float_range_is_rejected(tmp_path): + """ + The file holds axis values as 32 bit floats. A finite value too large for + one raises, instead of being written as inf. + """ + factory = DfsFactory() + builder = DfsBuilder.Create("title", "application", 1) + builder.SetDataType(0) + builder.SetGeographicalProjection(factory.CreateProjectionUndefined()) + builder.SetTemporalAxis(factory.CreateTemporalEqTimeAxis(eumUnit.eumUsec, 0, 1)) + builder.SetSpatialAxis(factory.CreateAxisEqD1(eumUnit.eumUmeter, 2, 0, 1e300)) + builder.AddCreateDynamicItem( + "Value", + eumQuantity.Create(eumItem.eumIItemUndefined, eumUnit.eumUUnitUndefined), + DfsSimpleType.Float, + DataValueType.Instantaneous, + ) + with pytest.raises(ValueError, match="Dx"): + builder.CreateFile(str(tmp_path / "test.dfs1"))