Write item name, quantity and axis changes to an edited dfs file

CHANGELOG.md

@@ -9,6 +9,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- In a dfs file opened for editing, or being created with `DfsBuilder`,
+  assigning an item's `Name` or `Quantity`, or changing an attribute of its
+  `SpatialAxis` (such as `X0`), writes the change to the file. For a static
+  item, `WriteStaticItemData` writes it. On a dfs1, dfs2 or dfs3 file,
+  changing an attribute of the file's `SpatialAxis` updates every dynamic
+  item on `Flush` or `Close`, and the static items with the same type of
+  axis on `Close`. As in the C# library, a new name is truncated or padded
+  to the length of the old one, and changing `Quantity.Item` or
+  `Quantity.Unit` in place is not written. `SpatialAxis` of a dfs1, dfs2 or
+  dfs3 file is now a copy of the first item's axis, as in the C# library, so
+  changing it in a file opened for reading no longer changes that item's
+  axis
+  ([#38](https://github.com/DHI/mikecore-python/issues/38)).
+- `DfsuFile.Flush()` flushes the file instead of raising `TypeError`, and
+  `DfsFileFactory.DfsuFileOpenAppend` no longer requires `parameters`, like the
+  other `Open` methods.
 - Items read from a dfs file have the reference coordinates and orientation
   stored in the file. They had the `-1e-35` defaults, whatever the file held
   ([#31](https://github.com/DHI/mikecore-python/issues/31)).

mikecore/Dfs123File.py

@@ -1,18 +1,87 @@
-from mikecore.DfsFile import DfsFile, DfsFileMode, DfsItemData
+import copy
+
+from mikecore.DfsFile import DfsDLLUtil, DfsFile, DfsFileMode, DfsItemData
 
 
 class Dfs123File(DfsFile):
     def __init__(self):
         super().__init__()
         self.SpatialAxis = None
         self.reshape = False
+        self._spatialAxisUpdated = False
 
     def Open(self, filename, mode=DfsFileMode.Read, parameters=None, reshape=False):
         super().Open(filename, mode, parameters)
         self.reshape = reshape
+        self._spatialAxisUpdated = False
         # TODO: Need to check that this is the same for all items
         if len(self.ItemInfo) > 0:
-            self.SpatialAxis = self.ItemInfo[0].SpatialAxis
+            self.SpatialAxis = copy.copy(self.ItemInfo[0].SpatialAxis)
+            if self.SpatialAxis is not None and mode in (
+                DfsFileMode.Edit,
+                DfsFileMode.Append,
+            ):
+                self.SpatialAxis._OnUpdate = self.__SpatialAxisUpdated
+
+    def __SpatialAxisUpdated(self):
+        self._spatialAxisUpdated = True
+
+    def __WriteSpatialAxis(self, staticItems):
+        """
+        Write a changed SpatialAxis to all dynamic items, and with
+        staticItems to the static items with the same type of axis, as the
+        C# library does. Writing static items moves the file pointer.
+        """
+        fileAxis = self.SpatialAxis
+        if not self._spatialAxisUpdated or fileAxis is None:
+            return
+        for itemInfo in self.ItemInfo:
+            # Write to the item's axis in the file, also when the item was given
+            # another SpatialAxis object, which writes nothing itself
+            itemAxis = DfsDLLUtil.GetItemSpatialAxis(itemInfo.ItemPointer)
+            itemAxis._Update(fileAxis)
+            DfsDLLUtil.dfsSetItemSpatialAxis(itemInfo.ItemPointer, itemAxis)
+            axis = itemInfo.SpatialAxis
+            if axis is not None and axis.AxisType == fileAxis.AxisType:
+                axis._Update(fileAxis)
+        if not staticItems:
+            return
+        self._spatialAxisUpdated = False
+        self.Reset()
+        staticItemNo = 1
+        while (staticItem := self.ReadStaticItem(staticItemNo)) is not None:
+            staticItemNo += 1
+            axis = staticItem.SpatialAxis
+            data = staticItem.Data
+            if axis is None or data is None or axis.AxisType != fileAxis.AxisType:
+                continue
+            axis._Update(fileAxis)
+            self.WriteStaticItemData(staticItem, data)
+
+    def Flush(self):
+        """
+        Flush cached data to the file. A changed SpatialAxis is written to the
+        dynamic items here, and to the static items only on Close: writing a
+        static item moves the file pointer, so a time step appended after
+        Flush would overwrite another. The C# library writes both on Flush.
+        A value the file axis cannot hold is only rejected here or on Close,
+        not when it is set.
+        The whole file axis is written to each item, replacing changes made
+        to an item's own axis, as in the C# library.
+        """
+        if self.filePointer.value is not None:
+            self.__WriteSpatialAxis(staticItems=False)
+        super().Flush()
+
+    def Close(self):
+        try:
+            if self.filePointer.value is not None:
+                self.__WriteSpatialAxis(staticItems=True)
+        finally:
+            self._spatialAxisUpdated = False
+            if self.SpatialAxis is not None:
+                self.SpatialAxis._OnUpdate = None
+            super().Close()
 
     def Reshape(self, reshape):
         # Set True to reshape item data to multi-dimensional arrays.

mikecore/DfsDLL.py

@@ -210,6 +210,14 @@ def Init(libfilepath: str | None = None):
             ctypes.c_void_p,
             ctypes.c_void_p,
         ]
+        lib.dfsUpdateItemInfo.argtypes = [
+            ctypes.c_void_p,
+            ctypes.c_void_p,
+            ctypes.c_int32,
+            ctypes.c_char_p,
+            ctypes.c_int32,
+        ]
+        lib.dfsUpdateItemInfo.restype = ctypes.c_int32
 
         lib.dfsAddCustomBlock.argtypes = [
             ctypes.c_void_p,

mikecore/DfsFile.py

@@ -3,6 +3,7 @@
 import datetime
 import ctypes
 import locale
+import weakref
 import numpy as np
 from mikecore.eum import *
 from mikecore.DfsDLL import DfsDLL
@@ -385,14 +386,61 @@ def IncrementNumberOfTimeSteps(self, time):
         self.TimeSpan = time - self.StartTimeOffset
 
 
+# Marks an attribute that was not set
+_Missing = object()
+
+
 class DfsSpatialAxis:
+    # Attributes that may change on an axis read from a file. Setting one of
+    # them invokes _OnUpdate, which writes the axis to an editable file.
+    _UpdatableAttributes = frozenset({"AxisUnit", "X0", "Dx", "Y0", "Dy", "Z0", "Dz"})
+
     def __init__(self, axisType, shape, sizeOfData, axisUnit):
+        self._OnUpdate = None
         self.AxisType = axisType
         self.Shape = shape
         self.Dimension = len(shape)
         self.SizeOfData = sizeOfData
         self.AxisUnit = axisUnit
 
+    def __getstate__(self):
+        # A copy is not in the file: it must not write to it
+        state = self.__dict__.copy()
+        state["_OnUpdate"] = None
+        return state
+
+    def __setattr__(self, name, value):
+        onUpdate = self.__dict__.get("_OnUpdate")
+        if name not in self._UpdatableAttributes or onUpdate is None:
+            super().__setattr__(name, value)
+            return
+        oldValue = self.__dict__.get(name, _Missing)
+        super().__setattr__(name, value)
+        try:
+            onUpdate()
+        except BaseException:
+            if oldValue is _Missing:
+                super().__delattr__(name)
+            else:
+                super().__setattr__(name, oldValue)
+            raise
+
+    def _Update(self, source):
+        """Copy the updatable attributes of source, an axis of the same type."""
+        if self.AxisType != source.AxisType:
+            raise ValueError("Destination and source axis have different types")
+        # Invoke _OnUpdate once, not for every attribute
+        onUpdate = self._OnUpdate
+        self._OnUpdate = None
+        try:
+            for name in self._UpdatableAttributes:
+                if hasattr(source, name):
+                    setattr(self, name, getattr(source, name))
+        finally:
+            self._OnUpdate = onUpdate
+        if onUpdate is not None:
+            onUpdate()
+
 
 class DfsAxisEqD0(DfsSpatialAxis):
     def __init__(self, axisUnit=eumUnit.eumUmeter):
@@ -508,12 +556,17 @@ def __init__(self, axisUnit, xCount, yCount, zCount, xCoords, yCoords, zCoords):
 
 class DfsDynamicItemInfo:
     def __init__(self, itemPointer=None, itemNumber=0):
+        # Set on items of an editable file, to write Name and Quantity
+        self._OnUpdate = None
+        # The name read from the file and its bytes, which are written back
+        # while Name is unchanged, as cp1252 may not encode the decoded name
+        self._NameInFile = None
         self.ItemPointer = itemPointer
         self.ItemNumber = itemNumber
         self.DataType = DfsSimpleType.UShort
-        self.Name = ""
+        self._Name = ""
         self.ElementCount = -1
-        self.Quantity = None
+        self._Quantity = None
         self.ReferenceCoordinateX = np.float32(-1e-35)
         self.ReferenceCoordinateY = np.float32(-1e-35)
         self.ReferenceCoordinateZ = np.float32(-1e-35)
@@ -541,6 +594,50 @@ def __repr__(self):
             + ")"
         )
 
+    @property
+    def Name(self):
+        return self._Name
+
+    @Name.setter
+    def Name(self, name):
+        if self._OnUpdate is not None:
+            # As in the C# library, the name in the file keeps its length: the
+            # new name is truncated or padded with spaces to the old length.
+            name = name[: len(self._Name)].ljust(len(self._Name))
+        oldName = self._Name
+        self._Name = name
+        try:
+            self._InvokeOnUpdate()
+        except BaseException:
+            self._Name = oldName
+            raise
+
+    @property
+    def Quantity(self):
+        return self._Quantity
+
+    @Quantity.setter
+    def Quantity(self, quantity):
+        # Only assigning a quantity updates the file, as in the C# library.
+        # Changing Item or Unit of the current quantity does not.
+        oldQuantity = self._Quantity
+        self._Quantity = quantity
+        try:
+            self._InvokeOnUpdate()
+        except BaseException:
+            self._Quantity = oldQuantity
+            raise
+
+    def __getstate__(self):
+        # A copy is not in the file: it must not write to it
+        state = self.__dict__.copy()
+        state["_OnUpdate"] = None
+        return state
+
+    def _InvokeOnUpdate(self):
+        if self._OnUpdate is not None:
+            self._OnUpdate()
+
     def init(self, itemName, eumQuantity):
         self.Name = itemName
         self.Quantity = eumQuantity
@@ -779,6 +876,10 @@ def __init__(self):
         self.fpTimeStepIndex = 0
         self.headPointer = ctypes.c_void_p(0)
         self.filePointer = ctypes.c_void_p(0)
+        # Items and axes that write their changes to the file, until it is
+        # closed. Weak, so that the file does not keep the static items it
+        # read alive.
+        self._updatableItems = weakref.WeakSet()
 
     def __del__(self):
         self.Close()
@@ -901,6 +1002,10 @@ def Close(self):
             )
         if self.headPointer.value is not None:
             DfsDLL.Wrapper.dfsHeaderDestroy(ctypes.byref(self.headPointer))
+        # The item pointers point into the destroyed header: stop writing to it
+        for itemOrAxis in list(self._updatableItems):
+            itemOrAxis._OnUpdate = None
+        self._updatableItems.clear()
 
     def GetNextItemNumber(self):
         return self.fpItemNumber
@@ -1470,8 +1575,41 @@ def __DynamicItemInfoReadAndCreate(self, itemNumber, noOfItems):
         )
         item = DfsDynamicItemInfo(itemPointer, itemNumber)
         self.__GetItemInfo(item)
+        self.__AttachItemUpdate(item)
         return item
 
+    def __AttachItemUpdate(self, item):
+        # In an editable file, changing the name, quantity or axis of an item
+        # updates its item info: in the header for a dynamic item, and in the
+        # static vector, written by WriteStaticItemData, for a static item
+        if self.FileMode not in (DfsFileMode.Edit, DfsFileMode.Append):
+            return
+        self._updatableItems.add(item)
+        item._OnUpdate = lambda: self.__UpdateItemInfo(item)
+        axis = item.SpatialAxis
+        if axis is not None:
+            self._updatableItems.add(axis)
+            axis._OnUpdate = lambda: self.__UpdateItemAxis(item, axis)
+
+    def __UpdateItemAxis(self, item, axis):
+        # As in the C# library, an axis writes only while it is the item's
+        # axis: one replaced by assigning SpatialAxis writes nothing
+        if item.SpatialAxis is axis:
+            DfsDLLUtil.dfsSetItemSpatialAxis(item.ItemPointer, axis)
+
+    def __UpdateItemInfo(self, item):
+        nameInFile, nameBytes = item._NameInFile
+        if item.Name != nameInFile:
+            nameBytes = item.Name.encode("cp1252")
+        rok = DfsDLL.Wrapper.dfsUpdateItemInfo(
+            self.headPointer,
+            item.ItemPointer,
+            ctypes.c_int32(item.Quantity.Item.value),
+            nameBytes,
+            ctypes.c_int32(item.Quantity.Unit.value),
+        )
+        DfsDLL.CheckReturnCode(rok)
+
     def __StaticItemReadAndCreate(self, number, ubgConversion):
 
         # Check if we can read it. staticVectorPointer contains both item info and data
@@ -1503,6 +1641,7 @@ def __StaticItemReadAndCreate(self, number, ubgConversion):
 
         self.__GetItemInfo(staticItem)
         self.__GetStaticData(staticItem)
+        self.__AttachItemUpdate(staticItem)
 
         return staticItem
 
@@ -1534,6 +1673,7 @@ def __GetItemInfo(self, item):
         quantity.UnitDescription = eumUnitDesc
 
         item.init(itemName, quantity)
+        item._NameInFile = (itemName, itemNameP.value)
         item.DataType = itemDataType
         item.ValueType = DfsDLLUtil.dfsGetItemValueType(item.ItemPointer)
         item.ElementCount = DfsDLL.Wrapper.dfsGetItemElements(item.ItemPointer)

mikecore/DfsFileFactory.py

@@ -71,7 +71,7 @@ def DfsuFileOpenEdit(filename, parameters=None):
         return DfsuFile(dfsFile)
 
     @staticmethod
-    def DfsuFileOpenAppend(filename, parameters):
+    def DfsuFileOpenAppend(filename, parameters=None):
         dfsFile = DfsFileFactory.DfsGenericOpenAppend(filename, parameters)
         return DfsuFile(dfsFile)
 

mikecore/DfsuFile.py

@@ -309,7 +309,7 @@ def Reset(self):
         self.dfsFile.Reset()
 
     def Flush(self):
-        self.dfsFile.Flush(self)
+        self.dfsFile.Flush()
 
     def FlushTimeStep(self):
         self.dfsFile.FlushTimeStep()