Dev 08 26

.gitattributes

@@ -0,0 +1,95 @@
+# Copyright (c) 2023-2024 Geosiris.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Normalise line endings and, above all, keep the binary test fixtures binary.
+#
+# An EPC is a ZIP and an .h5 is an HDF5 container: a single CRLF<->LF substitution inside one
+# corrupts the archive, and git would happily do it on a Windows checkout for any file it
+# guesses is text. The `binary` macro below is `-text -diff`, which turns both the eol
+# conversion and the textual diff off.
+
+# ---------------------------------------------------------------------------
+# Default: let git detect, and store text with LF in the repository.
+# ---------------------------------------------------------------------------
+*               text=auto
+
+# ---------------------------------------------------------------------------
+# Source and data formats that are always text
+# ---------------------------------------------------------------------------
+*.py            text diff=python
+*.pyi           text diff=python
+*.md            text diff=markdown
+*.rst           text
+*.txt           text
+*.xml           text
+*.json          text
+*.yml           text
+*.yaml          text
+*.toml          text
+*.cfg           text
+*.ini           text
+*.csv           text
+*.html          text diff=html
+*.css           text
+*.js            text
+*.sql           text
+*.sh            text eol=lf
+*.bash          text eol=lf
+Dockerfile      text
+Makefile        text
+.gitattributes  text
+.gitignore      text
+
+# Windows-only scripts keep CRLF or cmd.exe mis-parses them
+*.bat           text eol=crlf
+*.cmd           text eol=crlf
+*.ps1           text eol=crlf
+
+# ---------------------------------------------------------------------------
+# Energyml / geoscience binaries — never touch, never diff
+# ---------------------------------------------------------------------------
+*.epc           binary
+*.h5            binary
+*.hdf5          binary
+*.parquet       binary
+*.las           binary
+*.segy          binary
+*.sgy           binary
+*.resqml        binary
+
+# ---------------------------------------------------------------------------
+# Generic binaries
+# ---------------------------------------------------------------------------
+*.zip           binary
+*.gz            binary
+*.tgz           binary
+*.bz2           binary
+*.xz            binary
+*.7z            binary
+*.jar           binary
+*.whl           binary
+*.pdf           binary
+*.png           binary
+*.jpg           binary
+*.jpeg          binary
+*.gif           binary
+*.ico           binary
+*.bmp           binary
+*.tif           binary
+*.tiff          binary
+*.woff          binary
+*.woff2         binary
+*.ttf           binary
+*.otf           binary
+*.eot           binary
+*.so            binary
+*.dll           binary
+*.dylib         binary
+*.pyd           binary
+*.exe           binary
+*.pyc           binary
+
+# ---------------------------------------------------------------------------
+# Keep generated / vendored trees out of the language stats and of PR diffs
+# ---------------------------------------------------------------------------
+poetry.lock     linguist-generated=true -diff

.github/workflows/ci_energyml_utils_pull_request.yml

@@ -36,9 +36,26 @@ jobs:
         run: |
           poetry install --all-extras
 
-      - name: Run pytest
+      - name: Check the published test fixtures are present
+        # tests/ runs against real EPCs rather than mock dataclasses, and rc/**/*.epc is
+        # git-ignored with a per-file allow list. Only the FESAPI testing packages are cleared
+        # for publication; the field-data EPCs stay local and their tests skip here. Dropping
+        # one of these from the allow list would silently turn its tests into skips, so fail
+        # loudly instead.
         run: |
-          poetry run pytest -v --tb=short
+          missing=0
+          for f in rc/epc/testingPackageCpp.epc rc/epc/testingPackageCpp.h5 \
+                   rc/epc/testingPackageCpp22.epc rc/epc/testingPackageCpp22.h5; do
+            if [ ! -f "$f" ]; then echo "::error::missing test fixture $f"; missing=1; fi
+          done
+          exit $missing
+
+      - name: Run pytest (whole suite, slow tests included)
+        # pyproject.toml sets `addopts = "-m 'not slow'"` so a local run stays quick; `-m ""`
+        # clears that filter. Without it CI silently skips the tests that compare EpcFile with
+        # EpcStreamReader over the large packages.
+        run: |
+          poetry run pytest -v --tb=short -m "" --durations=15
 
   build:
     name: Build distribution

.gitignore

@@ -4,6 +4,14 @@
 *.sublime-project
 *.sublime-workspace
 
+# AI coding assistants: local instructions / settings, not part of the project
+.claude/
+CLAUDE.md
+AGENTS.md
+.cursor/
+.aider*
+.github/copilot-instructions.md
+
 # Checkpoints
 .ipynb_checkpoints
 __pycache__/

energyml-utils/.gitignore

@@ -59,6 +59,10 @@ manip*
 docs/*.md
 
 # DATA
+# Default output folder of `extract_3d` / export_multiple_data. The extensions below cover the
+# files it writes, but not the ones a malformed name produced before sanitize_file_name existed
+# (a ':' in a citation title left extension-less remnants on Windows), so ignore the folder.
+exported_meshes/
 *.obj
 *.off
 *.mtl
@@ -72,6 +76,17 @@ rc/**/*.epc
 rc/**/*.h5
 rc/**/*.hdf5
 
+# ...except the FESAPI testing packages, which are the only fixtures cleared for publication.
+# `tests/` runs against real EPCs rather than mock dataclasses, so the suite needs them in CI.
+#
+# DO NOT add a fixture here without checking it may be published: the other EPCs of rc/epc/ are
+# field data (Volve, SPASS, the 80-well surveys) and stay local. A test that needs one of them
+# must skip when it is absent — see the `fixture_epc` fixture of tests/test_epc_file.py.
+!rc/epc/testingPackageCpp.epc
+!rc/epc/testingPackageCpp.h5
+!rc/epc/testingPackageCpp22.epc
+!rc/epc/testingPackageCpp22.h5
+
 
 # WIP
 src/energyml/utils/wip*
@@ -80,4 +95,14 @@ rc/camunda
 
 
 # code profiling
-*.prof
+*.prof
+AUDIT.md
+
+# AI coding assistants: local instructions / settings, not part of the project
+.claude/
+CLAUDE.md
+AGENTS.md
+
+# Local scratch: dead code kept around locally, and output folders of ad-hoc runs
+_to_delete/
+results/

energyml-utils/README.md

@@ -268,16 +268,31 @@ finally:
 The EpcStreamReader is perfect for applications that need to work with large EPC files efficiently, such as data processing pipelines, web applications, or analysis tools where memory usage is a concern.
 
 
-# Poetry scripts : 
-
-- extract_3d : extract a representation into an 3D file (obj/off)
-- csv_to_dataset : translate csv data into h5 dataset
-- generate_data : generate a random data from a qualified_type 
-- xml_to_json : translate an energyml xml file into json.
-- json_to_xml : translate an energyml json file into an xml file
+# Command line scripts :
+
+Installing the package (`pip install energyml-utils`) creates these executables. They live in
+`energyml.utils.cli`, so they work from an installed wheel as well as from a checkout.
+
+- extract_3d : extract a representation into a 3D / GIS file (obj/off/stl/vtk/geojson)
+- csv_to_dataset : translate csv data into h5 or parquet datasets (needs the `parquet` extra for the csv reader)
+- generate_data : generate a random object from a qualified_type
+- generate_multiple_data : same, for several types at once, optionally one file per object
+- xml_to_json : translate an energyml xml file (or every object of an EPC) into json
+- json_to_xml : translate an energyml json file into one xml file per object
+- json_to_epc : package every object of an energyml json file into a single EPC
+- loadNsave : read a file or a folder (json/xml/epc) and write it back as an EPC
 - describe_as_csv : create a csv description of an EPC content
 - validate : validate an energyml object or an EPC instance (or a folder containing energyml objects)
 
+Every command accepts `--help`, and `-v` / `-vv` to raise the log level (`-q` to only report
+errors). They can also be called from python, passing the arguments explicitly:
+
+```python
+from energyml.utils.cli import extract_representation_in_3d_file
+
+extract_representation_in_3d_file(["--epc", "file.epc", "--output", "out", "-ff", "geojson"])
+```
+
 
 
 ## Installation to test poetry scripts : 
@@ -355,6 +370,85 @@ Extract to OFF format without CRS displacement:
 poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format OFF --no-crs
 ```
 
+### Extract 3D Representations as GeoJSON
+
+Export every exportable representation of the EPC to GeoJSON (one `.geojson` file per
+representation; a representation that cannot be read is logged and skipped):
+```bash
+poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson
+```
+
+Export only some representations:
+```bash
+poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson --uuid "uuid1" "uuid2"
+```
+
+**Coordinates are reprojected to WGS84 by default**, as required by [RFC 7946](https://www.rfc-editor.org/rfc/rfc7946).
+This needs the `crs` extra:
+```bash
+poetry install --extras crs        # or : pip install energyml-utils[crs]
+```
+
+Without it (or when no EPSG code can be found in the CRS), a warning is logged, the coordinates are
+left in their source CRS, and that CRS is advertised in the output through the `crs` (GeoJSON 2008,
+read by GDAL / QGIS) and `coordRefSys` (OGC JSON-FG) members.
+
+Keep the coordinates in the source projected CRS:
+```bash
+poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson --no-wgs84
+```
+
+Allow PROJ to download the geoid grids used by the vertical datum transformation. Without them the
+height conversion is silently skipped, which can be off by tens of metres:
+```bash
+poetry run extract_3d --epc "path/to/file.epc" --output "output_folder" --file-format geojson --proj-network
+```
+
+Each feature carries the identification metadata of its source object: the energyml `uuid` in the
+RFC 7946 `id` member, and the `uuid`, `qualified_type`, `content_type`, ETP `uri`, `Citation` fields
+(title, originator, creation, last_update, …) and EPSG codes in `properties`:
+
+```json
+{
+  "type": "FeatureCollection",
+  "name": "Bartonien Top",
+  "bbox": [2.3675, 48.9129, 30.13, 2.4057, 48.9198, 37.15],
+  "features": [
+    {
+      "type": "Feature",
+      "id": "02cc9411-6b90-4619-a9fd-a39ac332b367",
+      "properties": {
+        "uuid": "02cc9411-6b90-4619-a9fd-a39ac332b367",
+        "qualified_type": "resqml22.PointSetRepresentation",
+        "uri": "eml:///resqml22.PointSetRepresentation(02cc9411-6b90-4619-a9fd-a39ac332b367)",
+        "title": "Bartonien Top",
+        "originator": "Geosiris",
+        "creation": "2025-12-17T16:11:36Z",
+        "last_update": "2025-12-17T16:11:36Z",
+        "projected_epsg_code": 3949,
+        "source_crs": "EPSG:3949",
+        "coordinates_crs": "OGC:CRS84"
+      },
+      "geometry": { "type": "MultiPoint", "coordinates": [[2.4055336, 48.9140288, 37.15]] }
+    }
+  ]
+}
+```
+
+The same options are available from python:
+```python
+from energyml.utils.data.mesh import MeshFileFormat, export_multiple_data
+
+export_multiple_data(
+    epc_path="path/to/file.epc",
+    uuid_list=["uuid1"],
+    output_folder_path="output_folder",
+    file_format=MeshFileFormat.GEOJSON,
+    to_wgs84=True,       # default
+    use_network=False,   # True to download the geoid grids
+)
+```
+
 ### CSV to Dataset
 
 Convert CSV to HDF5:
@@ -399,6 +493,11 @@ Using qualified type:
 poetry run generate_data --type "resqml22.WellboreFeature" --file-format json
 ```
 
+Generate multiple data : 
+```bash
+poetry run generate_multiple_data -o generated -ff xml -t eml23.AbstractObject --exclude witsml --exclude prodml
+```
+
 ### XML to JSON Conversion
 
 Convert an XML file to JSON: