Introduce datapacks concept

dvdoug · Apr 24, 2021 · df5bd61 · df5bd61
1 parent 202678a
commit df5bd61
Show file tree

Hide file tree

Showing 10 changed files with 181 additions and 27 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -22,6 +22,17 @@ jobs:
           - highest
           - lowest
 
+        datapacks:
+          - ""
+          #- "php-coord/datapack-africa" #not released yet
+          #- "php-coord/datapack-antarctic" #not released yet
+          #- "php-coord/datapack-arctic" #not released yet
+          #- "php-coord/datapack-asia" #not released yet
+          #- "php-coord/datapack-europe" #not released yet
+          #- "php-coord/datapack-northamerica" #not released yet
+          #- "php-coord/datapack-oceania" #not released yet
+          #- "php-coord/datapack-southamerica" #not released yet
+
         include:
           - php-version: "8.0"
             dependencies: "force_latest"
@@ -69,6 +80,13 @@ jobs:
     - name: Remove manual import-only deps
       run: composer remove --dev --no-update "gasparesganga/php-shapefile";
 
+    - name: Remove datapacks
+      run: composer remove --dev --no-update "php-coord/datapack*";
+
+    - name: Add back selected datapack
+      if: matrix.datapack != ''
+      run: composer require --dev --no-update "${{ matrix.datapack }}";
+
     - name: Remove PHP-CS-Fixer if not called
       if: matrix.php-version != '7.4' || matrix.dependencies != 'highest'
       run: composer remove --dev --no-update "friendsofphp/php-cs-fixer";
@@ -94,7 +112,7 @@ jobs:
     - name: PHPUnit (Unit)
       run: |
         php -dmemory_limit=-1 vendor/phpunit/phpunit/phpunit --exclude-group=integration;
-        if [ "${{ matrix.php-version }}" = "7.4" ]; then
+        if [ "${{ matrix.php-version }}" = "7.4" ] && [ "${{ matrix.dependencies }}" = "highest" ]; then
           wget https://scrutinizer-ci.com/ocular.phar;
           php ocular.phar code-coverage:upload --format=php-clover build/coverage-phpunit/clover.xml;
         fi;

diff --git a/composer.json b/composer.json
@@ -20,8 +20,26 @@
         "friendsofphp/php-cs-fixer": "^2.18.0",
         "gasparesganga/php-shapefile": "^3.4",
         "nikic/php-parser": "^4.10",
+        "php-coord/datapack-africa": "^1.0",
+        "php-coord/datapack-antarctic": "^1.0",
+        "php-coord/datapack-arctic": "^1.0",
+        "php-coord/datapack-asia": "^1.0",
+        "php-coord/datapack-europe": "^1.0",
+        "php-coord/datapack-northamerica": "^1.0",
+        "php-coord/datapack-oceania": "^1.0",
+        "php-coord/datapack-southamerica": "^1.0",
         "phpunit/phpunit": "^9.5.0"
     },
+    "suggest": {
+        "php-coord/datapack-africa": "High-accuracy addon datapack for Africa",
+        "php-coord/datapack-antarctic": "High-accuracy addon datapack for the Antartic",
+        "php-coord/datapack-arctic": "High-accuracy addon datapack for the Arctic",
+        "php-coord/datapack-asia": "High-accuracy addon datapack for Asia",
+        "php-coord/datapack-europe": "High-accuracy addon datapack for Europe",
+        "php-coord/datapack-northamerica": "High-accuracy addon datapack for North America",
+        "php-coord/datapack-oceania": "High-accuracy addon datapack for Oceania",
+        "php-coord/datapack-southamerica": "High-accuracy addon datapack for South America"
+    },
     "config": {
         "preferred-install": {
             "*": "dist"

diff --git a/docs/coordinate_conversions.rst b/docs/coordinate_conversions.rst
@@ -8,21 +8,6 @@ Most such algorithms are not standalone, but require parameters to work - for in
 to be specified and these points differ across mapping systems even when they utilise the same algorithm. Knowing both
 the algorithm(s) to be used and the parameters used to tune them is essential to perform high-accuracy conversions.
 
-.. note::
-    If the Earth were actually the shape of an ellipsoid, algorithms could be devised so that conversions between systems
-    could be performed with no absolutely no loss of accuracy - systems would in effect be mathematically equivalent.
-
-    Unfortunately the Earth isn't an ellipsoid and coordinates are determined by individual humans
-    operating on the Earth's actual, irregular surface using instruments subject to observation error. That means
-    that conversions between CRSs are not just converting between mathematical ideals but often convert between
-    *sets of observations*. When this happens it means that the conversions between the CRSs can only ever be
-    an approximation (typically within a few metres) rather a precise transformation.
-
-    A corollary is that when dealing with CRSs that cover significant land area it is possible (and common) for mapping
-    agencies to derive multiple different parameter sets for use to obtain better accuracy depending on location. For
-    example when converting from ED50 to ETRS89 you would ideally use slightly different parameters for a point inside
-    Portugal than for a point inside Norway.
-
 PHPCoord offers two models of operation for coordinate conversion:
 
 * The hard way in which you can directly utilise one of the many implemented algorithms along with the parameters of your

diff --git a/docs/coordinate_conversions_easy.rst b/docs/coordinate_conversions_easy.rst
@@ -1,7 +1,7 @@
 The easy way
 ============
 
-When working with the built-in *Coordinate Reference Systems*, PHPCoord can usually\ [#f1]_ convert the
+When working with the built-in *Coordinate Reference Systems*, PHPCoord can usually convert the
 coordinates in any given ``Point`` to be in a different system by calling the ``convert()`` method.
 
 .. code-block:: php
@@ -17,7 +17,7 @@ relevant conversion methods with the appropriate parameters and return a new ``P
 coordinates in the target CRS.
 
 Although PHPCoord has knowledge of thousands of different conversions, this does not cover many scenarios. For example
-there is no published direct conversion between a British National Grid reference and a UTM Grid reference. In these
+there is no published direct conversion between a British National Grid reference and a WGS84 UTM Grid reference. In these
 scenarios PHPCoord can "chain" conversions it does know about to achieve the desired result, for example it would
 automatically calculate British National Grid -> OSGB36 -> WGS84 -> UTM. This ability to chain means conversion
 between almost any two CRSs is possible as long as they have a common link.
@@ -41,11 +41,12 @@ between almost any two CRSs is possible as long as they have a common link.
 
 Boundaries
 ----------
-Every CRS has a defined area ("extent") over which it operates. Some are worldwide (e.g WGS84), but most are regional
-(e.g. NAD83) or country-specific (e.g. GDA94). Trying to convert from one region/country specific CRS to
-another region/country CRS is likely to result in either inaccurate or completely nonsensical coordinates due to
-the relevant algorithms being designed and tested only over their defined areas. Plotting a point in Tokyo onto
-the New Zealand Map Grid just shouldn't be done, regardless if a chain can be found through e.g. WGS84.
+Every CRS has a defined area ("extent") over which it operates. Some are worldwide (e.g. WGS84), but most are regional
+(e.g. NAD83) or country-specific (e.g. GDA94). When converting points between different coordinate systems, it's
+important that these extents are respected, as any conversion formulas are only valid within the specified area of
+operation. For instance taking a point located in Tokyo and trying to obtain the New Zealand Map Grid coordinate for it
+is a nonsensical operation - theoretically you could find a chain of conversions (e.g. perhaps through WGS84) that would
+would produce *a* coordinate, but it would not be in the expected spot when plotting it onto an actual map.
 
 By default therefore PHPCoord will not allow such conversions to take place.
 
@@ -57,6 +58,9 @@ accuracy of extending the zone.
 If you are sure that you know what you're doing, you can set the optional parameter ``$ignoreBoundaryRestrictions``
 to ``true``.
 
+.. note::
+    For more on how PHPCoord implements boundaries/extents, :ref:`see below<accuracy>`.
+
 .. caution::
     The importance of boundary checking to ensure fidelity of results means that converting a standalone
     ``VerticalPoint`` cannot safely be done. ``VerticalPoint`` objects therefore do not have a ``->convert`` method.
@@ -102,6 +106,92 @@ For conversions from a ``UTMPoint`` back to the associated ``GeographicPoint``,
 The ``->convert()`` method *is* present on ``UTMPoint``\s and can be used as normal to convert to any desired CRS
 (including the base CRS).
 
-.. rubric:: Footnotes
+.. _accuracy:
+
+Accuracy
+--------
+If the Earth were actually the shape of an ellipsoid, algorithms could be devised so that conversions between systems
+could be performed with no absolutely no loss of accuracy - systems would in effect be mathematically equivalent.
+
+Unfortunately the Earth isn't an ellipsoid and coordinates are determined by individual humans
+operating on the Earth's actual, irregular surface using instruments subject to observation error. That means
+that conversions between CRSs are not just converting between mathematical ideals but often convert between
+*sets of observations*. When this happens it means that the conversions between the CRSs can only ever be
+an approximation (typically within a few metres) rather than exact.
+
+Extents
+^^^^^^^
+When dealing with CRSs that cover significant land area it is possible (and common) for mapping agencies to derive
+multiple different parameter sets for use to obtain better accuracy depending on location. For example when converting
+from ED50 to ETRS89 different parameters should be used for a point inside France than a point inside Denmark.
+
+In order to be able to do conversions successfully, PHPCoord therefore needs to know not just the geographical extents
+of each coordinate system but also (where different), the geographical extents of each possible set of conversion
+parameters.
+
+For a closer look at this, keeping the same example of ED50 to ETRS89, let's consider the far-west of Europe
+(Spain and Portugal). Here, the relevant authorities have produced three distinct conversions, each for a different area.
+They are shown below in orange:
+
+.. image:: images/Portugal.png
+    :alt: Portugal
+    :width: 32 %
+.. image:: images/SpainExNW.png
+    :alt: Spain excluding the North West
+    :width: 32 %
+.. image:: images/SpainNW.png
+    :alt: North West Spain
+    :width: 32 %
+
+Even after deduplication, there are over 3000 distinct geographic extents required to support the full range of
+PHPCoord's conversion abilities. Because borders are rarely straight lines, the full set of extent data for each and
+every coordinate system supported by PHPCoord would be over 100Mb of polygon data. That is far, far too much data to ask
+each and every library user to download and incorporate into their application.
+
+Grids
+^^^^^
+For more modern coordinate systems it is becoming standard for mapping agencies to go beyond simply providing tailored
+parameters for broad regions or states within their territory and to provide detailed adjustments using a grid.
+For instance in New Zealand a 20km×20km grid provided by LINZ can be used to make significantly more precise conversions
+between NZGD1949 and NZGD2000. Using the grid provides for typical accuracy of ±0.2m, compared to a 4m accuracy from the
+alternative published transformation. For obvious reasons these grid files can also be quite large, and therefore the same
+concerns about packaging them apply.
+
+"Out of the box"
+^^^^^^^^^^^^^^^^
+The core PHPCoord package attempts to strike a pragmatic balance between accuracy and download size. It does not provide
+access to any grid-based corrections, and it uses simple bounding boxes (the green lines) as extent data rather than the
+detailed orange polygons.
+
+As is obvious from looking at the Spain/Portugal images though, the areas enclosed by the green boxes have areas
+of overlap. In fact there are areas of north-east Portugal that are actually encompassed by all 3! If a coordinate
+is located within one of these overlap areas, then each of those 3 possible conversions is necessarily considered
+equally valid, and which one is ultimately utilised is undefined. The difference in resulting coordinate from any use
+of the "wrong" formula is likely to be very small (perhaps a few metres).
 
-.. [#f1] There are over 36 million possible combinations of source and target CRS. They haven't *all* been tested...
+.. note::
+    This particular example was European, but "wrong" conversions can be picked wherever there are multiple extents for
+    a given pair of coordinate systems. For example NAD27/83 conversions in North America, or SAD69 conversions in South
+    America can be similarly affected.
+
+Add-on datapacks
+^^^^^^^^^^^^^^^^
+Not every user will consider the above tradeoff to be desirable, preferring instead to have the highest-accuracy
+conversions possible even at the cost of increased disk space.
+
+PHPCoord supports this mode of operation via the use of optional add-on "datapacks". These are additional Composer
+packages (8 available), each corresponding to a different region of the world:
+
+* ``php-coord/datapack-africa``
+* ``php-coord/datapack-antarctic``
+* ``php-coord/datapack-arctic``
+* ``php-coord/datapack-asia``
+* ``php-coord/datapack-europe``
+* ``php-coord/datapack-northamerica``
+* ``php-coord/datapack-oceania``
+* ``php-coord/datapack-southamerica``
+
+Installing them provides access to the full-fidelity polygons for the relevant area of the world (at a scale of approx
+1:15000000) and also any applicable grid files allowing PHPCoord to do the best possible job.
+
+No configuration is necessary once installed.
diff --git a/docs/creating_points_projected.rst b/docs/creating_points_projected.rst
@@ -263,7 +263,7 @@ Universal Transverse Mercator (UTM)
 Although one of the most widely used applications of the Transverse Mercator projection, UTM is not actually a map
 projection. It's a *system* of map projections, and this distinction means that it does not fit neatly into the
 standard data model. Mathematically each UTM zone/hemisphere combination is its own unique projection and
-therefore to work with the data you also need to know which zone/hemisphere the coordinates are reference to.
+therefore to work with the data you also need to know which zone/hemisphere the coordinates are referenced to.
 Adding a further layer of complication is that although UTM is most commonly used alongside WGS84, it can be used with
 any Geographic CRS so that information needs to be known as well.
 

diff --git a/docs/images/Portugal.png b/docs/images/Portugal.png
diff --git a/docs/images/SpainExNW.png b/docs/images/SpainExNW.png
diff --git a/docs/images/SpainNW.png b/docs/images/SpainNW.png
diff --git a/docs/installation.rst b/docs/installation.rst
@@ -9,7 +9,18 @@ and configuring an autoloader:
 
     composer require php-coord/php-coord
 
-The code is also available to download from `GitHub`_
+For :ref:`increased accuracy<accuracy>` of conversions, you may wish to install one or more optional add-on datapacks:
+
+* ``php-coord/datapack-africa``
+* ``php-coord/datapack-antarctic``
+* ``php-coord/datapack-arctic``
+* ``php-coord/datapack-asia``
+* ``php-coord/datapack-europe``
+* ``php-coord/datapack-northamerica``
+* ``php-coord/datapack-oceania``
+* ``php-coord/datapack-southamerica``
+
+The code for the core package and the datapacks is also available to download from `GitHub`_
 
 Requirements
 ------------

diff --git a/src/EPSG/Import/EPSGImporter.php b/src/EPSG/Import/EPSGImporter.php
@@ -1245,9 +1245,25 @@ public function generateExtents(SQLite3 $sqlite): void
         echo 'Updating extents...';
         $boundingBoxOnly = $this->sourceDir . '/Geometry/Extents/BoundingBoxOnly/';
         $builtInFull = $this->sourceDir . '/Geometry/Extents/';
+        $africa = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-africa/src/Geometry/Extents/';
+        $antarctic = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-antarctic/src/Geometry/Extents/';
+        $arctic = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-arctic/src/Geometry/Extents/';
+        $asia = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-asia/src/Geometry/Extents/';
+        $europe = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-europe/src/Geometry/Extents/';
+        $northAmerica = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-northamerica/src/Geometry/Extents/';
+        $southAmerica = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-southamerica/src/Geometry/Extents/';
+        $oceania = $this->sourceDir . '/../vendor/php-coord/php-coord-datapack-oceania/src/Geometry/Extents/';
 
         array_map('unlink', glob($boundingBoxOnly . '*.php'));
         array_map('unlink', glob($builtInFull . '*.php'));
+        array_map('unlink', glob($africa . '*.php'));
+        array_map('unlink', glob($antarctic . '*.php'));
+        array_map('unlink', glob($arctic . '*.php'));
+        array_map('unlink', glob($asia . '*.php'));
+        array_map('unlink', glob($europe . '*.php'));
+        array_map('unlink', glob($northAmerica . '*.php'));
+        array_map('unlink', glob($southAmerica . '*.php'));
+        array_map('unlink', glob($oceania . '*.php'));
 
         $sql = "
             SELECT e.extent_code
@@ -1323,34 +1339,50 @@ public function generateExtents(SQLite3 $sqlite): void
                 case 'Africa':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($africa . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($africa . "Extent{$extentCode}.php");
                     break;
                 case 'Antarctic':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($antarctic . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($antarctic . "Extent{$extentCode}.php");
                     break;
                 case 'Arctic':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($arctic . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($arctic . "Extent{$extentCode}.php");
                     break;
                 case 'Asia-ExFSU':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($asia . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($asia . "Extent{$extentCode}.php");
                     break;
                 case 'Australasia and Oceania':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($oceania . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($oceania . "Extent{$extentCode}.php");
                     break;
                 case 'Europe-FSU':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($europe . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($europe . "Extent{$extentCode}.php");
                     break;
                 case 'North America':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($northAmerica . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($northAmerica . "Extent{$extentCode}.php");
                     break;
                 case 'South America':
                     file_put_contents($boundingBoxOnly . "Extent{$extentCode}.php", $exportSimple);
                     $this->csFixFile($boundingBoxOnly . "Extent{$extentCode}.php");
+                    file_put_contents($southAmerica . "Extent{$extentCode}.php", $exportFull);
+                    $this->csFixFile($southAmerica . "Extent{$extentCode}.php");
                     break;
                 default:
                     throw new Exception("Unknown region: {$region}");