handle PDL 2.064+ types like signed byte - fix #5

PDLPorters · Oct 9, 2024 · 8f5f19c · 8f5f19c
1 parent 4e62f7b
commit 8f5f19c
Show file tree

Hide file tree

Showing 7 changed files with 94 additions and 75 deletions.
diff --git a/Changes b/Changes
@@ -1,3 +1,5 @@
+- handle PDL 2.064+ types like signed byte (#5) - thanks @a-shahba for report
+
 0.75 2021-08-08
 - fix t/attribPDL.t to work on longdouble - thanks @eserte for report
 

diff --git a/HDF5/Dataset.pm b/HDF5/Dataset.pm
@@ -4,6 +4,7 @@ use Carp;
 
 use strict;
 use Config;
+use PDL::Core::Dev;
 
 # Global mapping variables
 our ($H5T_STRING, $H5T_REFERENCE, %PDLtoHDF5internalTypeMapping, %HDF5toPDLfileMapping, %PDLtoHDF5fileMapping);
@@ -181,38 +182,50 @@ B<Usage:>
 #   Mapping of PDL types to what HDF5 calls them while we are dealing with them 
 #   outside of the HDF5 file.
 %PDLtoHDF5internalTypeMapping = (
-	$PDL::Types::PDL_B	=>	PDL::IO::HDF5::H5T_NATIVE_CHAR(),
-	$PDL::Types::PDL_S	=> 	PDL::IO::HDF5::H5T_NATIVE_SHORT(),
-	$PDL::Types::PDL_L	=> 	PDL::IO::HDF5::H5T_NATIVE_INT(),
-	$PDL::Types::PDL_LL	=> 	PDL::IO::HDF5::H5T_NATIVE_LLONG(),
-        $PDL::Types::PDL_F	=>	PDL::IO::HDF5::H5T_NATIVE_FLOAT(),
-	$PDL::Types::PDL_D	=>	PDL::IO::HDF5::H5T_NATIVE_DOUBLE(),
+  $PDL::Types::PDL_SB => PDL::IO::HDF5::H5T_NATIVE_INT8(),
+  $PDL::Types::PDL_B => PDL::IO::HDF5::H5T_NATIVE_UINT8(),
+  $PDL::Types::PDL_S => PDL::IO::HDF5::H5T_NATIVE_INT16(),
+  $PDL::Types::PDL_US => PDL::IO::HDF5::H5T_NATIVE_UINT16(),
+  $PDL::Types::PDL_L => PDL::IO::HDF5::H5T_NATIVE_INT32(),
+  $PDL::Types::PDL_UL => PDL::IO::HDF5::H5T_NATIVE_UINT32(),
+  $PDL::Types::PDL_ULL => PDL::IO::HDF5::H5T_NATIVE_UINT64(),
+  $PDL::Types::PDL_LL => PDL::IO::HDF5::H5T_NATIVE_INT64(),
+  $PDL::Types::PDL_F => PDL::IO::HDF5::H5T_NATIVE_FLOAT(),
+  $PDL::Types::PDL_D => PDL::IO::HDF5::H5T_NATIVE_DOUBLE(),
+  $PDL::Types::PDL_LD => PDL::IO::HDF5::H5T_NATIVE_LDOUBLE(),
+  # no HDF5 direct support for long doubles, nor complex numbers
+  # deliberately not supporting indx as likely to cause mayhem
 );
 
 #   Mapping of PDL types to what types they are written to in the HDF5 file.
-#   For 64 Bit machines, we might need to modify this with some smarts to determine
-#   what is appropriate
 my %PDLtoHDF5fileMapping;
-if ( $Config{byteorder} =~ m/4321$/ ) {
-    # Big endian.
-    %PDLtoHDF5fileMapping = (
-	$PDL::Types::PDL_B	=>	PDL::IO::HDF5::H5T_STD_I8BE(),
-	$PDL::Types::PDL_S	=> 	PDL::IO::HDF5::H5T_STD_I16BE(),
-	$PDL::Types::PDL_L	=> 	PDL::IO::HDF5::H5T_STD_I32BE(),
-	$PDL::Types::PDL_LL	=> 	PDL::IO::HDF5::H5T_STD_I64BE(),
-        $PDL::Types::PDL_F	=>	PDL::IO::HDF5::H5T_IEEE_F32BE(),
-	$PDL::Types::PDL_D	=>	PDL::IO::HDF5::H5T_IEEE_F64BE(),
-	);
+if ( isbigendian() ) {
+  %PDLtoHDF5fileMapping = (
+    $PDL::Types::PDL_SB => PDL::IO::HDF5::H5T_STD_I8BE(),
+    $PDL::Types::PDL_B => PDL::IO::HDF5::H5T_STD_U8BE(),
+    $PDL::Types::PDL_S => PDL::IO::HDF5::H5T_STD_I16BE(),
+    $PDL::Types::PDL_US => PDL::IO::HDF5::H5T_STD_U16BE(),
+    $PDL::Types::PDL_L => PDL::IO::HDF5::H5T_STD_I32BE(),
+    $PDL::Types::PDL_UL => PDL::IO::HDF5::H5T_STD_U32BE(),
+    $PDL::Types::PDL_ULL => PDL::IO::HDF5::H5T_STD_U64BE(),
+    $PDL::Types::PDL_LL => PDL::IO::HDF5::H5T_STD_I64BE(),
+    $PDL::Types::PDL_F => PDL::IO::HDF5::H5T_IEEE_F32BE(),
+    $PDL::Types::PDL_D => PDL::IO::HDF5::H5T_IEEE_F64BE(),
+  );
 } else {
-    # Little endian.
-    %PDLtoHDF5fileMapping = (
-	$PDL::Types::PDL_B	=>	PDL::IO::HDF5::H5T_STD_I8LE(),
-	$PDL::Types::PDL_S	=> 	PDL::IO::HDF5::H5T_STD_I16LE(),
-	$PDL::Types::PDL_L	=> 	PDL::IO::HDF5::H5T_STD_I32LE(),
-	$PDL::Types::PDL_LL	=> 	PDL::IO::HDF5::H5T_STD_I64LE(),
-        $PDL::Types::PDL_F	=>	PDL::IO::HDF5::H5T_IEEE_F32LE(),
-	$PDL::Types::PDL_D	=>	PDL::IO::HDF5::H5T_IEEE_F64LE(),
-	);
+  # Little endian.
+  %PDLtoHDF5fileMapping = (
+    $PDL::Types::PDL_SB => PDL::IO::HDF5::H5T_STD_I8LE(),
+    $PDL::Types::PDL_B => PDL::IO::HDF5::H5T_STD_U8LE(),
+    $PDL::Types::PDL_S => PDL::IO::HDF5::H5T_STD_I16LE(),
+    $PDL::Types::PDL_US => PDL::IO::HDF5::H5T_STD_U16LE(),
+    $PDL::Types::PDL_L => PDL::IO::HDF5::H5T_STD_I32LE(),
+    $PDL::Types::PDL_UL => PDL::IO::HDF5::H5T_STD_U32LE(),
+    $PDL::Types::PDL_ULL => PDL::IO::HDF5::H5T_STD_U64LE(),
+    $PDL::Types::PDL_LL => PDL::IO::HDF5::H5T_STD_I64LE(),
+    $PDL::Types::PDL_F => PDL::IO::HDF5::H5T_IEEE_F32LE(),
+    $PDL::Types::PDL_D => PDL::IO::HDF5::H5T_IEEE_F64LE(),
+  );
 }
 
 
@@ -386,24 +399,26 @@ to the following table.
  HDF5 File Type				PDL Type
  ------------------------               -----------------
  PDL::IO::HDF5::H5T_C_S1()	=>      PDL::Char Object    (Special Case for Char Strings)
- PDL::IO::HDF5::H5T_STD_I8BE()	=> 	$PDL::Types::PDL_B
- PDL::IO::HDF5::H5T_STD_I8LE()	=> 	$PDL::Types::PDL_B,
- PDL::IO::HDF5::H5T_STD_U8BE()	=> 	$PDL::Types::PDL_S,
- PDL::IO::HDF5::H5T_STD_U8LE()	=> 	$PDL::Types::PDL_S,
- PDL::IO::HDF5::H5T_STD_I16BE()	=> 	$PDL::Types::PDL_S,
- PDL::IO::HDF5::H5T_STD_I16LE()	=> 	$PDL::Types::PDL_S,
- PDL::IO::HDF5::H5T_STD_U16BE()	=> 	$PDL::Types::PDL_L,
- PDL::IO::HDF5::H5T_STD_U16LE()	=> 	$PDL::Types::PDL_L,
- PDL::IO::HDF5::H5T_STD_I32BE()	=> 	$PDL::Types::PDL_L,
- PDL::IO::HDF5::H5T_STD_I32LE()	=> 	$PDL::Types::PDL_L,
- PDL::IO::HDF5::H5T_STD_U32LE()	=> 	$PDL::Types::PDL_LL,
- PDL::IO::HDF5::H5T_STD_U32BE()	=> 	$PDL::Types::PDL_LL,
- PDL::IO::HDF5::H5T_STD_I64LE()	=> 	$PDL::Types::PDL_LL,
- PDL::IO::HDF5::H5T_STD_I64BE()	=> 	$PDL::Types::PDL_LL,
- PDL::IO::HDF5::H5T_IEEE_F32BE()=>	$PDL::Types::PDL_F,
- PDL::IO::HDF5::H5T_IEEE_F32LE()=>	$PDL::Types::PDL_F,
- PDL::IO::HDF5::H5T_IEEE_F64BE()=>	$PDL::Types::PDL_D,
- PDL::IO::HDF5::H5T_IEEE_F64LE()=>	$PDL::Types::PDL_D
+ PDL::IO::HDF5::H5T_STD_I8BE()   => $PDL::Types::PDL_SB,
+ PDL::IO::HDF5::H5T_STD_I8LE()   => $PDL::Types::PDL_SB,
+ PDL::IO::HDF5::H5T_STD_U8BE()   => $PDL::Types::PDL_B,
+ PDL::IO::HDF5::H5T_STD_U8LE()   => $PDL::Types::PDL_B,
+ PDL::IO::HDF5::H5T_STD_I16BE()  => $PDL::Types::PDL_S,
+ PDL::IO::HDF5::H5T_STD_I16LE()  => $PDL::Types::PDL_S,
+ PDL::IO::HDF5::H5T_STD_U16BE()  => $PDL::Types::PDL_U,
+ PDL::IO::HDF5::H5T_STD_U16LE()  => $PDL::Types::PDL_U,
+ PDL::IO::HDF5::H5T_STD_I32BE()  => $PDL::Types::PDL_L,
+ PDL::IO::HDF5::H5T_STD_I32LE()  => $PDL::Types::PDL_L,
+ PDL::IO::HDF5::H5T_STD_U32LE()  => $PDL::Types::PDL_UL,
+ PDL::IO::HDF5::H5T_STD_U32BE()  => $PDL::Types::PDL_UL,
+ PDL::IO::HDF5::H5T_STD_I64LE()  => $PDL::Types::PDL_LL,
+ PDL::IO::HDF5::H5T_STD_I64BE()  => $PDL::Types::PDL_LL,
+ PDL::IO::HDF5::H5T_STD_U64LE()  => $PDL::Types::PDL_ULL,
+ PDL::IO::HDF5::H5T_STD_U64BE()  => $PDL::Types::PDL_ULL,
+ PDL::IO::HDF5::H5T_IEEE_F32BE() => $PDL::Types::PDL_F,
+ PDL::IO::HDF5::H5T_IEEE_F32LE() => $PDL::Types::PDL_F,
+ PDL::IO::HDF5::H5T_IEEE_F64BE() => $PDL::Types::PDL_D,
+ PDL::IO::HDF5::H5T_IEEE_F64LE() => $PDL::Types::PDL_D
 
 For HDF5 File types not in this table, this method will attempt to
 map it to the default PDL type PDL_D.
@@ -423,27 +438,27 @@ size of all the strings in the array.
 
 #############################################################################
 # Mapping of HDF5 file types to PDL types
-#   For 64 Bit machines, we might need to modify this with some smarts to determine
-#   what is appropriate
 %HDF5toPDLfileMapping = (
-	 PDL::IO::HDF5::H5T_STD_I8BE()	=> 	$PDL::Types::PDL_B,
-	 PDL::IO::HDF5::H5T_STD_I8LE()	=> 	$PDL::Types::PDL_B,
-	 PDL::IO::HDF5::H5T_STD_U8BE()	=> 	$PDL::Types::PDL_S,
-	 PDL::IO::HDF5::H5T_STD_U8LE()	=> 	$PDL::Types::PDL_S,
-	 PDL::IO::HDF5::H5T_STD_I16BE()	=> 	$PDL::Types::PDL_S,
-	 PDL::IO::HDF5::H5T_STD_I16LE()	=> 	$PDL::Types::PDL_S,
-	 PDL::IO::HDF5::H5T_STD_U16BE()	=> 	$PDL::Types::PDL_L,
-	 PDL::IO::HDF5::H5T_STD_U16LE()	=> 	$PDL::Types::PDL_L,
-	 PDL::IO::HDF5::H5T_STD_I32BE()	=> 	$PDL::Types::PDL_L,
-	 PDL::IO::HDF5::H5T_STD_I32LE()	=> 	$PDL::Types::PDL_L,
-	 PDL::IO::HDF5::H5T_STD_U32LE()	=> 	$PDL::Types::PDL_LL,
-	 PDL::IO::HDF5::H5T_STD_U32BE()	=> 	$PDL::Types::PDL_LL,
-	 PDL::IO::HDF5::H5T_STD_I64LE()	=> 	$PDL::Types::PDL_LL,
-	 PDL::IO::HDF5::H5T_STD_I64BE()	=> 	$PDL::Types::PDL_LL,
-	 PDL::IO::HDF5::H5T_IEEE_F32BE()	=>	$PDL::Types::PDL_F,
-	 PDL::IO::HDF5::H5T_IEEE_F32LE()	=>	$PDL::Types::PDL_F,
-	 PDL::IO::HDF5::H5T_IEEE_F64BE()	=>	$PDL::Types::PDL_D,
-	 PDL::IO::HDF5::H5T_IEEE_F64LE()	=>	$PDL::Types::PDL_D
+ PDL::IO::HDF5::H5T_STD_I8BE()   => $PDL::Types::PDL_SB,
+ PDL::IO::HDF5::H5T_STD_I8LE()   => $PDL::Types::PDL_SB,
+ PDL::IO::HDF5::H5T_STD_U8BE()   => $PDL::Types::PDL_B,
+ PDL::IO::HDF5::H5T_STD_U8LE()   => $PDL::Types::PDL_B,
+ PDL::IO::HDF5::H5T_STD_I16BE()  => $PDL::Types::PDL_S,
+ PDL::IO::HDF5::H5T_STD_I16LE()  => $PDL::Types::PDL_S,
+ PDL::IO::HDF5::H5T_STD_U16BE()  => $PDL::Types::PDL_U,
+ PDL::IO::HDF5::H5T_STD_U16LE()  => $PDL::Types::PDL_U,
+ PDL::IO::HDF5::H5T_STD_I32BE()  => $PDL::Types::PDL_L,
+ PDL::IO::HDF5::H5T_STD_I32LE()  => $PDL::Types::PDL_L,
+ PDL::IO::HDF5::H5T_STD_U32LE()  => $PDL::Types::PDL_UL,
+ PDL::IO::HDF5::H5T_STD_U32BE()  => $PDL::Types::PDL_UL,
+ PDL::IO::HDF5::H5T_STD_I64LE()  => $PDL::Types::PDL_LL,
+ PDL::IO::HDF5::H5T_STD_I64BE()  => $PDL::Types::PDL_LL,
+ PDL::IO::HDF5::H5T_STD_U64LE()  => $PDL::Types::PDL_ULL,
+ PDL::IO::HDF5::H5T_STD_U64BE()  => $PDL::Types::PDL_ULL,
+ PDL::IO::HDF5::H5T_IEEE_F32BE() => $PDL::Types::PDL_F,
+ PDL::IO::HDF5::H5T_IEEE_F32LE() => $PDL::Types::PDL_F,
+ PDL::IO::HDF5::H5T_IEEE_F64BE() => $PDL::Types::PDL_D,
+ PDL::IO::HDF5::H5T_IEEE_F64LE() => $PDL::Types::PDL_D
 );
 
 $H5T_STRING    = PDL::IO::HDF5::H5T_STRING   (); #HDF5 string type
@@ -1564,4 +1579,3 @@ sub nameGet{
 
 
 1;
-
diff --git a/MANIFEST b/MANIFEST
@@ -1,3 +1,4 @@
+Changes
 COPYRIGHT
 hdf5.pd
 HDF5/Dataset.pm
@@ -7,11 +8,11 @@ Makefile.PL
 MANIFEST			This list of files
 MANIFEST.SKIP
 README
-Changes
 t/attribPDL.t
 t/file.t
 t/group.t
 t/reference.t
+t/sbyte.hdf5
 t/total.t
 t/unlink.t
 t/vlenString.t
@@ -20,4 +21,3 @@ t/ztotal_index.t
 tkviewtest
 typemap
 varlen.hdf5
-xt/01-checkmanifest.t
diff --git a/Makefile.PL b/Makefile.PL
@@ -133,13 +133,13 @@ my $meta_merge = {
    'prereqs' => {
       'runtime' => {
          'requires' => {
-            'PDL' => '2.004',
+            'PDL' => '2.064',
          },
       },
       'build' => {
          'requires' => {
             'ExtUtils::MakeMaker' => '0',
-            'PDL' => '2.007',
+            'PDL' => '2.064',
          },
       },
       test => {
@@ -150,7 +150,7 @@ my $meta_merge = {
       'configure' => {
          'requires' => {
             'ExtUtils::MakeMaker' => '0',
-            'PDL' => '2.004',
+            'PDL' => '2.064', # new types like ULL, CLD
          },
       },
    },
@@ -171,7 +171,6 @@ my $meta_merge = {
       'version' => '2',
       'url'     => 'http://search.cpan.org/perldoc?CPAN::Meta::Spec',
    },
-   'generated_by' => 'Chris Marshall',
 };
 # create GENERATED subdir with *.pm files during 'make dist' (to make metacpan.org happy)
 my $preop = '$(PERL) -I$(INST_ARCHLIB) -I$(INST_LIB) -MPDL::Core::Dev -e pdlpp_mkgen $(DISTVNAME)';

diff --git a/t/group.t b/t/group.t
@@ -69,11 +69,8 @@ is( scalar(@groups), 0 );
 my $dataset = new PDL::IO::HDF5::Dataset( 'name'=> 'data1', parent => $rootGroup,
 					fileObj => $hdfobj);
 my $pdl = sequence(5,4);
-
 ok( $dataset->set($pdl, unlimited => 1) );
-
 my $pdl2 = $dataset->get;
-
 ok( (($pdl - $pdl2)->sum) < .001 );
 
 # Set attribute for dataset

diff --git a/t/sbyte.hdf5 b/t/sbyte.hdf5
diff --git a/t/xData.t b/t/xData.t
@@ -4,6 +4,7 @@ use warnings;
 use PDL;
 use PDL::IO::HDF5;
 use Test::More;
+use File::Spec::Functions;
 
 my $filename = "xData.hdf5";
 # get rid of filename if it already exists
@@ -35,4 +36,10 @@ is( "$xdata", $expected);
 # clean up file
 unlink $filename if( -e $filename);
 
+$hdf5 = PDL::IO::HDF5->new(catfile(qw(t sbyte.hdf5)));
+$dataset = $hdf5->dataset('data2');
+my $got = $dataset->get;
+$expected = pdl(-127, 127); # deliberately type double
+ok( (($got - $expected)->sum) < .001 ) or diag "got=$got\nexpected=$expected";
+
 done_testing;