Use generators for database access

This will make it possible to GC memory when working with huge number of items.
fossar · Feb 15, 2023 · 17166cc · 17166cc
1 parent 3d3df79
commit 17166cc
Show file tree

Hide file tree

Showing 7 changed files with 38 additions and 35 deletions.
diff --git a/src/daos/ItemsInterface.php b/src/daos/ItemsInterface.php
@@ -6,6 +6,7 @@
 
 use DateTime;
 use DateTimeImmutable;
+use Generator;
 
 /**
  * Interface describing concrete DAO for working with items.
@@ -184,9 +185,9 @@ public function bulkStatusUpdate(array $statuses): void;
     /**
      * returns raw items table contents
      *
-     * @return array[] of all items
+     * @return Generator<int, array, void, void> of all items
      */
-    public function getRaw(): array;
+    public function getRaw(): Generator;
 
     /**
      * inserts raw data into items table

diff --git a/src/daos/SourcesInterface.php b/src/daos/SourcesInterface.php
@@ -4,6 +4,8 @@
 
 namespace daos;
 
+use Generator;
+
 /**
  * Interface describing concrete DAO for working with sources.
  */
@@ -110,9 +112,9 @@ public function checkIfExists(string $title, string $spout, array $params): int;
     /**
      * returns raw sources table contents
      *
-     * @return array[] of all sources
+     * @return Generator<int, array, void, void> of all sources
      */
-    public function getRaw(): array;
+    public function getRaw(): Generator;
 
     /**
      * inserts raw data into sources table

diff --git a/src/daos/StatementsInterface.php b/src/daos/StatementsInterface.php
@@ -4,6 +4,8 @@
 
 namespace daos;
 
+use Generator;
+
 /**
  * Interface for class providing SQL helpers.
  */
@@ -100,16 +102,14 @@ public static function bool(bool $bool): string;
     public static function datetime(\DateTime $date): string;
 
     /**
-     * Ensure row values have the appropriate PHP type. This assumes we are
-     * using buffered queries (sql results are in PHP memory);.
+     * Ensure row values have the appropriate PHP type.
      *
-     * @param array $rows array of associative array representing row results
+     * @param iterable $rows array of associative array representing row results
      * @param array $expectedRowTypes associative array mapping columns to PDO types
      *
-     * @return array of associative array representing row results having
-     *         expected types
+     * @return Generator<int, array, void, void> of associative array representing row results having expected types
      */
-    public static function ensureRowTypes(array $rows, array $expectedRowTypes): array;
+    public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): Generator;
 
     /**
      * convert string array to string for storage in table row

diff --git a/src/daos/mysql/Items.php b/src/daos/mysql/Items.php
@@ -8,6 +8,7 @@
 use daos\ItemOptions;
 use DateTime;
 use DateTimeImmutable;
+use Generator;
 use helpers\Configuration;
 use function helpers\functions\map;
 use Monolog\Logger;
@@ -407,16 +408,16 @@ public function sync(int $sinceId, DateTime $notBefore, DateTime $since, int $ho
      * @return int lowest id of interest
      */
     public function lowestIdOfInterest(): int {
-        $lowest = static::$stmt::ensureRowTypes(
+        $lowests = static::$stmt::ensureRowTypes(
             $this->database->exec(
                 'SELECT id FROM ' . $this->configuration->dbPrefix . 'items AS items
                  WHERE ' . static::$stmt::isTrue('unread') .
                     ' OR ' . static::$stmt::isTrue('starred') .
                 ' ORDER BY id LIMIT 1'),
             ['id' => DatabaseInterface::PARAM_INT]
         );
-        if ($lowest) {
-            return $lowest[0]['id'];
+        foreach ($lowests as $lowest) {
+            return $lowest['id'];
         }
 
         return 0;
@@ -667,7 +668,7 @@ public function bulkStatusUpdate(array $statuses): void {
     /**
      * {@inheritdoc}
      */
-    public function getRaw(): array {
+    public function getRaw(): Generator {
         $stmt = static::$stmt;
         $items = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'items');
 

diff --git a/src/daos/mysql/Sources.php b/src/daos/mysql/Sources.php
@@ -5,6 +5,7 @@
 namespace daos\mysql;
 
 use daos\DatabaseInterface;
+use Generator;
 use helpers\Configuration;
 use function helpers\Functions\map;
 
@@ -281,7 +282,7 @@ public function checkIfExists(string $title, string $spout, array $params): int
     /**
      * {@inheritdoc}
      */
-    public function getRaw(): array {
+    public function getRaw(): Generator {
         $stmt = static::$stmt;
         $sources = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'sources');
 

diff --git a/src/daos/mysql/Statements.php b/src/daos/mysql/Statements.php
@@ -5,6 +5,7 @@
 namespace daos\mysql;
 
 use daos\DatabaseInterface;
+use Generator;
 
 /**
  * MySQL specific statements
@@ -138,17 +139,15 @@ public static function datetime(\DateTime $date): string {
     }
 
     /**
-     * Ensure row values have the appropriate PHP type. This assumes we are
-     * using buffered queries (sql results are in PHP memory).
+     * Ensure row values have the appropriate PHP type.
      *
-     * @param array $rows array of associative array representing row results
+     * @param iterable $rows array of associative array representing row results
      * @param array $expectedRowTypes associative array mapping columns to PDO types
      *
-     * @return array of associative array representing row results having
-     *         expected types
+     * @return Generator<int, array, void, void> of associative array representing row results having expected types
      */
-    public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
-        foreach ($rows as $rowIndex => $row) {
+    public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): Generator {
+        foreach ($rows as $row) {
             foreach ($expectedRowTypes as $columnIndex => $type) {
                 if (array_key_exists($columnIndex, $row)) {
                     if ($type & DatabaseInterface::PARAM_NULL) {
@@ -185,13 +184,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
                             $value = null;
                     }
                     if ($value !== null) {
-                        $rows[$rowIndex][$columnIndex] = $value;
+                        $row[$columnIndex] = $value;
                     }
                 }
             }
-        }
 
-        return $rows;
+            yield $row;
+        }
     }
 
     /**

diff --git a/src/daos/pgsql/Statements.php b/src/daos/pgsql/Statements.php
@@ -5,6 +5,7 @@
 namespace daos\pgsql;
 
 use daos\DatabaseInterface;
+use Generator;
 
 /**
  * PostgreSQL specific statements
@@ -74,17 +75,15 @@ public static function csvRowMatches(string $column, string $value): string {
     }
 
     /**
-     * Ensure row values have the appropriate PHP type. This assumes we are
-     * using buffered queries (sql results are in PHP memory).
+     * Ensure row values have the appropriate PHP type.
      *
-     * @param array $rows array of associative array representing row results
+     * @param iterable $rows array of associative array representing row results
      * @param array $expectedRowTypes associative array mapping columns to PDO types
      *
-     * @return array of associative array representing row results having
-     *         expected types
+     * @return Generator<int, array, void, void> of associative array representing row results having expected types
      */
-    public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
-        foreach ($rows as $rowIndex => $row) {
+    public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): Generator {
+        foreach ($rows as $row) {
             foreach ($expectedRowTypes as $columnIndex => $type) {
                 if (array_key_exists($columnIndex, $row)) {
                     if ($type & DatabaseInterface::PARAM_NULL) {
@@ -110,13 +109,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
                             $value = null;
                     }
                     if ($value !== null) {
-                        $rows[$rowIndex][$columnIndex] = $value;
+                        $row[$columnIndex] = $value;
                     }
                 }
             }
-        }
 
-        return $rows;
+            yield $row;
+        }
     }
 
     /**