Use generators for database access

This will make it possible to GC memory when working with huge number of items.
fossar · Jun 13, 2023 · acafe64 · acafe64
1 parent cfd7ce9
commit acafe64
Show file tree

Hide file tree

Showing 9 changed files with 32 additions and 35 deletions.
diff --git a/src/daos/Items.php b/src/daos/Items.php
@@ -155,7 +155,7 @@ public function bulkStatusUpdate(array $statuses): void {
         $this->backend->bulkStatusUpdate($statuses);
     }
 
-    public function getRaw(): array {
+    public function getRaw(): iterable {
         return $this->backend->getRaw();
     }
 

diff --git a/src/daos/ItemsInterface.php b/src/daos/ItemsInterface.php
@@ -187,9 +187,9 @@ public function bulkStatusUpdate(array $statuses): void;
     /**
      * returns raw items table contents
      *
-     * @return array<array{author: string, content: string, datetime: string, icon: string, id: int, lastseen: string, link: string, shared: int, source: int, starred: bool, thumbnail: ?string, title: string, uid: string, unread: bool, updatetime: string}> of all items
+     * @return iterable<int, array{author: string, content: string, datetime: string, icon: string, id: int, lastseen: string, link: string, shared: int, source: int, starred: bool, thumbnail: ?string, title: string, uid: string, unread: bool, updatetime: string}> of all items
      */
-    public function getRaw(): array;
+    public function getRaw(): iterable;
 
     /**
      * inserts raw data into items table

diff --git a/src/daos/Sources.php b/src/daos/Sources.php
@@ -201,8 +201,8 @@ public function validate(string $title, string $spout, array $params) {
         return true;
     }
 
-    public function getRaw(): array {
-        /** @var array<array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> */
+    public function getRaw(): iterable {
+        /** @var iterable<array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> */
         $sources = $this->backend->getRaw();
 
         return $sources;

diff --git a/src/daos/SourcesInterface.php b/src/daos/SourcesInterface.php
@@ -119,9 +119,9 @@ public function checkIfExists(string $title, string $spout, array $params): int;
     /**
      * returns raw sources table contents
      *
-     * @return array<array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> of all sources
+     * @return iterable<int, array{error: ?string, filter: string, id: int, lastentry: string, lastupdate: string, params: string, spout: string, tags: string[], title: string}> of all sources
      */
-    public function getRaw(): array;
+    public function getRaw(): iterable;
 
     /**
      * inserts raw data into sources table

diff --git a/src/daos/StatementsInterface.php b/src/daos/StatementsInterface.php
@@ -100,16 +100,15 @@ public static function bool(bool $bool): string;
     public static function datetime(\DateTime $date): string;
 
     /**
-     * Ensure row values have the appropriate PHP type. This assumes we are
-     * using buffered queries (sql results are in PHP memory);.
+     * Ensure row values have the appropriate PHP type.
      *
-     * @param array<array<mixed>> $rows array of associative array representing row results
+     * @param iterable<array<mixed>> $rows array of associative array representing row results
      * @param array<string, DatabaseInterface::PARAM_*> $expectedRowTypes associative array mapping columns to PDO types
      *
-     * @return array<array<mixed>> of associative array representing row results having
+     * @return iterable<int, array<mixed>> of associative array representing row results having
      *         expected types
      */
-    public static function ensureRowTypes(array $rows, array $expectedRowTypes): array;
+    public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): iterable;
 
     /**
      * convert string array to string for storage in table row

diff --git a/src/daos/mysql/Items.php b/src/daos/mysql/Items.php
@@ -418,7 +418,7 @@ public function sync(int $sinceId, DateTime $notBefore, DateTime $since, int $ho
      * @return int lowest id of interest
      */
     public function lowestIdOfInterest(): int {
-        $lowest = static::$stmt::ensureRowTypes(
+        $lowests = static::$stmt::ensureRowTypes(
             $this->database->exec(
                 'SELECT id FROM ' . $this->configuration->dbPrefix . 'items AS items
                  WHERE ' . static::$stmt::isTrue('unread') .
@@ -427,8 +427,8 @@ public function lowestIdOfInterest(): int {
             ),
             ['id' => DatabaseInterface::PARAM_INT]
         );
-        if ($lowest) {
-            return $lowest[0]['id'];
+        foreach ($lowests as $lowest) {
+            return $lowest['id'];
         }
 
         return 0;
@@ -687,7 +687,7 @@ public function bulkStatusUpdate(array $statuses): void {
         }
     }
 
-    public function getRaw(): array {
+    public function getRaw(): iterable {
         $stmt = static::$stmt;
         $items = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'items');
 

diff --git a/src/daos/mysql/Sources.php b/src/daos/mysql/Sources.php
@@ -313,7 +313,7 @@ public function checkIfExists(string $title, string $spout, array $params): int
         return 0;
     }
 
-    public function getRaw(): array {
+    public function getRaw(): iterable {
         $stmt = static::$stmt;
         $sources = $this->database->exec('SELECT * FROM ' . $this->configuration->dbPrefix . 'sources');
 

diff --git a/src/daos/mysql/Statements.php b/src/daos/mysql/Statements.php
@@ -138,17 +138,16 @@ public static function datetime(\DateTime $date): string {
     }
 
     /**
-     * Ensure row values have the appropriate PHP type. This assumes we are
-     * using buffered queries (sql results are in PHP memory).
+     * Ensure row values have the appropriate PHP type.
      *
-     * @param array<array<mixed>> $rows array of associative array representing row results
+     * @param iterable<array<mixed>> $rows array of associative array representing row results
      * @param array<string, DatabaseInterface::PARAM_*> $expectedRowTypes associative array mapping columns to PDO types
      *
-     * @return array<array<mixed>> of associative array representing row results having
+     * @return iterable<int, array<mixed>> of associative array representing row results having
      *         expected types
      */
-    public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
-        foreach ($rows as $rowIndex => $row) {
+    public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): iterable {
+        foreach ($rows as $row) {
             foreach ($expectedRowTypes as $columnIndex => $type) {
                 if (array_key_exists($columnIndex, $row)) {
                     if ($type & DatabaseInterface::PARAM_NULL) {
@@ -185,13 +184,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
                             $value = null;
                     }
                     if ($value !== null) {
-                        $rows[$rowIndex][$columnIndex] = $value;
+                        $row[$columnIndex] = $value;
                     }
                 }
             }
-        }
 
-        return $rows;
+            yield $row;
+        }
     }
 
     /**

diff --git a/src/daos/pgsql/Statements.php b/src/daos/pgsql/Statements.php
@@ -74,17 +74,16 @@ public static function csvRowMatches(string $column, string $value): string {
     }
 
     /**
-     * Ensure row values have the appropriate PHP type. This assumes we are
-     * using buffered queries (sql results are in PHP memory).
+     * Ensure row values have the appropriate PHP type.
      *
-     * @param array<array<mixed>> $rows array of associative array representing row results
+     * @param iterable<array<mixed>> $rows array of associative array representing row results
      * @param array<string, DatabaseInterface::PARAM_*> $expectedRowTypes associative array mapping columns to PDO types
      *
-     * @return array<array<mixed>> of associative array representing row results having
+     * @return iterable<int, array<mixed>> of associative array representing row results having
      *         expected types
      */
-    public static function ensureRowTypes(array $rows, array $expectedRowTypes): array {
-        foreach ($rows as $rowIndex => $row) {
+    public static function ensureRowTypes(iterable $rows, array $expectedRowTypes): iterable {
+        foreach ($rows as $row) {
             foreach ($expectedRowTypes as $columnIndex => $type) {
                 if (array_key_exists($columnIndex, $row)) {
                     if ($type & DatabaseInterface::PARAM_NULL) {
@@ -110,13 +109,13 @@ public static function ensureRowTypes(array $rows, array $expectedRowTypes): arr
                             $value = null;
                     }
                     if ($value !== null) {
-                        $rows[$rowIndex][$columnIndex] = $value;
+                        $row[$columnIndex] = $value;
                     }
                 }
             }
-        }
 
-        return $rows;
+            yield $row;
+        }
     }
 
     /**