Fixes for LazyList for all the harvesters.

dpla · Jan 2, 2025 · 51a47a1 · 51a47a1
1 parent fe2cdac
commit 51a47a1
Show file tree

Hide file tree

Showing 14 changed files with 31 additions and 40 deletions.
diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/CommunityWebsHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/CommunityWebsHarvester.scala
@@ -138,15 +138,15 @@ class CommunityWebsHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        for (result <- iter(inputStream))  {
+        iter(inputStream).foreach(result =>  {
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager
                 .getLogger(this.getClass)
                 .error(s"Caught exception on $inFile.", exception)
             case _ => // do nothing
           }
-        }
+        })
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/DplaJsonlFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/DplaJsonlFileHarvester.scala
@@ -147,15 +147,15 @@ class DplaJsonlFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        for (result <- iter(inputStream)) yield {
+        iter(inputStream).foreach( result => {
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager
                 .getLogger(this.getClass)
                 .error(s"Caught exception on $inFile.", exception)
             case _ => //do nothing
           }
-        }
+        })
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/FileFilters.scala b/src/main/scala/dpla/ingestion3/harvesters/file/FileFilters.scala
@@ -11,4 +11,5 @@ object FileFilters {
   val gzFilter: FileFilter = newFilter("gz")
   val xmlFilter: FileFilter = newFilter("xml")
   val zipFilter: FileFilter = newFilter("zip")
+  val txtFilter: FileFilter = newFilter("txt")
 }
diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/FlFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/FlFileHarvester.scala
@@ -146,13 +146,13 @@ class FlFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        for (result <- iter(inputStream)) yield {
+        iter(inputStream).foreach(result => {
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager.getLogger(this.getClass).error(s"Caught exception on $inFile.", exception)
             case _ => //do nothing
           }
-        }
+        })
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/HathiFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/HathiFileHarvester.scala
@@ -129,7 +129,7 @@ class HathiFileHarvester(
             )
           )
 
-        for (tarResult <- iter(inputStream)) yield {
+        iter(inputStream).foreach(tarResult => {
           handleFile(tarResult, unixEpoch) match {
             case Failure(exception) =>
               logger
@@ -139,7 +139,7 @@ class HathiFileHarvester(
                 )
             case _ => //do nothing
           }
-        }
+        })
 
         IOUtils.closeQuietly(inputStream)
       })

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/HeartlandFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/HeartlandFileHarvester.scala
@@ -145,15 +145,15 @@ class HeartlandFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        for (result <- iter(inputStream)) yield {
+        iter(inputStream).foreach(result =>
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager
                 .getLogger(this.getClass)
                 .error(s"Caught exception on $inFile.", exception)
             case _ => //do nothing
           }
-        }
+        )
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/NYPLFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/NYPLFileHarvester.scala
@@ -155,13 +155,13 @@ class NYPLFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        for (result <- iter(inputStream)) {
+        iter(inputStream).foreach( result =>
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager.getLogger(this.getClass).error(s"Caught exception on $inFile.", exception)
             case Success(_) => // do nothing
           }
-        }
+        )
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/NaraDeltaHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/NaraDeltaHarvester.scala
@@ -256,7 +256,7 @@ class NaraDeltaHarvester(
         )
       )
 
-    val recordCount = (for (tarResult <- iter(inputStream)) yield {
+    val recordCount = iter(inputStream).map(tarResult =>
       handleFile(tarResult, unixEpoch, file.getName) match {
         case Failure(exception) =>
           val logger = LogManager.getLogger(this.getClass)
@@ -266,7 +266,7 @@ class NaraDeltaHarvester(
         case Success(count) =>
           count
       }
-    }).sum
+    ).sum
 
     logger.info(s"Harvested $recordCount records from ${file.getName}")
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/NaraFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/NaraFileHarvester.scala
@@ -310,7 +310,7 @@ class NaraFileHarvester(
         )
       )
 
-    val recordCount = (for (tarResult <- iter(inputStream)) yield {
+    val recordCount = iter(inputStream).map(tarResult =>
       handleFile(tarResult, unixEpoch, file.getName) match {
         case Failure(exception) =>
           val logger = LogManager.getLogger(this.getClass)
@@ -319,8 +319,7 @@ class NaraFileHarvester(
           0
         case Success(count) =>
           count
-      }
-    }).sum
+      }).sum
 
     val logger = LogManager.getLogger(this.getClass)
     logger.info(s"Harvested $recordCount records from ${file.getName}")

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/NorthwestHeritageFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/NorthwestHeritageFileHarvester.scala
@@ -150,7 +150,7 @@ class NorthwestHeritageFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        val recordCount = (for (result <- iter(inputStream)) yield {
+        val recordCount = iter(inputStream).map(result =>
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager
@@ -160,7 +160,7 @@ class NorthwestHeritageFileHarvester(
             case Success(count) =>
               count
           }
-        }).sum
+        ).sum
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/OaiFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/OaiFileHarvester.scala
@@ -42,13 +42,12 @@ class OaiFileHarvester(
     * already be filtered by the FilenameFilter, I wonder if we even need the
     * match statement here.
     */
-  def getInputStream(file: File): Option[ZipInputStream] = {
+  def getInputStream(file: File): Option[ZipInputStream] =
     file.getName match {
       case zipName if zipName.endsWith("zip") =>
         Some(new ZipInputStream(new FileInputStream(file)))
       case _ => None
     }
-  }
 
   /** Main logic for handling individual entries in the zip.
     *
@@ -151,13 +150,13 @@ class OaiFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        iter(inputStream).foreach(result => {
+        iter(inputStream).foreach(result =>
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               logger.error(s"Caught exception on $inFile.", exception)
             case Success(_) => //do nothing
           }
-        })
+        )
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/SiFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/SiFileHarvester.scala
@@ -125,7 +125,7 @@ class SiFileHarvester(
   private def getExpectedFileCounts(inFiles: File): Map[String, String] = {
     var loadCounts = Map[String, String]()
     inFiles
-      .listFiles(new TxtFileFilter)
+      .listFiles(FileFilters.txtFilter)
       .foreach(file => {
         Using(Source
           .fromFile(file)) { source =>
@@ -234,7 +234,3 @@ class SiFileHarvester(
 
 }
 
-class TxtFileFilter extends FileFilter {
-  override def accept(pathname: File): Boolean =
-    pathname.getName.endsWith("txt")
-}
diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/VaFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/VaFileHarvester.scala
@@ -41,13 +41,12 @@ class VaFileHarvester(
     * already be filtered by the FilenameFilter, I wonder if we even need the
     * match statement here.
     */
-  def getInputStream(file: File): Option[ZipInputStream] = {
+  def getInputStream(file: File): Option[ZipInputStream] =
     file.getName match {
       case zipName if zipName.endsWith("zip") =>
         Some(new ZipInputStream(new FileInputStream(file)))
       case _ => None
     }
-  }
 
   /** Main logic for handling individual entries in the zip.
     *
@@ -113,17 +112,16 @@ class VaFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        val recordCount = (for (result <- iter(inputStream)) yield {
+        iter(inputStream).foreach(result => {
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager
                 .getLogger(this.getClass)
                 .error(s"Caught exception on $inFile.", exception)
-              0
             case Success(count) =>
               count
           }
-        }).sum
+        })
         IOUtils.closeQuietly(inputStream)
       })
 

diff --git a/src/main/scala/dpla/ingestion3/harvesters/file/VtFileHarvester.scala b/src/main/scala/dpla/ingestion3/harvesters/file/VtFileHarvester.scala
@@ -38,13 +38,12 @@ class VtFileHarvester(
     * already be filtered by the FilenameFilter, I wonder if we even need the
     * match statement here.
     */
-  def getInputStream(file: File): Option[ZipInputStream] = {
+  def getInputStream(file: File): Option[ZipInputStream] =
     file.getName match {
       case zipName if zipName.endsWith("zip") =>
         Some(new ZipInputStream(new FileInputStream(file)))
       case _ => None
     }
-  }
 
   /** Main logic for handling individual entries in the zip.
     *
@@ -109,17 +108,16 @@ class VtFileHarvester(
           .getOrElse(
             throw new IllegalArgumentException("Couldn't load ZIP files.")
           )
-        (for (result <- iter(inputStream)) yield {
+        iter(inputStream).foreach(result =>
           handleFile(result, unixEpoch) match {
             case Failure(exception) =>
               LogManager
                 .getLogger(this.getClass)
                 .error(s"Caught exception on $inFile.", exception)
-              0
-            case Success(count) =>
-              count
+
+            case Success(count) => _
           }
-        }).sum
+        )
         IOUtils.closeQuietly(inputStream)
       })