Package org.gorpipe.spark

Class GorSparkUtilities

  • public class GorSparkUtilities
extends java.lang.Object

    • Method Summary

      All Methods Static Methods Concrete Methods 
      Modifier and Type Method Description
      static void closePySpark()  
      static java.util.Optional<java.lang.String> getJupyterPath()  
      static org.apache.spark.api.python.Py4JServer getPyServer()  
      static int getPyServerPort()  
      static java.lang.String getPyServerSecret()  
      static org.apache.spark.api.r.RBackend getRBackend()  
      static java.util.Optional<java.lang.String> getRPath()  
      static org.gorpipe.gor.monitor.GorMonitor getSparkGorMonitor​(java.lang.String jobId, java.lang.String redisUri, java.lang.String key)  
      static java.lang.String getSparkGorRedisUri()  
      static org.apache.spark.sql.SparkSession getSparkSession()  
      static org.apache.spark.sql.SparkSession getSparkSession​(int workers)  
      static org.apache.spark.api.python.Py4JServer initPy4jServer()  
      static void initPySpark​(java.util.Optional<java.lang.String> standaloneRoot)  
      static java.util.Optional<java.lang.String> parseDriverId()  
      static void shutdownPy4jServer()  
      static java.util.List<org.apache.spark.sql.Row> stream2SparkRowList​(java.util.stream.Stream<org.gorpipe.gor.model.Row> str, org.apache.spark.sql.types.StructType schema)  

      • Methods inherited from class java.lang.Object

        clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

    • Method Detail

      • getPyServer

        public static org.apache.spark.api.python.Py4JServer getPyServer()

      • getRBackend

        public static org.apache.spark.api.r.RBackend getRBackend()

      • getPyServerPort

        public static int getPyServerPort()

      • getPyServerSecret

        public static java.lang.String getPyServerSecret()

      • getJupyterPath

        public static java.util.Optional<java.lang.String> getJupyterPath()

      • parseDriverId

        public static java.util.Optional<java.lang.String> parseDriverId()

      • getRPath

        public static java.util.Optional<java.lang.String> getRPath()

      • closePySpark

        public static void closePySpark()

      • shutdownPy4jServer

        public static void shutdownPy4jServer()

      • initPy4jServer

        public static org.apache.spark.api.python.Py4JServer initPy4jServer()

      • initPySpark

        public static void initPySpark​(java.util.Optional<java.lang.String> standaloneRoot)

      • getSparkGorRedisUri

        public static java.lang.String getSparkGorRedisUri()

      • getSparkGorMonitor

        public static org.gorpipe.gor.monitor.GorMonitor getSparkGorMonitor​(java.lang.String jobId,
                                                                    java.lang.String redisUri,
                                                                    java.lang.String key)

      • getSparkSession

        public static org.apache.spark.sql.SparkSession getSparkSession()

      • getSparkSession

        public static org.apache.spark.sql.SparkSession getSparkSession​(int workers)

      • stream2SparkRowList

        public static java.util.List<org.apache.spark.sql.Row> stream2SparkRowList​(java.util.stream.Stream<org.gorpipe.gor.model.Row> str,
                                                                           org.apache.spark.sql.types.StructType schema)