Package org.gorpipe.spark

Class SparkOperatorRunner

  • public class SparkOperatorRunner
extends java.lang.Object

    • Constructor Summary

      Constructors 
      Constructor Description
      SparkOperatorRunner​(org.gorpipe.spark.GorSparkSession gorSparkSession)  

    • Method Summary

      All Methods Static Methods Instance Methods Concrete Methods 
      Modifier and Type Method Description
      void createSparkApplication​(com.fasterxml.jackson.databind.ObjectMapper mapper, java.lang.String contents)  
      void createSparkApplicationFromJson​(java.lang.String json)  
      void createSparkApplicationFromYaml​(java.lang.String yaml)  
      void deleteSparkApplication​(java.lang.String name)  
      java.lang.String getSparkApplicationState​(java.lang.String name)  
      static java.lang.String getSparkOperatorYaml​(java.lang.String projectDir)  
      java.nio.file.Path run​(java.lang.String uristr, java.lang.String requestId, java.lang.String projectDir, org.gorpipe.gor.monitor.GorMonitor gm, java.lang.String[] commands, java.lang.String[] resourceSplit, java.lang.String cachefile)  
      void runJob​(org.apache.spark.sql.SparkSession sparkSession, java.lang.String yaml, java.lang.String projectDir, SparkOperatorSpecs sparkOperatorSpecs, org.gorpipe.gor.monitor.GorMonitor gm, java.lang.String sparkApplicationName, java.lang.String[] args)  
      void runQueryHandler​(java.lang.String appName, java.lang.String uristr, java.lang.String requestId, java.nio.file.Path projectDir, org.gorpipe.gor.monitor.GorMonitor gm, java.lang.String[] commands, java.lang.String[] fingerprints, java.lang.String[] jobIds, java.lang.String[] cacheFiles, java.lang.String[] resources)  
      void runSparkOperator​(org.gorpipe.gor.monitor.GorMonitor gm, java.lang.String sparkApplicationName, java.nio.file.Path projectDir, java.lang.String[] args, java.lang.String[] resources)  
      void runYaml​(java.lang.String yaml, java.lang.String projectroot, SparkOperatorSpecs specs)  
      boolean waitForSparkApplicationToComplete​(org.gorpipe.gor.monitor.GorMonitor mon, java.lang.String name)  

      • Methods inherited from class java.lang.Object

        clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

    • Field Detail

      • SPARKAPPLICATION_COMPLETED_STATE

        public static final java.lang.String SPARKAPPLICATION_COMPLETED_STATE
        See Also:
        Constant Field Values

      • SPARKAPPLICATION_FAILED_STATE

        public static final java.lang.String SPARKAPPLICATION_FAILED_STATE
        See Also:
        Constant Field Values

      • SPARKAPPLICATION_RUNNING_STATE

        public static final java.lang.String SPARKAPPLICATION_RUNNING_STATE
        See Also:
        Constant Field Values

    • Constructor Detail

      • SparkOperatorRunner

        public SparkOperatorRunner​(org.gorpipe.spark.GorSparkSession gorSparkSession)
                    throws java.io.IOException
        Throws:
        java.io.IOException

    • Method Detail

      • getSparkApplicationState

        public java.lang.String getSparkApplicationState​(java.lang.String name)
                                          throws io.kubernetes.client.openapi.ApiException
        Throws:
        io.kubernetes.client.openapi.ApiException

      • deleteSparkApplication

        public void deleteSparkApplication​(java.lang.String name)
                            throws io.kubernetes.client.openapi.ApiException
        Throws:
        io.kubernetes.client.openapi.ApiException

      • waitForSparkApplicationToComplete

        public boolean waitForSparkApplicationToComplete​(org.gorpipe.gor.monitor.GorMonitor mon,
                                                 java.lang.String name)
                                          throws io.kubernetes.client.openapi.ApiException,
                                                 java.lang.InterruptedException
        Throws:
        io.kubernetes.client.openapi.ApiException
        java.lang.InterruptedException

      • createSparkApplicationFromJson

        public void createSparkApplicationFromJson​(java.lang.String json)
                                    throws io.kubernetes.client.openapi.ApiException,
                                           com.fasterxml.jackson.core.JsonProcessingException
        Throws:
        io.kubernetes.client.openapi.ApiException
        com.fasterxml.jackson.core.JsonProcessingException

      • createSparkApplicationFromYaml

        public void createSparkApplicationFromYaml​(java.lang.String yaml)
                                    throws io.kubernetes.client.openapi.ApiException,
                                           com.fasterxml.jackson.core.JsonProcessingException
        Throws:
        io.kubernetes.client.openapi.ApiException
        com.fasterxml.jackson.core.JsonProcessingException

      • createSparkApplication

        public void createSparkApplication​(com.fasterxml.jackson.databind.ObjectMapper mapper,
                                   java.lang.String contents)
                            throws io.kubernetes.client.openapi.ApiException,
                                   com.fasterxml.jackson.core.JsonProcessingException
        Throws:
        io.kubernetes.client.openapi.ApiException
        com.fasterxml.jackson.core.JsonProcessingException

      • getSparkOperatorYaml

        public static java.lang.String getSparkOperatorYaml​(java.lang.String projectDir)
                                             throws java.io.IOException
        Throws:
        java.io.IOException

      • runQueryHandler

        public void runQueryHandler​(java.lang.String appName,
                            java.lang.String uristr,
                            java.lang.String requestId,
                            java.nio.file.Path projectDir,
                            org.gorpipe.gor.monitor.GorMonitor gm,
                            java.lang.String[] commands,
                            java.lang.String[] fingerprints,
                            java.lang.String[] jobIds,
                            java.lang.String[] cacheFiles,
                            java.lang.String[] resources)
                     throws java.io.IOException,
                            io.kubernetes.client.openapi.ApiException,
                            java.lang.InterruptedException
        Throws:
        java.io.IOException
        io.kubernetes.client.openapi.ApiException
        java.lang.InterruptedException

      • runSparkOperator

        public void runSparkOperator​(org.gorpipe.gor.monitor.GorMonitor gm,
                             java.lang.String sparkApplicationName,
                             java.nio.file.Path projectDir,
                             java.lang.String[] args,
                             java.lang.String[] resources)
                      throws java.io.IOException,
                             io.kubernetes.client.openapi.ApiException,
                             java.lang.InterruptedException
        Throws:
        java.io.IOException
        io.kubernetes.client.openapi.ApiException
        java.lang.InterruptedException

      • runJob

        public void runJob​(org.apache.spark.sql.SparkSession sparkSession,
                   java.lang.String yaml,
                   java.lang.String projectDir,
                   SparkOperatorSpecs sparkOperatorSpecs,
                   org.gorpipe.gor.monitor.GorMonitor gm,
                   java.lang.String sparkApplicationName,
                   java.lang.String[] args)
            throws java.lang.InterruptedException,
                   io.kubernetes.client.openapi.ApiException,
                   java.io.IOException
        Throws:
        java.lang.InterruptedException
        io.kubernetes.client.openapi.ApiException
        java.io.IOException

      • run

        public java.nio.file.Path run​(java.lang.String uristr,
                              java.lang.String requestId,
                              java.lang.String projectDir,
                              org.gorpipe.gor.monitor.GorMonitor gm,
                              java.lang.String[] commands,
                              java.lang.String[] resourceSplit,
                              java.lang.String cachefile)
                       throws java.io.IOException,
                              io.kubernetes.client.openapi.ApiException,
                              java.lang.InterruptedException
        Throws:
        java.io.IOException
        io.kubernetes.client.openapi.ApiException
        java.lang.InterruptedException

      • runYaml

        public void runYaml​(java.lang.String yaml,
                    java.lang.String projectroot,
                    SparkOperatorSpecs specs)
             throws java.io.IOException,
                    io.kubernetes.client.openapi.ApiException
        Throws:
        java.io.IOException
        io.kubernetes.client.openapi.ApiException