Package org.gorpipe.spark

Class SparkOperatorRunner

java.lang.Object

org.gorpipe.spark.SparkOperatorRunner

public class SparkOperatorRunner
extends Object

Field Summary

Fields

Modifier and Type	Field	Description
static final String	SPARKAPPLICATION_COMPLETED_STATE
static final String	SPARKAPPLICATION_FAILED_STATE
static final String	SPARKAPPLICATION_RUNNING_STATE

Constructor Summary

Constructors

Constructor	Description
SparkOperatorRunner(org.gorpipe.spark.GorSparkSession gorSparkSession)

Method Summary

Modifier and Type	Method	Description
void	createSparkApplication(com.fasterxml.jackson.databind.ObjectMapper mapper, String contents)
void	createSparkApplicationFromJson(String json)
void	createSparkApplicationFromYaml(String yaml)
void	deleteSparkApplication(String name)
String	getSparkApplicationState(String name)
static String	getSparkOperatorYaml(String projectDir)
Path	run(String uristr, String requestId, String projectDir, org.gorpipe.gor.monitor.GorMonitor gm, String[] commands, String[] resourceSplit, String cachefile)
void	runJob(org.apache.spark.sql.SparkSession sparkSession, String yaml, String projectDir, SparkOperatorSpecs sparkOperatorSpecs, org.gorpipe.gor.monitor.GorMonitor gm, String sparkApplicationName, String[] args)
void	runQueryHandler(String appName, String uristr, String requestId, Path projectDir, org.gorpipe.gor.monitor.GorMonitor gm, String[] commands, String[] fingerprints, String[] jobIds, String[] cacheFiles, String[] resources)
void	runSparkOperator(org.gorpipe.gor.monitor.GorMonitor gm, String sparkApplicationName, Path projectDir, String[] args, String[] resources)
void	runYaml(String yaml, String projectroot, SparkOperatorSpecs specs)
boolean	waitForSparkApplicationToComplete(org.gorpipe.gor.monitor.GorMonitor mon, String name)

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Details

SPARKAPPLICATION_COMPLETED_STATE

public static final String SPARKAPPLICATION_COMPLETED_STATE

See Also:

• Constant Field Values

SPARKAPPLICATION_FAILED_STATE

public static final String SPARKAPPLICATION_FAILED_STATE

See Also:

• Constant Field Values

SPARKAPPLICATION_RUNNING_STATE

public static final String SPARKAPPLICATION_RUNNING_STATE

See Also:

• Constant Field Values

Constructor Details

SparkOperatorRunner

public SparkOperatorRunner(org.gorpipe.spark.GorSparkSession gorSparkSession)
                    throws IOException

Throws:

IOException

Method Details

getSparkApplicationState

public String getSparkApplicationState(String name)
                                throws io.kubernetes.client.openapi.ApiException

Throws:

io.kubernetes.client.openapi.ApiException

deleteSparkApplication

public void deleteSparkApplication(String name)
                            throws io.kubernetes.client.openapi.ApiException

Throws:

io.kubernetes.client.openapi.ApiException

waitForSparkApplicationToComplete

public boolean waitForSparkApplicationToComplete(org.gorpipe.gor.monitor.GorMonitor mon,
 String name)
                                          throws io.kubernetes.client.openapi.ApiException,
InterruptedException

Throws:

io.kubernetes.client.openapi.ApiException

InterruptedException

createSparkApplicationFromJson

public void createSparkApplicationFromJson(String json)
                                    throws io.kubernetes.client.openapi.ApiException,
com.fasterxml.jackson.core.JsonProcessingException

Throws:

io.kubernetes.client.openapi.ApiException

com.fasterxml.jackson.core.JsonProcessingException

createSparkApplicationFromYaml

public void createSparkApplicationFromYaml(String yaml)
                                    throws io.kubernetes.client.openapi.ApiException,
com.fasterxml.jackson.core.JsonProcessingException

Throws:

io.kubernetes.client.openapi.ApiException

com.fasterxml.jackson.core.JsonProcessingException

createSparkApplication

public void createSparkApplication(com.fasterxml.jackson.databind.ObjectMapper mapper,
 String contents)
                            throws io.kubernetes.client.openapi.ApiException,
com.fasterxml.jackson.core.JsonProcessingException

Throws:

io.kubernetes.client.openapi.ApiException

com.fasterxml.jackson.core.JsonProcessingException

getSparkOperatorYaml

public static String getSparkOperatorYaml(String projectDir)
                                   throws IOException

Throws:

IOException

runQueryHandler

public void runQueryHandler(String appName,
 String uristr,
 String requestId,
 Path projectDir,
 org.gorpipe.gor.monitor.GorMonitor gm,
 String[] commands,
 String[] fingerprints,
 String[] jobIds,
 String[] cacheFiles,
 String[] resources)
                     throws IOException,
io.kubernetes.client.openapi.ApiException,
InterruptedException

Throws:

IOException

io.kubernetes.client.openapi.ApiException

InterruptedException

runSparkOperator

public void runSparkOperator(org.gorpipe.gor.monitor.GorMonitor gm,
 String sparkApplicationName,
 Path projectDir,
 String[] args,
 String[] resources)
                      throws IOException,
io.kubernetes.client.openapi.ApiException,
InterruptedException

Throws:

IOException

io.kubernetes.client.openapi.ApiException

InterruptedException

runJob

public void runJob(org.apache.spark.sql.SparkSession sparkSession,
 String yaml,
 String projectDir,
 SparkOperatorSpecs sparkOperatorSpecs,
 org.gorpipe.gor.monitor.GorMonitor gm,
 String sparkApplicationName,
 String[] args)
            throws InterruptedException,
io.kubernetes.client.openapi.ApiException,
IOException

Throws:

InterruptedException

io.kubernetes.client.openapi.ApiException

IOException

run

public Path run(String uristr,
 String requestId,
 String projectDir,
 org.gorpipe.gor.monitor.GorMonitor gm,
 String[] commands,
 String[] resourceSplit,
 String cachefile)
         throws IOException,
io.kubernetes.client.openapi.ApiException,
InterruptedException

Throws:

IOException

io.kubernetes.client.openapi.ApiException

InterruptedException

runYaml

public void runYaml(String yaml,
 String projectroot,
 SparkOperatorSpecs specs)
             throws IOException,
io.kubernetes.client.openapi.ApiException

Throws:

IOException

io.kubernetes.client.openapi.ApiException