spark_batch

Creates, updates, deletes, gets or lists a spark_batch resource.

Overview

Name	spark_batch
Type	Resource
Id	azure.synapse_spark.spark_batch

Fields

The following fields are returned by SELECT queries:

get_spark_batch_job

Name	Datatype	Description
id	integer
name	string
appId	string
appInfo	object
artifactId	string
errorInfo	array
jobType	string
livyInfo	object	SparkBatchJobState.
log	array
pluginInfo	object	SparkServicePlugin.
result	string
schedulerInfo	object	SparkScheduler.
sparkPoolName	string
state	string
submitterId	string
submitterName	string
tags	object
workspaceName	string

get_spark_batch_jobs

Name	Datatype	Description
from	integer
sessions	array
total	integer

Methods

The following methods are available for this resource:

Name	Accessible by	Required Params	Optional Params	Description
get_spark_batch_job	select	batch_id, livy_api_version, spark_pool_name, endpoint	detailed	Gets a single spark batch job.
get_spark_batch_jobs	select	livy_api_version, spark_pool_name, endpoint	from, size, detailed	List all spark batch jobs which are running under a particular spark pool.
create_spark_batch_job	insert	livy_api_version, spark_pool_name, endpoint, name, file	detailed	Create new spark batch job.
cancel_spark_batch_job	delete	batch_id, livy_api_version, spark_pool_name, endpoint		Cancels a running spark batch job.

Parameters

Parameters can be passed in the WHERE clause of a query. Check the Methods section to see which parameters are required or optional for each operation.

Name	Datatype	Description
batch_id	integer	Identifier for the batch job.
endpoint	string	The service endpoint host (no scheme). (default: )
livy_api_version	string
spark_pool_name	string
detailed	boolean	Optional query param specifying whether detailed response is returned beyond plain livy.
from	integer	Optional param specifying which index the list should begin from.
size	integer	Optional param specifying the size of the returned list. By default it is 20 and that is the maximum.

SELECT examples

get_spark_batch_job

Gets a single spark batch job.

SELECT
id,
name,
appId,
appInfo,
artifactId,
errorInfo,
jobType,
livyInfo,
log,
pluginInfo,
result,
schedulerInfo,
sparkPoolName,
state,
submitterId,
submitterName,
tags,
workspaceName
FROM azure.synapse_spark.spark_batch
WHERE batch_id = '{{ batch_id }}' -- required
AND livy_api_version = '{{ livy_api_version }}' -- required
AND spark_pool_name = '{{ spark_pool_name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
AND detailed = '{{ detailed }}'
;

get_spark_batch_jobs

List all spark batch jobs which are running under a particular spark pool.

SELECT
from,
sessions,
total
FROM azure.synapse_spark.spark_batch
WHERE livy_api_version = '{{ livy_api_version }}' -- required
AND spark_pool_name = '{{ spark_pool_name }}' -- required
AND endpoint = '{{ endpoint }}' -- required
AND from = '{{ from }}'
AND size = '{{ size }}'
AND detailed = '{{ detailed }}'
;

INSERT examples

create_spark_batch_job

Create new spark batch job.

INSERT INTO azure.synapse_spark.spark_batch (
tags,
artifactId,
name,
file,
className,
args,
jars,
pyFiles,
files,
archives,
conf,
driverMemory,
driverCores,
executorMemory,
executorCores,
numExecutors,
livy_api_version,
spark_pool_name,
endpoint,
detailed
)
SELECT 
'{{ tags }}',
'{{ artifactId }}',
'{{ name }}' /* required */,
'{{ file }}' /* required */,
'{{ className }}',
'{{ args }}',
'{{ jars }}',
'{{ pyFiles }}',
'{{ files }}',
'{{ archives }}',
'{{ conf }}',
'{{ driverMemory }}',
{{ driverCores }},
'{{ executorMemory }}',
{{ executorCores }},
{{ numExecutors }},
'{{ livy_api_version }}',
'{{ spark_pool_name }}',
'{{ endpoint }}',
'{{ detailed }}'
RETURNING
id,
name,
appId,
appInfo,
artifactId,
errorInfo,
jobType,
livyInfo,
log,
pluginInfo,
result,
schedulerInfo,
sparkPoolName,
state,
submitterId,
submitterName,
tags,
workspaceName
;

Manifest

# Description fields are for documentation purposes
- name: spark_batch
props:
  - name: livy_api_version
    value: "{{ livy_api_version }}"
    description: Required parameter for the spark_batch resource.
  - name: spark_pool_name
    value: "{{ spark_pool_name }}"
    description: Required parameter for the spark_batch resource.
  - name: endpoint
    value: "{{ endpoint }}"
    description: Required parameter for the spark_batch resource.
  - name: tags
    value: "{{ tags }}"
  - name: artifactId
    value: "{{ artifactId }}"
  - name: name
    value: "{{ name }}"
  - name: file
    value: "{{ file }}"
  - name: className
    value: "{{ className }}"
  - name: args
    value:
      - "{{ args }}"
  - name: jars
    value:
      - "{{ jars }}"
  - name: pyFiles
    value:
      - "{{ pyFiles }}"
  - name: files
    value:
      - "{{ files }}"
  - name: archives
    value:
      - "{{ archives }}"
  - name: conf
    value: "{{ conf }}"
  - name: driverMemory
    value: "{{ driverMemory }}"
  - name: driverCores
    value: {{ driverCores }}
  - name: executorMemory
    value: "{{ executorMemory }}"
  - name: executorCores
    value: {{ executorCores }}
  - name: numExecutors
    value: {{ numExecutors }}
  - name: detailed
    value: {{ detailed }}
    description: Optional query param specifying whether detailed response is returned beyond plain livy.
    description: Optional query param specifying whether detailed response is returned beyond plain livy.

DELETE examples

cancel_spark_batch_job

Cancels a running spark batch job.

DELETE FROM azure.synapse_spark.spark_batch
WHERE batch_id = '{{ batch_id }}' --required
AND livy_api_version = '{{ livy_api_version }}' --required
AND spark_pool_name = '{{ spark_pool_name }}' --required
AND endpoint = '{{ endpoint }}' --required
;